From 4c6e9e81bc182d80a788b383f8680e3bc6028a4d Mon Sep 17 00:00:00 2001 From: Adam Sawicki Date: Fri, 24 Aug 2018 11:23:37 +0200 Subject: [PATCH] Linear allocation algorithm is finished! Recompiled binaries, regenerated Doxygen documentation. Announcing version 2.1.0-beta.1. --- bin/VmaReplay_Release_vs2015.exe | Bin 154624 -> 163328 bytes bin/VulkanSample_Release_vs2015.exe | Bin 185856 -> 210944 bytes docs/html/custom_memory_pools.html | 2 +- docs/html/index.html | 2 +- docs/html/vk__mem__alloc_8h_source.html | 2 +- src/vk_mem_alloc.h | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/VmaReplay_Release_vs2015.exe b/bin/VmaReplay_Release_vs2015.exe index 1e08a58718a3c12f39acb9fb603d2d03c993e275..6f29d4be34688fee7f1787d60388f0e973c0fda7 100644 GIT binary patch delta 73354 zcmbS!2Ut``*Zm|sCl7Iz2>T*Dc3&NKBRf~oaqa5P;=}w`Z3_)EDI6a_WgAw9s#7uL;5YD zsQoWdqe<*O?U9^EP-N2KpIlL+!T*{mY@Qg>loJAiG(jlJU@wX78g2IiEM#Aa?RtV} z4!)S$)c%*KLAP1c=RWzEh7+#=6C7?SYpT8N_7q=Uu0!Wk)Szx^r+m5*11_+i~dwTh{^}3`3ZC~{TwT9)>FH;p~(KzY>4>BS!Qnji1^A`_IC3WF>h<(<>r}! zaDq(?+!Fic31U$;S4;85Sq(kL%{(7fDmicKvXS4&^Um9y<|DIK1uj*d-OLh=S+UKL zUstEG$n&+0uOFv+Z{uHhMlMx;{;2S>Q4(}vrv#11IPDng-)g;XT}vaPE$)SaPEgbhg;}Q^rE0I&VEnJ(M4;+lVGtO(Qo4_~Ib@ zEi_$RQpn709up3-ciQAme4QGlx&0EQ;tD@0fu?EJfSY^U@QhV~Pe}4%uJ1$NV!3Vm3;Wsbwx%}wc}JznmNzlE+zUHN&fh8| z*Wl_Zo$|_?th(*piN^_Kh}&;J&9wXIC{&x*(D4?}*V4gw-b=^RDsT-Q%imzbO^L!j z_MFMoro>CfxlKezI_UTbbR-iU`!*GpnY4oLBk#w~y8lhaRJQFKZSi347Uu zu#URl4^UCdy=-;Z5}}y+g|`=9+=5|m&62_g>uvzM$7n2O&xJoB6fsZu$xuvV35vIK zBcaR$%3pzU5TSf`Bl|rfUSuD!&K=tKU(5|e?PQqE^R@asK{X)iWU#5grHU2ZvOkRB z?7a?You2`Jm_=WKON(qaSY6X1(yj8eh)hfTzHCywSsXzAinZ>TpbO-ghrePO9oy)x z?4_o`U$Oj-?+SZZ*G|#mln>aHP6M@jfS`R?7~9)vqV`MvY+{L#6NN9?#>gkNyS=s1 zom+XIO>B1OU|~1Q>-?B*7bsEQE)dv>GsXtc3}tlzYusg`IPS~B$Ge0JVwc@)O;j84 z#xC|zltX;xZPvAGf?w4R^!ZiSFt)a97x68awyqn5L2O*NSH+Ak*v)R;y7c@4qKumq zVY2Ise=|H(yTla+aZ-N8m*gqZ*1hAE!i}9Qy zKbwp71w}{pu#dX86>sceN4gJ+OeTOq93V}H+EK`-$jN?k3ndz*Ryi#QrTKeUTn}r@ zOOS_h>Tl|8%|#8(Qg^_GFWH73?M2O=!moOWqL5Tr+_SyVEDVy+=kE^GXk@c4-oN!O zcB5CmVdG9gV{dNOf03x05%2%fPPRH`SJ;nWU!UKyffw`w&o)F>|An~;z{+;8X}#lG z6!7v(m@O!$=Fje6+k3YYfBBM~?7dkWwuU+Sv}(TF*Eo z_C=p|;>F#pywCOlDYBrcTnjEO&T5@LA4UqAD@gJFg`ZO|r_jve4($!Q)XyTPYHn)$ za`YDYESSE&i|y;Xy>s%MWzPQ7|8te0a z3h;sb{VC!3iO5rVy8bC!6WdN~w3BU#-7Y@4noW&s)uJhfI@_cM%Kua5ifb(zcCdHj z3gfd^q4%$a&| z1lExk{HhihP|&83UF+XYeEW0OJiaiz;nTo3@)qIqwAANc0i0Dnh=uUsPiW>JQOaIm zN_>0qqtDnw0}92a`K)L_tHzT*Rx8gi_QQY(aoeY?VM1Y4^E~u5`9GZYM*95zH6yTS zJJH?+c?_JczyDL)*)Iuk4If6a^4#;R!^08coI;lNaG_YX^7(B<#AdF(pP4&&IZEIm;YBet?PkzC)xeoq`M zp4`IvmTdt0kVO=_;4Kl9qPk%FZMlD{Qa}9;$u{JjH6zM?O@307@6_asnv|)@ zK{Y8-lbveviJE+*CU2|Bt7`I+nygim)s#&27+0v-#cGnRCUezfx|&Q;lW}S?Tum%$ zlB6c_YSLRxx~WOW*Vv}Pf#Qxu?DN5oi(N(5P8uX$*~mV&w0?L33I5EF9?yYQi7>1} zs$2$D`4m!5kq=<*FQ)p>P%p_1Q{@GIH#MR@??uc4XTsA_A}f-Hh(pw#K6@XK(f9Y% zqo$`bRBx#1Ne2Vt>*>vPhYhv15j%8M&Uoc!k^T_93u+#!~!)h^Fjtn^5}?k^q=rnVcdu; zLHub6+cGj$+_;4OH8Qtt`aBLV#ZT4QuSoJRog`0+h=x;anp!zv9(#UNOpr;`h|5AG zxdqm!#QDu6=iVZ8_rWIi{is!~XFkC<)(hE&^pxOto&$7uw2b$ElODtMpAUxj@yc_o zI=xefor-$!+AMmEK)2-R%zBQ+jqeniLS;R@cn5L-WN01k{~TL7zEk7&C?jNz7O*Yj zJL~VQ1ALSh{`NZd%lPgg=c!CZah4u^D2}aT-6nKu{65O4hxGYu%7p0PgE3!fO(_}*Hz?rt zvWxAX*jIeS#qJ_$?_zx>^${llf^ptku@t4?pQ=%wWG%bQvPiZc0e~LAk+RYgLG?2zEtF||gT&cNL zeVhr^f$DIt-N>Gt+Cx-x5eZn?l=Vk2(UWyTZo;drVc>HjB{?lxa=(}e??Kre$$pyp zyjT#y7G(6+UK0hjH6uto@ER-4h-jP(e~;u=^d!5Pksvux(Sqw9lZpXj)%!-yBvwt(&o5E@yW*XZL+#w+R49yWe_= zO`P6cY|3*HBVPu)TlV|d{lr+8-F3~u?ujr~HvKtqOI!BHjF>jMyX4EI&|J^Y#SD{N zDG`w=WI_GEU(Y_C5h4bN?AVO9y#;h!nGBCs@@O%UudnCSR`%pR?mtcJ-La%VVHd-_ zvxjlTT9C)uJ>Dr~ES0E=If@>Aia8{YWjx*~v;&p-bl>CypgGq%kG=JHr^a`j>InQe zlbwFNbLb^xYv8}740TBGXPhi}W~Y!(sjP=rTendL|_&Wc4{o6ga$NfCX;6- zm?o_OTQkU&=4{!J#UNCUMu{_F%0nVkX0~nl706OX?Dcpmr5eZ&^H}Jt4q{`GCC}>7 zW_BDc93`*|n_!tGw>2VKnUM+6y8XY9+1gpH#6Op^_hztxpUi#PrBIJxt)BuQ7(32?u5P#5HX9{C%N(~phQ;rnls@a6p8jGAh(4JB5ClI z#}jkYl@k#WbBcJ5@ED^OvpG+65j~4o!4m@;&dUeXFORbCpJ*#K5ZT=)+BE8dBD9~l znuX7cX+D(Lb=z{4KgNI|xBtVd+01zffvr}f*p<5dVCoKyCXp(PtJ%(ZOUJHVC}{GF za$3j;!v(y#3cVY@#IU>t&BbDzW5)n21!dsdV~hu^#WiR+%2xZzI;myQcHlYN5cP^oduLn2&3RGQ;YWMRkx{8q94nPH-|fKAEl zCI&BH?#xW_U=H(ShB!XT;cI>}VoD_xQ`*4uQRR~JZyr;UdEWUqk0~uzfy-$m7Y`RS zX6JH~hZE2n22Q!P)8qLH+f`3xUq2Giiy91tMIIXDF?O9#;a7uyARNUT`erE+i3d}$ z=xW&dr}~OTlh}!;UJwm-HY2NB$`3#S>Z2jK$1L)t6uIJSsF!AjzFi7+YHUPf0s{P} zkaX=06IU;dFjWPkW-V98BTCg|b}s92@#!pn`=HvAKWUj|oO;tvvs6DV(usi zoWmi!-s#~SCh$tR+JZ?JH&oE%UC3#an)gSJ-zNg18K>lcbsEHnvO|qVnq}+0@OaF* z%VRY;2^jSso$R~pVDa(A>{j-U_5s;^@Janu zwql+IJ;ZzTXl{DpcT$P7l6*`Vt{0eNVK>oaVOtk=6%RYucMHchU0hiMuq_zIVo@J) zoWvF^3KuUw!8R^xuf2jbuV_)nR`1S%6?4tNQa?0O%0HCTB<7UIxbq2icTq3V_5|y> zc$n!ngiAIdC1SWl7NHOY8hzerqSYPn>~gkgaZB;(hgF`x^!lO;_Dm}s|3 z@=K(mfLu*=6H$kvNVVu4Dw5QqXtn4QD(a&a#i~VLFK7Lh^l)sYnrqvr?^D&(_cU~b z2kM)*I)geC(R@MCK?2{a7Qr-9(RWm|O)WC0MR8QLQ7sChqWn`iq-`guJWnl`c)3ok zIKQ0zv1F9kWEM+W+F#6E&XzAto1R9{&u;a2dK`LJf7SHg>=sv%4%u zWRmtha3;?X&0Fa5GOl9ymqkdjUR}o<@^#z=5C9J#62}3u(vB4Z zeV{CI5o?(jEnH=T^V;^kLXAOQBWi4MHK4|Nr&s@zu(DytmS~)yYcpC6{VL&g!DW6fAzXSHr{@NeBedGc*|LN!^ zpIcDN6?OCWJwyS}Qg*mW}NvbwMM*Ch7n>VD$xNo)g>m6ORu8ayVkUgF)1iQgSUw#zLc-hwG@!9U;k%4Ugv+?5Su}oYWEryL{z1PNwe~e+X z*7g-Y8^hi}vTO`Hk0f;rYqqYh7%+xekenIKRv>wGG~0<}`e=3yNw?9g^K*U0|BPaf zAlWmDy^JJ(6g&A`KhZjhHGaOI*kTk*M)K`QwiL-5BiUz2GDosMpYJQij$|F#MBE%8 zEjaZEdz*C&sJLGX{$kC=PX}-@_A83=_WdT3}rR3WWwvVY>8Xh6sZZZc5yN@tq=BA4PD@tH*S|2X{5y@7p@6hD)em?COeSv+lK1#U2F0K!4dH7LKIr(`|xtly(PUYv1GQ-Q$ z#oyA{f|q-UpQo|+U+yU`Nn;mYju8i?F~ckU#DFwrd8M~_ZU}QAd1DCs49T1!tO`lO z5Y~P}jHnyJ#%>rYl=G3>Hl7{a&=r}#HuM&s@6Ebw>~8cUdlNUgRGqJxRehZa<>yGS zUC6pV32UbueinA|U`hnGawMa}7fD8Kyu)Nts;XGw#(84ZZ!G-P?mg2E&~lVoA-Pi( z?>_oCoYKnFCgdP-eog6Vm> zGoj)%SB}kZG!d^%W~*LrBd7&K*EjS-j39&UdwpQ<)Bnm286@=j_QV=?ddfa_)=p&o zHuV$#OktUuvczXf+0UC6_iy?R1&lA9`foHoiZS!iIF_D4IA=oQ2~ZNk+P&SN`7pwS zg|NpeRTr94RUlJo3YyHuzCA)X&S}z&tEFkp(>|Imk7eJzJxJ`I$Z5JWkkhnxAlvxP zW1{;Srzr{d47@ZoE#ovzhOB)wji6`HRB^0^rYmDrnvUnGG;O6SA5Do9+2`-tdLR4O zvBW&$V>z0W8VS|xi9Xc1ymmC}_a4SFfn~loPG{5!kkS03peHb78ODDM7R{=;s+=tJ zoZ(E6juPUg!|M1&1M+#Qj_&w$Bc8u?1qRd}7nlGsL#)Fi12te~BBv zv&#Gxs>sGru}@^T(wY9lp28t!{;)Ito%3O6%LFVA%86VUJB+H=*mWMkUj1-@=|Q!a%moNAD7(9zTXpvc#_;}(^0S3Y1rU)?JMP^ zeM80#bG^`^lJ-0m6KT2zWN~JVV`n}J*X9@n_V-6U#9k4s>*f$#m>ImeOZ$n((UjtV zh^UYAK1+Cxq31aNh~w<}%^8iGL#t`qyDOaivw6I@t&EM^(jn*o;%!FvSh|St$0TBQ z8TLT|S!L{nEl&x3S%a~fJac_KQd=1# zu;E97XB-3Cx<^ubc)s>XY75VMkEAxKz@^ITdhA|HszN|TAk>YYJyk)8m6pJ?<1QDM zJFzu%65FZf*bYwqWW2Viin9OpUiDG_$n&)*Pw~7L<$e{oRN2=Qmt;=v4`imoNJkN5 zrN$jHvHyLxl+4I)DAX*+R_hxHn<*5Lq>Nd30F|vlrE(EIYRpXu5lRz4p7esW8HNXp zw*es37_vk-_A>D17wPjqL#Gr~8YFt-v^!A|>V5tS&(Ofsm?A_x)iCacUhJpB?wztx z<4ouZl0>_n_ZqN{AEw--DtxC}Odn_7lElnV(M6ea-2-4om)N7spr8^+mLm z=W9ju8qa%0^qdM@s;p~5=VYKiS~(MpT^EHWH^Yo84d)|n2vq}W&fLJB7&A3$%Ikl_ zj*-_>q{$kQ*^q|qr-uZ#=CfhKSB2kwmL`b7V_4@eHpgA*R-dv8liA| zQ6oXOp%Y#8`#y{n7q@E|2EiyFdD!;7f#QoEcDFcJcdjG93K+)n_eQAoKi=c@NAID& z4f|>D5V0VP^(YC6T|&+4oZ!#l`C7S7T@W#8kC?uAoHA}={VYE3&LV%A7uwe33oO8ee8_Bf0z!ZezEObZr(ydhGWwUEPXB=`-6{ z@1q?$&;>GT04<`Y_>+3)+fZa{{9gU44fG%fNZZ)*qx~Cg1Wz90#a0329BrOGe_IZN8H*N*oMt;QWloZ?9F z?d(_8`xw~B@hce@ud%DghcpTaCN?Go3M~1=PHibt&k1Af_Q3jhH}QNe-gP|h#p_go zOO=&Znf2sop#|<7w`==*bKV8aD?(8BT{`~s3cGkRL~JgwJ11Rj2vE`85A_h8dtCpM zZ92sPzdY43_D8kLntSu7c)pgZy*%%w>N6F%RQdEz7V=G6`c%qWaNFCmqPpI7??EbH z%{A-3JYRdwx+BkfuUQAHz&JxvuX{K9lkNE?N;C-U@;7ZA1~1FGe-O*{?t4E$MRGNC zWbJ#TV#i^-mf^`7wU=BGxPAp_jl!9{y3Z_1qOQ|vLkl!|R0XvZSNka!$z3e|$q!em zuS(iWk}IJ%0c&qHlU!~3^X8LCr~7n#kF7tQ-e?-e$76gK{@3l(5pDD&?RhvJlH3w* zW&cnixy;p7koFNJ{dd;?%xpoA@oT|$oC#w5n3AuIY%1ECctK;AiBXZ?D~r^zHM{X;-0D?6)7Ko!lg7O7O}F{||wa^L6uy zqwr>l2~+3@O4Vt*lb-%<+HT*(r|s_nM0|I6B2QiD*{BzwA>HszeA*f}@o77ngS-Sk zqxdMMEv;j6z$>q#-+I$_HI=B-cG>G}{C5LIw0Pyaw&@MMQagcN<9%ANhl|VUBL7S%*W}Mslgg7)cq5VF5;RjnLJMIhuHf+3jC~rBjm6Bm8x9uaMJt7Y=MR z%FVL+2#y|=M;Nel%#@<`s=Xi+RJT9m4JAh-Rh^=L>HQ?!*!Bz$!XdH#g(A=qS=)eZ zz5Z}uVliQ{>F7d-7L)g^()Ii2Z-hB;9e*sECt_nbI2Y8Z`;l0jgJLt$493+8OF^57 zT}C4IA)330EMV(##4}@^c;z;?=(B0@S;N83IV z&;q$q@Ew(LKx;&7DEa99j#1Gh+8ORkoKlByTW~8!wEpw!xTm3x-f7*X;5wQ|tgBzOSb8E0s9!$~iY877_|O zFenm$n6O~gG_`~U-I&p13Ly1g_{&&=D{&1+lbCxi%-+r-V$Q6{vNmsjlyRbxYK`BB~3yuWqVZ_X6s0 z<4wob{MFk*(Fcq|nnt~OKtMx-DR+`WqFrQlM z@nTGR&?#F#-d1_d1_(wufKUH2C(Zr648@SCaC0>$KU%1O=*F7$yy4;CK;w_|DC5Zhi> zt(iZR$6Xwf988lBz0{6YV@_$BF6I;lb`1KM;qr^409;StuIUOh0Ij?iU8j`9oKk3` zR8!3{aT5bVP3BrI^q*hGtkmbp#E1h1%1SSzfr+HtrMdrT>z=xU!0Eyg9Z-<0|ri_w)1kQc1q53_i>JZw+*q#*FppP3oc=y+Nmy zSd{7gr^gWtlH_>q_AG!Y87nBKcM=DKVcU$|{zH#P4)`?)JCGbHD7Q(Q>O;TlgI2-d z2!z661TouEjF-GMwtVI`!~_XsSNUmLOus~&H~Cg_W(y8-Smo=me6TNVZxRn>N&`+)y#+m+Bf;BSZljfxROVf;1ZDj# zYE=(9)dwy6z%*vLrP7p>RHnSkQ(p@Q_ZV9Enz)VC*c^b28U(QqBKYMq;s@l^fC0f+ zKV=ISPnnWh4+BT=iLZ&<_+C8(4naA`8TS#y!Sj6J^)ToZs~5x1JVg`LY|>DU@HN4E zU-d8u%KQfrg!`Z!wHV-(7!<_lAy3TAZt{mZX>%7FQchnUl9_$(4Qb0 z;TuqmvdhcXn|xp$_2jOv?LXzd{(C>z{||)^=mBJCQY^e@F2k+#CLVa{`=A!R#albL zR)5!|z7R|QdVm@GzaooNzUT7h$dbIs4*P}x+JOa@T;hJd)^1}?JzUVgNe?1xkWfk)7*H1 zcGopj2+DRodi9_ud!g$O^EuE*lK=X_aqC*!{UeV1`XBLN|Mx!rw_%P|X7HZ5lrd2layy`k=?lB8|JtNdHWT?aNZspt-8b97V4wwhTnsYKCag& zyhb^^t{V3W$@%22TIFesRl%BZLMRprxa-$&vpVLKjZ#JV5wy`w>%$ncdYb|0dkqrf znnn~gJx&JQd|)u#u!+!&`@nTx@H%i{Y>G~Kl6OsKTn^>*pDMaZ528D?%_j)sMcsYq z{=jhEhwc?GcwKagzYosY5mO%G6p^DfgptK8PyX3+`i|M4@cp{nE?c zipGm-bsbb*O#j(h!_?)5a0cs8D=5JaqB`J(uLGx1dUCZJ9g$&?^~wu>s1gi)5SgnM z*`EkB)#*b}w$L~y)!lOnxIe5C7jvo(D8`I$;mOm7$2Oen&{Xr3sg-G9^JoEHZ47sq z;Bi>wO1Ofb2p0JIPEJ&rGKZ^0y$MpjrA7iaP8457%-ARk%0FCi^}s*yf}?9y+}Kbn zfxKHx+_TCeddYb_4zA^6$~=rV;V)A@eGu;>zK-h8EAI@UogTPLq-~9NM^ov*Ff$>r zWx*G33C$m5k-uFH7Wtsnqq!U(4N|AD(T0P(I;4X|E}44ZA9%s*z=1{Ht;%8*ScFfZ z2oIu~SBvW4Z{&a1NJ&the-PEcT2u%g?)(AUB0hylsNZ-Q+Q2P9JqmT3H4Yy~@8HC* z9-6KXqB;MODu6n08YPAI3y#hxA6)hg)RhO3ZS*y+H&C4p|Cf>4g|YQU=zRXKst3y- z>^`b~_meI1*dSU)A44B%Bg=!0Zhq)x$V-#A(Z8G5U~?@w6rh|Gp`#tIiMF@5m9E&8 z-7FDu0(PyV?K~{UCwzB+@UAy9fDi_oZfxJ;Hg>LWlE>tNuQg(!BwI2Ij_QQgt#lVC z{^=eTsmy~VUnj}qptr+fdF6n!-||ZTGTyQw^-6X>dTL1xC5bUb6gHLI<4p(6*!2(@ zw^0&Q(7urnDwb_87}!WKIWoB`3}=U5GL*%KJ^-oRi0q5#$QrD`j8?IiA>Euq<^6#4`E??%EOT-LJlKMX1UacmY1TpWb!9!a>qaJ4GZtp9vSs6{SvRtk1Mi<}f~i7#>J z^S|MuNeZ&WCF%6}xDSVL`r;sTEXgBvL|p!Celf?Liz=KM>hs^ggH1kxu535}=#S0# z!WlC_eICjYx-&GMq0c_AqqjB=rNo&d?Kv3LAWQt=tT~dW2=6!>tnx_18@Qgtjcm-Z&-R zZm|1pr?;kx=rR%ZT1zRAgE1g2fejxJfq^u5_mU1CigNDECCYQK(S44d5O!K9X>oTY zeYH@!$A=Pf(tZ5Oo+T&-6Nx;@F5=egjiRS zpX8oEfi1*_tvt}>N&;fD&JtB*IVeEZxcii3K-k$ON-nMCpq3ZO=5%^tYr4K&qK1TG zbteW{d^hqZq8KJLJ78iDuQi6s$8BTBjkBIHSdtUl{M~>pn}(%3vr!3CIz(7=f3>;OgAr#xqu}` zD*nMwD!%PUM{d0#g91-;A-&H+n|#=cKx&5M!j}l-F#{#pucsA48=i_|8CPlnRJf-O zjt-t*$Bpav^pU9r1#4Oe!JVgai$|Y^GR@FCQr}bN`{PS|xZ3qRM>}R1wtu=5dGg?b zvKB&U%0Q5*&+CCYn6-n0b)=IP_W~214XS=liKUO%%E>J*d)mlalkcZ7|DUyC}?36QiO#C=|&;6;M2DR zqa)GcTtv!v9+sT~QkC-`H#^HbP%RTTT{=mYbdn2G040(<+$1LjIjcQ6f$k=;@nzWo zS5x#KX9eXnLYe*dUPV<=dP6co-zjvebd10DtnZABQiYFH=?xmmIVV!kH%kt(zq%a@ z=AqQoamQK7KxcNOeJ`QE26lS9r~)f;C1VN9?LA5L*fZL%nvypl?Ttrt!`A zTV&!25?TuFC!)s*u#?cuUN-r*-hc^>_d#jpipn-%CuNl%3$jK1s_&e7Ldj}O{#Ys| zbTX<^(~+92t|dXPC6O@~Y*D}Ej5kB9K^EBtgOyrQu(KuBT=!6uYjU;ZvYmiHbV>?M zDpzVbVSyf?u#Cq3gmM8qVF-2CFT>)UVgbt8ir5*`6mYZ@|&jYmL;POCf zg|~O*^4tH`Gj+WJUCToa=-qAh=u?YbgYf)wj@C2CVC}pd128FKxJ8~AZIQEME%K5? zi~N%r`vLQ0^Q4+6B%P8kVm$9^_g=~({KuqAG|DW`=EHq}>Q{~cz}A(iMO z37U^Gz1a?SX^h*@g&5~Psb;PDKOivt3xfZt|2>%D=wFZuzGx-vZL7n1uoREiHuNiB z=7)e*1X|tQzWnYU)HkP~AV}ye+$$&!5<&yh^?BIn!hT`##Bs_knBJQO*Mfw$!r1~t zu+T0tzo-*@IPEdd5ALM9AJ9!#do1bXOnT>isK=tkwTWWs*3E)(!9u6T>H7R90YX?- z2MbApuHZKr`P zpw;3|^GMpV0)43PkuG0{3xMFMU~i}}bV?L%pWT4-Rf%6KJh#V>%D%=C{A>6v!EZHw z3_mLS5x3_)#_v7+%J3`4kIFUy`AO90AkD+?dHgnYDwx+s=+_8rz|Rqqg9^5`5qb;h z1wXYB77K=gqPBuD9CLgKtcge~fWPDts{An7G9-6er1J3V7+6o*%z~HNV$A+``0wyi zIESxpD@^L*Av550fz(4bI$5*#g6o{=^Nz_e)@&$P(oQh-hRS12@*XtK0?0DZeZ1}h z1+j~KQqqhpvdU6W!HIT4qB#4Ng61Y6OuV_jpsz`YEXXhkA-%cBg~=_C$7;5hUr55G z9_}sER2o#buVP7F6ok1|u+=2I*tBvV2*>vw+-aFgx0ee>v=@RW+`s==$Zgzl<;^z; zw%-23=fWf9{!KJLZddOEpU1YO;Vm@b+mb{$u=0Vqf$`EWXfT*9cszQ2&!$)(G8L^B8{O87+^%_ZSc5pV}D&@>1k$<{hzkNY$ra zO_gU&vT3j7pGtvMiaC@f=jaS+?m)~3+|b4acX<0TMX7jW>4l>B(wrvPcsRJXVV*pt zgcfsyO`buf%x<7V&jx1aT_GotOj|g<1ew8)u8J%CCFfF|0V-(0S9-kJtK|No2@a=m zXMsB$1SVY7*qf0njjo9^vVl`d(AP{Wv_xl<7dbL4TE0AISmO6BXo{*#GKT3={DJum zdlgmArWuWmP-LpAG^Ut8XmEpwG=2Wd;5+6*j0dRoc}_l9BX!(*TIC@Id_fB9CAJ7& zw&^YkZFX*RAEmi1{u}*D#~;X~LQ}e9gJTKB?k2}Crl<-#95Z$}d-eIpP)|%*;>_6K zd`&s6La)t~V6UZ=`8vgLco%O0eocOsk%b-2&F zjCyd!;whV+9JCoJGFjq}XJIw+*yMAz_tKCeBA?I+42hCZh_z{;{a%p6EQZ$v_{al zBG9PgM=iOn4<5dHa+XqAY^J9xP0jO!4?X`5t?FQ4xRPpPpKwO@)e8yujNwkwh3 z{sfUXWU}H3ep@}2Ri_~)o_SX!_{cwH-XS41%X`iG)FMp1y;iviTxCBji_qXGx-NwY zqb>H$VBSGZc)>&;w8YIT=)cZ0JQ|)RD6DH<9y~@G3R9hn5OQ;$X4~`rt`%jeLmN2E|%(?%O=67IDi3n)3CwV3z2o>BD2b&f(;wU8SAX^2eZ4;tN^Cr5Nr`Qf$kv& zNn1+0OUW6hvCp(nsG`^e;H*DkXR@O_*euFNEK!Fg0pEU=lVhd$Qb?ni@I#N2C2t;C z$tHgTN<_Tpn!$I3`uw*!N!WSN4G#zw4u`Iy@;Q_eIj`H~QVW%84`WAfjX#+kN$4KH z^)ILpG)3sZjqwxUg1fyNjW`jcA)1Jf72q3QaLPd-cH+F0I75XiIoc*SvuafDKrI+1;Qx-Tns&>GOXC@#xE;is?Ys<^L+Ce<9r~CHq1M1*d$auxHtG zx$Q@+$CCr646;HL0fMo(hhun7`YfsB><{z%7s=G)D3zR-P1M3(Nj<0ob@cxvJZscr z7!le(ldo!ZJ%)j$H-4euK6ig=2_&I+w&$Yxw?T6JpGN8be~_$lbDOpqU+U)2ARVC_ zli*gLrw2_|gwRNHY}!%_Ml=4DS94BD!YDA41}#0wRl$WvISoqc%Ssm60!xdKj*te$ zL4XF)lR6P1_b4b1sZ(0CuhdhD5EBYXaco)xq^(~$+CfBpj(@xc!IvbC8B5b&o$r|a zWIBT=_!()HPvNQ;`47=v2k15|!EMzN%-11snkG-`2e+qd&tbB)Ct^yyaP~V%urEZs zB_mBv=?Aqq%*_+}aDc=CSS~y{V=S0BQC4{}s$n=Qpf%u(CUq_}MV;59C0aj=v0R_b z`je(!8;b`UT=}>qs?;J3fwCvT5h$jKZE+=GC4o&}ii>K%FA{e9WEd9cOlr+6c2rc^ z2yvm8CX4onRSvermt==f*S4r~n=pa*i+C$SZ6(w%YM}90alDmStr6yp_Ism(hWa1$UjJ{a|d1~&C8FWmDy;;HOO9TBao zB>9B8&Yk2=%l+LA!MGnncZV!4Jn}Ipg%gA^5NVdL3_wS<*NmXaq_l>Dj0uSJP~hW7 zYy@=bs*XUzng}}G@GWSA(xK$=C5k2O0j}5*^}@n>3c<&r@d>&wlhramUypa!O+zUr@k`Rg2I}s1BI3 zS>+gr(u&j4`0w>jV#o%ah>~PgDA*GI3;mtE^aC{OiX^|l$>juq7l-`4myxOj>I=Sh zEpc^e!=x%lN1U=6ioVav^>qk_$=b)-(-1#`clm90622Yb!EJN61kn#twhp4z=fCBE zQ*I$8uQU<3{j!fRRk~JruNAstvbme|vPEI+KIKBT$fxNp$?P+5O<=)@22u+95?uMs zkVq3CcrNt;7+z1#W9q#BJ{uo<1U2NK(7OV#Ww>uv&qLAYPXD?| zH?zsO6o1efbsU(L(dFc7V~(lB_^{)G0`_WmvxiyYFJ^b<$2=+yFN$@lVX@Wr4?qVY zxJw9*)gt)o93i;tLqJB-4z@Vuo4wkW4vT{Kt?p(I#<%@EInaR`p1k;GcoFvD2#=76 z3DjOmt(r+uR|uhlyu#CplVgNXH%6?6AhA*>)~wX{AVtJ^5QO2;=Vf3;`gaw~2c9wWE;$e)6er0!bQMr&Piod9L(qbve|8swv>i72?@6S=&;a$Dc z&AU*QsWOf-5STu%?0@NbaQ&WRYbZSeN+E5%^G)6Z2C#Ajo_uRLQil~03{wgm^dOim z!zkQvuH)$kk&BP%oYbrm+OZ@O0N3E?DX5bOtElCaGaDo`%P z!*1zU-2`g<6$U=btnP#z#rHgP(_^1>>r<{U7Wp43>JFrZ2;u91r5$hHOL(xlCz8HM z?#XzUjCRVGEVsU~gkO-bfv1yM>?vrgY{5(k7s(LBm)j>JIFTx!G{+Y&Z;&D*L|*Ky z_FoVptvAbSR%NE*4uST#=Ld65NHmg`*@g5{{6Xo~4bV|#qDp7AIRBTbhs@49{@Bar z71_%{mHjymY?H?tQt@`a?7&qRS^%(f!Is(U{@^Q_6_PebP^9=<^IIYqk2TAV z;X&XpCI0Yoe{7lMlf>y-mDANePJN8(^Ri(5t3p-Y^iNNtTvH!=`g~gWL5zJJtq1QD zK~cY2wZBNNSgR`od%H~R?J^^`3zgjfXqKOtB)JnD*S@}o&rokvM;70+BSvoRr%jTEr4$cI$RI!#clS||ywf4LWUY}&sp_{!z2TUNYG$kia9iv}C= zrFfw4L1b9nqdb;dxGh#p%O{B_{Gx*dG-F*o5gk%Rbc#fjY)0(XEPrE;FM))h3ne6> zAZa~RVG{&|x0UJy$ITW`SVQPH&@&-8j z#X%mJ)vwB>w;H(e@9?RF*`27K2jFudsI6*C1yFeX)1xS>H&a7WdVPKp3P6;7txqc~ zQU6$7(a?U<2-u&@NXfNO2eoJoNUIzX0kyca=N=yU;TNVBg%nrO>ked&G0$9q5u^|J zWLS+pV1O}Qw4*U`P-}5NiGcrQnv(mW6H?S=iezM{itz>shonSHwu6*{G~(7Nb<%t* zg@13=4*G98G3~$T#EA!V0{dNzmgG5R(O$WqpsKbGuc5e5M+c6RxK{FJu&SFtB}V)O zKBFuLlOidP)WB1DOol!W_q$;o=h7bn8f(aYQo=OKMJ*2XY5yx}@8UIDe^PR&U;F^A z)$OO9Z!tR;25E8}eQdI1Fy}4`MSx`k-ZH==VxW2J=n<#&X6=63C$B&jj&&p=d^!uR z1n`YX3Eb|nhO7@%O?(CzDX;p4KHtIOc0dbpwscO?{)gU@3BSdO;qd^>ZQNAAY8&ns zEX7~7r`hC`U}$hrPH%JWLZYZKr)VKjp-;W@PZX#$r361f)H`@v2=h&y zJW~8wdlPUSX~So2!;sTEKSpCJabTyyQ7lOGX=iV$s4j~ccM)~JXGo9j_s z8-8EocMLx&3pV4BD-nlX`1#{mj~|u& zfgizth4eUn7rkY{$$~~nLLNVVJnQkJvfkj9;Qf)d#IFN>D065MN7372h#s7@&^2C5 zB(6|EPcQ`<;Vc|)s7{%PZSCT8%?=aBz$OpL*p-L}n$U?9NyRoxB8Ou*MIKD4F;Eg5 zI~>Y+iUtYZh{1Ubj3;V*K_-jbAT@5O5M*%&3OPv*>=SX%C6S-rhj6V-ayJQzKY}xS zv;O0m-ZT6vI>VogZ(T>Fingme3nF!yjW3fCk>s_qh$~}LL24cs=8%`kxi3TW< z<}#dL&Wn1hj>t!NHHY18pob2cGqA>`#66O3-V+4o%<>=RugKkkclBo~mL2DDERu8w z&n*812Oabs~YFY<*W*wv-;_M10;<)mf9~m^9-(y`>7Y4c?Q7@ub$E~Hu`nS7|9<|Ef zSzHr!7S|ku#g!ez-{iw5uQD|b3&pY<;|>JvaTcb!lCPR~&@k9sY2^T~u(*V^GQ^Ee;8e_+GeCdM+aBl6C6U!s6fj^#;7=iR*<8_06me4F?BJf-9q`{0{$+ zFKxkRl=4sQq7jz#sg7?Y?M2>IzQ^7HXR_#7saRO2M{jQ$eKW~WxgM>oxWPj1CY+cd zI6|lxqO}GoR@|EFOs{uP`%QJ*OAk;~HHNmTJUCzM_`!cjykGEwWY9#P_riV;PVs`)*6v0s@1I z)bH~G1WNBYsbTJkwC|y|N3e$C{Zx;8y%qESwTqW|7iJyBpVBfA<0My>)-P=- z-AawNxO+oE4%niK@f{6+l0nzndz>(PI_V_d5$B=Dp>ct_Y)1p^*o7+mafH2=_|mM4 z6iPAd{Y=^!X;RMpTrJ$Cj6ge@>J_4}LuKlc}a{v7CV0S+KfEPx1bx`McJksZaol2OoYI(1RQ>p1^6;QXKNiqMCZ>8J~lqXSvxl5@#S z&HNtgaJvp;s2*u~ZwrKQXcz%`9M???h~nT2P9e$E%#W*c0$p#jquw^4_8_`U!p@g; zyOG*HQ?uX{*5Krfof4PnsF^#+!L?{lIG3br=(5BglH3Hhgm0VlPt%58io$m${F>u4 z4Hg9S^($AREJZ$Mi@KHbc#3?+s{>YfD(0s*lFCmDq`1y@W6fekAxXEzgu~uWs6W{8$Rx^8G-l~?k*U6O^SPn3x8;0o##YV(_K0gxQAroqgijzJqilUQ>aBl zCdq{>k$58;H_s&(g0Z+gGQ<#zT`z+2_(e+k6nr8Qr!uY~(Q0ub9wiC4! z@k_@q1HYO0WmbMfLk_aq`N(PsvdlYyPQzb9NOhYVf-nOoY$xq|J;v}lq~*-^lJ!~* zS)dAC2N4M>ML245B}GaHlcIA8n=6S(QE9u6B$e!iXa|a)QK1)S;&C+^R>3}=lw2)n z#BiSr5r^Sk^Wgi2hUCf6(W>VCkL(0)6U}8aTQ|WRz!FIJq%rZ%^K<($x?-p#4hKnc zJfHt?x;!~pDH?Kx^5A%HC@%LL$^lAT+Qyi{hiL@c3xOax?lHz}qQ->Y1@D#aAz*Qf zhIVi>>Uy2QZQ&LXU#gdl?iQ})M{f^6Jz|e?3VpEOhb@HH@ z$hYtmxdzRo>zt|3)=Z15LDc}OdwC`qqDXScEzUz8dS`;BJuIu|#h@0rJW+~BSaWLC zz#=z9ql1HHVn|zS@GeIUM!62gRj(EN6DC9yv)>7^&Ct}^5-a~OYhpqAFL+{ z`o6A>4OflH^56`ipCqrMX(-9P6-mb5J{XGo|A{(zEbMWaGRq3etS0a%XbS=-dh+R_LrtT# zG$+-oeB~XR}Sq9Sv#(!iWsA*9fqm;cw^yT?^oZ2#jk4{SuP zTLlGF+$btuBSk|^-6-frLGc=y7c^Azxz`pYwfvf3Mf?pU-~1X0J7C)~q%6nP+A_^El5Rx$2$o z_$NnS%5+v7xuWG?;@EJ{z(V*E`Ay_cT?6lxof?LB=W0>7eFjmRmB-B zCmi3Rv^kqn!*XGJBhh&ibI~K6e6W5H>g9UMyTOS;JyAeG`UhQj7e@igKulZbayr^h z(!@pc7|mUUHnTJ7KFmE3I|vVha}pZ6R=EbqOwKlrt!T1_f@4h zmrla*T%vO}UjD#$OvXG~lV*pB(A*@aZ^Dr)UJ0mTEcPG8=O9lEXo1C+GZh*4!zO}n zLiyzei3L+gO-p(oA*FC6jLoxSFa^aX77U5U1Ykk_@I^0Io0oB|PxqQD9k9KCTbP4a zeTRhO#E(u965|)d1wXQyGJ?HPyfKm@n-Q)2F)EAN8delgj+J0C>}pbrl5Zx{Y=gB0 zmb8UQ@8EvfcT!-n4hzXjoW^4M!WDbM`_X)Qe#m_fBRJ&~M|yao!CJk~`wJq67J2uF zaa49FCM>*_mQYiUU0<*4FRywND_Py9BS|=w5G#(4XeE2w2M>Gs%i>sWmq*GU#?sV^ z!=c+=^IO?IyqetSVs|t-JXJ1eTWCfJK9uXIvSLf8k8+EK6Op-rt*RTtqR9ctGZHx2+*Cm3P(86kebx} zdrD6_l?-b3KlCr`2vHwh?U0@H0%=B?k)!)+$76R#jg}qtBbx%A!)cWCsC*YEb*p?CCuvU#O?qWe!B9y91=1u3(3}eF zK+qH|=SaLVpc0F;Mx;1@pvHu=VkPn=h$MmREfhCSd0rt|^C!9o8n@J<5G>RNKrm(& zJOE)dg?J0@tpW+vc&26~Oj>R(459-JDI@#%)$ls8hk#YX{DpQlnjMxx!dUkSW8 z^^xy$`ilBE?Ybzh7Dz5i#nNyFLSY;mW_&Dig@hD~OoYB1;qkt9jF2g=>Wh%brSU3>p=>vT+d!k`MZuc)RR-`8v zjyj)QG^&C&nO#r!rxXVl(5!|1M1TumXc>mSrZj_gBA-nSmt|5{XmQnw&t@l3+QQT-X2;7~YJ_pq?piMU|y1{;|-8t%=*FJk55} zx>GHwk;5zu4yM+TIA8BBKDcNV?-1kRqqhHc|K9nfp$Er!8q3q`J&k1t@L~sQI2xpeOj`}*h>O~?QYm|=mck1|fLfs@GEe=dWXIy=deCqhl zAXmv2sV?Tu9Bp#ci%;r0D0Od$LaAE<_1M+b5*JCsN=F!5&<^pJ@0Ue#S|9V;`ezqvHI*IvweBHcL+7~Hlw}rGi>@U@0 zM^U}Ancf>9jlmv`InXFyB0qGe&gmW@OX)>&_ia^MQEdq;9kFadbEM+;*HsB1tElu? z(h!KsL;LE1_Eid{;QA-H929g^Rgkkz{bEPptn20jl*i$|PE)^q=)&H2U!Cd7tiy(x z=B(>O!B)3rZc>T(RY6wlSNJ7i0)Z8i-dTh-H?r)0_BxU(5NSR3Ed5xOINl$kYm zs*9_~j%N|#d>INH$t&NXkp$0@=T%j#xl`ZuarJ6C+5Pw*;7WCNO>`#X2;cO>UTpmN zi?XFJfBdM^BuI~wSPzpu7i83&%)d8Y-11mMw>;*WVPhGB1Ig2_4N!r1mc{lymJfL1 ziFGYmK(sm~H?v+!5Uw^N0$gH)4d%C)_-$eu+e0}EUJ+B{ZcwxD$eP0y;gRURT!0#u zYb08lY=-f7#B%-X{@Mbm6R9O48MdrW6nciccY)&(4X@JEJ&U8K(bL!X=uj-@9lW=P#nd_bSV?o?5NANOEG%pZoUBQMDfrTpN&Ij$3&wbq=*;~fiSLW{ zhI(f%>?ig``vS#b*{7*|OSDtk@K!@B5rlGLY>&0&m+wQFi{a<6r=B!UMft%20!LRm6N zd?`!9zGVW>^-bjy%&8o=h9WJgeNrvZL7N)jMR6essl8BKpy6@~qQ!y+yVx=DMB6H9 z@t{GMBNdze1F=se7;TBXe=S5^+yYpVws1 zO+zch9layKu}fH4fl0r zKUPz7W3V^R@u$fKyQm^-wU4Zer6jSzzi^fQ+W`NSp%R*2_n8`^6L%C{zT>X$EexYEaxXO&oC)3?V=4OZ*T2J;z% zXNsG>Fpcb8hja~+HH$-y&}<~ckghi|;6Q^&*8$*Y&+DSgkO*qigU0I>gZT*k;UqqY z|EiZK@pc_vc~NEfWqg*>*kooStp>sS_25DLZoSIJqdS%a)-KlV$HVj-0k&T?E?YwMOmjL2$jW1A-xic**jT+EYpDuSB(_240V zhyB3s1kpt=)!WQlAMZ|_hAG`p~F0d-kcr+^m7xI$<468+W?o@#z=u~EyI z#Zt?0P~BQSTr9QRctL4dT2NO@Uy=OUoUN8|g_aeIrIw?Mj9R8Hl3M-&s$0vYMN&&U zx0WJ&kJDJ$sYtw`vY(J~g_aGA^b09`Y{dHytEzQ3>W4reTh)AhQX=mb^a;FfeP2JU zuS?~fd&W_HQg2#}FM1mF^+7fa`c}!fLSO#F`VXmmbOfD8W7EykiS7r%pzj2{ZhhzS zq`uLmO5e;!>gwx+3>oz0$hbnEJx{+qj3;?bZou_H!+Gl_8yAw|DtDj6$?8IV@o@Cs z5(Kz4?p`Q+Z`oF8RO6RxVJdl6MVFwziQe08seW@fpFjAg=O|rhw-qaqE<^7vKtc@Z z`Vb3LXb|Z-4P5lza!8}5JARbDH;pHFy=vn6jWi_ma&B!xX?m&&U74$Q8o_%dRwKZj z&|AQ9T5F*#g?2Z5EV;10EJs&k&r({~MOPuF>*P@KtP&oP@CA$Yog?@MgL`eG*r>qr z<#h`*U!<$iZpVI3dg*VNY?bIZridn5uj%=q- zlqj`t<=0ia6uB{!*9k~;m)E=VWpD7RqviQ*sU=i$3N2CdrIr&^ca*u)j=EZkL=I{* z_er)#%eidQGAtM*PgY)^CzZ67j6%uX*{WkxIg^qvm(^AB3{qss$~W^oO1_?_UmFS~ zAbFG=w@W2GB%@H0Fi$G!>Q?go(z;4sMv4qdZrD9aezwc<>JFMm%RlByElHA7Xc=MG zi&A(*k7ra){#sI3$y-Q~L5Yt+iDuVl-N%z1%@8*!c>nLIxO8lOx+FW?lXTS;9aD>*JOz87AiOt@Z$A z**r1%AQ4Sw8)v|OZ90Ia%lH6r6};*hihsMr9mqUpDJB}se0%Gyo(kwOHX`& zkL~#5926f6EOKUeqf}ObMzVZl5WTwOvX9ZrAK+*CTD^EYAIcBv7sm65CI=z5 zCe%EWl1j&4u)We7PT<3uOe>rlXDyi96W6207md+sLa#rhPn*Cy_}L2gV3ETL_mt@+ z6L?>(+YJ38AYZAsoXFegeJAoPZTds{j)}azwmwU*oXGzXxb-2CxuMt?&Y743Z zFvlTACiMTeWX)6&5m$N3qH(C08rOzpS zX6Xx+K00ATyh+kWhC1*iNMDTfc}ZWa^qHiO3~F%ZVCpZuWa@&5Zy+a7ZC2Knr}Ww3%R!MP(@6B`6I3?aCg}fUnBG1ap7FRbaMaKz zL!9e=%W8lYHKHdGYG8oeqO0&Q8vROPdXsaRU>2dl;v-p^x&XOkPn)q=Cdd}u=@t}F> z9vQRK5AeCTO-=i>yMB+IclZASJtTh-y(#peo@?hTdIsVCBg<9fVgO7_(tMK&&DcB| z0`qfZy-@<)&Vv{6j$Fkd*nk+x24iruerX<$2*0{K(fK5Oi5yq;;tr*xJ|E=3GBXhd zMVa$>!vBNl?j0+{e#4O1WLOzNHQq(A#^8{|_h|f($lz7GL@2Bgoi@9kkd6B&GqQPe zyd_nbg<}!BCDmDlGbiT&?;|cPn&T3+;6^y64mOX(#g;Iw!I#ShSn!21oZgG0I$YQG zxi4MU_6aUt%JsA14oh+B`D+Rp0;cr&A$ZMJj8r(hZ)ofDYw(&4@X)DyEFQvgJ3ArT zx$a1oH=d+yUbxTa@R=+VP<$g5;{~I7oEQiL zprWDi;@qqz^!ZVgoRfG}D1Mo~Z9Xo?v=zR&6Y>k+EEE4UL|^qLTgc?X7ST56Z!mGj z!N(NQ1cWu?C^HS1m)^xk$X$3=^~c-iwm$R2vcJBHV@X6o7wteCKknBpIk-*k45=)< zH`;mp3dby(jEsq4@nS-LKY9n`Jne&uEY^1Jqa!`7{VZ@Cd9SZVBzb^GSgO-II-#(D zQk(32;yftH&iz#Yu#Wx}M24$nz33z~QhzIlcMZXDZY4rpFSbQcpWE|W;jB)t$>CuQ zl15V8^?GD3Z`#HTk9!3j0K6F2ftkd2&TB8w)YEc#xNgto1A=3}`p#1?V%b_51Yv1(<#w8lgYGfaeDV8#tS%>D?D{ zThM#MJ+Y4u*9#Z&ukJcMOwhz=@fmO%pOfwxCEt;u;tA?a;Kll|c0A%hSsrh|wWL?{ zHy-9I+66cVSn!xbBcIh*ydIi?dUCEinu*yx|B_Gs@3@04RnJ_+`$p9pGH6dRHeuy= zJZBVM_QqR_yY%CWcq{&h{;x&66Q8U%TFjHlM%OpF6Rwj$H^p_=DZvBE^@kSo`^ZG0 zPcQ~6hZw8|W3bBe)%}<7yMq22;>l{mp}K7ekMU|btyrJ8gpYMRfj~?RbguD1%R1$o zX^55q9HyYdTj14hB+wt}qf6K3iV<%xhM6ogjfeg;6~IJzaO~B_`Hn~??5z=MwcES~ zV=xf+py6WgVK99WHnS;u%2M9OJLeCq7?$)y1FylrB{uV`sognESIz^_;cZegoAVD` zh>VM2PbPJ{kywzs1Mj*eIjq_^&YT-wzui@8lw}%)mWA z$<7N&@$Jz^u$Y*DYlbI?uSmk409MJ}lT`^hDr!Re!&98>TqUMTDE`s5GLODoR)`@a zY1NI~sV9O~<9>y?K6_X`cULiubGF5)OMJz}pt=1OhgS~CX0MM({k zF?=W2oERX!Vs?WptT1~oFb^pT=?VMs*|;3!w5v1Js57oqyed~Trh%(j9Oa`qT?7wHM$MAEn5$4l^3c13uM^^Kr+xQYT6paJ2TE2IF^P|`*E zP^L5K+d|yKQ#kdkuniOSA+B(RcIxSy3L;&;GXQ-LB3#+r;D!wUM#G@X1|lv zXOHL@D}KC%hs#y?>Y>zGEJ~`C&51jkg~|_HeMzOmNgqk{SxV`4pI7HXWw-YVQga$6?}PlC-;D3nXIzf_cOL)Ws`@dEb#)EzK!P3m(%R_-_qnycP( z*X@=&vy|+SSEp*1J5cG#MQdmgQ9C|j4N-r5)UafAPqjh7i&LF@Xn_Fxso+%~prWa^ zrrvi)zdwgZm-@ruH@TAi2JZQ=O{hDbY_G>J!5wtiEJ7NWD_b$v8wvKe?G zRByDJw`zkV(xN`u`6Z=thiFrEMzN44dfI9p?!c1xu0&_yK`^6w{qbb2Oaw4?2ByjC zrLj{i-|7yasN`HRfKnyDDoQhz>;v3m=a#ON={j1S;4+pC*>^lji!W3*PJJ)AX1)fa zikf37ef}A|BGx@>Adj*K7m0lIH`*Na7LkQz2c7iO`*Bnb^HKSRhrJd9Jaric91n)_ zSWq62sHd2a71+8td6%eaGSW~}(8bV2euRbd1#chA!$pktn{t^}ZY`Vng| zMkp+0((oRVt#Hi26en)++>iH>iV|>7Ck7#0v6e`~#paa4r6q7X;mR;CZKef~#a7fl zAxv^$Nlv!YvGj+#UvTYoVgX){qf;D<>)>d8xsx~P)CSL-($LBaE#WOX)=n&VD+Yje zaN=c$$N9f&NE7|z0^T`b_hqU{5-~uEu)Hj?K;`Du8ZQ{wXi9xA8v+E^*%3lf??nbK zm5xo7L2jDQPZKqMZoPpDE&D zU2kFak>va>VdYyiaZob_NOImD+^h62GK&d*4NPj7IrDEcOrPWC5)~%fEzOA& zGd=B=x}s|WUZz63CH!x`;f_{|cewDOYihMxfS$3Qhjjg}J!Pq0GyT+`h<7wotE~T+ zmps9XaUT|J3G%E3x%u^mfPVz)pRdOb=2x;3*JP3;!zx-{Y6iF)*B~k8&9y(y2uluR_iO+Lzcz9WvI}qPkoG= zgDJk|ZhX5^MTjNzZ{km`jen1d|Ndj#KSanUA>M8wNlHi=wJ1Y*QVR~K6>@HfP}OQ_ zJuVXE>lRfd%f~Xb9utW=Fw>o%*+NuEWIZO5WOhqhtR#IygJ!)%O{taSD-+fHwR&8& z-Rg5GruQQrhI-xVO%&bGBJX$T$t)L|{+!-2|EXqbR`HDUj z)CVKCyJNSKDd_V)Hd$3hme%dq3@H#xx7X#;&GaXFWcOMDAIc=8E%~Thi19kkMW^vfH0 zeCr|iPzG?U@I_8=3NE#Cjw!J@D_qN4)93PVKT%;nz27F@rdM0=VF6J@lRh0J|0wND=`l+XQM=k%5{=;UR zQGB>j@A4$3oq7MzQ=jB<+8053(bK%8Uh*W5(0U&^aQI0c#Iz35 z^SFM`(}+6#BZ?ZK&wCnCJ`AAvd$pFadcqc*GuWEzXSU+Ji5}49xFL(GO@lI+5VE{EU8jD|W&cb`nw_f9QOW z1NWwj8%tmZgY6GG2*D+P{b+oHsp=6tqIGjMrenlHm>A|^_p$-)fd$(N2l?Vw4I3>e zVz1)0D1~AxGybZ{WyOLWPu8elERgUn74{%;S4nUqD>i?znY@zjZ>Lvy%qfqz>)0ar zP%=LG0*yG`{~+ODqsK>d4ABmB(x;d30`UU>l#K?|Y7) zb@Yls%J7{n+_Xbm!A0WIHL)mziiuYaq3L@J`RVc|JaOXW;9O2yd98>yu3EwOJXT?* zf*^XBwpQaK3pKeBV9Nc$)r-hZv(t~>x^&`-<6|*L>AayRBpFwIcn90O>V!Rh$tO~O zXFDIHwQsLCDCI4Ah2FN5uW^*qMD-f_23q)cgolcy+IiP~IPw;E1QdF?MxmIozdg%! zf=05UK!0>(*C4F*T=Qhi)rg5L+_A#%DghBsOgK&sYy>H;DJ1vPvA1hn=je~lxuLCu zsP}z|e~31@hLHE42omo*-~(uHK#EjwN}}}6F0QjdS-IK(6?`$c1M95OS#` zt_;;zJkL8g8c_^uw;MEbSzP}_)vrD83|#y=?)$@Dou3qY4aYM$u^DpGG!X&dK@LM3 zQ1ulBy4pkTRxF^99lwLLWq;+GWH^BkFzCdP2Zoa-3u0rJj9vA$?&m_IW9}V z0oq(0@rN=FY3i!Nhd;+qMx8fQ=!#@#j#r~bi^&7D6=U&Cv z93j(r^lB71yGur2FqR|u6kZLsFz|k;fA<2c3?AFd*w1*g`p6OUwBL@@)cl;Q{fwJ; z-!ornkA$NcyFY^W+0`h@PA+7iH8jRufi<}eT!+wF(DNP#pwI~jrQOhfP-lDK=#^2V z@zkj+cqNRwgs9Jz0iwvGrrMm|ZS}o7cvEf3Yx?^;c=I^@HL9*Rsh)xpj~1BL!0(zy z4YDBwA-nVij1bo?NPwqd_xD^s1?b^Bc})DbSj#|fC-lnpsJdIom1~n!L~#~@#z@$5 zLBrLHSEu7Esj08(5AWpN9NWO2f-h*nxiEkbtj3Z#@JT8h3?$XTjZB*a-YD=HKpJ4u zkVWf+qi>+6Fjpm70(E(0Ff7G+ zD4Tm;WnRqniQaq{@2P#%Mjy3{+j*{jY!^?*5nIcb_+)-acf7<$_xb5%NX6$(=HjJj zyiYbHl9VLjTqy}B>7C(2L%o~uc!?Lm5?c7nTk1i(c`wIO5`glv*?z0E6`B^17nz)J zR9H{o{j!71$`~M$6;4^GX}b&i_vAN`Uk(aq`=UTs`g*&*p$&wUz8Yl29L1exy-Z<{ zhziC>_DoUewgejqh)V{1$LMEv^JbmfqHUw3y`3MN?y$1$!Bl%Sh zLxh^p*IVd~UglxiFD>*=FZ2G|t`_>Nmw7CTR&B{g7LFr`> z4Q~qgqH)H*(TtkwVl-N0zrh=5e(W^fnhShwH^{rF81Ud2k9v8E>IH9(;5L+U%)h3N z@H?;{vXbb9rt2?BRx9wEh1{6|z(o_MAy6{QUIfaP>PTS0KNDO?RVxGi`2pk@| zGjP|7Bsj&};wIuLKJXbg5fOiC8ILq#_(OeT9fRJ6s}ZT(tZ3ymp%2n#wDYx_5R1(i zOdwb=X5jG)rU&8r(lXvdn-Z??E8}ej=b|~;3jMKz;@cfJ8^C-7Ul=R0`MPG&iUWrW zA{?&>qN&j+hE`=O#1~?UOp8TG0=P{XE%nG(dHCSaeVcuxw1K9hqiQC><(sT#E=6mmMtf+r(j_me0C2|4IvN z?7fbsH5aXo^rm}xlU{GGrBa}+iXK8NKKCcRt*PjXLfbgq;h{48VP7h>Tg~+;dwIJ? zqj$)uGWfYf=aG}9gdO^mdtn>#z$^L(dwKsRpS^%~ny|9MyKQyzWfpkt`1)QI}6TW*|;9M!W&$#zO3iI#=C{n=#db7=u*i1IVVlgkcA|m zp+&x+m%qkiyH-DsajwGmr0F6MQ~oUgrz=4t?3{d>UV&|Lb+$L2tUBPYkgmyrylo_|FmlK~437{X914@<5ud)~2DH zu2kHkpV-ga1U{ZltL3CTRP~#q^c(wm_kj-}(kWYChE-+uy*G}u%4tv&I#|%02{&>3 zST_9k2!E#v+)(B_AO7~jzw-C?3*d=vXVVE18OJbFNK?U-UWiUr1!FKkSP6 z7vZNeLwMd5d%iK;_e#|*qd)9Q)z7v34%_l02T4!gdU0N^7w2WfF{Y0aSWG4MF$pVL zn?*tjB%V+B5{XkUxl;b4;IR=sQwc(xSd8oJ%Bln@895TQz7O~JkZ@w&X~1uNCFoTo z&`haSN)z4jg+R!=F2aPY7!{^U)Itf4`(A`sl6+qW7@*ifo?BKCP)&GMoQLT$(G|~m z1J7XtPq~3-pMhtyfoF?>r&#g8e1~1hF)-yBm}VISr5Sie8+dF6p3VlISOZVAfyYuvq8MZe@={tkVS+{x8J!oAAU6R^F;33LjUPae2uw^CDVLng` z`azF+3p3i*UM>UQ| zF?`Flc2xT*ko=xe?UL}TQSD>l!IRxbSCGn+rkmR_!BF;3n*IfpekV+4{0YUp8*?SF zpZ?QZys2X&QF3~_zQ@|u<-%Wz)ASf0hhh0tLq@c&~yM1p^ekspNg7V2P5qPx+rxaFvAAN9C_{ zmQ-k0FkQi(3Wg~7hZ1;L`70E>px|l+-Ewa#zcyRS4^S{tKGcmE$2P_XC` znu2x(ixn(UuuQ_ytU?j06uhaR`KDCRB3zb8jDoQW#w%!3FkQjwFzvw16MR!^pPcO6 zsf-=C?c&)yIOU-^QzvI9%$hzo`=Oky+4|h?_!|deuktWoN3|>)X6`ICRw`JfU`DLU zYK^S(66Lojf7#tKyu62m`SB7K_mz+hlCWCA44d@ZC3G-+{!TOs%Mc06yU8rGrqW-j zpuL&&SKpQiVJZQ!;gX)AV7`KB5t3e}U`cc3k5v4v2x6#+wMasmf@M+CZ*DDNm`YH& zQe39s>d$5P{}W6@Q~tjd(AxBPI5wVO!L5l$+AKg{_{#w;`d`2D=nnC~K%pN!z)}qA z(rNd|j_syr+@tUP1@^CHLo`Mm+BQ@Z{B}TErO}V*ZjElfO7N51n}GHCNQ2cT<)r~q z%8CIkfb?_A!viY2mL$?6$Lm-S$_8nSmY?)$tdWgxCDNi@Ok(|f)YI$#1RFv zfG?vSAIUB6@8R=eqZz9Jp?*$?(QaTQ1$ppW0O{vWAki!Oc?9~yU)@h1_bV=wtL*FH zCq61kDv=P--9O^bFz~tM65M^ShtCWCgYa7b>F4I7_?A8%J}TLK_%rZ|#iUZX28`BPMh~BDd|iR*@KgJw zpF1P;k&^Erg#%KJ4&PjvMAcSR@~4fr-ow*SbN20rV5!DqhT@!v64rvGe?W=>pXG1(WOrBXp9H$l|NoA$zQByX;lpD+%N*@V zAdbXX75r6z^m8YWV4abW^v8hF0!Tmiz(AtvkAyCaj22?T&n@up_;&sqK3lz!H64D3 zMTqkVq>{Kc(vuVNo5A?s3`o^!o&*{*a%xyzc~$T$y;v*gvwz{uyBtiZrPUS!Wztw$ zB(oaxOS%5)ZQkSo_*j=S4gpEQc(}Ok8jEWw)w#oPRLihijQ6WFR$MO(TP*B4*h>*s z>Iu`E|HbEc=WAB}SX7h1^S{W|Tou8`qC#~0ZSJif|BIXb@-mpJ)B?Sy zZ>%S`VFnzUSUULKKb(Tp5N?I5zDwVAn@9LpB0LLm@U)}knR*yR6dQ1gO6=~RQCwCm ziEB=&t3nh!3qjfd;=O5*V7?pWVeq>pF?a;CPzA`7StOS;huO3vxBzzZ|JUnLnjS?muJ#y{E<%e3$9Z*mhPek2j4_?%Ml5W zDsIMFNMV1cg!qj=BpgK}T2o6iGDyFXqY_6||Je7XV>kXB_)??P$EGhB_K}bNVGUpC zh&{zw*PN+&InlGl({JSk&Mb)HrbWlbvTD#uu(*)4JXQ=tm%d1g^goL5Int z>qD87b0$akAQGBB9m}w;v9S+Mn~p&eep;h;9XJvfmk&$;ipR8iM55jVEzH9>1}HOM zuM>G3(ZcG`k|`>))T0qGrXFn|Xt9EZAylR&SqO};!xkv3ZLm4PM_Lvcq-zH-nqIzxG$OpN704B^+HND`vO3OyiGV;6`f>#-Y9 z<^oMrhop5U8k>q<<*DyDBGGnE5N0nGw>wzYGOLH1Iv)rFD%sh0lIvwbj@P@X}yI&7R<` z3;I`HT5m@h!ZQ#S<%@b5-Go(PUbruuv4VS~{JMVhym~z079Z9a@9GEUG+}|~e9}l> znp<96V-}0!@;tcnN}kcrgMD<}TkGJn7m2dA)lYb9?HZ@WOU2gKjE#Xyi`Q>^Yh9a4 zIuelM!}8lPwg(Pj15Da75kCy^O{|yCD=tzm|Z6wFM+ucPn}-AesmZ_tThPH=i_0dl(8h+~14&4``6)6N{TQ1x zM;qBVt+&iiFLW^4uA;okZj(5Fzvm9^JcH>@|P z!m6N`;57wrD(L-{q?;9Np-(z^PBe-WZt-Tm8@%ma8Coo*wE$@yf;f~$`&W9mhFVWYzN*VItVmA6k-t%d z4~vKkWC0_6?F}+y`DGw3#mh(9=vS=LSOQ38Uj|5JUatI=fVA}aCm=l^{sL(HrA-j& z(7$b{O%{zKzP&!oPm7>JpW&xPwmaV!8Kf5f@kH+dQ&3?KVyq_`q!dp^4+8CcU;P#E zHYMKhRwfo0*M$Z0j*M;;f}F4YP?@Yh;*qXID_Q(q(1lrgh- zeU-5mfRxl2K+0GgAZ5&YQJ>?lEefKB<7%f0<8%Fnzt*oY$tnkJB=|^H+C_a(fYw#? zTpA8VKH~hel0wFM!@{JhzslQne@*zEjNKv{;@9^ZDdd4=vC0e!VQt{>MoCsgRjkiiA_5FrONCfH&aTo-$x&je~rhW%q z9~Y>#)-G7}Wr14L-Y%tMe=#P-$1sX3e=Ph&y8wT9*9NR5_tktWeac1qOcND$CQyqQ z6rZ39=Q8@>^W(p8DjHON*B~vTXGyKBpZVU+_;@(n zJ|(NZKNev`D@o9oA#OJjSLFA1l*3K#33sPIpnm$_N4z0%ShI=>zwEDVe+oTuEAmD( zq`$u3jBuh+obZ+==Fk7&hVtkdtVN95qT-fsL!~~)82M%SHP(EqjOE9f-HgW2#)q}J zZq}NA+(`3{YpxZSGS=~Vl)IAS@$1h8Yppv*52UnFy`tXZ{8=^3&N6l}Ch6#OwSN7F zV6EvL{8cX@f4e>W9>3nzthMV`Cgh_nyu#R%aKhi-hqb@%$09EKGvByIEWeDg-{IW6 z9>2c9thH_zf^E==OaWec;G1JI-}(s`|T* z^CvpaA3SzHW3R&%tN0$jKCF=zDF%$q=;ZkPkLctX7#OIt&r&c?4H&D`fKhx=f3}e} zMNIE@#LGtg6MqT9>9_l$9?@7Ew}tVQ^o~g$DicHeutJ0C;T+mW;VpfUNGNdq3Y(T<5xVE?-^_SgNMiC z*R^I^J5eS#+v;7LY26%`q00(g?w=L;qcW-fN!I-6U*sd?1VCyD6i&ZXK$?P#<#3bM zGcA2s%S=BO7T1vZjr6h8)Km#N`iXE9ha`;0>4z29P63ra`z8ImW?E#VT`5@lHwr4- z>g~d`$lf~vgIPNPWfF9Vr7VD!Ci(a zDz4G5XGCjZ4jzFC0Iet@)0agc8xcrRM7O3@;UV5E1f}XTQWH(uidv=akpDR~X2#sm z+Efm}lL6`XI3QtfDmtfDidKi(wTmKdtffUo`LU?$3=? z1q_#2HTd-pBDA)NIBsP>!`TD_wi7N#;6bN3woox2#Iu)9-YS5#%4y76Wj121;>@hohOp|URUws)%gt$cla6_>z!{jm zxfbuR;AyrWTyz>G7!4sOh}EHqKI$v#xw->J^p2`Q`TL+fV4^@fU?}lAVJN*ie(HND zBy%UutZ<#h!vg8@anvJXqr|`z9*h<}Qi~N&S*X6W&T6g%q+hx6^B`{)G}5mM>py2(BrMb&d1@^iAn`Q{_hI1}aOKbtCxuri;blrVjSY4c;Y~pLeGN$1UnBMJT3{hZjj}|= zD>uY5k5Y{kkbaK>61HoUh$m*4JjAP1e4R&2zTtp`%>bl?hKs@oKwbA`bh+ow0gH1WE`kw+T|Iluwp;$0;%)*Qd#Z1W(szw9+C(#g`#2{jAd^ z?-oG%eF;d|O+^py>Ca-~qFES^U;)))WGi<+(TP16$axq1q}yyrgE>pd1*G4;8B*@q z8C?IfmDVzXiZKI`i)Tv4{<9?GxLNuit+YrH9*^*R&`J1Rvt{^@+4>BN);f~nn^h+4 z2qT_pb0p95Ir=V(lHuy4u`ZGXynBR zXZbRGA0YkCDu1K4eylBe73PxapsIjMzjDv!U>4$0zAb3b^y>~t*mOWD;YEPuE2Z4w zt0YezAYnTI!Q;=)0HG(ArEB!BqqWu%C2KhQ2rdtnB~hSz{CZ?tt!YOskk4A_k6)`#X$zUv>o8`m=d4=E^!W8>+hWg$WQM8CxaTVxA*v9E1o=(7Jvw*5hdu9pGTmnN2VQG{zzesomfi{iswc zkT6nIjHW=)JsXuhE&bZFGNHQw2|Er*e2rT8iiKM0X%uLSbwSFQL4ZZ%5* zpKp}{e%&f&?GD782WD%D6u2Ldez@U4C<@r76zR)bYR$wjng;d^Mg9|zeodd#JH}ux zus(<33vMSYx8$LI|zF~*VP`6!DMk*lDkM5Rz z_1RY>{S82(hYz4tNr)E3{WMRN(wZ43S{VC^yK3#(05jGTD#E*ONCn>ml7PfR(m(5v z{!|C_6?#xCM?&QuX>(R~n6uEg(C;#2Q>)|m#R0%&@Vw1g_&Ww3gP;52l%tzG<&aO% zeyqHlvydYgn3YU}-@(EI@f|N}A!?`2Z@zb-wXg_8?%2_6y zo2QOn@mP*=mILSJk$w!jf}ogUkK?fn&dpTEPdr$nTTgH{0gjgFqRrhwV|95b9v<7+ zdz>|ebD!49Ui$GsZ=CZqWPv!?7eJ=3q!kQWI94WY6Aqr=ugQa)ctcCt{C~sJ$y~we9WDY3R0$DGHFfxT|kw7 zSV=g@zJz{8Q5halta`c(C!P&6VMFpWvNqL2y9OFrJ`@;e-_3%R9B5Sr+OxCqJPlfw zLEaWS*5X+Zc-}xfY{#)WXcf~uO1=WE;wLm81C7nY$__NUfz}!{J7@)a{=zcI@oQ z1Mtq%%+#qllgTD$(%`9cre;6%pw)^~{Qrhuz9mHL{}aDSDA0fEr#<4pl+YKB3?Zz5 za}`cUt6LTBvEadm0#@g4>mGv7V8vbq%O8Sea5sSy{2I;|7B~^} z55w7kmjkYCBjXkWhD6H}wE(8JmCDipbKAj=3HjsU@=?_aDrq{L$)&pyBdcuxGB~dn?+23jW(l|0w;JE zTrqH?F^tiWh74@xJc)(`9)io@s(=&R3|9@D;QMfFfF@wjQ?R-ME{$PO|6~ngw3HzW znviW6#6c&x32q5+g0I1?16~f8_?)zkAvnJjBO~Yp*TbCx?y-E)ghkBJ7tuRFCwK`i zWFUHw1Bl!Fh2UK;p`jsw;5IlLaDwL*PVgdJ2IwBs7c@Ya#PoU@qao-7r@$2eC-||# z3Hrk%hUf$v!#PUeG1|(I%}jO~hE~uCE`mD`++!o73EP=-dtqw@I>FE3ykW=Uv5i53 zUc-U0lh7|uV7!6XCF z0T*jSWWXM0WdF7T-c~p}%h(fe#9sn9v{DW@X@Gk^lz187WjNxg1}r(J_yNZ_K2n1Q zVC#=%1S{Zs3anhKm92v4GKpCCne7 z$AeCA+LuTaa690juQ192PXkQ6i2Apa=OSi2IGUZx0iUnJTmX6*;Gb}0PD60cC7A94 zC%6pmAn<&^pW!Gj`x-6(GO7}EJK*>KLK=Zr115hXXU;Uh#c7k?SLb$ zVDdmf2HRi0WY6REJ!AKRPB08E4S22H zjsqSt=y@D2131B*a5=>EBc@|GN`xJ7pTf%ly{jeN4EXI&C@t`lVb8)}F!BM<2aNv} zjd!S4$Cid{YW{#L1D&9Q-9T}{L+}eYYFr+p8mtX}W9%uo^WY&^0#^;3VD9f2OprLE zA&t?p&;?EnEq3f;Zp}11ES3 zt`ayiNn0ET8RL*q&JMV1zzMzv=LDpaIOrvfxr05{1SusC;Mh=!X8`^NM-gz50gHp9UO+ZDxX^$d z1V6#a%{XJQYcZJ52(y}~aBRquy$fJ_xOm{W*nka=z|J(<1THsVFT!<>r|}=ilg(v) z7|m+P{AOb$4kZwo;10Mv;AMbhM6(&V(U^veY?@k7m7o*s0(TxbE=FKGqhQ8OaRGf> zW1BM#r2?4cP#!xV+0z6dz-U)PHaGWmKn_4BNcJ*mz>Rh@WNR}FE)#Tu4PY9x1USJ! zxXr+g<}zeka|&)J=ng`BJEMWXLof)gk|F>m!cno4Jq<3lU{#o2On^P3*d;38#dS@eQ`RC?d{^e*dNCg8!QGMfgv$d?Rv>FeW5*OYS#(?&gK8U%>Huso z5s3zF1{{}xxUf_(S}BnA!jqFxlAsgZ3s(x93=teXrgOFr9&w|jg5CnK*28_>u zvzh4kaMTV6=FCDD1)UE5GiQSioDTT6&q0yihZYX_)Lb-D;B;u8I1k+zI33XU&PE1+ z(;9F5k9sa zGRJGEe;}X1g~3C|z6~Ek&VU<_UyaAK#v@!h+THy)N(lS}_rVd1Zr2sdc6w-;|2J;bkGSt4L1uo!E(5K-~_`@qVM-c zA^^|B(Ntaq=%|o}o%w(`A;g=pU@{&Z(h=fhxMBz*xB#vccs}6QaI`F|2E_eXY#-A+V3{tUMY_;tXNb7&vHD**oqM}pW#oF#pX2?lf<;1)Qd?*_cAaDrX` zfoU4_SinUJUj^tmh(Ag-!NN~aL=dzN@GrQJfz#1se>mcyGsCS4F9-Zp;kXx!4TBSF z1wcC-t#An95;B380?zmx)drb#CO;bv3B~gtkoVxOBY;l$hh9LD0?z<^6)x;SR0rVK zaMhsG>zMuF7}^cNui(tU>6MSrOQ=FTz!2=Ka4TRIoCOap^ajFaa=^)IX!_Svky?z^ zbQDd;%!Cu9W91f5Mv#t;iB6D?g$XCPM&b0Hc$>?z*kb{gDm)+Xw8U|JTk`&~TC&E= z4-84y4)D9Hktl|zs_j+VtGDCU%2MA_OKD7LMrl^5y)>^hzqFvVxU{VFU}<^j`O>OV zSE=4}99}!HzmWF=vuJDy;%05PZ_nGFzrA34@%GKzOSbRaUbg+<_VVo&+bg$|Tvx4R zb7`269$Ok;YAv;urj@4G)3mv?q;#j!S5aE2bXJ$%EM?F8K5u@$U{A@OoqJd_OqB8^ Z>GYDkl9H0jlA9$l+tRnuy_G)v{{WPDsEhyr delta 64691 zcmbTf34Bb~`#(PSCSfcwu?!JQf*|$~A`u}&ChpXPQ2P@5z6FUrCOQcj*Hk!V>{PY1 zsPiyE`6w*`dHIYQ%S6% z(Hs#pn(Z!XE^96{751^dLYG#lv7+WN(Q`yiA$T++Ma{Q0T#OWK6E)cg@+ym(lvNGt z)C~6*HE(#OYc30#%RZ|b1XXLDI&NMnQZ^rgoB%u=VL@X3$WDv!2qw*9gn6XG^Ix1s z6W4m|sMJv)Z7Yv|l0=OL|7*Ii>0(e74g`QSH9?AFYsKIS87_ta>@zXAEkJYjL6pby zUmUnw&l)#%#uyC;Ple!oLs%7`7nb2YM2(WtM5B@H{-#d_jdaE@86+J9;IStj>A%x{ z?deIfo(#MH^ld@&`RAWY)`wxr&TATtZeSibE4VlzW;?_@)mi+0P7u```ILk6PblWk zM9pXs7szm)7NusB%vbB3O@r5Yc{+O$rFpO*klMl%=q!ZeWMr_XM^rph@*- zKM9)I-Ja?OcL=zLLfiF?53;WVVyk~Z^zKAIf+x^#9bnC>MpvIq^aP?$Rq5*wu(Yb3 z1A>X(alA>l_%fLLA7J@a+xk5DQDA>meMP*rpBbu+5|{61JF3Nt@o(h*TrEX_uMMrS zC+g#UWE0s?B*o;qCkhd7ph#>$LU=o$E#Q$q-(&CBZl$&I$S3cy?`w|~qc?%ODVrSB zsqw)ND96w5QUPs?Lc>k=d4gnh{xhcv>Utwjyzp-B*&v;uePa)?p5Da@>(ps;%CP>lZ1SghO(pBSLdn*n z=z&%llFf?hzZ`moSYEOFo}va(3de3WdohlUJnq3E^SBGgVwJfVN0((RDI`wV!B&NY z)O(8;s6cTnB^-l*<1TP?Cmfkeb1#PY2-=M(hf{xHJNvppxb{^Z$=c4UHk5?zY)Hc< z+Hc>X^eWp~cEg3jo6NUS1F=6twbx6gILCCaLbKCO*XLAW^s^%J!{pfc{2x089gjC zBu$rpFQ8J1p5Q534>=C^GiRILU}=&}xD9F+~)7{5vz z*wc1d0V~!EnzU*L-GMmm_!z&X>sfZhp@vsbd|g(}axT{MJX#)6-2p>ouzj|Ujcp%Y zeHo|UMr#3`vM*l8_O%ZduWn*rw0~dJ&0>}gwHj=JsdQPtfs5;P@aP@NwBJymn6M7Ni)zx7L!BE68TD*KLn}siR3YYMyC) zQ*|c!8VY@913MYHuTEeJ1iS@%r~*ijD&Wi-wz6Ze_-sAf*KuFVmNUS1;7`t0QI~bs z-_7>m4bpR$;HhQ0^9EZH6)e73$M!_+6WdH@Bcf|n|A!IRwaUd@e=uf?t|MyJv3H_# zV)eHnYBQ*>xBr#|P{yAxYY zPWVYB1c2br*l(SJ#W&WnYB9Nu&P;*6zRyXYv!*WVSMZtTGZ+Y4ucnrNUU_{1Q(_v3 z8`rQ(opZ&DlUZKpS``ylsflamv+p~Hid)}c5kQKkVY-ZO`QMZ~oy5?5tHUTOxJfYU1Pk5aTij%||ue0q4?%3JGxE^A@opmtu z5ZBq+VgwWH>Z;BLJF{bh*Vr$C0B8X9tc{9`Wd`U-ZeTb#IO1rxfD z5{Il{W4ot_zsu}e_i^GHnZ?GB6k}wzH9k=+TFxHDhl@{^vk>ECarJVx!Z@K^^l}LG zWjV$`RyCoW1A=~SOuee5w5ZWR*#oY zqwPojmT;FFJQJ1*}t-u<)^e=(sNw0|AR7r1R^jE@5OA0QQbc)ggiX~~$ z!MG)T|EIvlnO+e$FC3#aCVtix%*L3?)bTsjyTg z|KdywDg89k)q(^((*(HHJ6;HEFC@LnB}MK>9>q{vQbv|r{WVH&ph|kC2}vDmjs3Go zVCJ4J#Qsa!{GJWN+JYChJ4B41+;*uvU`G3Xris&K(kjjxYek+DJE$B)E54SPmY`kv&31j9D@RMdp zL_t5YvSGcV#Xkh*=+!~I@fy3>E7Lf64jkKS20w(yJu{wHc-%GPiB*{k6yHJAbDBdX z8sZ%TnWJ}z_`AS9?Y&ogDzM2(ohm6y1kHo;(AwrTwl}HG(8V>oi)O3DE+9U16Yr|TR6U4JH}Rs2vxX%13mpXct*p7xeP#*b=22`KLQq=2Vre$MYa= zeFnSr>asdvukwY>B`M}G%_-6zNU}^1itbfr={K&Sm z1Dg>sU{FKd@zZzXxU?%V)BpK~00k5vdJN9Zip+ z5K9xC!82L(;HGuk6RoXFZyU~l2#v$dXR<|un^vp}8Yw$FjO`iRO!o+rkhB&q{`)EH z$HA?G&Js;kahe|8DvqYGRzsRrTn8FS2pqDx?)Ec zvtuZ`J)~Kky2KIgQdf&2RJwppe$`W0qoFM-KEh;&WY-_c1`VAeuA0P74UH5BPhu|- z)SkpT4C^4?naHLN`#@|rk<}gEUi^Ck>x_u&y@s|l=G@=y_L z>!r!T;;}hwXL1XQ<&6zxMak8~l!2`Bh*tVKp0!g|@-~r>r}VhTdH(WW?5c05?kwljNS^l6_4aW)I5HO6=~4)ndo;Z0^YRKC6Dz zun$HCiu)I{f{~#WQzolr%jnOZjf@qSEMOf*wG&Sn*`!ee#5H4B{-}=PurcgU1a-!+ zh|!Va-O+5y=sjZl(JW|8gy^c$=5B;+HV2 zNxrU}IL-DZH}qK!a^BcTF|!XV8rw?WhI~9aJ&6~07ZtZM7|6TX95!@ZYwWSv|{Qx%uwYf@T*;^BuR{VL4S_9jAv9BgHt9uF2 zV*Ya!A%VnyY7DDAv1!l_qP1~(>sE>Y4(7g_#;_q1n^s&13OVb8L?%y+4KYtevBuL> znine;Q7TqHP^_`ZHwC6ltY7m3z*71gcRKG&#he?)vbvKRi{A*W`=mDY7SyMK;{^JJ z9Q4eR-5eUOj8B1S?S4C_u$7Z)i9ZcvyC$`*b9Wf2H;0Be^#}3zpsHl=hn~~!|H>5h z`=r*@Mh~U*6`^scH|qWW?9AFv9_Z6Y(XiJizv}Y<;SZC0RXOx6P5H0W+`Ela!{Vk4 z5N`-<-INC6`AO{Xl*mRmsQaxZ$(1C#L6B@d*l&uPUF(4lim8r@?@eMgrVbFV3vBAt z2I7cGY~|FZ?pXIp?B>)Vk=Fnv;-qBDD#WD7EdOSW{Tn1Ptuo;HOoBS zhDKbWGKV_#jfSu()0&IV2Q$aCuH}=*f$MFF-I`Wkye6=h)9O_S0txwhPGpUyM^x)V z#kD79D8KYaf$V;*C$fptV{4S32(rz%?~L)FMiWN~eiPZJ(-#d`hLJHVFSWXiZ8-nc z6ChoG1l95uJQss;Y8onFDv+VvPJIM;O}2z2(B`>m!72^ju3zM)RT{+P8A(;&9Vlqf z5W+Bc)`akC6WGHU9cqsus{ZOXGy_4RA?ETdRGI(p<5{PahGN`6mYmW`tU8d{Q&PlZ z{g^W)$nq8jM~wO1v8Qx`_LLZpx40o$|KfW}Rvx$h#rKrvsLVI0CObt68iRFlh?4^d zMF*!m|J3RH49ivL{YYOj&>Gs(HOajKo%+xLwEY_37mo?nQ(e{g(6}?k5TvqoGb6ut zArERJ#lqe3yt2DHyFP1zxVSHin;q0=G=^tvzw{_F#hjrM_NBPFMUlzs_oG&pbHM2|WGs%-q{gDw zuZ&^e&aN#E9KxQ@KG?tyvj!A_+;0e4@%=3*L&TMMdpA^Z;$>w)S9UpdSVUA33IjH5 z2KC1lS%DJ#qecVAmruhP(9%@m&x(d*Ve;dDdo&x7HeLL+H@lQpr}LHGylnBIDBMXZ zTs{%ZEebLC^TK_|8LWTv!tLORa-m87T$$LGHJLL<4C&3b%n364QVMz@cmcQ3@4YDA zq6kZ9F6IXwciHGVkH|&TF>MS-Wu2B}JGDCM0-A?2-?`Jorz6?yxwFM*->~p`wU~2m z8*$4`JQ}JjyhJW>O_DDtkAKrJ%e+=%VQcomyq4mj5$xM}1F8glUd+(uZ*&&(JBYPf zv$^vdiFXp&ruhwgR$|P{o8P3?mY(RvY~wN1_X?A;&Zbt5xZ>1*n8;qvZzmcPS=$A@ zL(amu)F;G;CP~yIdA7myAR>9|VLs-(|}v{xkssHA2@+N6>KiIjDPds{xyGgZ38=~^}6Y%=?0 z;j5yriFI4lNgSEX7BA{KE{3>Y|G?>#Pv{1oeJbgi7vMm_|Mvx`*Hbt}Axhp;U2=hL z$l39dZt*$bYeRX4AkPEx?miSVBW8_lb%qLuw%|XOh#f(c?DbO2taW?~wT#%67g%z7koa*kwjjN8%cw7$U`uw_U_&!1i*FBQlQP=62zxB ztt-baEh#5XAIbujgzGDx1P=L2NxrBw_#JIF+#37%I2W}yqa2&Dq+GME=#Jp(%I$ZZ z&WzZJ8HZEdQr@VPz!?rA`DX4i}O6mtAukso)O7f=AB;-I7qw_Cn zVlj=`(i9(PYH#M1^TmgHQ4`qWSu&Wm*?ZS4J=@8Ne~Bu^L`pe2*>~} za4@Tx87}0p9+~wc(aVeJfLDPsn{4GMv(D=B|8^MJuwsidBqtBQf8iYxKrT>cAlsN3 zEoQ{AYnkJPV=Oi+Laf-9&B|&~<3Lw*57A>o=cV~ca?yiU*4Q^bV;^RPh;;_AFS8nn z15#K~R!GH@u*-uH*4RdrSfhTz10R#K9N#bWZ%8`jt4g)EQ9R`T?I_P5IFQX*8eVx$ z8>Dha?RS)Yw6smcQNrKYgxujvQ7+x_=rP*Vo)KzpbOF?aUGoT#sQE zTco%(h8?gC6%MhtH-?C9d$JD8Tls&AJ;%(=XlBpiL!(x2s>!mJR}=5HVC$EM*OalB zNqhF;Y2~SZ9Q2@_HFo@`?Dp~y@s}hf$n&&~Dhu$CsmIt7IZ%A-Hrps;hoKA0lWPeF zIp0@(*q?GkF`y2szam69z`Cwz7=6H1iI{eD!%PlGY=5E>eaI=f7erv7QLb%)9cW*c z?7+!&d$SEII*W^=*o_tasvmLFo2;>O_p^)kIyLKHcvYr@c7JZM)0IzRPJ7eJ&zidH@2gK(yVt`bKKa-B ztBoUC)L(~02)eNeg0w{7_17Sgt$Dq*nAMn_dOb!oM>26`xEK`4+OLcdf9${}t&9}+ zbYO2HNbA79MiAS9Rn3VM{W>rcf(z~05(I17vriEWZ_j>1(6~KowklHmC4!AY@JR$) zk03pQeX**e7$3nZuI?ySieTLlTxrJ^Az0szeS%noC}A+`x9Q zR{r}+d%^FlnwZ*ga#vC6#XVTc8}^1zUOPnG)|#DOJE``|kEr9?pIgF3Qj}+3 zGRe_fc;D3nYs=AAsBaf`$kEwFthBB|<)U`vCRDVGu`Fa=ZC6sSb(O`S1U774BXLg^ zwq#x7%5&?Q2tHMcdIo!QtDk>5AA7yAaW=9^mU{|=+78eX8b*OUj5*4P_6$*|3-cO8M=PC55F z`o%Nxp;*e1jb8bHY*f!xO(BYWk>zfhF3RUvqs^_`KG;RWk+D#+8x_|&`VuCwM_Tbg ztGE(>xS&?ziFbGEDI)k9JP2HHKE1thH9{8#sakU zWYmb{`wi^wPJ2cy>#?Pyc&!P`*fK)A^)>r`OYpe5?_-Mli!Kx0F%~a0x->UXU0qYU ztw2VMT!SQeD&Mw)G>k{p-X_H;o%dj)v%W^%irrt2%nxR&qf7;Q9%qd$+`*mW{WmL% zw`19|t@UsSO+$aLb>)4rM-azOZtdEB$G;Zmky@NLVv38?R(2Q162m%f>nQ#d%2Kw? z5}WU4KWtmj>9ZxYVN8d(|BbOP>dcKXfu4bJ&-P-B@5iu1Z`Q*pw+G|iF6{Z6NjT+} zw$}|zisoQOj=+fcFx!o$Q#4zID=VuTvXk4ph7B zX=B-d9esqY9MfYg3Q^x2(~6~TOy5MZZ+9e!?Ls)FC&3)kpDu%7{Vp!Bo{w(eymh8Y^ug#Z`KH&5L?; zV8OcraUkzORI3vkxT|yfjsJ>hzKX~f>rEJMA`rd#nj6ut?bw-JabjX!R^i>r^`g;A z%yum8sT0E*`0LZfX8U9VH=8f&%5AnH%%#a}%QgT>NVTOT9}Q~G64hk=iR?Dny(p%8 zudRTqDDO4Hzf<0;TeB4g2jy}GIy+2+T%BF*wrulzoyCPg?B;u;<45eqh`{DyqmHJO zqgGR|vtcWkb(tB|4*49Hrj^%@QLDt0WM7F*4ULjU)rM);$#opsiDoLaT#E{AHIWIL zwjA|1gFCXTyBqnm|4YOE+TBKcQh~L6KS*$}9`84AaB~Z?(mo(|)UUgqB|PWSbF|-% zEo}AsBP)vVYFhR#sKEYyf3VnQ3mdqnao}3)x9RNz=pw?c&Q#cRtb_cYY-VrlnTab7 z>}x{D-5}o=h)rqeNI}Fv?seP>k#-8r$*6%62O=-9QT&UcMk~KXeK+<)mN4x)B zN-Kx79NKN0s4N^q#=W&#&{$s(KtA!iMoiz&C;aNp;;m?A`>?P0eRbCRe9-t)(29MD z;yXB>@VMth>TMo(ok*=ynG2K=f50tIJO}~^UuptWF*iFxQB5d1_>x=89#;o0iPhAV z-!*LbM}x(3Tw~PF)kh2(}=a$qNQ&mjeC-zw`+ z+}L{{Y8GRe^`(S?3na3V{)q#ucK=F%sC zTd{YbH7ie-Me{M*QA!Y9d$b=Sp|0zBKc3-z#)v%ZcB)&rtQxzU+q&r#q*!BH0+N`f z<5K-I$T$&zAj#iJ@;`9TaycyO9 z$fhzEC`njuaz6-HuGYqii?$^kC1ab?yi588TJ22!zv#Ms#Gh2Ooi>eN6D=1k%p}Ar|HSR zR4OU)7LR)*Uf^+;#5|R`Kw0(+-1Cf4!CD91TYBMX<7JKEAbb|`_04Oz)ciJ_tLc3x zeZ=F6CDj!2xJS)p9(SoZp)waJ%O93iqucLSCs5ElyOCHY&tNTV1cj9XS&PHFqVHEI zspD%N_vkpw<1QUXROSN3`hXipY!c4Gj_gW)s4XevTORia`JBgHLXN4-1t)ZjYp+lf=WV+2NxjY7Ms2aFo@M78Uk2`#8UU2Rm!~**GDKIX(-< zzXv|+L-$}R9&3R}(*Kx3Zdw3^b!7JCv1#oiWC%~g-mQN_ z?3XD1NPo}`z3C6jfk*bgia=4|?q=EV3r^A>U(QnVqJ1nsX>#}FRS>jtdAjCzRb=`3 z!R195Mw#*>+jqQ%*#AfNGCxCmrX0TpD6*{Mp=$c}A9(tTACO*;-97$_xKdi6?pb z^1(dr!Dr%e7rxFabAjUXJ@Dz-w$stArvS54|H(_P;UBJHiIZQDg^TuXD#cuyiyiPi zl&SI815=Ot_>7o0KpbbSbaESOUMZ({uFJI?socf6Qu9%$a$Um%Lt zzGVl`wHL>J%O0H@;WOh;4NE#-JL=UxNwud0-Fe(of(RaWm7s~rT%i1Nlf8L9GGw@g zW+2!!?=TcklKhQ1CNHg_Ip$(oJ;~(|NDs1q!D6=f^G+33piEBvwdWd^|9RUAuYen_7qgiC^?6;f z)nZorLdOcf|3xfaI7{;;(YVJ5PyHwzaq-&w&hY}-GkRj>3Jz8IPI}|7Xynp zejQ`Wb@uquD-|lfAdLMmGIziHsZSn4=Vg7=E^M=Un|~dTd-P`WxJ&OsmAOFac8!_8 z=-1^b<+WG73Ugr(=>>P2CfI3&+vJnmJj4J6;plk5QtB#l>Q(mL7eOMnjsN_@R*x8! zg0D((Rie|7kG^8tu5iX9SDHk9qlzq^yqx2458gvO?!voAWiC)szhXgO_8c^j;wIby zH!UeDaS6PO%2<5)nl^HAR8xEjygrY+E?-wwnQ^kDUIPE(OLp|j79tw+&tKNJJa!d1 z?n^36iCf^Uh$I)mdvtTj`7WVXdpQU0)Z#czQZONOAUB8#i16wA6ts9kXAsqq>5j^3 z$}jR&zN1NG{#{>Ow0Ch?mB1^k)78l$7~i=XT=9{sFxD^FrK`2X$zL$# zs;$D8Fo08E?~x{V!L@aQh?ZaDMtbGLv#4SGi5B+hEVlVZvd>^7{Bxs?Pg8{9U$<%S z#{(#|8H=#t5}vk*Hn|=c&9-saC`m`ruVX;V__~2lF2WsOukx`VSJXGVd-h2YG$-&D z3jYs#DTBS^aj)WKlMuA5o}g0mm;IvY>EBv&cnWXL-(n8n)HlHa!OWX&D||GAx8}EI z@YcL$25-&roaJ>KwC8`0)=bVV`v-xuWNX$EMQzP5r?bJ|b`_Ch!?*PZ-E&zj=NwtM zgbSn-i8?I(-07tG3?z@k6V-p5Ql~1V>iZEv?$DnCPxbej&H`@L4GbZw&vR6B6V5ga zYb9HUYSUT#t#0D+Yb^WLVzJw`TsQ^`Ako^OabRdR>ipDKu_kFbKD!^OvybN~GLb~*9zSM26*L&TBo|v z;bH{(qM=lglC!bl3QK#^u$CWEa&QD&K`wCW!zco#s1vNq(=Ijgu#+aqiCVf+;e&>9 zO<8n;O;7sh~1-dZA(V zXLZCMcCz=Lb+>HAo(QrQVo-Q6y;KgXns$WJ`j$$Ypr-Ahw8&CvJ=L@vN((BL)<#Vm zi!|Ig(z2<~+gsuWaIdw_4+QNuUFI}QcrXG6n&AXk_|}L+B*3~T=XWeq+UKM-Hatrx z5>o5)QxN@mT-sLN!+Oi$Oh8vcIbhl36Sy`|1UI5fc($q3^cRK3>6X$+Cdt@U_Sj{* zWR?FMP<_@#8t_l?qsIbCeyq$r z#?t=mY6WWI!9^Jo=aKI)KHT=A^^8R;OjBU zx7+D%0xpnZw*4_TB=(sJk#ee3@hk%cWjnIE4BQlR5pUqnNqt1U6hsmolA;u$i!T7WGZkGLyF@Uo%r!9t}$ zSt0+tse*)=f&r47hi~4HRG5$aAyMuXpy=sl8O$|NlK&5L?S(ipmqpzjQtdVA2gf9$Q2Q-=p}Z(94Kh zaA-_@qG5y~8K)&D@CN9xP~KnmGxXYi@G03#Wa{EsV%B$>~@u84I&1dwT)(<(c-(Mkwk;t~!o?+fFA5%L?J?v9@L8kt5e!TqY6yO<3Y?i{N0 zgkZBA0}J>0RbzpPVY$RY)xslxr#HPW!pg=%ll)8_2A!ufI)`&(KTB<7k}a-2ucPt= z7YNiouYVz?H@J_f>^aDBDfh_TO5!;QZho89u749#cNM)wg6pVY~ME{}cRckYCE=u`cX$%i6G?8;xDxrxf;g$y9QT z#oKwCxY<46P4c490Ht0r_@)4Fc6kgRA=`Pd|F_96U;u z_o2_*Ayh?If_Mo_PpeT=JF>q9s0as*C-2n6=w%` zap^efa3vd}Rco)lU&`8>9~GniFDJLC^Vg0_yRzVa`3R`23j;N&DHyFVzEzH+QHw99 z(uS8Z>k|hRp$C>y6qGeC>FSW0Zj$>4NX~Q0X8=v@P>Xh$>S~7z%Gz$AD>p3t25jUpAyzQ& zla0Jl2}(0JyT{Z@u@tY~QU{3ofA-%V-2jqHwZm`saXd=RvZ$Y~ zDMlSy3bj-g^(oFx{$CPyoN@$JqQXA?Qz_VAJWyf(pQxuoaw*hJU8qNuu5F8;bn(ER z4KsP*2nr)Bm8_ZoH+MeRb@aX+zz<4!E%hx&Y4yeX7s{gTzOe4$XY?&qKLE!c3nAs3>aIDak4 zeNrUZG|{oKqEM&Sr6c^1x@U+o4TEHkB)^Jnn4aN0vEpW)XgwJ|l+COc7#SClC)sd{ zJvi`;0cVs{+!vCq50(J9YYVhIplgtltsm~D;0f1Q9gcuXLWl*cX3IP(5vXB&FyF`> z4%;t2PF(u!5C;olCGLqISxUj+Zc#MzD@_lS8)BeuOpUZgkH>=bN zG20hvEyjo+Fd|?-J1;(aTmj83Xu{8dJY7xAipHzTe9@BUwP-1zVqwv;wXFC>F7fEg z&fXMTZYGIZww&;qo%}X!v?%G1e=14G?YX3F;j098ufKbFWu+b4bd)``j)T@97&Y z3|-UEFQC&KdF?XDm9=i}62JqL*CAO2zV3T>Zz$?;F=#6?T8LDJ=_Wep%8CBF4vbO<)JZwG^FZn1}P*4^9w}qw9-=hya z*p1xm|84UnP*BR|on7GPlr^H|RW}~HKCTq{U1Y72=o`DaJvKMU=~|^;G3rJ@tuD6m z)D>>K{}VNaL)BG2f4vOsWw;v&YbyC=t+)l;E>HOj3cYRql=PKEo#N*9pjM`;LQD9c z$;C%gv@=G$K!KvDjdEylqa4T0`yY)GEizG_mXt`l+A*%_k#AY_-`iF6&|{&Nf7)2U zR~5xdg#Swk3QH}2Pubd8?vjg^|NM4ItK9yfj8%paw}tu_k6CErMCk{~stYE6=XSvx zY6F^dm%AX|NW6%vWdGUjMTmC!;0;&1KZ*klWAwN%T^5d?c(wjQ6=f+|Q|DHwmO%*` z<(Di$aM==Ua&ybr{q0cmFg`~Mm-_?Z^7_r1qCk~M&>m%XcCN_N9;LmgLX+t}d00$~qUi{0v=Evo#1+X1(3DPld z7|7QbRbWT7AT70^tW$JOKv-w1SiPZ+ECqFqOQ1K@&aPQldtGoG4w(LRL1Y=uGjhbQ z-TXfgcW@)-fQP#QC!@d|BUWr=Pj8H6R%|x4(B2p=ShDe{;(ssMcrFg`&Nn3imY9jH z;e0d`gGFg3rdsqfjyv%&&BUJYS=px0edyRQnHh;a*viR<;f7&+Blh#p9qV;MNR#_G zwZduw`v~JkB+6glb0vv$jq;6n`De5I!kBqBt(G}@RDk2QPKa(ga~@ieE|Vr*X4#Mu zag!V||9f94|AjA|dUsSe?j2nNgh3rr66JGd95RliY1kmSe-BCaZEME7E6Iq{TAOjJ zB*(WU{CM68U*D?>Gj4T!9w5|iHi1vi=$maJ<8>C}UZtF`Mw6MR%UaDf_aBG}lQv!+ z-rG^7s!%t+2Vm+laS=^}ncZfS5u#+{3)7}B3(i+U^USfjJ#(>nlPAFLty z7;o#U8t>tT%UBhGLa4EHp*JTca6j`-!773~D~Z9TV`*Xnowe zI4;HDl^@+}>qTnnX5_JVn0oUcINVf6k6J=A;iY3{Eun6WBwZ#oLNksBBstU*a z+CrvJhYxP74H9ZLo&?#GQQlM1^WUTvKbveVLrvDdAm}LYq8fBG!EwX{2~C7sj?qCv zOv9ydWuUe-!&HE3viEdKJ{KIvf`r}LrGs4hX4MgTB`<6&Xu9JUjo$?PX5vRQ9bN@| z)I`vn$L|Jyckv@yyD&i`;P)KQ^$~{QN3_TIH301-!mseV!)cECb%l-Iw@4ts|wLpl_5U$*+Jxq;ASjvI!4-;lzU^J64?aF;qy-o+BLz^-yL1 zoBxQ_#lNebFs$iI?j7^zXN~d87aEFxFLs=yaMuopQeOyjR0$U9w(m0! zO+_Az{`NS(wT7Et{5z|OxHMz(7MBn8f$GTUNDda(R9XKP`R#RBVWM(0-64bswJi}& zvmG6RE!rgK+2+G!UrV<60dY|aa4SCk46cA7%@87;F@$AM$!&&kq~jvV{5bq@$!I=$ zhxvo>8%cR^#RiWk?fiLWWH;Lk6D8ZHl=IAzZ2U{zv+Yw%10d;BtgJLGr6u{;*28t9 zE87X&3EN>Bog~}g#}vTP)Xzup%U+H&qY&k{g!mnMjR4;VdE8fp`r|oAUP`$^s_o3P z@n`B_`bhFp5{Flvak(+6LMap5gK^>S(b4kaaV)rc{l-to6a09t>CBbo59NLLDgyLIVrO8F4czNYY_qhjY+D zYA(t&!Yk7lJm<*E2sl1_HY?1k2sR$Wb=5P=2rK<3$qp4?FRgQJ2=L{2r2V$d13^#H z$Zq~|)UDAypzskVK6m){5F!#(NKW~-2UL~6uD=au#NHeU|`E5`0~Zi}(w{Yt_s=es zuRB(m0I@4iwf92LJ`88<=-wr&5%f%p;0mY~Lo#l?nV{iwvkh-#(7Y*tt`@nE%I#A_ zb1+-O)kS(gA%JhB**1mJ%GD^-CX_kG6fmdaK}xtUX{lscH&8%XE9d7c7BN(?W$i;8 z^N*tZ`+gbU3s5r%vF~YE)$P}s?8c~wo0(_b4my$wkJnn@pocf}YSPjrykmGa%k~0@ zig22ukCJJ0OV^@l19!d zgdla!lyw=8loZBIktipI8D(R*0dFqhBE}_bR%Pa;1>%B;DTb?e!I_nPKL9W?j#4o- zskQO_J)C904i}Z7O>p@{E~2-0s$=xWUAB=zS{3VJt&pZQoX*-s8E6*cIG-f6vh+p~ z>EkfXq%*jZk8+-sY?2ngfOuO<;P^9fbu~h^R6jm@WH1PEHYr|`k1Al4Y~=}dh*|!G z?1OI%#>;OP;>k!~gTtOPpq%5o^>(V7c)5Zh|Gtj_bs@>MP+w}fEuz^rOKZrxS3W*_ z0kn+9y{@c-;FoOOLNEg>jL&{0>VC3F_3$>yF+B6dXH2tZfRZR@QH3PN7(;YcVxd)j z#4|%wTKlwlm`}i~QfO%C;;N%w_R4uCXF;Ofgir4P1xlG{*@ddvWyTg@3U0}obls0j zasbaM#ay55hwNz=&_|l(qf|&5!oIjK#hjZH($ntSRf;K?g`&K0o;En~7E6)OtnVJ+ zvlS{Uc7P`$3+5&K3e$CLb@O?jAtSxLCJi4|n%^YH`j5qDc75rKUrjO2Sv8%fy?F&K z`Qf}g5c&xf_C?&%{>-nHKeS4q^$e}bii%Li%Bj?wF1{4dg75#)d> zavC2(O8H@9?1N!hlDPpgJ-+jpAM+Ma$I!;@J}`R{44dTbK1ubXurSiC$;&ptiDt&x z`goj3Cs7ZOC|`<~KPja0&?|)f!^wLl(P@Y5@*csEufTf>y}ZXF+2B13w1#Kj2IYlj zWOsxA%qrzS;qlqsFlKi{>1TzV?p9a8lq5uLDLSS0dAtzkyuSl06@y$kH2=P@+Eet? zWy+vo3p*Ywvn0uS4&$Vgns+>AzBXe9e%ZQ#7T&&d5ZNZle`1Xo&HYigv@0^hUCCp{ zkN=mR;s)<2_95dz{PJ9tRDkCjs3_1;QCMI_{Z~JM)gsl8#jB$jiTw{Ov^t^5+;FX{ z^613W4eWhWx-${toEhDF;377fx6O`NOF&O z;dst5sDnLdP0(P}vEIXe3+JG2c7fO*jD+~d5Kk5fqazhO<02%VwqR4yH81F`bvpRG z1$|yxCBqp$g`mF7BiiaEdMBQ9Xy|$aJO+6lV>m|6Gah-J@E`(t^7zZ~d z^CX4H-2$ye&eV?f%7KQMi?iE2)_u4jFzw-Rm@o%x#}KpK7y_pqi8ZtGGmbNLxTsmx zh}+xKwh5{A9_%(aPjW{xSpP;(h1CK#@qJqUn5-7`&N_qD^6%aARDh+|!Tx_WvCRq@ z!B&YkF?8GHkeJMPx3qqt^9c1VzL{ILsadL1n?aX+nM=YHcY!wU*?0fm+|U-ou+>3L zq#JK{p0-v=S#gHi8chgBKG2gP_~S8LlY)~T<0E{r-|WLlmFfrkL~%TGw+4FR1>bo=9})tvcQLC0|JS-;#TN%9wFHMhy0;*?^p z&4%%pg7zw~FM%_5){|_5FitCVP#`SmJ^S-q@{3vQ_j{+#;OX zx_gjqc+R0k%&0{9brPK@movy~2?f6Ap{z=PvNL?%3JOTUCj09I#i{RwoY?PEs$6$k z0309xN3~YxH}fI9BHvW6;Jo~x0+^io-@r}2JWrBco@PCnM0Y@G;TG3_0C!4$l1ne2 z!&RcF>O**!mMA~QC;G`BMq*k|9h@J>QX%3O&SS5Ya5r&8&%FYvS~w4h3%b?-d9 zZ4*e}Ucra$0SsR#t`Kie6b==EFWzp{0%E`+Eh3*11+SD4QZQDLWc1GkN&vrkp~TBe z4|2(7LIIEld}PJOX|g2&PVAa@T(D)4@#1Y+w2D<`UxUNQba4i5^g1^yT}d4_h0Nhv z4{)F@QB?p-RpE~mp21mwWhERIC(rb?ExyKuxm1gJ)&a7G?#zzF8lb?v2H3};`u&61 zuj@=2W3O5M9?e>}KY&`bG2$14d@J*cYdpH_agQZ95_Y>s7M{_pFDX;=sT>{6@|*w@ zF5c3_!!(h~kbv2;1?Sdg+q4^}jfI%VS+I$#Ua_M-bL=E4dF46SAjkZ^MmAlLyMd!@ z5wlhtWI?t8-%ed^m&;`^G4tP9sw0$80=z^EXOPgV9ApOvxi!|^4{jnfeKtn znHtE$ph6`Iaq6t@5Agp;Q*D1(!8(lxpdLE>%7IQ*UeHD78eJ zY379&XQ>u`U9+w0x@I#Evq=5w-=nE;z37r=0{|c2jUS`pmgaTpTRqIY6TU>u?)rBh z`%%C994hIqW;j|z7UmFC#kplInZuP)C+=~dmZ*4Q|A7ouvLK^MvGcw)+(O&~xo~cE zwR0Vi3#(V+10VXNi3dGaFJW?B<~`nEOLng>MgfDp|Bd@-o%kCj*3}@e{kJH{HsGd%4SYT*UP><$VV{z}?5lB?^$*0_87>KGuaEyTy4l zK+xdeb5Z$yp$e~c+2T|x1&#_MH^{jnFLyH`cg zJ>&D29O{3L&nD{;-mc(9*LXEuQf8LXf@@@6YItRspN8JJG->({h37DjgFSSNekO|`f#i4QC#>2R{g z9>rkWxv)0JysiX5JLl#AKiXRgU|08B6Xa%NIU~RGiXI2p%+7NJ&JN(%?so5Mv|i?! zNA|@t^x?1<7ZKj3DIpfBe3GhOMDV(Y;OH<^7+}d?>dAZx&pGlVDS9vU@N;mQb`je} z@g#(=hT`QO>P~Rw0wwx7HuiAa@Uiohd_jugYp+??J)lfpa;K48T9hQw=DTalU0sog zxDN4fgAi;{*WB1ue+<=<>^IoYJ)^EA+4}Frlv<`kF+u*gXreFQN=LUG46y|jPu2f%q;TFWqJ5sv%TdplO_L0U$f5xtf50NdCrjX@AyhS zUy^2b1CwOmf-}Z50*5zmWW1cBHMIE0mVQ3omWEfv=HXA>rQeq1a8nDq;Beh!EpVD` z#(UTaH`@}QCsw*_w)Mxp-A%J?xDLBHNwoACCE1e#X@e)#f)8^f-ZMvwlCIl1d|8sR zg7VxPQ&FWnL0o zoWHh*Zg3hk1|i+ohz);}kQR_Aqz95xlRYt{SfRzO5!CQ9>OhP#$CT0qqGXQtQsfkc z1-|jZEUhHW3NQ-`xE9I&ibsoE35cbXSnNg=>XouuKrE?6Htxicrg5iSV2WAc>fOG5 zNdt%Kfa+djN2{k<4aGCH$y(GVKj5=amzys+xee|_n@3vr{UFE7WT8QWgx9EIdQQZj z52Gh}AXepmqiDfw?}OE!FT>Gel#pD#7HIL_=dZ;tDNGpi(TarCTH$#k7|n?b;k=*bIo7KOW$q4wH}lF!X#rG2;Xh^4PR{n zH?L9L@N()47dT!YFGO}9q{`s>hv8Yb-xzEy6!HuG=mQ49==|{R{u#7^o^O$n@ZAF? zYXChh3QZzg#h%GR600(DC5Cy9 z^i(0J0~wbJ^2?5rI^G5#uYz2h>(TLKuA}p0VL(uG&|EE5_4~7PFwV$tgl0I7jS|{b z>7wT0>blK!Je(}Fj8s1n0ev^$DyfeFo0q;L5aq6{!a0s!Q-pp&L7N+s&CuxTyp)bf6v^a08E@co7B~h?6K3~lyNz(cX~W(GF0Te& z2$5d6DnPXxS1Ua82A+prhWvKX07tdyg2AWiQ;lQ5bO4Q?T@3Uxc%1sXvmEktAv%5# z5+H^H9foJj{6lE=(&SmWkTe|A=AoS>Z6vHKLJN^oS=NgpnLNUQpPuKy=a|m-n7@T` zqXI*=mNe)Mi1h+4%<=$ljAss~E~({5 zK=LYF6)!FAQdBLoXLxJT&rr)^2i0AcRWn@edXuXxv71V2`2%9TwA5GgR-k($-Dap- ztkb=%%bU|xEz_ww$zq8cOKR~esin1=muu-eUDd*-d28t~P1UjoRJWE7 zr>R=r+2GPLd_zesdVu#TOBXdS*D`vV3JFcJUrfpMc_@gfRbT@5O5nJ`HaNQ`q(I! z1H+$g*{jw$rp*z8EH6QjPsGb7jI#MOE+KV3t&B#-c)2&(O~-wp7w&tXPYJ%T9RFyj~wo@=%=_y zH$9|m7pcHQ+Uz2I;vszml1s%_5A`cfh2A*D=Ar(iQd?^@0m^I-S;L!F$tz;klOZ%c zA925@!;w2z80(YxSmOwqCxo~728-()tam2jNDGrME(g;WoWA?_fXqWl(275Oby;bZ zdv3&l7A*=oSX@9Z=Kgt|nsZ)mWGs8>+^_)!1+~7DX|O z?B`IUacW{!H72Pss~SsEV~J{PkQ!^M#zv~K`!E4uPgG-f)mRE*WHX$~Ms?Jh}Bv;zFjID5C^w#tXU=`cZ|~Qe}(R&rs}LOgtX?gRaLVpPUS;z5RRF_ z*|chs+$~Jb&L-EBS6m{uR{EWEG|d*m#pOdBgR_OMzNR6#z`8YC2opEoaGcGCx(_AC zi)^8tm@YY5*@P})tmK$x6S@lFjy*P^MU{B85sV;w6p#ivezM`x_ytG6azS6^`an<8 zUjrQx%Z2_`-cV_Lh+a4Fz}n@)QK8Da{XL8q`X3mz0{7L$=aCMFU5Kdk1K?&Xh{6f! zH;$`z;k9a2yEITA6UUAEa}sI2T}WL~JO@J?w)Wo1)S)L0$MCdP5fnnr%_XRUoW8?5hley`u_ z_516)U$1pO&wAFgo^@Y)?e(m^&lI10cyny(xYBYdjT3jbmC&KRa17sHT3jytvOif} zyXeozk1KsZ{vBXk>BNK~Z;l_auc*IM{U>Y8|666wg?C$t*6YqScUz~OUU5<;$g!#Z z(v|g6r!6L?WtaG=?y+&%C0VK1t)sh8$Jji%!W?GrbstAq$jh-;qcc@1MI~~{*p~8#d!T?cN8-p%IZ>k zho$*rC)D^$2m4FUBnO$XlIvnPw|)jq`y;e z!z_J1{UVav7km-p8Vf@?BP}cTeFct5HwUCn{Jz|}+S6gSWYALHFIM}K$VIxj1>gg}N!wIam5?b!Q zKy;KjkF2&jXU@YD8fAAumvh>*jGf!;(n~Y5ORvewE?wl$F8y2Mfv3<}3Fqi)>)dv& zJ+a$e;AGuntr&j)`J$ee9ZdR9o!(>u$|N+D-VA8wrXP#W5cAxD4fo&6o#c!2o!<9a z#qFHQVZjF{J1^a9`P(IVc;hEKt?siv8!&Z}meb?@5@^4*ATM04T}-H+L`F>7+VOrX z#j-D7?+kOSrF~;d$NIkGgPF`-#WLC_-@)r`v*^#IckL`-02F@_Rs1ujeJ43zIM%3? zQ>OvhzepRi>|3koqA|;U;o#Jh&geB(PwRGP)*7q7b(vGT#>$S(?$7&qxha|1BWfna zj@|Bjv&Nd{WIbRFlQ|-wC!7gPI8~AbXXOLdpmt0B;j*vrJG&p?6`gKQ{R8aZcOr=S zUCaXglA6@)s}sVJK8NOj{{`Jk-<908A#;n#{&jj5o4~xXzMRBlN^c`mI{y_sA?>?P z#ep8GC>a`o;(F;*EhePNlOIQRK9Kic$22%^thM^aT>DEygFY@JKW0C_B~upft%djV z_Yhrnc*fK5I3dHABPQ=1pZ?>xm5ZO^h|KuXuf~;J`^LCoPs@5JcEwlxUcmv)ZZpw@q!f$xDRyPVlj!Q^b3E#I>bV z3wAwK?@!&sdjG&_qL0W1`S=`pUY-;KSQ(FA?Ry}_F%0UTM-g+N=Oc5V=kpSMpeOkH z;!`s5pwS&@Vrc&3jV9ASERrXa(c??^vrD|1r>c6++pYHok%7M8n|n)pU;9(%_B(6lC6NyRp+iEm6z{6I}` zFdDn?Pm_k_$HlJTPYH;8X!P&2QNWX+RQ(I4>bKI~o~nPKyYq*Ktd= z+EH5s3uzIWNj}0m!F+^Qz8ck}0xVB5jRwI?`25kX-U&BZ^byk`xyL3DP3kC$!0g~nlxf=T@mQ1N-^uXtM5o;&R+m6G^2cDGRC?>)aiy2P z!NJ$XPlds8qE8$dcu)IfCuM!C!DXZ zH4~0}I^8yt}?%9~Wo^rl%Lm=|2`(v=)s zfaQvp>z|TdH>P3F_z_!8Z@QTAA@8q0N}Zd|bls_)iO*0qN$9eFbZ#tcV`ut}?@)yz z>3;N@j-&gLoU#1fesrHxETiN#DRbE$Wr-NR*BxqlpSx;wc&uCtT|T+=DOvsTHg)Xs zbb6fhiE-3^voW5ev(ogI)|>rF5nn7VQhL^0@7dF1og^$QuLZwfaEjiyJaSDs(WJ5) zqj{hFooo)Y%7~Pc%zy3d%zKn|FdI?OzxrQpWU3RV$(!BDUL7Qv2~BZ&s{KoRv+_$X$}YXzEV#{Km#;DV>BBBZW0wwQ z)gwhV9qOuVKGQCD3eY_|a&b6;fKK5iPYR8Eg?R=TWJl$NvED#2g(N?niN>B&>9DPuXRk+fxdDik&)SrVg$4I8h!x zA6@obN(J8&tjdq$Iitfs6!fp-9$Ot+3Rc2r2d3^-fTdudpo(fqs=jy+vm|DfV zl-v}usk1Q&yd1&zC9-43?P2M*hCWn*TaT+`ow7!j+CC980{VqT{-i@b(w`|mS>sj~ z@t!B9KVL9)zVz1_$eVv3(35tHj`AR+?+YzCu(esvKa5+sITaAuOf&OM;`^Ch1V3!^Z?5$BlY=JlB~>S%%Sx0SsrOO zclXt{3rQ!jHXC31^*HjLGk)0S0-myCSCqF71*c25=HZ+&%ycy#$PUJ6mFG6eKvH7{ zW|0B2){A80&0Xu2$$^ew{5&bk9YV{CMo!&2tLtUYvWcU&clBq>`5P!LI(L9@8xsxQ zp{2rUx~0ACYRO=;O1?zP>VF4tW;|i_unRjlw>)9>u@kN~HTX~6C7RY?U{0ve<0Y4# z8r8Pr>=%J#)j>*%X{UPbSN%n=W)?2BzuW`nr}i6}kRNtH-bpB&o89OWi*aY`c_=9bUy`Rcx{K zQCKpD{rm~jsF^W8j@zv)Icg+BVZl68@7c}LmYiP88plXID3#=?_ioLk$u=)$d>1Ki zQuyRwE;3~;b=i4k5jPayvh-KAc#1dnU!{+ma<(4ZzV~ke#Fg7W~|E^*K9ZkD`TXh!Xrx>0CufeHUk9FsKtqo(1^jSWtbDtKCN|iLiqdB!%nr>oAx(77t zguk|xxVjT^kLS%g8995#7sG5aVa7{MIU}FugBh<~uKU~NYjo76<9riqAyc$(CI6oO zc6BppS8T3sk!G1XJDe%!uTNXu*-QC_lskSUJ2qKA?y-tym_+_rptF*DppBHGX;zN0 zgfLRRW?89dCav6Fk7m1Qx`}g_(LDLeX0pz>%!#sOs=Irk@=fE?pd-dcnJiE-y4TGB=@zu9c zGeO9(#x~7U*$2~qcY@GDx{jKG@q^g0yf6rsn-#mA472^Hlkn z(asysTYc@v(wuLfw=$FcX;K-OZ$BxDmASceYK5Oy-6kY4oksAcOvz|x+y<-HpkBza zA}^C=vMkD9)ypXkbj!{E_Fb~hZrq6HwZ*L!8?0+Ad;4onOr`ajJ@L7%`zytscl3bs z=}T5WJ8g?|@0RftjaG*JY>e~pChIh(a--GNp7_kx{Tr=zmi^^^ z6YImx605H>c#{=xPb+aIY$9rxcO+_8=k`rRd9;nh-)Q&l>s+`Q3vNPJ=k+a^l7CKh z9^PzCYgLg-c9-$rVNT3e>jC?-)1CEOsf{-ijMy`$i&!tGXBC0_`#7&u5xBLB30%|1 zY4({P;FMqLY<|xv>F;(sFxv*cneL)|m<*U|8 zJLW&ms#mR^_WoYZ##gO5cBbug-exTb#7)zG^2HzG@tr04X4jLl2rM0gEp0m%C7vYw zrRlQO-JA1G@^z<4EynN(r((I|`Q@V>S@mOGR=?;IxIeuBgr}^c>U71vgZ`RH+@@BvMs~AVlvs)0Iuct?U_lA+( zVZ=-1tIUlt!sL4|9fD84h|9mxy$r2_H}SjQkPC#|Ps61K`-kLG`Rh9TKQKiEe~gk+P_3y) z#ydOSu=)lPB}P{Ik6Ann`htI>G;iH6a7;@-#}1W1XwuZ@=K zEQ|30!3S8+P@L9P4TK#zY6MOuX@Pfu;6eoUM%Hn<%_i`>=&UW4PBC!jzThmXF1VB= z#KgHQQyWxjg3lvJsXO~i>+P9ydRwIk>^F~n&7QQo?g@@ID*uI zDYC{YJrV4z@lfzv)^Tzaue1viuXg3-j6=bKF%+Ng^|T~~zY@;3G8RV${{c1Ry%(+E zx9H?Q!;_F4VFaTQe1+is`d`+O-RJb%&S9%}A5!WVR{ibUwp*R;kq}cv=m4Gk8^mbIA@XHS-4+Qz-4vC^?$Fc{Vmv>m{f!PG{T1Fj?F`}`a$E}!4q;ll@im~k@r)s)=%w?bq zK0em6E9n9Chc% zMG?#2TilKgxQXb1^tejPh&HxYr*3N{qaT;PUG zE#jq+m+jPt$gnwAsVpUbJUXPBzqTri`9KnE=>0W2r~A3*2`}a3yI3;pIk5Jv})-{!E+cO!f5F9 zz1~56IH)D=J47J$#Mo`<)W3@}cds?t{<5>PZm*SQJ?DI|m%HJL&d!OwR?iVrB_dCM zSq+`0Kr%jB|M*F_>ZC~XQ5F8Ti(bNAy&BM;AdrE;Y|yD~XJ<-{m22T0act**!ueP_0TVChRIh(ClU^lDio z=407y==5<%XY*UunfBU_&e69xW$$PGGj zr6x&f`_6E^%1ym2W|SNGBpF3=u^aiCaO9ldkIZJcq&)QziOgD1k7b!9_@_9!iM)u# zlH|x81nWO>i#98lvdDWRRP5*$e}RBVa1J6r?zDC;yn5f^ zbggCCQFmiBS#0!Wopv^cw>055MN2tSIy-CkTYcK!waVPrVjmk+g7IiN5{GQZL7O|mE%1Bww0cEro(bI z_LVQ<7cJNuIiuVt-5YsvxpU%eD=m-;XIV}3-pDQxG=utae9%`C*&bg}$7MCXy^$yH zR*LSZN+N%S(u*6~l;uvAnH*#~7#4Y)Fh1_Jv!VAhapbJ`CjGlb|JLc>jb1)W>W1D` z?XdEE~XZ?675SN~?}-(>w8uYW((iB;&|_cZ4r zO+TW4*X!Re`Aux-eNcPkC|yJEpr-q=Cfe^@@Q!tZ^@6kK9qS6~PUnnT7USpFT35uc z=iW~#_3v}~_uM$Arq)VpmoiiCA$R6bSx2I$I8pCfy<*;9D9hyWOK91DU+WBg*BWxp zJw)2?Z~CA00nH2lM(MM61{uyr zGA`4}YdY0KclXe79y-E9pO_Jjd(=bMdFX>4x&}HBN_ZO{xy6Pq_^DC{>s-FS^iHNp z?j?+z}u zdJf7+CdpvA)L6VIE4y=^D5*gRx>)FKhL&~Z5!G7r{lZ`8Dnc%e*;XCdb*U*iIo7Ry zyuB7K7vZ3X9{IDb-a1j3Be``GT^@?B6{4@5j<0pibHm&W`y|6fEp>RU==TQ@AYq?) zMhabaXT8wA3pA~TQamyV9+_Xy56dWz%<<`AJ%>Cpp9%tWLA1)=6hrcJu(Fz8NZRKlyoOeKk2-G(CQcW(O_yg8;sgWO8Jpv1A+ zhdP5lWM-c=)Vbh8>-5WqB1W4O6NZQ!Gdwc4&MfBL0hkg}pd`q5ml*k5Xe?xV%S*|b348H9><^*4QU&ySMsQuwtu9fz?v)r%p z6e&+(J7sTaZ{&JZGBL9@iM$U|u04^rwGomVQRWg;$f1ha5Ib=}Aths7k!73N;cX8yoCPIaPd8aSKxlvABs4!nSZR`CJ^CyZ#6l*SZ)=FwJcuUo-xD<4n!%a&gnRfMGo5GKyWycX6VNLtRX9v7L*@ z&N1=pTR85h^OxWdINTs+&wegT*0?Ba>BM&WxdzT)DaTr6~Pri+tY%yhA_29i}4 z^S!FrVQCEU=fCLs<4pMdpIN?`ib}(rA%XWKwdfzHiLrGwKU-L$rtm`JkDFpJtDV7v zubJ??*9}fxaLt12Zd~BIe97F)3yZG0Zh@O_)$hp9nO)$UJ%9dnb7vdj%vi0rjS4Gv zzWvO4J&^giNnpu`23NYc#>FZZYh0{zvED`Xk%<@QVycT-F6O&f>|(i#6$W!vmCLAg zG3cT?WE4mI>?Yu1f{Q*E(_GAQvF?Pm^$&H{eaTxBTG)k_vnIl3SHpR?rQK=k%t*U+ zbdTi7dlK1KP&J)wy`dNcxgV*l&bIO?XJC6fxnE7Z(B_h7B8ej%sJ5f&MiL_Yx{l7$ zSo`8Y+Ud5E_N)PgUk|2&atS|_W=Jl_m0R)~a*{%IiqTgKO5Wmnhw~HC$3w50pRDs^ zM!PIB(t6n{6O=0?C?ai*pyo>kefa&_7nT$LlB93}*^E{d`16~|iQcU4VYwD6mwO+K z<|P)v`fg!CN#F#2A1GHSgTnWF?{i9429vbrBC0y}e%FX-VqZUkEAuU-ZQx zx6)HH0qOm}rLZ>bzbnZahNEOKWnq^SN z+ekfex&2?paSn{;)4T|78z{f#(&HV!&-F{qNr~HmpL!-Pj2|ioIlk_zuaKNzlbKM` zuOOmNi-ji!NwWD&nD3DbDf}%t-|xs(4`@~p{O5-f4k?rp7BmxFiNAU87gdG*nkp>r z=S)xF#+KRl)B-gxxy*n|xTjQua!I&ctI}`1R8Ta}z-^1Q)wTu`8VYL*AA^HRh_h9K zmJG*>k)Y$m9zfuLaG=vI(OwXdV`o{bk~+rJzHgRZhXhw8#XE&v?FeUgq8-;d$5!<* zwqiNEDk)L(heo#xmovdblLeKBeCT3Ryg}$JTz!o5Ojo;Wn_9x>5r^fkt1t3QkBV{+ zEm1{0ba~ayYbWZuN$%>1!s1x8)pfHY&OYH$5l3w`S1Ib25HD22UbS`YoPOPm+I*t2 zme!fVNQJ(jk49J!ZYod4ykg^mT}z38zQ`U@I39EAv}3v zwV7JoxI=}f=!<#4K$E5(kLyGJl!fOhb)l^)q;t(#v`FMtJ{e&47+C1WC^q+zfmdE} zRl%Z^o3u>1xv33YxM<`#jY5^|f zmvdIaL#}g@J+SNY?Uq{8!d6R>l@TL7UU1tEXJ(Q;JP=KIK4D4GG)ZnpRYwM=CDJM5 zjsB*7Mp-kNL|>HZz?)$)MIBYldr>)}FDIlgqk~H8&K!WtbM<-sR&12hCE4y9wUA5| zws6i(w)=Fbbp^AMm70pHjdyNKwg+}Ld@@iJrHcC~^#YEt9m)1>I=-Lyz3lYiaXfJ% zr5#;QPd+K|vY(w6n+12`7t{!y>d)!ZL8Ml zQ8|)#KNXzVriE%Vwq;IK8lSJxJSREDp3LP8G&V5lDtb%j}pL4Ow#a%Aey7;M!K^Kp^sCJq7ZCy;* z<=oKQ{`%r7p2?C(C-QiV`tc$jYI7OP!XwYk{dI&3PvRu{B2@J1h{6{6cADh13LO)P zBY8~QPvDh1*>&4_$eq^4T4gz$ z`r2oA5MA{Vwwi*N=!)Cz?CEO{)FW4JLb@Cot?h~urH0?hbit;9<{fWii@&Z^sau3c z{N{ddoWvfHs;%{-6?oUeHhuhWNzS1Oe__!sE z&v2i*i8S|nB9RH&i#Q5}s;NT4=<28Noxk+6lkFL&IVbzsorljm%@kxyId_p&OxCVH z4ZrYa;7=UbQk`Z++tIaA)w+M?=mtB#zuooRDv_6|?r=+c;5|yc$$5=~u8!vZ^3}}M z$Vi;T?h&fB^-oKxhQ{B3uA#mjAzjC;jcb(p36~)}D!RE}{=7+ez8=m=;->4ky1YMA z4=1c}cqspEo9BPx0K4nNO^s=`u^(T{jp@(KC~lg~{c(f~PvRt=7OC1;zgV97*nqz2 z-Fuq+RzJ$CWRGDtz;EiWgB@Mxtv_byI66bGDAntDoZanUdpkO#n_a$6seVsT@2;M( z-$_1$J->CDFYQXZXS7lEPt!r4QOfXH4>kIo1!vfuPm!;Cmf7IBuzc9>Y&gU2b9S1~ z7;%ozY$uS1R>0E+iZFI~@kv$$jqCIv9?*5vpz@>Al2G zn5#GJccvpdB zj!HFz%5Vd+K>CrIee z2j>ceGr=re==xxfvm)K@H}HAFBu{%?gy-L8F20YmXNWy#`duzR?OS>}PWzeO zBMVyb-)!9PS)+zzn)qEA-}g#&_v>`6?zC)A+qt&D!wRM#pbqDtuy4N*(@AgUU}BiDz*UmjFu`ky-lIknHv@0 zDaBIH;*Po*@cI*bw^AK5x~h&=2i3N|b)8v}nOlD`2_*Jzt-7#?@3JtizFl40+Sb)Z zLY4+H-V%Afi<)AmN%i=t9_d$E)uvww7RP6sX>4`8VeSHjItq%gC2dXHwO$xiXcFtJ zx@t^({{L>87S;)sXyUX>e%%AqIRkY zPj}X6I$)H_>TjvbaJu=VIcPdX6RPH6R6rA1>4bEq`~j9K!#$^;fF!M`Zq1n7JR60a zw?^7s15))qx9am5X;Q1`Z)}Q?UwFQOCj1squEU@Vh6{rud`BvZ9+#9~jxS~udLG?Rg;piqTrI$mFi zx1M}F+H)BG!Iz8#Op1QzcmgRj2xn zb?s|ot78h==C`3cN!&a)i{*PQ=Y0&N~po0C`{7bGnVezPF2EdakizG$Vtd}{fUEHt6mwM zRmY8b8mJ8|{FZMg!z3Ph_Ndkw2!uy@9SS9(Nux~$3K;N01>Aw(-WRQWxouQ826@bs zR&^~4qw=Lhbv!*wyjoB1AeyRl&9ZxRm88m%Nju*}{~VO-m-EBn^9c7{V8Y)4<@)vlr}Y>+xrfBhB3!Px z93vBTp^-_t(3v>KO(OxB9QcyOh$$w#=M-nD=#YHYx3U#0OO-{K$o%;tBlF5dVI4Kd z)WetXb-5<|ja=s!(nuB^6@<$&Pp&$`MefK{BNuUTSkDsV(k?dPhe5d-JmEQn`=*)j zgP>eLObf?PB|I+Agzp07`aIA1a;#e)ig0~hVq`u9<@(1ZVVOhGZk z&T;Fc5}6(Fe3u%Tk{L#3-3;fkkjzSCY}mDgNqJYzG&0Z44Cgl=nL79qey@D`j_ZY4 z;qXktYvD`y@_ZBi=lpPZ9O2dQCA?_13BP-GI6T4tC8WpKcc_aiuO6a!qsQkDha;VpV%W>^&mruWXL4{jkNc<`bP zlK6?p;68r~4{o?RH_uJ|2xh2RO@!K~v{fku9hq%AsO~)V#;p!d<V2{>EkDlqj<}W))RUSUoO{;~PoR@0;;V zoEX)5b(C5B7PM7yD_FMU3UnN%aO02E3@2^0ouG@C=N7N}k0$x^R+{X51PXO*rNgnt zWZm9r$gYJa*(~%_rTA~bUtD5z^t#jN_^Qn4P=7M~f>mbFK6#JfZwDoA;#gU$bheYM zwsyGT*j#0;vV1!eMO!-k^f;a)-3Z$%OakA4qTuXDjsFTzD`aio#pO+>rad!Y>ruRMN@l~$Vmpx6H04UkAK|L;kZrwvNX2Pjy-t{1 ztrEC#@SUCnR)q86cpl9qE#FS}Kn9GGrKFbN05BrKO#XR1eM z&X}-{W8=vGd)&3o4)Z3BhpuIA@$e>|%UKn8{?TE%WAN(W)ntWv_1Sd2d=zGS5{aA0 zdIMgnM=%{;^}E=bJqmN-`Qg=icsIbC2QR@B_kMVl@S;7uE$}Mfarz*o^Zy--|G#7L z>i>?#KgQse>;Ly-@!9Ti`2X+6;#=@EfUgy$cv-z`2hmQteXJB<3IyNBqFe~1C zjuZ%1;+8-Qeu-NNT@QZC{`?xD!2|Kwm7qh%0|IhhK+X;1vLEMz7F>wSfEGN?9^E)- z!6ezMgZ7>UkTU{xJy>GG7i_>ShgLn6+JU-+WtJWC*$t92!V7in74B2GfQF)PirENp*1op&E?m^noPc18>6Zf*(GW zVC&NfQD@N%@C6fbC!q7e&A4dROtS5pC0oCe32+5Y)>46Th>u)P)-ZBEoPIz+_*esH zP~hc{!1e=Q@F*@Hx*oiH6m1V(4!(a89T3`k+`xMjL5?VNyM&B#3_)=JCA9x>JP_*E z>6j@Jsb28WOUWIy_xypJL)bloCtdi0Z{g-c*MjG9Zb15woFC{m3%d_|A9ycLnyCUT zpG_rT_7@yGM~(?Fn&t$;X9(mh!HjEp*dajhCR_!y;NOa98fd}(iw!>woNxmj6MhbO z;fD~A!EJQkS_-P!?9t^UQ|u1?S+ReRMkTpi2w3 z!PYK(!49~950BRfFUI(Vl~~5$3*L;&gAN&xt#nxUg3(n< zWkJV*H@(iB0bLCCe8bEWKJeM?=u8SQ$v_<6&&v_XjLWyV6XIkt$p-_t$`m4lXC5%) zI}1GN(&`94EKqKpjN) zPh3Udy)GR#q1)P&{uTV?C>1=0TLUe}G9SdPhYp+8ZEa%z1%4HL!7KhsvqBew=YB>s z=p1m|=d^#lcs}Qjhm*TgHTXmwGg%*+2K+Zp?DT>QzECO!TJSbpCUh})3@35bm-PI@ zG$s5(@cXZr(xL0Y?61w8GY9+wt_Z&RCcynTXi8rQUU-C?2LbuuV>r2a)Pc$08rpjx zKn@06{~a}fyx$ZEZ~P~TKo^6V|Dy9kdyVvBtp5dLXbjm&v1^j_<|SoQydt3Y1R{)Iw73&!AfrScf= zHQ|dXKmFfy5QGGe;_9L6!DGMD#(l}?e>k~>llD#nuXO1`@IjZZ0KdY?N~s>4c#>Kn zZVq@YPKKf!3D|Iw_D?0iYyX#X0WBLCF7O4r;Buk8CjjJRz_V-+6~PyL6;}>j4Teqv z2zIjAScNY*#;dti@-_S)6O{+^pe!te#J#07rE<29{|$-Pf9+qM0fZoyHwanM;{ zRWB4kSA$|qFNF3Q)y3fcR6k2S1Yb~$<(nif7&5R6K8UM<4+hj!4C8fp1TV$?3LQ3< z+uE|;{R~UB<&_e_Je&_&a3(GTIv;F5kTCj$7|Xj3BGd52;JqKW9$IYJWohgeL5rPw z@?bJ}2J^pIp|4A42bh;y|yn{1W55CcAs?X_Yc{R0fyUj0(mFF0dGkBnC$0uptF42Fr6UBr>!ZCx=gA z0}5JK~#HiT!V)}u+(FJy$My8>~SlAjv;2~%+4E}m`PQ>Lv3(j_F z!N*-%FkoR?5h1V1MNBWZVN)rDFZeocIkebW7L{1)UTCqG1n;49L5pdl{897_qrJhm zAEQ$;0v~TmeB$2 zHA{%OBI-G;JMaZN;GTo_fj8mg##0PtKFsJ zsQ*fchK(db1oaZ7!Rgy8cp)wielA#!(<@!@W1PtFwO`eHli{a=|G+Ir{y12+nUMlr z4sOCpT*1#>x^6S`|DY`NQYXN%uNrz1Sd7z)1@OXcm}*dz z3+~2cK#Q@Yy49~ zrh>QQX22J7Vv8M=0@?>&k1K}e^Bd|hTpqN%1=ApWI-uYqJE=KnIcfU=E{ElvAYY=? zbU^K8$lxkiD5imVxCEA5a-cyDGX$BE#ULgIFQEm+z;z4NE+_^p;R}j^N@&4pE-i27 z{SzmJQEzeE#R;7QKKzyyA)c-89&#lLJuLESA3yYX@bUV` zPdu*HC9LzU%UPGVE`Qy;b%pDetSequv2Ojk%5^pCYS$fFx8{k8C!F+K>~x=;bB!bJ c%I7PdU;jKrj`8PJEUBodsI8EX5-qX*AN9bgX#fBK diff --git a/bin/VulkanSample_Release_vs2015.exe b/bin/VulkanSample_Release_vs2015.exe index 3958098c3e1dbec8ebe9e517dedb1712794bb1ab..4686fddbbaff9b3073f6ab3fe1681e2ffc456f6e 100644 GIT binary patch literal 210944 zcmdqKd3;pW-S|Hti3By?L5W6%GPbEkV>LA{q0u@6GjgxaNEDE`ps}I`L5;~oVi65a zqM7G*Ds`!}kF9ijN>^#C6mUxbNmyjlD%L(WF4Y_35?n~cWxnsvId_(XMg6{B-`^iU zUQF(G*3bEz&wkE{T(ZcM>+yK<_$8Ab&kC;c&#(S|!JizD=aBKM4)HuP>X~C!P zpSkmdugUY%D&13oiaSqK-#6bS*9&J~RVQ@}?@@Kg5!?&p{&& z@px_+;qh$NZ=ajRy;t4)>9)S-)dBi*`s!H(GzFOa%k|VQOl5NO@>Um;x=1?rjof;| zAC2&wC>7M_dV1vQ&-D}>DPP=wEx8_(_gnuq!c(7mx0kE`krAHzvfsCi@OTTdUktrJ zDSCBtlvi`Ecl#-Qcl(h_>GODMPimNTW%Npqr+P7ODbw>)t_@tX{`qP3NjjOQ{ZKym zfXTsJ4;D8_r8BMAkTo)$OV>3gJ`$k+d0|1)E6mpTtrkMI~XI?S&8xKD$?OcdTT zXN2bivA=@W$(A9|bRfq!Nq{}>Xi zk$d@P#=|8AW_((SHyjU^6o+~AO}%Ia-7YunkIh81q-Ld{OtR>EjUzm!y~A0ROeRAW z>&3Qp0@3#5n7$ydV$YnB?STw zAE}jOW<@G-I8V+D56n*5pJBR~5@B*jt>Fr%D>Ya#=~aRCdL%@)G&CNGb6g zDZosAa}p(}k`VcWAxM#Qwc14-V+ z)>dX!k%C8#nfS>vg&20Nlyp^f9v!$-fu{ASYE^hlUHB4A+#$EYcCxI z_sOE&P)O6>r~pTM54&+_d(-dkKExe8;L<-FS|fn3$BCLm10k9R%3Wg)Y_qC1(!Q`UsUuvq+A=0>bsjiOz;j+f#rCkB8>Z;oE@xn-bmJg1 zcZ(T6FS{{Wz~?pvKk#XkfPSWFPx6{&o#t7>wi?dd)j2*oGFf!BK*6*FgJxA{1JBdd zGd+%dT+qmKNIl0sKJI27Ds|#+=iktUiVicLtKbo*;9=O9sXtIGI2{}GHcXM0Bs+#R z;C1*_b_1SM&zT0SRnKk%{;m^uJD1Y{vujeZA(#=HU+m>UapQ(u(>{8gA~Zv$M>hV; z9NXZO6)Ds+?ZITxQ;?|Ej_6^OQb&ovH#5XwOniP_<_)Ohg%G*$jje3a8D5N*g50qU z$z*kLY%m!;%Cv3ylq@=$zS> zjGiA(Kof=B`NN4|NsZjqn2Bj6enkSA^U0!DN94{R=V0N+=^81prck0o|+vpX1JZT1C`;#P4xl0*RuPeH9~YI zls9<;v5#|_4m4x8)Onh6&E)13RHC;#=g>v~a5G(P%8z}P6FtI3v!MO7>TVOL2`r|& zuhsA-i|*2I&xQsROBFsow+jI6dl$(iUQo7;ZwxXppHv;EOERiq!GkHlVI+w>ebkj7 znVp7#VRlW{FreRH-dx5wKAEgtg5H74uNVRV(_TAWI-aQeg}Lx;RclwsP|P7-17PYI zx%`tyf>0G}&3NbxVNU=EcnkG9(X?k&hp!rOZXz$otmKwf=BxPfU4fUTR>n*mnTcl1q73&@3Gqf z{MTKCNfuqm{E-=5U@YI!3m`qQjmcioD$K6X$wGia6}!UmrsIO`O+ZLtS+Z!H&Sdu+ ztvg_Hrak>Qhz>*^!kS~;xm?~QE`fJ`^0{%>Pk9R4k(0w!8yiQ5;s1ytiO3PoAgWAX zvDx-glTRuz6QLul*vDSem=rpq9e7PYPIB=*dh(EO{`OOW=uDNtj4e1>=UA6ltZGqh zp?J5b$Pr=6h3oJqaqxj=Vn&q0Mw(2yN^S<_rktArxhdr)DK|iIN?o$(o9ij>xnuT-M<{2#)nWA&Kqj z`cnfCTP;NI8||TiWYKwGUYF-^iiTd-2^LO2!dM;}=%dLY?mVPog!&*3ZUMk)gB?09 zP}Y&#XdwgaT4eLcGPnFD>_ zfO}7dEGCO4&{^cp!U^iBOg&K-=wRBRlNFDv&M>&Mc7={mEOrEqg~LsR#jX|>`~J1^ zT(;GW?N%)IRrMU(tyt_o)PucMIMglH2#-*e$QfqECg&iY!ifh<1a~d_(o)NAEVt~} zm}fJE(4_diOd0r_nl+#ZwE8NqnY+!b+R!-4SPsI8*l61H8@_fccc9t6Fc3J>ZK z+Et!TVZT6{WVKIZ!1Vhe^xKsu)iixk=$ESXbZRX&CloY(E4EwBnn1E@wM@H7)k#JT zDwBkU#*#@Rlr3FCi)0`e@Mt3+qc_0UnTLkoSPo-)6u3w$&kIWdiK$FwZiwK{)iOKIxc9x580yEO-Ogis_j7|v^!vT^}Z*>!VdUN+bi8G z58Hv7$W;aB%5+1r^aT=DG9YwlhCghNuo5QJ?7&cadB9%R`!~s(B50#bp=e8Z^?yOp zd;#Oh&1rN$5%e`G^oZsW83~y?Lg+lxY$P}< z#H6#`ETD&ypV=}VDg@=5KGGQP_oU|+sV&4lQlrmM3X{=d9ww&0Ct4Z=S8>^}A}NZt z_`->Z11)+}ksw80=6z3-mk0zB@?zY5oT8hwgW9C#bPe{*_sp^#(Nm-<_vMDLCdOfV zjnXrcMT@Uy3Xw~4n^ilEw$JF+y2&@8V^He^=to;8Z&Y{MUNYJ$Q5&F=J|p&=^xs~G zbk8&|Fz);T&sO5%0-@jYqT#AFjpHJAr>5H@9sejPBm53BFr z8vb3;zw#`N^F`5fg-?ivs-<+;q-`MxOsY30lxj@6~7d5=ykon^61*1){T>3p+*6bxxG~l#Bbo45(nhLh$ zbz+m`dsG;<)`hbvnG4^vev8U)f9wdpX@y@5f}-$$R~3>?lK1GJ+qGK|@?Ny62Irk> zb_Kj-hmzb<>}?o>2DDBJV29qqxr-NFd zi_iBq6v3$0i$f6>R8?9p@lP$O0#NDAnm$MYUrje&~^d=dT6a{tG zg|=!6x|PyX7(+3$LrCLk@-g*t7-_@=ku3T?pii|& zQO8FpH&{9ECZlzO46#teiv%m-K~T3ItE@v4!=LljO4KM~Xxw!NPa?=3uBqWx=M<(+ zs@yasZ0gBwPO!bacc&0Fp@i=%>Xv~9!c~uOqJu@IRoS41j-rNJg&M9Peasn=sv_z; z<)yx_A*S5}xl0zcUZ%-ot9rUwp8l32U}!ksN(8dV-d08SreCheULa);3909dIpQ?+ z?3yD=b>ePk%h}jHm#^s?u?(d%(l3PXGG2ufmzC7-MOKq}HJn~#3<*sKR3vTWPUqs! zfQYstR3uwFO#1`V2qFphQf*rYCcXh^0(w5W{!R4s`yfU}7R#Fo-fiRWyhZ&uD## z4`OPNCMXK^;U$=cc1cOq^2aBCO2zWPABDox_#mc+HOh_^lw=uG1Ia}T)ux70ZsaTN za*Yf5r;nm@VsQ`?1KNM7s}zs zZzwR!dZp38@0`7u@_x#&WLwhyOFd^$SMUQjfHvh_uM>AWV^n+cs?0cLCi2Qek1yu| z+4*Zib1SuV1GOR3cZI+BAEdjZD%L~pV{1#-mEQ@*mAWNgaVh?J(Y+l*~7%vkdCoAa@u^qq!q zU2IX{wqA3^AAYa@nFDW{+ulM4y7_Rkt4ra#-GwnUT-7lz@7%+&jK!fPwVhLiF! zWax#q=p#khtMP5VOPFL`pL+HSXIr%jfa`G z={4=KA1k&ZT$;)DQcH7*Cpsosl$6wxtkAUmk_=+=Qq%PU;2OER3R{;{S(7Y!P?Cyf z_-(PvDVs%pEd)~Q7m55cMRejlmW!gL>$w+FD{ z@IK3HHC?%_D#a+6m|N0}6;uSagZ=|IfFzWL?ain(O40>vv>ZJh56&DgG`2noQ8r~h zR$g)W%+%Pa`MM9+*lux+)iRTu-@m?Bf7rc(`AEGPR8Z4&wP&dO+cLoug_K$13aXtx zOn4_p_V@a|e1{uB%yW|7WYLj)W-y0R1R33&c!2TgRCGep$8IU`h!dlmF@mxUp&&G4 zB_1r*R{7(f9+(!GuGRTJmiXUGCRcQf_IR8qm|&LmAfnZ2KKNkCAn(oiBPAYO;AZ^s zk^-*A@sE@&yt(IT|oC}VKd)5kQsfX?dB>4Zp{?9BUNAzRXg%fLINfbIk_2>8_DXK&God_ zza5l6=c*%556&Kx`@q63^`smTo$rkdRt%=Pi^7U=7h%RqjhPXM;p6Zacb|GwdE z@))HH`h!MksE0q1t^AqZp}YW-5_#VZ8l$Jz^C!_l$GNr%sWV1Dv4qR}#^^_vagDdF z;ICQrdD9#b+{(Jl&cQrrT2AAo+6G_tvDul-vuxz}E_rX<@sMOl%*ZjzcAA}^=SjA0 zj&MELLhd@iXI5=SgGf0Ng_il9i=QVGB&=Gs4mh>&L-t8VPeS{s$8@sPY5SMbbGM36 z(yQ+yOGW57XjL_JI!B%+Lmlh#DtD8WTqg)+hO7D;$7LYYI|AHB0w08QB#TBcg2*Me zmQ<3oBv3{A#sg(&CrlaP>FcZ59#~P0)d_48mDV{TzW~@0Nnt z*`B7O-ME0Cl0}{ma-aW;=VZ|%cp4{+JH}cU>r-vIsukdwX5MiJ(dCoXqZ*CC0ZP7E}%Ly)!FQi&mysX&C9Es?)fy!+O#cY))v!`4^&EGjRpjJ+nQWxV;oJ)1q+v`o&_6%hp3y4#D@pm0>cJ+@l9Br zgChjdI(yF!=XP41Z;cGln~K#H+pU3q(|*72sB~(xbN2||n4RyQEnckL_mf56r8mms zD&r$mE4e4tN}9Rd*d3Q3joxe~4t^S-GTxCEGZ%&*w!dxK&`bNQ`JavOh$K~wb+jEn zxhHyz{XjFMIxLda-SWzrJUfTS>lcUZ1&h#w+a>+2;l$+PmfO8LVlHe1J1i)kJ*6P| zKrMOtE(_Z?6_BtgU;K%e@2B(^_fq=T_FZ~CrAv`;N$T6;oO^o?*m7dTg~(I^%|0R0 zX_keSSUH$2a~I%s&$T9mJPYzH2pS5f2924xpja@a7TqCP^tggFC}YNYVJLz-f<+pE zTC-5dXnjTLnKPGZ$0r1$QuLt`LBMjQYS1d!7y5zwrc~M~Z%UIzbHi@?@0E`1v;9ye z{0)a7z`^C2j6zcCrNU~7Nf3G`;W+Zoz?3h+1{>+XCbP3AZ zg?rVrD-PeO6L&k`DpxGZWmpfL$5WVg)I?~9>S>)-wq7H-pshl52Mk|r+1nf+v4moQ zN(eU><^J}QwN+Cj2=sdxXQSV;=bHhew6UK*^Ls)@>7=dvIj@61Q)`*R{FEkNEqu1@$zF+bV#2qhXTfHgU^g9LaA+}lfhEX>&~q#U zD9DD;Gq{J)lRF4sgM5T2MMMx-zhCZfFo4lwhNy+#b)%b{@iE-x!e8Z3?UdqpPO|9d zsS2JJy-ZiIr&NU-14f*408_8stjduIDR!U5y)D4ME9A#*>#5j|y`&ljVLGAjjPL~G zS5J%2ybOd7BlPezi(ehTxm}?ev92&`Ri7I7cf>={E+r(0!H`$cqi>rVOXl8oNGzGx z^u4Z7voM*yi^*@1yGe&Mkaw9RZ6kOjGs^%_xES9=svCJKd70IGT(;bVS3e5I979IN- z3&J1tSoodmf1MeGBU2^&GbQhL5SC^Q!U|0LYCyoG;#j4}X!~CVg$G}M*y!|OOytMJ zH*<0VvqNf{><511+xjjTc%}2L{5h*<2@{-LP-TXV*y{>9^GZblhEZ=OPHM7}0Sd%E zW^um`%9dl)-$V*@&Ng!g z!&Q^KO$DNs9{(iM$Cl8*gKw9D;N#ylym{Tm=n=@F;chPi<+$SoTFUWqUM44=$y&y){0Q<2xz`4cJHHt=KsQSb8QG+w?K9gysrq ze*Ji`ap77aqsqwy#YJ2TA?{I@J#lQHYGQHI5%I}+NTquuP;cnBG$1g$ zBVAQcHzJ_h5Q-lXu+N2xoLd}#z62^d2X+Q>FD#B9fe=6YfUlZs26~I`tJ2Sh@WZ(! zOJRmJj3Uf%hh~O5GB(`&uNaM%R@xAf$sTUCvPx`jfOae{nvVafpf0ZIxSu6 zHn&znK_8W#Te;}z&o)rN@$+wE^{sy-6aDMoPuch&j}-_zLif^7#{BU1TY|?mm)Ix5SbQpg(*drgcG%5 zhgq5|+T?fH^ar9>??7iBpYdx;FMZJGeBae(6srolMvuMiy{pE(( zvbR`hE7@H6%M7uV7`rBH6K!Rm_3wA3%R|*tHMCkSU3A$VLkog3v|F5}@2jCLH|`J_ zLxwh-sKYjP{R($zJKUj#Pw)ILwMh7Kv4EAK-K#R4I1@2=!B(6Pj0%irdlX+%;aRlM zX(sTnkMM@61fP|41Wkn?+>St zlI=(bqi9uS)Ya4&7OF7>yLi!;3spChU1C+Lv=v!4s|LgN)r*Wff}fxmrD+;Oqgn7McfQ?oUU0;Q%ab{+;9&|lw@ zEE+5RI8kvX36S?WU%Q#*?N~@|2kQVbGf(p}jel3XOwB~Md+r0w|>1&5D=^&|Qr!%Y>q^eO(IM7gC@V zdlzg^AyQ}S`V0mJkV`?S*5AXQBN$K1ls+LSl1(v&TM=Gx{wc!v!nGpP43SK=i$X+B zCMxmp+Mf(l#!k4?X}wEqpUetV;!KlkTFxBrX&MM8^6!;!9dX}FYg~U=WwPiw&_*m9 zm15Z#n~Hsdv?PmuqNXjMl12ZcZVS&4>jGUU$JAdOJ%-qrVnG`OQK$lLB?-A?b z_^vRMJ*t@ZmB)6-CpKf^yvK zoLDBt)X_&ESC^b|E* zOD~#C`22QKz`U;w<5Cg>4HPwOpN`Sd>7LKnrX+wOTS)*e`^P=(^qwy>RA;g;Dn@i0 zm|TpAmc03B;)U$Uwec|vlMrYeM)-|8dq>C`o;RsBY-^x5Er1tIrFzr;T6$g_(#ZyNU{uLJhX)&mFo-q{>?zj^-W`R=)JDc2eN>J;R?OU zaD@zGbcA(a2a^ArI2>9Qj7)aNy8DwI5U&%7%Z8u??T!`KUUSVgfdL0ApuM59cYN-) z#KHIEt`C-VVh;;+_KfK}Hd(YzrwuIZN#;^eVIbKZSlI1ha3qmRcVIj4QDU?aj<2`PF3sN4TbqJYFU7?2oo5%UfBF4uXx;tz)oxwVQ zyXYgRI|;l5y+@x1DtaSN@j`U^^Qy&kjiypTnRmIC=Gt38qzOtKi5@v@_}4jigayqg znjm&WAA4%C)#;3oD7~>jA|20bhwh4xX$RkuI4`;z-jjM@pGT((-V?8mBy2Z@SEYhi z6J3P)j>fAB!K-k5J~ZY_;nc-rX=viX5m<}9XI7aG6yi3 zs@RNg?_Kmoz z8HAvBK!LB5B9Q7XNEU6u(#o3}d6Sso3n!*X8rF-b_cID)z=UX@L4st_Lf*nQfKhBpY%i4vzE z=8!V%UxOpemEzao-`OxIaqaYN)m_S}pTiop^^R#pfN4)+Y|WG0GwlDO-oDGFRB_D}NDLR)=B&;5ntKr85Mf zr-RyncUFYR(xu`PTljW43Iu{E<{ zIcR@L4_Aj@e^=-^@d5-`exZUoAOzR<2rl0ay1yfK9&F3=iW&aO$=-+&d|DhIyqYF) z9$S(R)Iglzc~?pc+O|qt<83?irH4y6pZW(-7zMZ)nwHY+<^R@H)Pyhcyv4ctdZzk> zhZXrOP5EjOgMRY6IUcaIT436jk42@#Oj!caqp;@%?9YN+OEdWYWV~Jf zL$FO0CYXy(_fy!OSPyak)4}$+CBCP;05!$$u#j!#5vlcQ7jT=LVq&!jO_2A>5bnIW zguB>#IZfMwiE9a9M@$D`LZz|mZ+6{@EWYa8je&Qywux^ISG{V)6bVl}*+Nla`(10` ze@&a^ck>1#*w|ea{iiR}#DDykR?%k^I6qA=0TVOowO)>rnJjt%uO@nVvz1&+m_hS- ziGxqJ(*4|b^$qvPJg*EI%jtEv>~m6D1Mfu=ryfCo!~v2I*F!kB9~ejjp>~Q6Y<8{{ zRownOGUxN=aPH?;)j)K#4B8@Wno_@2m248ZAR4gdKnxPM72v;c?co*?0Eye%rJMF- z!uhlrV)_5l$DQ$XAD90iN1Xai+o(l2Ze>6XiW*?^vD63(e`K-H(7~DifecQFIGQda zb|1~w5B+oduH>#mO)^>~(%tY9 zzcbSCn&<@R1ztZo0eXSg#J4g99l`JC+0Sz|&(%EF@?49IX4dyzHl&_ghSVc^7_Q@x zD!nwKOD_%S(n}+{^wN+ny)>drFAb$ikJKamO{te*P(~Ecf~5h^L%2eT8$r!C!)0&K zdu!lrEBCqK2>BLssYD}XII+A%;FU(B)cDzlM-sPzpCeOaxBDf=?lfpX-zahHIJ-er zD{)ah>p0M)g>V#|aS{|!@H6%{IQq0DEJAQ?uWe{}P&Hi59!(qTAeD6x8>Q#g3Q;sc z)+bVEVA)Tb1Mh|RK-;H8nFMzis|zBUi@m>7Xv%DTZ72>G2|fuPcgy_F#^di~;ql%P z_@jRUgmQFrv7<3bKLVFE0t@y&HH#(~%h4kh2G3q2b|&p^+>x~yL9{U3=Lc> zK?8eC{KTpEn4NFrWoE=)C;zR~Pk5v?@bV%SBcEXbdJ0R79gP>C`IYkGCySmxfle&O zK|Jt*Ip?3?)4#=Qz@{`QX7J8Gz=KlaG?hQW>%0R>uuQvIGXv!FWYGzwniejy{c&ruzR676qv8X@m z72g&S1#57JIzS(=$1q2p-U|f#t zVU|RUzG-!Snyd63u{|duOPmr!i$%!h%(0}1>T(T;B31ZPgfXY-QJ#xte3Pz<5&yPh zx!IQyKQWedCvF%O9z4v{STlX5-LwQdgPI1IU<>a>)^`q>DF#N5Di)LXac;4Db@=2; zter7qm-K1p2gb#0y+9@Q^cLE9th904BhXgfHQUo$d3TAtdxCeyC7Z7?joVmnSa_$r z`yKBpUXq!LOCuRO^XE$8+u|If#)EYXBJusJ;f)H+;JebJ6%!m5bx=m@^MuEBFnrjlcQWF2RZ)$Utmu76&ob({Wwu{P}=vj zZ?EK&a?xX-Q@4dP#NGhC_=d7?v*twGxA9>D*-^=@;?0Eh_yFf3p?7HfI3CO3E5o$b zd3D3o)noj-){SpE4{r{N>bk}%v#j5YeO}N|#^2-SVGQjT)%fotM|c9srvvQN()cZf zn!S%eIFd#0GLoB;MQ`d$kG{O9FWb35j;CSrM@nWtkRSop2vlgZ&f$oR0&KqsCRXAK z2=flDk1O%yQsirK^QGnMi?%b4Kf2AGmIcm53tjOFkY}eT4sK6mahFrmNS433*XjyW&R1B44P7-MbQ!~ zYLBSP;vkf8S9IVIzC6_6l^=|+UL8!-AHxGhhlpkK_(>@Wy9kRSL`IhJf%$qLEtkAn z#g2!T24cyaro7-N2@Ebgd7I!^s8-otMbpJIhvb5j@~$?e zrH)M*R4)@NkWx>@sQyPZRB=%AmSSpUF)SSR$9R4zWJjT z^kVAN){7SIAk1w`RKhld3eZUVNkI{Gtu|3JxggJB`QEmx`Fdmd6y+EiC+6c*(A`UA&qQgTfHrw=C5`LD&`BG%2!F(2 zC+`(=lEMzG{-r%N=*Z)Ug|!Ay1_obWK86b z5}`&T?&r&80_viU83Lig*Ce1&$K*(fW4^75_-xY`*8ThM45|CSYgERyWJCye>%B9p z-f>y=Dj27Gv7}sOjel{Srw^_3I8`TJY&v}&x!q}>lxe?TxE+Gz?N0kSOgz@7GT5nU zuLeZeD$`zpwFL|dB)b$n4m^1bNXa}#-k-l4x^62wns5!vBVd29MCg1r*Vuzsh!a8h z1+x&0oGvcq-@zX`C>UaW$3`h8a-(pD_gw#)0gj0eZ~!7Ap0wm+)WJj*cN+V>xQ;@u z;vB@uESt=Pj+6Nzg4eZbqQqyiqgiNV3mKj7Hi_oJ2YdRBRB2G9bFO%H?802`)0w3m zs(`K#ui;inRg|z;j0wa+d*j})pz1tBTbZ8Hsx&wRl-Z)kyI zrN9{!C?nQEjf|KNg$d*NYvO6s>lfJ1ZEneW#-?A|WQw(k#s7)vH{z$f(Wz{d{++dL zz08%rEAn%6ie1eFu4%q@`x($2qCscgNDD{0bK|<{D)+tQ79MkIs))1)b4U?CVDn`+ z<4)ydJL?iM3XVLaaRkZ;JmxX>Pl?B-oE!YD(;wpcFszx2AP`sRDK~fVyPF>#TS1y^ zMCl4W<)^#A@9!;KA;7R@buAamlar&m-87F$_k1^vw-;QSo_N(^*Ee`%YPRz+LtTDM zN$$LB95(MP`xBt4=cDPF;h*NHy%Nuh_89D!xj? zZakON3)c!#s5Rs>dq2bGkoFC#`7QnywYk<(*?%-r& z!?g2_ztdAC8j8|vCZ6ze5hmzg>E-VS<{#3_E^3|41p@p0OS`zlL&#?NetnB@*CrWV z-VKFnA+v;Jk4HmL@#crGS_IPxsFi@BL;#c&ZNa;$fP^dM6Z=;jDTMrWiI-sm!s)e0 zIL@^b1i*uRxSc-L@3uG96Oj{%S88rCv0&ps;9w; z={rQZs&r}=l0gg(t3_lIu_q%h+p0KuiSkRq%9mPB-gYVH&r>O9!b56``%+V!5Fzm* ziel@!ZAN}GB@`31L+OM3Xrhwdi_Q2Br-hX|~MI`^lV?oiZVy6OKdPT<6JM>qXv zEz)!h3+q+EJ_<$rLGFl${1mLVji@6Hf`l%$I#nuC5KBT*00~j57+3vbhOP{k?Mg2h zy`9XK{j%BlTArEv@*=Ho^_zAhW{@h>DUqjlMI*z#@yao#omMN$Mfplbk4y#DA>pjW zOp%GHlM-O6YI$JUWUZ?GShYkHiE>2-(-mYX3c#7=X}XrzN(2XKyh{xan+E=QCu2ZV zQmf%**$N}}MMCI=nEG8_JrYNB_tr;JHi&|;M%BhMPDo#}h_4Cf#KLX%( zVv)=vc>DE(np4Om)|{ddPBFhcVqaCMOl&TbPX}oX^SAEM_B0O)rPm)V$QmZd@`cluOs|}EP_)lVmx?ZlJuUeZ9aL4tm7Ru9zGZq{6f^IXx z4jXnCDhvbO#T?|#{nAodd870doUxY)-nK21w{qNYBh!uV+~vYRbXFp?LI>ORmxk=7f@(&nl(e!ZvD{ zR|SG3FHB)n-N2!&XIu)-`B3QY6tJ&o0UgJ zFL}%?mI>M{z3CvLX{6m5rap0j^{A!vt7Y$P7IAbsqs-YI{a{|W3?q7nmlb+Tb^UFF z0wvj0OOw$y8FiO%B;=bws}^W2LNI)*17ysfBUI^rolj`$uNRO*`Ql_hcs_MY5vWTw zPA}2>%4aOo)gMkrEWbvb_daly^|wDYP6Lz*Dd>B=2Ks4JUbZDtnuvn zD&tuoz|R^_Z`OFqx9sse>M6$aG}d@CXgZ&a=X3LAJo|7_$#|*|;b3C^7%}eo8MF-l zjdj5s=D{4!fFf*uQY}{3JR+P@zTA*c#Uhll=lu8tMf3&ylZr(f*5C)&qHC+EoMsUl zoq_<3WA6IJ+1r!|B(8u=&%#6SL<*3Z_`SgFP5YQ1z_TX}Pvv6!((?oUngM30(w!L* z1ntbUc$T^Ayo=|CX+m;=;UczQ&@jA4!H_1q^MJmDtCA{mKbW{`RxmMVjL{}xLSp?J z0?}J37F5&_9j?x1K{R;0Fm#30Sh2noSpRai3)WOqqOE}^#t0?o3N7_OK$TxXiF61T zSpYCo0Ptr6U^_-C=bdX%%@AI+9lO#=+ybABy`9ta_loT+gfyE25)i{$(7Ru&{Gt3e z2VRjKqdR+Z&0N`swcTiiv#0$Ts!vv8asli@RX6szb5}fy{N& zRt+`Uu4K?3la^{C8>-HRVND8#)!8uo;bXw?cUQYGB!i%l)vVE|>F;Lj`gUfd104~R zK^U9tmR*}&w&F29xMyv4*{4X^5z2-Odl@0JNa=m^Fl?=K@iL2+8Ex;=^e~~D)!F`~ z!0c!CF~1xImoFyPzh9L_tQjbG`Ou!!`le%t4%F+J#S<%NnkQ@oJN8MkpA&=bP?G(l zgR@2|&4x6!7UQf&euSBT{clZQGK6~hnv$I8NHot{e_wuVR!QwsUz*T-6~Y~=&mPC- z6pqwqkK=VK8OOG(q3PZcOr6w-}J9s<*U zqO7N3x*rPt%Wr&T3T`O+0DEkmR|#SFxD6ANj}oBM zfOpM3X4Su&MkI$~)xQZGe2sZore2>AZNU(1&z5b*0lx<=Cft#DfWcX~w`X z?mklPp)aLn)^iwh>~~vTN7NZ&Yt{R2*o8;A?LSnv|2!`LFX7)atou$@_vKr5_n*Q{ z=KS*#)qVMj(yCcM&!sjhVWeK06mhIO#&{t0HBo3>~8>#xV;cnFs#b{VJSb#OS zS+q%S1jBHmom&iZp;D}JZHom!L@;o1t!}_;+`+VxnI;ofGK$4z+nA!0dVLS3XsuDP z*7Qm-$=Hd0X%?pFS6`6TYNa~>L^0*1zV%7&kvR? z;l1nyyHs5tFImPl9RGF63V8+(AEZCZAyq$~7cMcg3t#>t6MO#%~1!G>0m+(fM`p zanug6IubEZ7p3@WcSBmqdBgW*eOsg8=3B{gOmRWE5ub zVC7X}asE^>!&GwD)Z|&^oe}XK*^y142gxp(Fj;%#3l_b*}-jMJBst zYMD~P>u61l+TOMtrnbW;9ZZFTNz6^kvL$N;c|^OEEnW);f6vGsCT?onN997qP3Mto z$x9hBmS3ifN2H7`Kr*Y0dbf-(+m3oE!ya13iIjmfUGJC1w0y}99VZ1%99qz8*L<}O zz4klG_+0#f8K6Ammf`)9?RZGaST(eaOQnp&{XWV7^a~`#4l!FN?q4dUf{#CWUE@`q z)LiGBdk$ai!l|T`n4&s~rW~&VLA|L!(2?4&%hHVv>?851a3j18LaeG&nx)a!O7X+( z1)nd?qS+UwDVfV^v&@fAk8ukXzNf4PF5$*bwhycBDP^=9t8Tgp(U^Gu{LGfSs8*bS zM9vSj4ygoLPQaH7(tS~Zc@oc~BIlJKug3Kq;CpmR#6)5z6#0U9{^nm~gm||26!G*C zCZG1>4RPW5+3O(_5C2C_Do{@RtxRs1M>{Y}k;Qq4Zm7`bzPybqBX-j z=X}}$p`7^JEJAs8`*1ob+p0_lLo#IYVnrqumbsCCI-C$PsZmdS;2GE43aF%MtJplj z%GU^v8bT%A)Gm?i@c19}*rCVO4j7@TS4?WJL6-Wvt>6WVys5;ty=l^JO35ri& zA*G*oD&sZbS1!49caR{QI?Q4L2?%s;yFVlzPj8o0N2uf?OrVP-2u*dg@OTPM7X9~U zI$~CnejV00oF6DZ*Y4j)@*v4o9GiOghI&`+7LnQ66Odz|UXTuesHgT8Of?e=j%oEy zERc;p+8(8Ycz@W2cv{AS8l0_McmciTo?y$gw2xw+2C$yA56>5lyDaRisWeS!IGBNQ z28uBi04%#{$qQDC5)UAMe0tw;=!xci?0+qu^wdN$XBWc+d2RnN$ognnZHL)+2jmB0< znPwug*!jkFGMpcZf3_>M>_F9}pajfJ{7@=nQN-+zxcSMxcdQR6+J+O$#S&$Y848d6 zp~pU5SbL&J*goylU;W>p2TSjq4VcBO6nxoOUjbOOHHB%}_+hyqugnwZD*OOY|Mfip zr98tv;K}(g9|+Ax2eJfEKd}ZrAHFxXypj!#*0APHJjk-OwQSlFme|~z1j(l{;d;RB z9q!}VnZ`#wBu!)*83iYJ#1bK?sPlv1C zin3|UTLqj&9WAmr+L4n9Bj=MqN_wZ9Nt>PV%tUkBq`nCqtgxUN;SAp zHCRt1L5}M&YO6qoRN2u#>y5}yTeAd&qgLdRm!cEWWtOB7NLCCGrQkwhOpbJvf+ag4 zJyN(I#bWr%rJ@Je|A3WCTa6j(W#!M74~&a9$PDFv9rvq^OI|cC*&4nIyg@0sdN=Lk z9Cau*k=i4M(5lu(OHrx-}*Vm=^hbO`DpYJBH8L7oRolzgyot?;|d zzF6rvp;XYIyMAPDiC4^UNCiX&s}q5IH0X9|(C*t+^r>tAc>-QQocX#Ca@S!TlNLQN zHki|NNTM#MZ&bxQefdua7pO?4?aLJ}wI%(=ed|*6x9ug4RQ4OopC>-wF$S2RYhs^_ zXmFBUgf=&93vlv5?DG)~42cA0;SH=*78RkWrfior?D%I6Kp3*W#E|uaZ4PbG;!-B; z(X{h@KLk~jb}73)2cAIBMG}}>(%hw&#E4vy=Cz=k8;qouzj%?NJgX%`BGRsTeGk&` zSw9!%{V)_5-c?JfOn0Q0!f&egH}c;8Sv{{cYnLwvsV_}@QDRUBMM6Ki^<}HRXj5NX z4^McIBtd9#ylpLa)ZRi(9o$N!{kW%pCfL8xc_yUG3yFMJ37$v<6=6xBP7c>n8J!h* z{NXdYTBy7t8DCH-%9n^f#Zts~L{4@+bw{U62w}C{hVO1X#i!E4!YS0cLW;a=p>V+p z@u#og*TJuw-wq+$byP!i<%(j`r8L88!L!=L#j|Jwt5~6|&8|r^g6~nUSuyRx_b>xE z2NJMH+m!3E+Bvv}nkANUotx7QIn}NT#B)HC&i}CF-+;b?R3I1xWmopUYT2AY+x?=` zu~tKbv51dr5vxPW@RGI@!P#}Ezm0AT%pnr=9rO1_XFTxatPZhvN2&%IR;oG#U#q7i zN}h`3zMfepY7M*|E_>D5_I5b;Rjc#u1C-tBUw~v^4sNAYD|tc324)E)W-cdWjD0?A z&qtsrgOql$4+U- z(^4kZ{TGBnzB^9FLIRW0a?Bsz;T0BA8IBsU`il#{*=Q?aY6qT}C8&m$*qoaGS^M5* z??P*DlQYShWeePH*#h@_vQf8MFeoRs%rauAq-vrNflUZ*yA+F<=}S9X(_{YFPZ{&) zgVN5lueI9w$8>3j3ddt~dS8_MRRiu;mNMIBZ^#JAL#09lIXb)lLHGuPFp-KG)h{s! z1tQO;8zUrruY*9m$6g0vHkQ$0gK+HIS%aY9Es-Hc+Yh0BLm}BXLoZ=UxP~bW!$m+K zYu?QJ2{628l7gXp61ltD`O&mA44D&vF>8PteiyMnViyTHP)Cf3Q)E_29Eb8XjXRbC zhtDkbTS?8V7XXORO~zOW%p0Ajm1Lr3R#?xh!dDkVY=jDGVxuD#1NOgC{aGOmb#Y?Z z(Ej|Etj=l`RG0RlRXd$i$snZU2{BJpTUe%9k87&vo+fe(Z*Z3R;aJ_^MT!()Z`kOJ zp}}Nz{RF3z9m|A{qS~MB`Lo=wVtc+3OSKrDWttgn7c#^2I3#}TKY#&bOlqvE=bH`; z+v=pqn!d4S)%(#S!e#F(!#W^KUBpa@`+(Y+XW(kSuz z2!SU^CgdmEz(dp|v^VAU$X>Q_!I88IA*Mz#)JjkVhU(WBQ(vm}rA97^`&AMI{yJy* zHo%t^l>P(o$6~7-tWdh|t|$;7y{7AwSP z!oMIhjR1MVcWZEXSdTO8^3M<$*Sgp*=4v&SBTKX9@x%8E|C6vg+2cueRlipziF#lI zpF7pVf~B(iOV!Iwb7Y4iwY6C4lq~H@w%AC3*T{Ls9j~e^Q*&HkNe($6YZJ#USh?%a zZ`c7KY)tAvk+F~u+ON{cSt~-F#tJj`Cc*J9MQ`J*f;0M>r21=zRX@XZWM`^x zXoBp0wWT>kkLP?KLnXS<8?(AD#AFZ%|aMPj^E5-G%O}_)kF`%MAuVNK7R9slYi1 zPKc4}^#FrrKOwJV9FAs(Aw{o;?C0P$t2VNqT|&j43xcIBI?5(WGYiA_f?{3xD3uTC z@UX37xwRK57ct3sj1iy)qgC;2&e~=B3@dk6N**wi8_mSc{h6?YPwvTrBBaxW;+}B{ zt3TJW3B*AkJcZj1MRy;tJGBTvam3}_DrQ=d30nU{DqA786wOnhf68t}kCzHyTyC<2 z`>a`YF_}1fRc6)w@<}6kV}!fd^^0`i2wnAN(2;yX$L@96YCQ-?cjoW$Nkl}T0isdPjmoCVMu4jS`I<2E% zr5kRa_VnG<3^l=nG1M}tQX#>n%#&l8@j`oGDa@43PMAm!rtVAb_KxAQHP{Rn> z`#ky=n|6NuHu|8{@+AuK-&aR_+DDo8#BwoAN&VOLrP**+I$x!?HT9M38lFrxZ7{}{g-ClDXrCzrx4bi_IL`ef2gJHXdfBU5RQ=c#ucL$GIO zq9<}-K`9OON~4d;YEQbybl|nV{Gk0p-*@+tAM>utKUVs)@BF!RNKu$}sLRqIt8_5R z2;9XZwM1RzXehO{%2Ih$iEDXL$FH6rc9OOhK6Hg<$#=0r!J29VHpk$c^_0yFg4lLd z$Vdm3LmgYw7Uvp7W0v=%g8bao;z+vZ5R4X1lPec9xiqp;ZltAtDB&Pi6K<;TP@Pm8ESbx@n(Qt6vQAi2r>ilCOYMvD z@OpGR?@UBm(GSa<9(hQako*z*KO+3r$e(JRq{o>VAW6XfUt~HTr&D$d%Q0iO<#Dd; z_kkD3%CZsM0(N=xhRT)vroB@Y+vVWYmVaW-iRb-0``ujLu_yd8GoJUN9Is_|zLIa| zMtaP6VhQk6G1I@gW~$Ig_Y*6iMxy45(Ts;QuSfe z_YrWdw5cPf@j7AEe)m+>%;B%Bt&>zr3jc)^p1256_!ZIJL_$s-Q)}g}RZ{q-VN$r? z`Rf%jU5OSI5eG{cfDtjs$I&Dr?1<1>fke~6y+}08&R+<1KwjA@hM%H3bYb~N+_IFLKA0{AgB)0f%j=Ov4# zkpckwk{wbF$D=j;W>{z0FQ7$mrCjt30deYtOfu9ef+k&;HJezNE-o+79@*4c|kvw%V7i}Mkt*n#%B&WqvggOmx)+H=X9s! z4BX8_&%j-$&%pgy#zDs2ug9yXKvjh2@MM~`Pd8I8`<`tPk)hIzvxr(vZVBD7&#Jyl zgpQ8r#I%(DE$z(>vx}umQvUd+V4Y0FY|QgZITW!~h|_x1n)ETHx0`X?q;dAES;=U~ zKd8%j#k}?Lwv>BZ%X3)pKZjn-0fXmMN9=!uZBB^>Ym7VJAx8v;1a;8(lenN?beaVF ztW0UhNEJwf!(|*rM!b`$M8GU_t>D?F}z8Agha7h!;!00P|U7qeMH-q%oW19`Df5isduu<+WC5( zweVAq7tbS=u2YqMOMPMYH>+x$R60(TTh?!}30x!3h3Y$d!AUV5^^sMp5U5|T6gl=W zT~BhnBH2)?*-YaOJ<)S9fJm+lOepE}(}G;-UD7*M^?z31GKb*)Ts;F+_YmA4s%Q5Q z+=V)Ew^J&0i|R3#+I9%hkO%@@+hr=z(Awy2m#Xe;`N&a=^tMZSd$K%BOmZx#dOKcy zk0n)a4^i*yfvABKpw7LU00x;kG`py?aI4 zE*N@x?%a~B(*nKXNekFJL`{OXew=1aw};~+WP2K0Db3+{eL>dT)i2Gs=}mw47pHIQ znN6;$Z9?u+1(auQArh&#{>-hJxvkFJ)}&+uNCZ?)1s=lfu#%}mdF{ZMft&JViICA+ z1LPEaPy$vrBFR>M^exoGkDc(=eZfYya=x#e^Bb`ZrgLePw}D=D+n%zma!iTr;@gvN z=M_|KZA48`PgZWAG;I5s!X44cVcRT-RCP9u;*8$z-v0p;ht!u@XAiu!=O=`~e_!q)7^0-aR<#kT$Ts3Cy-s0)hmGvt-<7bGOrv*n2QG z`Yd5U8GxjAYo>7-*o5sTmeAzc2szSD6}l80dYY)cQ*hoFv5yGZ6N~l9qTTMnqO$dJ zAd%;l!EeY5vU{LN+k5R4NUJuYj<88{Oa?Dxe|*iNeeIrrc99)31$`ab_2(aH*cu<&v)KFEz-A)9ES zq)qKX&UP2EtV+(vl4-ub5eogKZwZ#IPI((|Mb@*_UoA{Uu1ko=;ONXx1-pRfa@1qfND~BqgmfLv`D3a0EA~ zMCeexpjVyn_1jY>W+88T+UWbPpq4=rYJcCA`>Eiwy;rawCxD#Nm%V?{9L(Ms*jEz& zx5IvJyCov%qRp_=dKmQvBg~r8Fp7<7 z+)?XBlMUwzG_8ZCaZbkuiK`J#g7T@Z5dNwfZ`f9hC^L_YJJx;-lfpkOWaT}57&tQ{ zbr`tO`VVlAgFttp&3R5E4xMy)VfQ`R@U)yHBh@4MNuzZ6+cWI+fj62fqa07+N!#GL>dc3(SO4sm9vOJ{FIDBH0xjGZ+E#wyk&J^+y5RzJgcP zX_HH=98k7uq@e6bQ&89Bd~aFGCaa1vtUy!IG$Vd4OI^+XNn8(J2r#|fi ziz2xn$!Vhd8t2A0EZ%aq^*?u8K9ogeugjrq2?Z*9&_sg#-sIOqW zmoeT;ztVW$NspANa-(zlZkC_2B7x^GKjas@KD6?$IOMB8-usZ>)8&F}rNC~VAQO7k zvCko&u7+I6io!F~qb-#3n_>v59U>WKcHNde)&?hH%#z+foz!^;E`Jw3n@I`ioXw=% z_t~NnLm9UJWyLQou&`KI@y6oR*O1wh}d8 zUd*V7O>H*lQGlL=*nE!Hw9AR-UgrX8kwA>Jr6k3;F$HsgSa-{XlpuS~UM;)ro3lSw zm!DXPDL%f)*)hWKeR9TxUHI!JoR&}F?CG0pPbALe{Ni|EtUd8yedkM8`r*@VrAJdK zbrC)2R{SL^EtdMEUT5bqP-jl3{zaG<6V$v z#?SG}Ib*eQZklYCy7!aDbVREtBJ_JN`=QS*S;lUd)k?<5A2*BnSHX#SVxBI%;#vF# zhfYVmjXjgB*cG(jpa5+jb|Wt;1F=tYnyy5QxHPJlid3RG$}Wwvb)&sSneuf=-j zI{0aAm6a^wLz1+$@~2MzGz%E&Y4eDtd{(Q~04W^wMtjD?OW;cDq__H zBm_`4s|InwrTWCU;0n0S?|bfjo>>Fx`|JPr&*w9ld7kG!_nv$1x#ygF&bjA`))gx! z0jSrE`|&0B=W`xOd`?mPnj!A`3LVe}CK&|E#gIIwjhM; z(~0HVL+g#tN^`_Rr4!d^RmU9v9`JSebN^X<{Qzqd(Es#mg)ds(Ex!COX5i~+bU4Y( z8;_cS$AzEh_|AFTC4k2y#4m*IWF2?{6+5wy3CSy>H__!z`ypGk(}^PxE`q;fj7M{1 z_ojsLByw`0Hxuz7E38{(v0OnjPhTa+3SW#AL~}jLO)Zs{ubx6mbrlwK$Oh@N43&K@ z>!(2CiN}B+UnlRwB|XN%u_~u}#1wPt&fjQrs>q5xggPjJW2W*g&lNk^O>ei#Dwv%F zT;=3jHm#w*ode!Q@@U6?eDp|`ZT8PjWgj~top;y*mI$(oE&knxA4vYj9qPy@c17ez zFH+XZWe)%E|+s7t~aYccBLbnpxR~9+bbC}7dOQ^Do6BVR#fYm2^7)Veb19x z0`X*Zur;|!RHR!cpCx?m93)q;vhOxeq!B#@sF znTxSw#YbD0B@)ZA`7?Mc5H=0=mtY&0EkSp48rToaCv*7B{tXFt}YjgAtUD4y;$swN^82<-;1HI7(!v7_U>;ZfaTg92n%DO=mBqqU! z_vlF=K|PU#>MJF2zHM*@K3*7#(v+)*Cm}3KS>W2NITun`mFoC;S+Z*dx~keu5svxzyWuKVu?>iCpI z+s+~LM$6jtFI@LGfe#h`eC&Z#OH5BIcU7q3di&cVINhMF!kq1-CvGI)K*AH9Oz;#_ zPyOktHeaTylABId8%3b)zN+_Og4k;W+l+Iod_n)dG;ySE0{#8a1hD^8Rn5+rV=tkt z?3$HvK!>9(0!a5j6ZfS1Deg)4Q{<%k)lPa-L%wF)e+#`5 zdOJ-St%R6w>hTr`EG#8b`yWGzCov>#ueJ*Q&}I*zHDl4E3e5-6)5N7|J$2%SRiv%9 zP8KUECM%N%r>VnZ4wnt8?DDS>T%V}V9zRxOL`}ImfqZw`ri*C=;M#cRJ#9+=uGrh;;*Z4m_C(96M-2LZEkB)lil^3Nw@*)Q%*e zB~$aJ>_}JkiyH8{(eV!vw(!ChF^Tx~PL$jev0uV7E56$}&v6{mNZz)-#=xkTW+Aep` z^~(S0KKVbbNFCJ23th%p3As~KEO)LeW#?W}*{{gIE%I-Z{97acy4eT2 zi_iPa7tmzJZ&Phan=B^~fg89RcCMDUa*MsOO=1<`9;|~Bk5Yne46URa=q5xspi;D# z2y^2%I!1em%rAbUp0}3>53{D!(dKqOI|Ya?`PVD|Hp;&~QO;qr8PTC(LU>8T5}hKK|B z|B{}5CHxn9lD33edihQi-pE~{C%JOzX#(-dQewL*aU~^$p8ET^Yt^8dwwElGIvm0h zx^xL^iTb{0-d2GxvR-l)Sw3u9%*Pf09a>%IP_~K~ z4_mK@KukA|HOOTPYm8PtlC0_yPJX%^wZx15^_Kpav7Vmu_xGtNB4_i#v*J?dDT|?{SD9a#M7PF7Fgz0?-U-d zXf)Dg4?CPkPRg!+q-tJZ*;rkq#4YmLVY-Nq`~81YMGAG1{oNv66rmC+u{!u?*^<;B zQw8_c1-)*;+fxMxE($C|^`Zxb`R?Nurzz%t-oVr(D7#k3i~`zO`gM8iv@=i=Smr=8 z;FdZjqtwL|4qKmf(#s+TMZTUO)JN#14s`FL?mYYYLs=4p$h(3bfbWll0^gDt*XS{| zasS=``A@1J7#CQES6X2Hyd(ERl@yQ%gzB7N`p##-{Wy%B_FFem_jsO&CeTr^wn?0H4Bx0jwV0>Vi z^YCEz;bHO+0&2<2^TT956f;RGK7KVzR;evEr@SXvoCe=E;48MTI7BT~dqlUK%x~B?dBrZvX+6*D0LzbTFznBZv-_4;>=rN1 zE&fz)@g+H>bl>upF9<$r>isMjTzEkC28Na*wsVG%W8j>6KUn!p&$lU%Zs!7sn z2AO8}Pm09A-aF^or3r1-IoVn0;Wzj=*$*7fA{4OpTSE z@(f%LlPhh~7mGEd!k)(j>y&T=Nkscg!Ip4BHqrC{P!;pv@kj2$6ZeA?_%YS!_a2)- z(aSM#oIh|%VA+21V!Uuf^6@?LnDtQ$I~Q694h*Oq9F!1yldfQaTfs4@I5{~Xqin6Y ziQ~`ZoQz@O3z|!>8U!5!-wiBtIM5H1wH%*p2aYmVTu&zcI%GM!9(|@!!*8`yiFl~d zzvV4^*a;kgy}34C4cFk@>g(b02)HAU*lDt?wAjQV4b-XzBE=c?|31C`pQP8nAX)zvPW^_epG_NIy7hk{VJH@> z=d}8_Wz>H`di{~~`YV(5mpb)7a0~XSL=VByy8e`cVl4b{r)sGvy(Kr?S!0SFkSRA zs?-p@U47!#{n7xub=9#TjUwLPS`WKxtTzt;1XAteN}WiPHvNapmP8H zM_T1x0+vy^E6Dih=j1jv1H9%M2Ij21q;tl`yuA5*?&PzQ&#Qcf5TaPhNA%|`{nym` zZ>J2MOv%CY-=M4iwgO60|1B43kBTnV68@P=_{+Jf#2#uLSR3!lFMVo)Fh74UZ}W{d zEw=jiOqn6Av`fR-SLGWMeGf;1lNe5dDW)oJ#cnIKx7lziY`%rhvaDxL@3adw@0=u+ z4V)ZMd#-u@2lx5M@Jbq+eLo%ZhQ!5Na$XP4359rmBcsrJ66 ze)%POnl$IAkZ*FSYIy^wMMPJm>6a_qhil{^$g0r#<&JFqa=Qbc-9x(E6nuA3bDup* z;3Lyxu73Gn4lE9LZ_>^6+2=@eFx^=~G4)9e^IoBASHtWPa$v0_!>5%nL=GktbW$dY zRxn#c3F*I;f56`25Yl3K!@d{Em3Fh$&rPaZEZ?GmQ3=vPS7~3HRamJB6qfIt!jbI4 z0FhltS`yJ_7gIQ;khpzHDVATiPfD+RnnhT8pJel|YRS0}*3?m+sa_O0iJvzf=vE&hd4rz5P@-Gy znfLTXs@66x4_NE)MM`Y!`^2jr0V+i4t6+aNgw(0{iA)my&|h29m`a~a^A`I-W=>Ln zy^Uf@7Br~;VfCN2J#t|ja)IM{3Xl4SFgkhj^CC|ECEaD-!1btd;el%d%f`t2tatzJ zmg$o+7jbuBR$$pa`sqKDPnXM60eRqr=WHXJXA z!QpD-qhj5!6f7IxF>v~hSGwOC%HAkb)Y%rI)H}5{+mh5~pZOcwYz3wurOnQQ#TMKD zO>;d%-#Q6dCqrAVZod9lBGEJS8E)pI+ZGEiRl04tquWk;OX;?7g>==nB}0vRUB<;# zZ8yq=tFYlPb`-WhsWQFxZ41GkR&9-i$7`*2d%A|!mr?aM(P}GI)k>><9MqvTFVb4= z()9Z8$f*CDXti5zN@}&L{*>lnEL@;<&5h|TEz4->Tj@yp3a$2P)soU`b6WZ>xhc`q zed#Uj%xLMGXthICOX*r|qW_OdtF6M%j^;W^e;uj64&e)}cD%kS(qCir*GT=$qpv={ zTfP3V{(6@$-g{GDZPQ;{^w(znY@NP(LVrD~zaErll4N35(xWxk)oVu<3)RJ28EMgL zx8A7rT2VoFtJjL=s{gPam$K@c$&$-nT9!!I-zK`xp8RM2o!-X3@U#4T=mq{&FXG>$ zzvAEAKk;uRaU%9hO#~3chnDd+9HCTwv5_iK_!I!k+KTX%F8*ihP*1(f)J6t2N!1 zwr{s=ley!q%uPQzTXze7#kiZ9TX#=n0W6M9`!0Ew+Qr(Zt~IX^$7Zj(c6PBoq6_!h z7v7*#hZ0s#*r#+S8+I8{(Mowrd@b(Xw~GZ)Zm;SJQkhUXPHw3a=8=0bWHHNXP8f#S~DO%|U7kUi?C! z{!=I+RJGA)BUq9Z)0lA8W@FLb>i5~cP}!%Uhu!_2A@-#WvK1#nn!1j==|#v|M-kc4 zzN0^DGPHe~-*iKyY=u1EVTYFK4WyS0%Pb~ZPZ`-~cIGz;qE^5#6NOLSt`IdfKT`HQ zuaoK{v|}LiJ=rk7$!4cJr&yE6E^O5FhO5?)&W;=bZ1rDfSKvTl%6b_Qquo#w-Ghbg zadO%98e@hSzHl!ANK<_ht=3zNx39X`Y^(9&0ZS5O*gBn*u`7&@2UJhjrlEyQdV-lj z`fRk_MCq{gZ3;w4)bzExz8YqiDBOe-5NejLlkQq~J)6yZRo72K{bI(@^{(b3!#CU* zLonNE^wyN#PLa;~b2_Uff&TCM1Ks&Y#!TklGM6aAWip~N9;uBOVBHR)y2-jNhdHI_ zPtzWrO12MtYP9{gLe+?H)dpkHula3SA?R!sdUgKp^4Z zo7*81@Rlzm0pCiJga9beU6}>loUpZ53OWH5x|ZF}X|(+YOgvGz{kCrg;C7Je4(2dD znAt!j<<%I+eS09&+^1E80{fKRjY?|7|DTM{o3PhJ;rl=RX5A0`zZ;*)3c$K=2;h>h zJwDm`O1#c*f558RoiIvEkp-!vr5y5Yev6hO_cFAU?D?4~A5YQpk^CV_KHewFPQqA8XXLBOi~`g?sH6qyHiKc#10@*Zo3h;~6aVqIdjEK{|+h zd_fA>2h)76d>kcr_aq-5LA(D-`S>;YQ+I;YA?Z8{o{`O?#xwsp6Cgvpy@r4%{tC4` z#iKl5$)lbcDg@w&vU?mJWsTa&c@)A55a@g0Pa3ie{xm+xpFHXOX|e*Y?i&JEu7Fbv zw0W3maZspYn(098i9dPlQ?OhNDm4b337V$9{=rT)ihs}cGlUAYHq8c#ghmJ zxO{yJq9!m;%R|;Bw5P)%_I5{EoU# zam+F5+ToZ5x^S=kr-nUp%xwHP`j!Mf^LKbs-x`vMoQ~ZgXSeeFLGTi`w7w<2>Ou9b zp}D+lI3qJOi&W{}6+TN}ZXHT(s`dY8? z<~8pphwByIMU$N<;Ugpq=J)V5cCyl~Sn00M-Sb@3O4796g1Oj#9Vz(v7Lp8|8~WXj z8kZ;>5QBx>W#L@*Df#<7ve3k{S>vdA9((5qUW^yKtB+$56(U~nrM41~SVPzuD{jAD zX={25vru{@$)`lIC2ci}7Hg_EqBQ)FXQRx>wA^H@>5MjuQHw~%*Oig7b$a549zIzA z4at0g6-@d-ypH_F{~J>Su`V31&KJFgRg`BFdU|G4&LXE*t?$+npU@ks>J1w;-63mj zh+RTr_vmjDY~@-&-2SaGA93f9q8~9gY>Kc14wY@x)|(;Oof9o7V(qUww=%R`&%i&9 z)a#wJ(jnkW)9pv9o{i1vnd(;BYwogjqw3b?jBc$-cI$|}GP|Ybv$E%iC%DKt`p2sF z>B!zN)*V^Ucc~@xy$J9`;r&+8luh9eP_7n)cT?e=Pa{fW)>KgI=favRr*jc4K2bKqee822H!C#{jG^xjp?2z7T zfGXCJ$HQoEr@++sSdJT6ohUp6>k9MBalrCJJ9!Byd(>BR>|L{rAm+Gbn)+hbNY+0T4tdG5a%BYLO~cd0kP{IAI&O}d?qQJ~`0tnQ@Q z_Cz00`jo<_5Vd=hm7?eOLl$^Rr|0L0&q)++>7dVwLdsT|RokKI?U|@qqKf^O6w5@7 zidk`q?pjsq3ROyoZh{b8{0NPg@9&A=pfx=|T!K5^!O-^{4EY2@5ZzH=$-ZL#H$f3( zXbKy1DNotslLoVJRhQT~ACmE$7B6T;EAb??k`>CB{7W^~;#sU(A-iNObq$ki`&8v9 zyiQ%WeX4R4UZo3f*Z3J{@m>7}40J`0oaYun6N&CUU@JwtAhK+oD2 zJd&x}i@=kOe)tbQ=C!c3cPMFil3i(xx|nw)?HCE-zQhW&p7^&@!k+gdY?Na>Pn9{N ziy%_cOjIFERKF1?8_8sVcBL6R)dxNwlwFOuDAWzC=IrqjZ8;?^p;z&bEJL=-ANeuW zcpNH36hcD`-LzBn_{2~2VC=1^D>WEjs@|st<6U*_48|)OHOaxa=@P-EtT!@P{ih@< zaR*>V3KVAmp1|3MFJS}tFBlB}zthiq?Ki)lga3cYEvql2@E_6mS8JF=VYE$lR_+_J z=Am_bpfnB&_*}!%8-hzYe5LJ;?WTu>Ngsw{H~3`hw?7o)6+vsrMQ0jyJNSg0Moe6T zf*?+v^e(Bar|+{`NB@46{(P#idl*Jh=xMFc)4$}QeSqpr%KsqBzN*VwYkzsM^op25 zRUU8hKNn!0vK^kiK)Nun=Z zM8z3@s{O&oQ@KDGvhTFYc*aLLz7NeKj!5GX61D~n_u4g7pKji9ht)n@9wf&!9RS^b zR|R$ZFM|}iysIUPSBgKGhLU|hUtOpAeww;=`d+OIC;R^Dg}m&}&lG<|UhQ(H=jrA$ z7ZL=O@W=0BSa?!a_Jhq$uX;$or%`nlEmg=Z;f*XO;Eh-OhYg*OU zu*mV6MKY(g*G}W#)s-Fm50?RjzxoG}R|C4!$uSUrHQH)YUaeEt&KNwV3n$0mybELu z(*M3nUb&Dt?DA*0D1{dBZw@Z6npFYD2Re~gL$l@8dGa7_LY%6uQ-g7`x^@QRXk9ot7@N+Qm%paG z8tx3hSbUj|yu!bnt1o`>d0SJ(~S*!B(Ol?nrJ zvQYHz0|c20t5{@}92MOcl zmLMTh$TVY%4WTeeRxxzwGE@e+%5|H9*t6C*SuxTiF+Z|@`Z5~_uH(xY6R)36|!KJs@sM$BoB5wpfIVor06m^Jmn z$4s$gigg&XFOpR_nrS^Dtu>gd{r8DUht1${y}77lY+w}g@B3!_NCUCR&5g*aKK7xr z%EZ#+6-!T$^${U>rC2v)8w8-0KY{~V4uK_{ZdRf&oL0Dq!i$x{$s-Bq^YdywE1vgQ zLb$f05Bc3`#)nQh2Eve#J(m{|VqYwqq|;}RjITGf)KFVc50vwc_Ld-r zb!P1-MKxU>LT<_AOx_XFPH;TUObh;nG__DN7<^sSXo`Zevi z7CLFpuPphY;0ZEN;dnhT=diwK~`j4DBO|$Y_nW&pLY&T2X(o= zzro)1u!JsR7uj2lCotPn&r$CT0^s3Sh7Qy(g>a#9nCU#36kO5K&yaYBrMG<$pSg*SyRkmXR4>N(t@Iy*J6n{ zPm!hg4EC24@Ek#yAAqNX#B(47MGE&fX(gR_8D}Z~orzuY#%Sjad{hu&`g^aW-qT@a z$dfvd)xToapz1WhdFGC9^_kB2aze}C zB8`b&&S>)Xvt+<-|8&dbj&Ee< zZpJtL{p)#Ojqf8=s>XMr0RENZyIcP5${R&uYiDRA^e6*$ro6dO+HtY<@zqjLN;*>qz537ZUpgKA zGVO1-_{@g)k*m_$eoQV>@UBjO_8Ym-@^VLdKYk?l9q2AOJPSH4+WYs(f=*Ly&aV_>! zD?d?JL`Wj%xdVoZ<93Hw>k{Bc`&UHWu=>rmbqVLZ4^9r+WVB<&T8E5~nILm%49*^0?*K(i;3J%-Vz z76k-ly?D8(ZuYmuEP{?*EI_fpP~a(Y;5ka*sh4bX7S>MzX~8-H=~VVs&XCN`Yz|s! zpX&fcK?#VT_@sS&ouHMpMqtCQ^^SCWB;ngH1z&}R?;R9W^c@W!o1yaT>%@Hcwc&el zZ7zI|s1>E^h-}sdzS~dRBYfz6ZvJNFk0jsO2PlYkIbkIs3ri&~Eu3!gv-0gLReLqw zQ2bP{e=!GO^b`!gZemx+I*;|TAe957?K>DpwGid}K2#UEt+*R%8N5Q49PzP>Sx=Rs z(hP}JGig#*P#MqN!Lx)}lLgO7v$wN6GBg_{`K~Wz?b#X{?1E+B*^xPk_zW_3MT#hDSEmB z-F8Oc%z#>Cg{@aP8p-mqA8p1pPI}gOd*Nw|+T0 zIzD7!0Cb0Ui2dVUwii7#^inKC{iLwZqtL-Qg)Y;DCQBi)0aK{>`r?e5^6gp*$-wv* zPa59;kd!hW>uIyS+J+%{7A}>4lS#_g{9OX>jE?`JP(~#~?8IsX&fBPt16O4|drd#< zm1k#UJzK4xEtO}-Wj*_oes-5UJ0R=Xt@>F5&k!iyOJW6;Z1)dWeQYf36}F+5%ZiHC zvhxZS&M3VUc6UKvY!3vCa%jrj0tiW`ky7l_j z&-0@5Q(!6oI}v!QA%|U@tYEA0LeL}eJs~qbX+GB9{!X*UOstoS(v0~$#2)Q*hOWGX zLaDma_HKT(Po@GqQ{h0hAm$6feI~ng?U$}S|C7M@{E(P<#jJ0%>mWbEW3Voinel27 zVsa9*zepB*%u)6mcms)z1H%%9oAITRM^oe`kr7*)uMyOgn{B;`*6Teyiu?1UA9E^v zkr|)ML|@a4Zys9(^$^@MBJi+qq9AKp>ly0Xd%0X6^PFHu4EUS|urN!_ve z2;)qTr5QUr=bp<8^mrlvw5Gzi4E>*iqyz+8{Yqz9YhNld#i~-88BK7Ksr|br1RQed z#S8nfPC`&U1;Y;RsVU93fALvP>3MExT?}=<0EL9o(8o-uckD*{X_e4)vMpeuhJ!ju5yAsGqx|iZgXmG|*VWcq^wGTLD8T7HTQb~3ddQe+RGcaagwB=s&;qa-cuOG|AFYV(@eO0v$rBX{(k z=}wOIC0XsQRa;mydUmf{I=yGLwm;e4A7qTs?v(a)_GCeo+K|kO=DbvHQFs1j>nkjd zU^nl>WA9cNT5rFh`kbLjO;9ip7f8-E%zQ9xS81?|6vP>&KkkzLNfe=5EH7XkWdB&z zySMeU{gK*gx!%4$x21>Ocge4o)zV+orrV5`EVp!eOXn5X6Ov8EoTkR)Hg&4|a=g%! z)l^MxQ^juSjHbNyGSz?S)j>{Ef5^qiYwRFa7>O6&hY>f0n0&iWVLhW)|KpZUHHC8M zvm?4G0=MiloVHHLZEL>!cDyj0)z%fcZH3&@8EqBWuP6+_PoB2l_)yM1xq;yDoBI?4 zTW_z-#ZR7FQM~Zq7=P2d_^~3;j4p0|Oe5rP7?qPmVvlorb#ZR>zjj`L0)L)W{m*l& zZ*ohgDhGqb_Fk!8`REmG+dFe{a;*C{xd^h_Iy|?n;cn@Swo2`LRK*H|zaux5v~`Qy z78(aik*rbHJes789&CvzY~bKUn@ z5ix@oup!Q8Cf3?NW$+kcZ`u%V%kAlb$+rs+&VqHHTv*>DN>gY)bBHS)RNRzX{bSAx z4DsJFXQrd#?`p6zP|@j@&Ok+lU8*WpXg~6OjrPLaw$5ZjQ*eZC3-b zoK5^AkejiN$+uT8Pon1qOi`2)*%IiPf_C=^BbZhE4ySlpIZ%3@*Z!WW+7OO4={?=r zxZKvxPPKMac59K`)(#(}HG37K#-YKc^^CptUER{()UEL{t6l5Edk-i3bsMJ1GAXNDv=ca$Fg&*|NY>PF<4WyUWt^`~O5I(-sU_^d-wR zXSpXqm=>vP*8hza({}qc1@L;iOaYuRpXNFcz*T<5GJsb!hB0{9vlX3W6t8rOrBSsd9%L0!`Pdb>Hd+4nIc$fOtBQyQmrW|SdE1JqEUg^S}!V~>hc+&kJkp8c?yK_0Hw)Bt6)w81G(InyeFkWXq59s13%SBk!^8$3C;!AJflgKFjwQBx%H z`9nrUzceEErJFNfTHw4iIP<=rnvwu2_>^f*y9QzPs!aUh``F; zm*`e7xPN39EAAhO-V{T*JH3U7f(EfTut^-t5*7#61tSt0I|oJ*2%ab`!mHd#j*n9N zJmg9zE*d}Kf)#ss;|=8^jSqcLwL$!hFca->6nm`n$1i*L*y|rZyLffLhtf(X*tmJR zvimai)?xm|gISBQmkds&@~hl_3?Zluz+$FnM;K>mjPXlMPcdZeH{<6wDCQ22r+leW z->!AJizkt^9TVkKuoHm%8L=jUGyhCr^BvQrybRaO#Uk)arBI(fOM*zGRk3@P25{oQ z>dFJwPFcZwgY3qNomdj&fN@DGaTndu@l;{!9ACKVWut>p!gDaSG)UalZ7bW^`6a*F zCvZD#olzXHddUlBJh;-ht((lJ9eZ(~0@3}$o@=oPP0r`QAG7URHdYwbJy>}LIC0>? zspAM7kW&B&b+0ZDN#>2G6(XyPLY38Cqhpm^Q33Uu))^R+HX9urO2wf*@m3y%tJWJG zzvc(A9MmL%peJCSXdxemd!C@-8+nP*w8eXgY22`;wlYu}ZH8KkLS!AgT+*wO#*EEk zA>t~Y2}u{$G@q<$80`JtwwBVo=p+;Km;{S79?rfbcJ8QDX4KLZs#?>y!K_-~je@cT zMbVM9RZ(BGU}^rqkYl>U9syZ`b79M+LQ>4r6UkTkgfDFgZMHouJubh|>qJd>~Zhmw`+cm!YsAp;Z%40H~>Udu7>LH8K*7<62@tDYsH`irIv+&i+N<{A&xj2%W1i4zkGf?AX2~S` zzT|%sl!W8{BC%YKAgnV~=?xjx8%YZnvNnfe-13GhN4mFSBL@x%kxs&TI#mvBDiq_E z*R0&jDQ8NC9TIqUbVctGM5TH39Dklz7Om%N#Z#A0zzwrLzuH?%w*th;|#j9vH#TV*6ILHawS(ZH+#wneAOfkK#@4nA52ewd+_5HIV)+SE*iOdgLeYF<$?_I(6HPQXTl@}B>je*tN^n?_|>OUti zdk%5c91R1#nX!MAY~#Ke`)5g?d|L}{40CTIzr)sFTlpdhzR^CEvf#93;#LvUvdu#d zjb|cGQuzhrWA`^Vv43=h>FMGWtpZLEU{6(q_9AaW!H1+eG%u)i$|U~}&IkcMA|u#EP9)9W_KP*}+H;?`@&;Y>n?!`AUG}f zrZGkm`%Eb&QW54f^E5%A^i(?8@oi9Iw%zDUMExS{ToQ9g&e9|pwlukAQE14T0!!Tr zG4)Y@5&ZcvSKV;uf380454Bw*Q+o>{Q#tYHT^eXxpJ1XEE3x1;EfdB8hvANC)fHRS zMQuA1&12)EDyr5ujow~s{I#*Dc?9>OZy(b&P(}<}kp#C=m{WRYpa~o%9A&^ODng#w zc6lkO5^9Q~zgmipqoP>Ey~b3om#M5=>o|Kvr2E7ZgWf@upJ*m^Fe)-Z28!``GCZ6R z9~Ob%C(*sKS;A;v05yo%ua_PKq+iYFgsjsQtwigDPMW7W;8({-6;*9$s&c>|uD~BS z!YGxB6(0q3KUAV-!c+DUfS>5019ET?0*AR`3S**9E>!x)Mjl!aiiQwPzApznuH;R%|*5s22 z1M8xqNZA|4VZn`d<4Ex*1ewchD7S*M_y$$i6~(SAowzkp_Ga_ha(*8{y%7Xla2s)- zvG3c=xSb)-2BlK)d%LI=h=RyA9ACw;IXn5ek-_)c{}O&IXC%`;(~6txWh!m7w}K!! z{9~pZ30NexMpdm+fz&8%30R8Fh01!Q1$M>+w>h=j*Y2+7o2Z#Bf>a(4B#dA;$5wMx zS+gg>zP7_|fYu-)LY2!2QznS#I)6qcaCZWLL=$eT=*OWRSetkQ$;Mg>#wzQIo4*x7 zIt#ay2T>6Jgt;ec&+AAdkO6{l0Hd|D!P4%=$$?&k|~9o<)FwJ|Gg^ zIBeZkvINQ^s8uqa+*NWP7wEsjM3B5Fx$L0p*BTEor4{<*FQa{=N95EkC5tH&LjO>+ zeuPp6W1fLyWy)(_`}`1>*KXt&^C*Dis#t{4(pbV&J$UMd7etV60iP39orbK zP~Z~X9zQh^^0486Fy>EOFy&x}n0H_TrR9jqe+`Um`*bK_Q|(<*4PeLB!q|0(VHCBn zIz*9!FoDeNFREL-U?dxWjJC^^l)bp5fjwjq&l7A691^MudW}VOJOB~!(Sj24qs?n1 z-AFU}&B7&KSHEX5fI0?GETOD$!YD&3>ZE625`45c(0Yx?&tXzi>op=kKNIXyB;e^$ zqdLGfqpIIxsTg2!(bY66T-Fu+jhw1ZV|rNbRKv=WQC3)SF2aJL9h*eWNv>Di{)LFp z0uSJ}R|3H&kPu zF2MNwqXzckj7g31=I?~9YI@i*E7Y-65CWn)p}R+oB@TJ}lm4GyX#;qmi-f5ehE* zAQYQa5^DPl(tVW2jF=IzuAG%kzN?J(yX9>bO`!zYIY=V7)tvBBih9{bm`=T+2~V3< zPefIEm>ekfToub|@7gtH@JvuC+iP;sOmM}e-jmQW1Ayi9feaXPV4%u!^Bijv; zs#QkEe9)ub3|XT%K8>`I449{fDE}g3(Hdf9boXv|x~FHQ$PQvHy!60mzn({t`1C|1 zejY@zIqcaInee>qC-5`!O)?91+q(Jse{ z42;uI8#f2!YHxSK868J+Eu5V8qqn;YHt znZ{=ppNsg&y}woE=iS6-KA*e!+{Z`mH2~bNd0xx+3_cg~k$WP~i)KpnmKNaj7v{KS z;u_;HV!cF3X$f1O29Nm!8*I!hHDhLRLgY0|@cf!0NQFab0{IB^ZAxH@wCoZAY{q8V zOtj{`+OCzLj5PtHw%70-hjNCX=TK$iu;@lsBzne9Q0+6WrS0rw!FJZ*n!ChN z+4+0v4o-csbg{vglFH6SgMNfOpn!ln@p%yT=uz zIrs_^aMT9l&kF2Me+N`mFE^2!_cLr2YU93VrdM|Pk%NV;`vis`QUfK643|{}kSpUm zPy>FS>W2m&CCJU+BO|_rC57N;8~6!pYY+OMbvBFB6)O9J>{<-+17r+2KI7~vwV zkHyF4srB(%(*u$F3Lr~oecY{b&#*qOW_>K_wXjyPJ|^!M^@gfeG+lyWBorr=m@K+I z$RYH*aOFrcJZZxiWg6epCM&Ri?#fX7TKEN8%Ic=^h&M5-@Mg%fP669y6G?SC$k@MH zkntP@++aWYTNzR@dZ`*5d(@!z;1;10?7=@q67RttEayR%r$vNs!4&+SMEELGupfC| zAromgrr^=Q)~Q!$(~6{Ap_p?7zoX=QcH-0=voU)eUkNC)@zB3Yd~=KqR!@|l%X$j| z9y>i511{Sdy=RJcDR5b*&k9>>IyOICL}zNP_XEo+_z^UM+k@6+vm>7OJ2uy%nNM6Z z&1&|Np<^tp02VW&WG-gYiCsYgRB!95>TMpynpT-=0&^gG>QqbA zHCK+D_eQL1>O&s2+^-K+Ju!c8o;1jEU>50m<_$*&;2d$qyGw&|a3wL46WSI{o>wPKRDIcK z$!1j`?=w1nPC?na$6=2|WTTjM`DDki(LsnSJPQ4;17AHnmwgId#i6PVjoU&NME_i* z>hq?#a?W}8t3yMcGn4w`p89i`)MOnHs@iHSJdod!s^^ThUkgDX@RfxeIAvTyWb|!Y z!Fqa}CsOsV#y^==M5}X@7)8P75Tj$0!l%L}VH8x#F=vv49AHdOoj1+ijYhZKjGwWs z);gnTBB_Gci*ljSyxBps1Qv&_k+O|&;m!OGA|^R(_(4;l`8d#goB-H$+_Jd_Nr<_<=uc+zbhpR@CVPpkD>^12Jh#-Mq|wNY48Qi;0g-i z2GUO1^*-9T-bW(^V_@WXV`OW%lkCF5N4h;JZRK2yxqTKUq_A~?7Xnc{F7U80xLOri z3(EW9Zfio;^@ouys({@rVm7p%3R%rZ`m_%MA(_WsnIOx<)iuS?LZUV{o%Q;3AD?nQ z=tBiOs*p!*3uePhZlnQr!C&aRfIAN zCuKiwtIUhe4T(VOE~aqfyRet0vIORJ^1HJd%|{McH5Dwaz(qIBL%}s>>e7r@PXh?a zD}wbxkab1}3IPyFW>w;a!77TscU%8YxGT`y6G5uVT@T6p$Va)UsZ?6~I!(zyhvJM~ zJ}Pe9F(p_@Hd4}rgk^@4Qv`{Amd zrXwMmBX@%zqvKUzPvWSj@vZ+Qj-oA~1mgiq`F}NCq4+(HvP3hBDlnzgg`uH1i+E8s z+G!I!ww(aa{YZW#dd;@Cisgl9zf_~|@CT%@CYp#W_vO$6hiEpN`P1nNdts|K zDKd!hQ#L&(FU=4Zj7S^{Nj~cf@owR1O7pJ8r$aPCO4G{SyC9l%NNk!AO~k`~^M4r& zX9!3S@=vqXor-Hm4>a&?1T_?q1;h-cEJI5X$~;Yi@SN6HT!s! zoFk4t%?cllKkgXf|7Pb$TZkU0pJvq_4@q#qCtQg-0PsX4M51|Sh#6BBw&w5@Zb?p; zIQ^2z+^Xq?`8-2X;4_~o+W4-_xdTH9RzTJUPHPC|Ple1m@qm2zOz2Ro8tDNerwM5u z>)V<2(K#>_=mmw{oxTGRY4`6AMCi0(Yk!V{XgWY@j+FI9^3PG#;-tn&xl*d-3AB!s zL4tYm0mE46XCw!H8n#}NSI<}chY2YCm(DufVY`KN$EE*D=pU0V^soAa$wKDeO5G>L zqz3C##9FDkBTGVtLNsg$z9{;AdL=VZ=KR@mewH);BDCKty@kKz&ObqI#{84lRo6ph z+e7({4n}*xXpeNAg&uGkZ8PX^5@l;hzZ=1iqO-e<{u5Bt~&QTcEYeF%8ssZRl90&H`0Ww>fp(*%@AFswvZD6YuE ztgMVqOLF8@jqio3o)5)q5dF_{wgWnDpHwX1iovjz@38e8wxoKFVvUmBLUeudV~K3K zB7fV!6lvz4Bw>|01iw?QQ6uq}N?jawN_FB4t(a=njzk&lk|ZkSVb4yoihF6e>8JfO z!K+xst6ntPq_qh2QRdnzS!E9b{-F-|6qQxvZ3s$FGP^!*7CO`FvA#Na*M&Q3=X(0qX>@M#pba#zP`*`irG>^eq=G zjoPq9nnuDiBJokjM9LVqf0Rg{L)O1oK-oWd2F(OMCLr9w?dWxEh^^sWqpBNo;WThZ};cocWi;ZuEsuE3OB69ZL>TuZx8n$QjiH^-= z)3CUW6PC&X?Te<laQL zrBlkQ_h*ODX2Dq2k>aUoirMyAf(>J0XUk~r5~samReS3M!BRBZFK>-8%ifa)MIro` z-KH7@qjxDgGLqV#l+@?V=`YzLHdB#~^)QIR$%DN@iadH1dAKCukcnh$S-72zUj0W> z0a!KK{vbcB4+#Ze)kHYQ-l3{pO?!i2pc0U_KPyBoNjCo#wuafSFwks#W--b3>ysH0 zW8qQ4tXc#h2`>Xn4^@Yp+ zEe01NH7CP1lu%-6m%oksW)_30W`G(5h^$*AW_POK=S000V zC?jiJb&y#&ey-4p8J|PSNm(1eEN+4cTTNE%I-X6|#vy2S!VqBqVpYc!5ww(dU|6j< zx0HL}0P~K3b;~0EIfQ?DHY8pNBpn2$B4=A9dDCiBJOiH&6w5>LX8l`yE3P}M51^ty zQH3McnLgQmEC*={(`UWH8rji(tY%hzp5}bptlBYeUuE|xN}C8+-KM-orS%VCdHTemF|_iqyqo^s-O9XH6Ccn^xQy$ADe)%~31_S5aA(JrAW zDc6Cxf6`qCBKUzN+9@*NJ{hL6H^QEOo3V;;te*8;1M9g4&dcjRl7a#2%GqZ8r0@Lc z%Y=9_)u=k|qaCZ5p$VfS54wpk^@0;$ml(W4G0uc65~qs? zV*cv5O3=Cn)fe-e7UQA#xM5hEjYV4ospxeroOwl)rd18Ib@g5qBT4h(PBKyTIIV4?( zM*D?4B~XL` zw8?mo7O@;IlI^m>gPCBlu_Zza`)fX-Xo10HqIhO7Wq2BjUyYCMEt!j}pa-+;EwMz_ zYohqIphj?{nFx+GapiGzjf~$q^icd3^iSw>7igXx|&|Su3G3p{g@|O;<~_LMu;+ zf|M1n>G^`lx1mDcP+rxEo^ZUtGcOdt0h8B!Y{W8sjwAb*>`iqX*`4xJIkH#D!uTdG z(Wpzs0~aEMQmsc;qHmw+YrJ0*;g4uguC)&6>I$LwjSwK=1kahCk7bATgdGYNwf<*P zLxQVA6AJbxWo&d>#H!sN+U;tdr2NiC$4CWGi+I)(>^If^=T!tk%r)BH<%R~ScK;lh ze$NJaCYqi$CSeMr=yPeL>WRjKxiZ@R%FA+x^@&1?u#zfv7%r0SvbsOokn@%*{!{52 zncXD&h%i-IDYYcb37;{mE=6;QcuvLW3%*x|CghXsGy46Y;4a$;uv9t%CT{5 z(+ZUmLkfvN{3Gx?pM-nlGfubq{*te64dfR6g*etyVA_KUfcwUjCPMV%NLih~m& z5WEU&ntv!Gg!182sXqL9?s__84GUIYpI;Q>Xolu$mWDo|C|MF5py*M>v#lc>FJSz2 zt`m+oAmMl-lWSww5(0g#Psks-n=9G%)Ii)&+V}?qw*BSMygYEujH?rif+?fpmr0D8 ztGbdMKbWpv6=UnJN5IuqUN9^0Xc4^(SnmWLtB@Z?@ZBKFN81J@4Ity#_$02L> z$#|Q#HeVRAo=nE4Fn>=@wtsT6{Yk3*qYx~D>O;V?FHp6v=|J7OZu_l)z0=xmNNe+t zIH=$f8|=FagrgCshD-&?^@86lBVL*=D?c-%;n8oc7ecXf$%J^hFBEH1N&fxhQfpxK zhT$6TI}uYHAs&p?Y!57xF^I&b^-+3TC{|-PPiebmR-RIP8IeBIgF9eda<^SKjx21i zUcC#0^JAU;8*-W=NBgXvJR2n@um1)9i2_ZU4LmE$Wq3^^_`K8o@%p&WD#0@k?@`Eeqv(irdfElGT!eHtg}4 zhh0U{R4gi^C5ty_Ckhe2)#Cz^$_9sOKB6$Oq*c`q#AlQ{S7~GD*k_1eRZbY~V%Q>< z7v)*N$f(UTYZ;I=oM^@qvA|$-!~SxF%4B&GgMv{7v%d#RT%|911dd1c-?R{q)B>nj zUBsTApb(^m`II>e#Kq_Le~$sHwa%R#h}R^@xifD{P(q6}yjbG3DJon!W5~AXRgldP zTUF$2X#U$xf+ol#v(BxcQ&MDT>msaLz5YG@IyM;rMIN)|s$uXNexYgzc*V*ztsbMM zs4ODAJiQX*TONu}f;f62W!<6fokK;I9+BGb0^jpG@$$TOYUNn3M95EdBLI3EUlP4E z02*1$HuM!aikzi)XWv{T>oo~pCG3F4KV)QtjiZ4T=LQ3dNKu!VeKMp9PhOLhHJ@Vb68-;mW4^ zrZT0EOoyz+yP{3$xb}Ux_9bXY0zYsKVIS&g?Bh_Ixq0)x*O~6Oa^e6EP=%TN)>(=y zx7+{)OY!-e`D)0USDEHt&RE^(I0AVenAe*ubA_ybq2F)deYU~WKP+BZb|TDOCR!~A zZ?#=FD^G$T-Z8CFY^2&%Y~1{FDN;t-w}O302Oq+AZ!ClhF*%=#lY1l`6-8S$SAgl7DM^x)>efvn_Pw*Y9IAravY)+NNL@+x1{BSR!UuBw{)=tX)o?{|w+p!u}6 z>u_;|U`Y+&UX3WX=aswgfe6M69o-qP$}94H(S6c~%dDF0YwT-ln#JH{(E@za;g@bc zCzU4Dj3eMB!om6=7J%(;sU@^591HoJ)7`N}9;j#pRxy_2AeXlYYU5L5WS{O-$(X#{ zh)-=u6s|gm)BISFZe;J3(Vi!qD8{yi_|RL3;WJ|xTL}u8>Z@RhivgZbscF>WHO2c9 zn_8}K`}t{do86GtR`9!y&n!OmeCG0L=>L-pH&Bgtq1qRybcQIkEi@Pp3z@P%hDht?Tv=&<|I@AMb$+Dfd&ZRs!Pq))#a_ySgcwg(U4_9q=9bH0RKprzLvrJ z(PCaCqJ$h+F)~=eJLvrkQT#G8QkjW*ZFb z1~RKk&3`nlL^D!ixT2{9o;(t1q36;v(Z~!Os>=u%FKFS5^y4)|l!%T*pU_<;Ou>?< zvvgm+fQ5BWr^d_7iCQlpG8p7Y5)~UG#{04b)MAUsE<|i}_{6UMUkl>mfuf1;ST9x_ zfDY3q#8)9IA>&2L*3tt~X>X6=n0sU^Z>HB2w1jGiKhCyHPzc$sm4#BL3%er z(%_%08lN(^YE|PwWvl3HbU(IfHP8%W0vBb`oA`M#edKV2nmwfaB7j@!+Wv;u9^v zs*P`CgPeFIAx#Ly#LwhVdD;EDwfO)wxPo8z?bGCKYxMw~Q@Qb7Kf8b|w#u|!$p1n3 z*X);q;3%lK1&irLRy_&iKtA$MweTq1?1eU`O^ZcKz9wNa@ z3tQ11(N@C7l#M~lEDl-I!R9nCQya5@2miXnaKLt^7pe5N-R}+Yx4sEepXgtF-LbJ6 zIJ9gynqFFqC%&gN}^>VPi%jK{B)CBGwo07m1Z;GDBm4>K}Il3zVc%zbT)fcsdTFWp}h zpWDqI;q5eGt`6?N%|th6kx5{qT4V>Wwou=?0^RnoKrT8Ra-`nm@ST3^eL&1tiG zyZeWR9H2+xM^tMA(jlmV-jGh+1Gl*v3b#v zk`^t||1&R3<*HMC^F}Tc5c)f~h9->%(OM=~OiZ+|0kb=0G>~}4xcBn=A0BRKO9i6nu>f`&Yn(Q2Qjc!~XzN*g81`vzWLwP%+-6q59N_OF~F@ z#253sGhj^?Aw40{6^KnnNV~#(okoWeZ|6(Ffpe%A;ayu&>YhvE$ZbMs(&cuowG+YZ z4XmC_CQs0ltgQ9c}op^#p(x_Vq*{^F?z)h_L|p?Re^&>|f7;#e2|${x z=x^nu;+|?kS*-}8HxgT+H_-L^@Y=-}N-GD-F6D6S47hq@DJo}q?c%8wh=XEv0_0wH z!yn0HJ2BD!1Xe^vT#FPnohQ!0+KPFTj1CMsvM>ZPScjs1S>9JW6nAQT6w8&~NIbMN zBr2s>AuASq+>A#a565}QtlCIi1*+;8RR?~pCNU61*T}IYbP@kN77Lc+Rcq%RG%a3m z3UQ8&g=nQ@=LkR%Z96@UFAlJ;VZr&r(q;h;4p_CcfJ?8p?;Z~A-7qOHdSpk}H7q~< zID5bJur*@SK&a~F#{H-!z4+A>@91jTiVoKCOUB5weyo`lH8Op=Yl-M(Z_Ddni;|Zc zDN25f-@B#c{4uG5Ka+z0_mun>5*oO(s9LI2l>E5|mf+0!!REVAQ!R3}!lAOXNe_aR zLED}9)||7fr`6Oq+V7#-Z#o}F(`P*&jP?yeRoQbQHj4@2mkD7WR1sNZ{H$wn_4wF1 zGAsJ;<^^kJTRX1h3A7!Y_>Mq5OUV~Va}P`od$lk}MXAQkvhkHIIdH2h&r0eGT0Y^h z(9Ci*%#mCJ?E2S;BU&o>o@|;6L$k~`D-VHDl20@bXMEI_m`qN;!#6eC3^MUxNy5;X zwJzIyCXhG-zS&$!9q(ajQi%m*oAuBT;eit()<-J2z?k5zI>A8X9-Uzz_&`{cUdd6M zWVoQhSXhfJ;eA<8)Qj7Vt+Ss&!6}NB1Kx9a@0`a|>~F;A6w!?4_ihQc4&RI>Rs#F9!jUH%14aXz5iokK`ZWbcvfIwoE8h5rF6dP?2ln_P#XSi$sui5og z^w43~M|=~~6hLZ4lu(7*J%0VK@Cqzd|i|NpoL~S(i6M*Ax zP0w8N%@virK^YFvAc-g3oyM3Z?SDQp1;?HI?hIQuls*a&aP5Oy8~3ioQ7)(6Zm?fh zY1|CaQAA~K2zM9;js>wz3E{_0&tf&9Z^N`ESSBEQ4>~7u% zjdOLVN)>K=M+L*z$_i7zjVCos7K#0*#z(EeZhb08w~F$H;g-|B`au|$jZpl@i>cSX z_6p1{GXjgRI&R=p4a5}M67>sMwD@T5-6;&xfmilBrGPvjY`rU_M(~SM;bBx55G5m2 z^%tq8>4G$SrdwxxEX#~)szs<`=Yds&3}}Em4DtHJ8MtG2a!@(0SzSxxs)7A8UY<(P zAxM36&5YhLw&239zkR-cd@lRz%B0Wv9( zEIp!d#SUdzl{B~ulE9qvU8NoW*K}uvS3j`}8SY2f$ium{F=FlHi{PLduS6`1BW%RP zt?*J#C!F5LKEPgaoY{MYL}nX{Wt+RqdouPYq?5%*kf*e?Z@XF=I{JwC^p@KAwESA? z&9o(6L)XdlCEOL4mE=3n3*ps%mD?sZYZU7R_6uG>$7AO~?&md#x$VP%_2Z?qB)Jw^ z`3|a3pncip8;IR357?75`B*Is(AZf(l#210juhK3y(P>_Es~tk7hjou=nqoF6Pv_1 zIej6RQu-#rnxh>WA__Vb!}t)I&6GQ3g`lQTP*Q7T1d?kAX3+Q~~L6-l43jDnR z>xK5sgQ({HBo%O(#ykA(ltuCe``TYJF{}-N)*C0G;{OquYCm+h+&Vr?z(a^+^`eP= z^nH=Flf40zBb-F0Bws95CczE1eOFHLRawPxJ!VH`wp}N}5ouedM(Eg1VK_wf(bRm)5RyS{t3)+FKWA zVnU^$-C$3=Q=&X>EXtdA2=A}Lawrg-1ag6T1cNeVA<3j1{7VktZ+ zRk+dt{X4m!pO*!Tfc}_M+Zk$HGkW^Htl}W*AvMWpaHZ<0!d34bIvMW<`>Na~2QEr= ze*s1dMwI}6+%N}U!c=Lyu{>|ycY)$W2Z|Fx3u6y|+hEtzgdUBa+a)UK#$zG4-!UO5 zKh7yXit?)e8u6xu^EW;d3*vhFhpOdMz1Gl+uocD%4jZ`K9%Ou`rU7`cq}-+U_1HqC z){|B3894cNR`De9{7vCcA><*zmDlgdMd=F{rqJQGb_L)`@r`QWGFtoZtm4Wox555| z210F*%UH1J?+}#metB+lS7yDB`9+znh_ByfosKWYuz&D#0XC!6-sBjOl z^w*i8#1oOKdj$$(;dRJMNv^!XzAP7vH!eu^8X~wFB2X@7HUZcNKxFzw4gV>o%tpmV zHrW5IUYtVy*kKFO-g$#TNHbkO!1*37r>Y^BV5b}tMM|Irp<$4zGAd_#5LO&G%;rIu?J8LaGCi}9w7O3nkBV?Uo%UMg9Q`9|pLJ^@A#b*A@yxRPrFLclUux23(&%&+*6^$=wTCEqW z))*4Rw;Bvlwhvor)myyPO7HCpy|tBI->9u^A}!tc{mnej z?k2?Q?Wh0$pAXq*o_S`@%$zxM=FFKhXY6dr6o}b5xufZ=zP7Ohw0vDxX41ryO=+E)reBQMhU_;Suk0Bq`|Zfbz?yRvo~ZUs zGcXWPA4}cyiiQokt&@?BHl`^_Jw9Sb%xu?6HVTp<9r_}aIVf$CPG1B&kCT9z(M7$U zgR^#^A)%YSBaezv?`qq~>}JS~VP;3;BlP8vPj)|M2$xYt=P8XY9T!IJD4nh}kEZW( zHf8?$p~Ne@i!@Al1tN8g5la_+k-^Q#8l}&*YC*U`6T998!|4s!ka&vSC$zvVh@vac zG&7LX%-aNBq}!;G+9cv$;I}c`z7ZPTOUj=?rQ4@|?TI{K);M@`EHgwIxabbp=#=#= z*bm~GE#vJL*~`5gW|Lp<{&r|VA#~@dw2j#>{^-K)3mM(^{JZhqm$N?J{y`QMI|!rR z_Brl84ywkMMR+wwe0tasbCMDp(~FC^@? zZ4DDFB8aWa)A4*f3!3n}7vuT2l$R35G2&lP(*7lyT#ip)($(&bbJ1?&T$IT!vI{-p zE`6-h$C~^RAEEJixOrS&UM)zq6GwbxNfr-AYnEF)1gA%C1ZXs2@e;|uHZQ_vUO2z4 zD4svRB%VJHUnh~j#B1YaK#{JJv>=Nb#A`qnao@9XRm5?GWnQ)7?zC}m)Cgt8*}0#N z3fts$>hn*k&%dHD9}AlN^Gag*XP6RV$uqE*z^SY8pej#kYGiUwRlFvjNVPnQHxA0c z-(_{QwuHzo6DeE;5y>5=W*^FdvC_RPw~*bj+mHxf&n|TBJ$A}#fB;$f-<>7z&*YHI zgRD>>@AyHw(Qm_xV2=g2A=M@8MZse)xX1J-%~*zZv#>>~ZAA!(L&e}BL0YLg7Bz*}uOCp#Ylcb3WEo?gW+bA{5P zH?wQbrI3@j ztGr7II0nYLaIJ!@3)k?BC4OYjfskaCdH2_gkFj@Hm$A=f&l&AaA?wWv{Z4IjSqp37 zzG!7%R$o|%r9*QW)&p$^QiI^qZg_zT?ATu9;I5J-BDUs8NMv4!QE_3vR)6wf!?+kW zHHwMjv0qideyMQ`P$E~;G5705a?emWWd|OX+^<)~+%K`OdH*Ss6StHPkkRNnv9NBC zD!nNNP8^h!#@{PpbVB9Khpf2R{a~91BT_z9!4({~Zg76#;$se_BpW(dI2b(w8U*Pi z_j79BZ#n#m@lFj~7;QT~KWg6(-nbsFip9T59!#(+!w9HdT0W$T+_;f%ICUwF7QpP6 zm#0kRuS}#)`a&(0*3sCN&3ZYU**~7dnRtD2W`1h#7qrG9oz(7~WYSSlgBhHs#FAH% zzpD+g2<{gqdsiD?xDTx7`ZiQ*sYjk?P0$;4$Ln}Xt;rl??nq9m-~deA-H^I;P*uY` zLrAK}{9X;$Kqg5t{gidbc+meDJODQ`^JIE4jRU+HOwOrVgRGT*pq3BvNhd zO%rr3z=H8gPQU=Q!52$hOVcq4WfI=YD#^p?8D;VOZjC-Rm?N;rcL>wtey>J#`W4xSxZ2OTPSb+mo@|Nv|ytv(id#?@9n89({ ze(?r|YX^$&Zc?Wx%s}FM7ir~KZza#1VK?@8)y6X#pew7X&0B3nMrF~ALs)H~Aupm& zf!wTDo<>>T&~(^b%v~+$#;UO1eODA)jY=xG^vA>+d&1Ehagxld`G{n`3CX-KM>4NU zhi(}bHim`HW7&4Ct6sp`Lb{$WY@AudR9F9~lfY9(;xeKxE90+%zbff@K9KstahKWY zxDZ1Y<&MtS8_vSnq_Zu>XzY&FOkB7ZzR=mIW}y=4Vu5Y%;{KHqU|_osBSflosMy8dODVh|0=&{(GI*scf5`ZlrxO3^yCU| zP`iYLa)d(#-<2xytHJwkFK?&PzJCNTHp`tllpSZd@pR|l{nZ|)bZTdoExX-NnD&5X z(gSCdVS+0MU>UR(rjDAgrNBmz-tr5J@R*od4z?NFqfq@rP+jk-)Z7nz2a%5ZJRIua z2uJp0YV=s@Tf6DQ^g3v!jEh^QfDI?^yli>{$xH9?5syt0~2ZqUuYvhX-0fJ;Xvp#JQ@@PW^D|QJzLy9Wvi#J#@*N;0 zXD>*Hj#UU(da-06BE4mDHk#?ba1a&3E?LW9hVEo!*CAgy%>GJPW)g;;L=Z`}{y=_O zP4pLS^zRfs-$awq3{SEe`J+LbtC{=B%}8aVoh68IdMKNb-RaP`-e!JC>B>YZn&gF} zA52S?GZzHU*!gxe_}L+5r?;KpXJX#Ce{{0c#0VF{$4gCHOo z`+$>xw8C_k)d6VrlyL)?Sy~vj^~u}&A8+Nf_YPkEk@nt7KEJ)U5#F=Czmb|9Og<;pS!Cb$45*jyGornx`rZpVnP z-s|c+nf0-va2TVj$dnHKfC%4DajfzTzsKMH1U-H;S$zzAkFchJMx+ZX3C*Ae^|JVx zL3P+1Fg%z*wL7DbN0qx5zgVQ@kV%A5l??r8yoWaT`yZ;b3Z+jTpj^`XxKHkF-S(@&f;yzea#Gyi{BK^Ap2N3cFULY zCTHW#vC26|Qa4LzC)=&1}c#OqArLR1IJL` z0_^9BY;tE#;L*t81s)CVOc5Z{7Qc0f!%9+Z&N~u-YVel|WsdBc_0;5I?vG%GDoVY* zlCGIy`4{H5q5n36Hp9OLZL3VzT(`q?&2ocJEgx^##spe%uU*{aEFV#7&L6o0^oi+n z%j>!H`6zGy2z_oKpWhdo2!8~9{u2GDq0b%P8%du())34x@jugWz!@Z51-Or=&pJb$ zmNr+aJDIs0OPszZl0bt?=^{|PmqVRVH)R;xTj=drEEvYdZ4v2EFK&rJp2k{|NnblFvt7g78Ps&+ky38v6Ogtt08@YmAy4`uU0N3Dfj10q*1J z=M%U0``dHddk1g-NPBN3pWoiw2!BL-ccOMS?S1H$k?n0WoFUWI_gnOt_Ra#_Ke4?( zXHlP>zkb2nKhoYGlh1GO&j^1+dq3m1_tBe2wztzTflPZhTJ)Lro&~s1)ZP*E*MLaB zXT<#V#T!Akoxi&Hn#&?``p1h$s1dbab_9L7{`=dflwhcT6D4r40Q9L7|pIjq6V zVHg~h^HS$VBzjfzY9_xb+_UjMn*V4f`ytyBhW}K2cO?HAFe5F~Nk`f~H=T4JXEN!e z<#y(R1^srjWkG$a`I7YmJd7gLpUD`HBl^oIOh@+D5_Cphf5A8DFI}%{MOF1zyXmjj zU(NKFib{tDiBNy(?e8p7=J<=09D!qitbTvJLKvac6FBzD2poTWk9hh^4NC2rr5CH= zG5z%`eWjm<`Ko8HDuHIQOYg3PUfY)vE36dd&X_=GHQ&)f5L#tIg@nYl5rZw|xse7! z%M=xvN~qn0s_DVSZsd3#&2FTM#{xHUERQBPQo*Cq_I-oxd%FHGBB6v*9^95RiO1ie z1MQ0#=2W&~42wR~-e(6sQG4xqwaj@_&-SlOet5K*6=gp< zbQ8sMnT2c)Qd?Yk;VK|DVJd~ZyVgYEcR3ll)sOp;jZ^NQ`Ee_3+yvt8^W(0uaVUL5 z-G1Bx;xv)PlG7&c-S5B(Hbp2+K1)5PVP7s*H8v+QVDcdwI{~E5}3Xg12r! zMVC5+1m2g+^kqs{YJ4_oIZ;+4ntCgdN%-$x!WXhp|K&w($wqZ~QERhN(q*cIA7`Up zBg!k`rflp4I96EVQC;fFZ0zBFY)R_;Y;3jRZlgCM%goYfdoAhw$%yBeu3EF&(%Y>( zDD#77g7=Tx*;k&wEZ-fsEyXj8eiye@3HCoat}4rudRy+#aRr<#-mx5+2JgkeNUxJz zSZeNs+ACPX5&4NHmjp|~WxCooqHU5(Bf+A5{phN9p;>JsV@k}nM zda!|{+7C%R^e_pH!NWZZ_Lsa=uM?yl&4lX{_blcCly)BB;O&%b>UKgJ(B4SQ*d@)Z3c&1Au@jOQJotKuPqv9 z3Twg=BlQ#FRQZY%{jl`^sh^3z&p;*RbTdzjcWk-*L255xH?NUy8*Z1 z(h_b-O$i9x%_`1uT{SL!^Dw`}sklsB9u*aY(q99<#K(EUHK^%8*PquK4!a-L*W7af z5Eq?*e-lz8A5ZFaI^M%g6PwXn8oA8Gsl8>(0+KAG9yXNjsr`*IwQTA5jmpF{?^{$y z+c}y2n27t~Ga#Iynz2i9M&XHr7>NEQbag zoWzX=Y~t<}f}Xg0rG2cjkJUU9cdy}hknL^vgIS5+zORl~3$X!XpH2PyU>OP6#(N2} zP1*MON9wS#+WbeA*sGL>QF8m$TtaC|=E(9z)chib)}@?dE3y3j1>?b1dSiX< zJ?e_!^&PM=olC|D!&I?jgg@8Jkq1qX*3}Oq6Fs!{Iw&%@R3p#Zm<=x7!b`hc!ZEaH zt=V`PaXL_&lFha?-=Q@tb2cT}%t;&M^-J$Hh26-Kjd}TIQ1{e{YP3FOOshx#{0whv zQMKwHM+2uf@u%~sbU!8zcWTtJ>*b{Gd)3DEJiYVTw2@^D$;CH>-q?ugE1PnQIP?SBdY;?p8ZLwgJbRd^+xZ7G244 zZ!uFkN38W@YPK69poE*mqSK+XpSEve`>^`u&~=v8zXx8F)Dr9H8LjF(4u&5}{+uor z@tXVM%i6D$&Iv=ofaN34IEf{Ofx3a;i?Jk@h;{P$JNB)}zWv<3mDsoW_D$|GNO_%o zlcNdVF0yYG_N~^wRoS;r`z9w3q`b+#$z=j>bM2dB->$ZA4ff3|uhG8EwBMTS+mZHd zfqgsFzBTjKO2-+wa}Ik$>Ch&_SRl@1=t3fx?-r9N9r~5x6CLf!FZ0u*%+xYo3}`&@ z7B_5Ad+&LsoSB+Z%y)|))G^+>#gN^qP=1S}nHB!`GMm53f46v=S?#}DOwFwG-z~0Y zI{tf|&EMd^TYSxI^xrMUW;Xfn7H9Hjbn*Ypz1c(4eTAnZ6LZwE{2$}S8Czqu_q zk<)Z<>#pABdQQ?b;1!8mfp>NnwjQu^>Fx_)aO2wc4tKotzWIuxX}4HHWa;B=*tR2G zZNqKx(4vFLF%G9wOz#W0Kr`I2iSzzIBRaozp+2^Hr}K|AeY~_>11Ribv||-U60DdE ze1#jz61`|X(-(vT>W6;-4))1sfa0ojB70d(XMeWT^DINUPhkX@xBYkg)Vm#qS0=3M za$P&VQoyjWF+(?rmQJP$eMH7w4i9lim-{I}BQrJPka7Q>%KLpf4ceO!ff{x`Jg#-T z@h~vh;8)b$tc(sND^s1e3dgE-_iLF-xBtnn^Rm;*%Qy;?I@YU}=1g5zt!n0MnU=9# z%C_mtnO#!!HS_!vNZgp{?=nZ$XitV+W@w?gEFy0{1J;{QPoxV0$jFY!zT8M$pJ?mn zcErA($iTr5-d;;oioYTLhO5K*t!%+X49&9P78xEa=GBlahqXw=<-|ODb?4W7Y{?>3 z90uu1Jq_p7nXyaMyNUVL)$?(9)IS+8?Y|04u6I?MSd`*vs^^d6V^$T zt*N`9Z+GMNPYII8^bueX>FJ1%@d>c^AB4hQr98pC$@IT9@X|Q}ZNMiS{|t2F;nnjS zSs;X$g^vg)qIKcKnf1CO4{v^LY*y}v#V$lQuPVGTeAu{X;^27V%A&}!uNY@yyZR5= z)cz93N27;F5|@`n6LTx#2@LDpNRu4nx%;J-eLbC?oh-bcbI7drZZZpqg5GMGX2A64 zug%$yg8-?nS?|9_cm*Ue%U!-p!d2?>e^Eq6_w(7uvuS^)wAhqo_txtm6eRbTJtg0K zU*H;X9s#4Fd+n&g%M)HU_!E_c-Tm--K2R{VpzQ!d;#j$Gn`8J~YYDG!6Q)jReT9?y ztrIlTpN)2Wvox=b3s_ODQvU$sybT%k_VRW}O)-~ZV)q;RFwb4yBQDbZ(q8SmjoX|X zC4=GC6zRBM_Z8H;H=3@heS*tHa57+-my93d97)yaCu%K_H0WivRv~Lu#?H1>({J$i zVXl5-oQp}ukiOTb(65)o-Hq;($me=xWFQf6e&=XIrkzU-ncA~|wvrYrAU<1HcZ;0- zLRH3aW@*EGOykZRxZ8C!d;tFrQ#*P$b-4ohsqbllb*zm{5v z{9mqo8%AAp`|PnQ{_CSm#h1>d8!UNI=@n`>>{#8VHAB0eBgl?>=$^9v?mYPkKetA! zwYzOUb47iT=WX@`S@+7aH^{|nZAS}N|2~nG5PSm{b$A*E7t<=Vo^_XR)%2CR@!@oO z=kz1;Wshg=xGSBcl_VzN$cU>6yJQ8m4NFz-gB;u|9eR?TU9Kf7p_UoG#45?!T_3<| z#JDl?r)4KSp!Gw>U<|SF763>rP=P5cIgRP)vDB|05*%!ZMe02>s`ZQ}=T|>eM($jDM^**BHe0IG zyCQeXZf$1Y2KYcvnIgYpyrN)#L72a`(dJb-|^#i4P^e zNpGH8JtsNg3-RQY7$9GSgIA+wDr}eGalBvyC#lL}$#0YilU!|8q{A!FOas{s@tkG? zax4gqaOzw9WC2Q(@^&R4uG#T46{e59J+Aa#=^2GiK0Z4s4zKd_@;Ih}(W?%PP_3ml zcg;bZ)95khG_a(w`VOa~Hthswu+|4U`hXdZlbGRz)A`|axGW2dcC1 z=Al(E#UMwRV#EQs^{ASgY6P|0K;;}|*oh{>Kt;a-gJLz3o<&l z3*yOSA%k1=^v^C&1(@=*DZNVc$7bZwckzVD8|JoTy_jA`ycls?mv=vnXtu=LruLm< zG>T1Di%rI{ign-8)ewOxII+S9@x6AH10&$PN;I)oG{Nm5i5VsJ?h?rs^-#R2N^*(E zVGc$V%W@bd3JhG&>M`AHM(nwSwKh>MPDqN;X#2lP+y?E)fQhzt`*HWr+7;LAZpY_g zZwBu_^)Pp{KO?mUP__~F0tK-D{o`qT&GFFagvSD@YReH>mEG( zS+HJ!jdg~vL0U%Rs27tvTUVtL*t3tY&R7QwWp1A?_ z*OeA}ZRi$0=Iuwi_CkOsdPrqH{U zw{u6^PqGZDY-5%o-Mb!!6n57mJ2mEvsf^JghO|YxAt2a^PBc$)Q*bH9jxg@S%H7+{ z=-|?TDVxF_o70in&8?rWuier*mwSSSg1+`x{=@5V9cE!^+{U_|>#=F+2oa!si*=;FkB1|vtPv|y*~#F1EO z&M4ef&pI4GAm-~d2DsDg6pWF|(>bCa@I8FWfZEF2X8mi8&+!7|xZB-Xg;tfqrSidv zbzQ-dQ`iY>=tD6ytqgTg-GIRrSIIwHbfFXcW}n7*9Sb{TX}Fm=+05M>imQPu2XEv8 z_PnK=TK9IioBPJOyhs;RmDg5<=BC13zQOLtY+}XpsM(J>5fU`pF)6bh(`mM2fLDbp zJC>B2bUT<`^oQ!*%@1~woZT(atr;NRwv3|Fp@|Fu6cS8{Au$zD#3{4h)Ny#*W4A^d z7Vk$B47+!nn)6B*O%qqhuAhuo0eS4*n(AgF0E++`xi2@%Iki`mw!Acmv%rJICVq4@ zQcB{8u)A^aDks^gd8B$iqKutJP~!R0!U5gc!Fdqj6>d^nc(|`gbSl8ny2LWAXPsnH z%4tr;q;6zsm$TBZW7mV$7+LoZw2_tmV8EO~i6wDJiX}aa(^}H+VMH(gG<&)vZIq&_ z$?Udlcf46pZi6%PU_-V6r%<9_`spg&ac?zj*X`6ih!aBYVcsNf?cKXM47I8CXkpZs zmz`u`jf9|@;~&<=1B=aJC@xytjr^S~e2=?8TW9D1Anw+=sqU?o8mE$LQHKl6WjnZP zI~f1(9Ny|Wd4!9ojUZCW6^pXe8J`+~uaX>;|LFJT(@tHo_E>7apNY9!iei;!mus9a z9l>XJ8{nB^@*Q{q$lB+BlJozJD4j`QIm)GV-6C}^rr9)QCt8w5^x8L?&R!zLqx-^` zD1Q;1rH}HL34?NcG}O#!Y&n(_)TQm$mE^_PDdv2pp>65Ur2RIu%HlSTKZlbW2Lr#8 zbd>GL-JrsUniSN*Lu2@huB*PhBm*s)1+}xS6zfJu<0yzu4h_n9cOFq&(|uBGX7Dr8 zjldYhhBSg!R66Mf*;_Z}8W7S4kQ%FNF3IK&3zUB8J~fe56z9u>^{grA8#Pes{Ap5! zfleoR{8mTEgm#iQ3X#=PruD0>cDZ%OL#C}pc>5t?C;uCbSxmf{gWnFmg5QyQH|Ers z>3Cmn=5OH`{kk1c+$84?abB2{#M?TM;3lpEV`w+AXo z)*4#-zPg1X>T=Qr@1K4+XM^!&)fIejh`V{N)0xxhaR0myryq|02o_EwbNpUl_KbGi zQ!8}eNwi4bS&jV>JdIkEh11{F9*E&*a^xj1(}(Aa|BnQWz*!MXB0x^z*f7pVx$wZ8 zBkum8_+c0MkCjAAqcq=MUFEQ$?Qj~{N$eNAfA-;Sx8H!khuzC&(+a?cIVF-sijPd5 zAMfuD|5x{%O6Nv#Qe(o;#iB+*K>Go8t@>C!8jKI^S6{wM+c0!RKe& zT3+R5oj&}3Yv&bywzcI;;&LZME4iFP{OocL5^r31Fh@VTVwJt{vq#w*_FUfW{H^%e zQ@C_XxsyeYndzgRFMdV)ALJ8z&4)p6AY)$<9TL%HMk1QYJhz}k64kyEs1S~4Rt@go ze%;}D=2`{5)*&d&tkP&B)01daQDzyZ^f6uhIujH=jo%b?(*k>pPXoYcea}00d->n& zwV`|1Oc4?oy*4M}M%!=~!HXKwE>tCbD#A z+t)Tm8qg7LP2GU5ag@)cb)`c2NQY@@_G*>}XzB($0 z{9N7d6>vHQge8^)Q)kEaD63*rPE)6K4SS@^%Sm@OJBhN{fdn^YC%(3p>BU3c4a^a&aEIvuRA}`U9nPLbcOIlq z7JF9%BDFCx1g3L}z31RPi%r>LhbT4*!I|xm3H_HmJ_+9_%bu? zIzv(Bwf-4j&bf!Rv4tj>ZO4=CJ_)EgsphN{<`ERq;e z-j0x(Xv*`woy~euQCm}gGK4lF_3uZfHp$N36X?opGQY1$dLATpswJ@=^-*%d*|{W^ z)7L7!a8Iz|ELgv;|IU`3QvVBeH5-7zR4wX=QH?AbnUNeYuB1^KT>49nN!}l?!xAY{ z`+z~~^*=)lVlCozQth4(47Wvw zCgn-%V9_KD!5pUa7(z~OnW}e+=rZGf&w@I03NoFC$2u(vX1S9F-0w1Os75e#-ON2& z5*w9~^^Q@{(}$?BUrOC|7aJ9FbN6b?QdQiJtQ$-XPRjLES6~2HPjzxYF9)wQZYs2q zGkpef`scQL^vU%msE+l`Ir@=|m{&J1x0GIWa(6^U2 z5HGhkP3HOA@&$Ujfm2TLwrV=9W2m>;TwfD&KleXh#kE;dLiI+RzDKIBm9=LNv(^fH zkgViy6@RPQC0}6_XiAnz!z$M38}-A6Ap%wXGQ=_mDXlE^vsuD20-m|FPh%3Adxae@ zh9iupfHBL#M`dW#z6#mu4#QGPD@RvmC-|S?bUAW7OzZqw;&TpEXulo!PWkODEa@#pluT zMvy)N#dk3%J`TmI(JN!&=!k47nHHUovd^M*q8VMIknlR&fL_0wEKDeWl{&D1ZoGYi z(X|*1wwm(TUsvH*2puMR{#eymISQtF_pq&gHNG`t3|xL~R2HwwAC6V!nrvF67!FMt zF{ZwJ0famPmrWv>J;;@l>&P-BH{?3w44N9=S-&AKzx6!Zro>owK$Qs(KmE59Hcbj_EU#leV>)M;wf}xHt=PkO#!s{Ok zqX%T+Wh#8PG2d1X2F9xJK=H^n(%`3T2^YdoxnsiJ8Lqj{${u$~EI*TUG_EGRf7k^+Y)n;jdsLUL$`n83}Kxo~l{9eXq#%jx?Y+5tmn!N$H zc=Uj)6*cWaB+VX?EH*@<8Z8{FlwVF5i7~b4xJ-*yO&_BXSC7i#HDc0O;OAu1N?va_ zgdUL}nh|uqSY17;36XXyJ1?UF)7{%Y(C*33br~A}v0-Rf{c)8S6TLQk9!m}EtJch5 z4j7*ynM>Gvw*B+?G____mS2p@LK)-yaTFe_jB7?^5XQ+giTo6j(x@|<5jjZbDK^XMSdAqj#b9JqcW(B#hEgGV#s$?wU&&^LK!7~ z8KJRArNLwvIYE_Y%9u9R5SctGONQsPj#a>$VkMTTjp8{+L3y*2g}=8ZXl-~7&1E?M zW>7e4Ci%}SunhHlneA?i^;_!37z>w;%0gv4hVw2g(r1^y9;>wDvsqc?<+97K)B96) z8iC&Os_PrkeEGmG84K87kQO{Vi~=?pVBz%oNbQe}Tygy`cF8WL4PS3%=t<>H|>ib9^g-5U>!5(HD z`FG@vYD%PiL~NMiE7PG;TXItJskY-+F4rdN?d1}5*-WUybg(Y!UY#)?#Q$x^eo*>P z>`MIeh0+d|amG-aaqe5jqcaT{_vzuVOZ|2GSHpzv)VS5SS!DNYT-bFArJ_2LpWxD0 zX{6qaoz{`zjxFOzGhy%`g*tvy4G!|#_6lQZi&N8%&H0$ggnRN?=rgTswc}MVB3;ErSsBx`chJ`IrzY~+9wwlCHBJCTdn(A zx+_w!sn!(T(%JFlw7GeV3#pA-yhO?~q^F3NX0ezS~)73-Suts(bj?FYwP%oJS3 zrsiN^skzE?d#0Au%{w*6fGv4R`CHf^_X~m>a3bDHD6)%dfUeMyIQQD?;_g5^@kQ-C zaApspi6vMWzgEvKgp}5C_BGNs%T`_n#NUAiaPfxSj1IFTA}{jy7yjO?o}S-&F@c|A zbNm3$oA}e6Yg$jCh>ox|gmc`)yt`Sz+{ydzdm?u>6y8zW6S=VzBWIi$pUPYMW?9tN z*R*|Kw*iGQ04DrB6lC$IdFf`xjlUV37bL zrYPtfF$%dLW0ZIKm;Y7Gz zh7YKF55l5a`lt_HG(MoGV(C-GN&O%(5C}+Zuebk zxq<(l=2ItIW|JqK$~Bu5aP1W$N%w2lL6bNwjzLy24u~=yFu1a~fc|#z$7W1qyid5?&t?eqr$pY(Uxu3W*n{^h&CtSu! zavt@ulv~fUS_+3Mku*>4NQQFN|Ea!hxzsz{dSnjuZeUB00f!{*lkfi+<^cJogUh~6 z_KzXqI;u39gtsO{!rUI{lW?^q;a5Bo#>#I?>PycX5nv3V>vv^7JBcrIJCkg-jBgA)Y>T*c=Mlu+M?*Zj_j3(t zkD_NN=UMD7J+jWBY0E}i_xDj|8R)kk`nxo3j{M}%bda`LemjDu?Z~pfb+BJHVCaQs z%)pAmRd+kJ+wdMwKbmumtTm%-S;-cn&gmG2kD{TGu;V!HhcUcvM{BF1!D-cOR|S_! z-K^;yHF&!9OWrZQBm9X7^T+n_oJz)H!BIlmQ#X~Q@C5pH?YQPx=#kyJbTlj%LOZOF_~1I#MKN!6zdw; zIk)i8_G&}1X0O{^roFeo;u+M6USu~;uwjCm$;>kRs8RtYD5c*B{IPK>{Jk|M{)DkC z{)A2Nw+q|TKBx9eoK-ju^M&A2K>PUH*;?ngOtkVwjc2m1)+6y6TNFoxD784W7Ueoz z98NzjtnF_uv`mNQ{7@*$!QwDj>=70raDzoGfdi?xzoTTI0Fy@xUKW#LXK?#bnEby( zPe0A#Bh48g@R6Kw9S9lscM(#MK?uSeCOuzXl|cw}Q^FZD5WZ0$5RFb{mMW|RBe-&X zH1+(7Pl%NL1v873vB~S7P9KPBS^Ahx@5Pdn-X#tj#7(Vb{y1ZD_Qn}I@FEa@pt10P z*=)u??VW1oSR`UF!uIk80~L&Hxk*Vs0h#0pW)>r3lF9$0@lyDA$z$|*IpPN&g^Zl> zQkZ&}Y3UOoV%HrXj|h(mYd$Cu!?cIkqq0SYjYz&}{j#htkd?dPLC2+9ZVR}ThKs}LfX->Ea|A?9`(b4fX7yLH&+A+{S4$CyUz zX3SD^aj=~Ys&puz9cH2)n=6bJu#nC}5jglT+^~*Z zdc*)jZV^#7mt9}R+-J?wVJWEm{@>=`XQP#yon70F3v?Hoz0^rgK_7~MqHv-hoM@nj zl*5b<*-KIMnf&P%iPqVoC5TkriwO~j29wtfq3F=H4(=qz#^cy~!Ve*MJBhOb_3WTw zeP{j$B*0x1thNOuiO*VQPias=)VbTwLObO5KI`j_*XbD=4$Pyt`pGE33e@1aoZj*7O zyVcn?>{N2W!jRMP(vTM^^%Btqpz&d+avipGPRGEIQ`>zlCw8}_zQ(#A&+CI=lp)W4 zo_RN&&_i-lA=T!l7JcX#9(N7%g;`hqd@o@`_G~bpMscbpFj9xpbJBnF4_>NwA!)f= z&wv_t4boN*H3Zbr!IJSbOg3dzvC4H*9@qKuDtHA**b*!$2u>WIYSlhj$D7p80D@K498pFpJ|;J=hr#UdD9jG~nC$_x!x_w8oW(4bijStwHt*v5bb&!ff$8TFm?h$F|| z5%_G~H;Yf^cYpkg(V-uOP&r7$n&d4b;dy*kO976Mv&$|)HnS#(y8Bp>53R4{y>#fL zTQ$1QEi=51$-G{x?i8Ra%*_}JxP;hn(SqlKC4A#xPe10dMfvEPo=(nTE2HofcQ^Z@ z?zTj11CS(jV1qrZ3n>Ug7j@#Ljb;KU*wj=1XdhGz%#$ex?##${oQ|#`js6DB5V}eS zIe~fB<9FzH)YOc8@U~{6N;uxIhOsUWx{av>_6Pqmqs;Ahx?cv)3*vM~({nN0(>S&7 zESxgZ9-nz>$VtwoXw4|d0!(tXeODB7orW$feMUCp&*?7VBh6Z?fhjQGTs&91ncI15 zw_UL}=iZpYwzm8^>UK+Ij#2r{Xmak5Z{N1B#2Cb&n0sbXy!MGJj$}T(;wYK4cRb8} z8{O2Zw{=1gyTX!v*l7+=h_Q6)_4oRNxaj57rLy(!7qwCE@U zEYt{nPVMI4(oKvSiM;qgVc@AW)lLWPDRp-&c{!UEFT`r!4lZ>~O32$BY<4QQgBX0y zLnpctW0TyUlto1!wGBm!CZn^15TA|}AP8|FtB&=zH{iw({tJ31YEJEI=N2t0_vH)d zM!1J0hvX?=9S*o?&D^3ZUJ;(nr7yKFwB2Y2nTDb+KcSwj;7zi4MwB%(;HdGYHkENv z0P+=*xIgHIG!g3BVGTOwM#iBAT{L+Z$b2GIYK9xjW`^6R8DPT?rYKT zYSHkDF}$9S)jp+L8<|>5Rz%%#j1{olqS4jD=z179Pqq9d${joIb1P)y93C-0F%W3} z#Hey@YdK6=LFr~?)X3TwE%tEST$8PVMcl@g&GFh77M;iy)Wa(r_e)wNt4sEd*LJrY zq@}|VkT(gR<$MwA+5|pJ7)r!+yv7KAr)3M-+iC#RdM!s!TPMI!9rpkyv2Qf7KMq0Y zTvK?*I#XcOS|pcssmDkWC?oK#oT@_=Z9vN-=Ui<@{}134bV?Ry(m z2}$wULQEX6{jb^KgW?@st9sSp#E>_J&LAaYXjJd&yemsNOSTAdbRmL`q#DUlms(0r zo;57uc7CVHK?Kmqpo7ImcfDYeA?6mulZA(Co@me@f|$m0V%P{r$Or|g2bio33YE%( zel5#6fl@W?j9`e*j3-Qv^5&^hSfLr(e2p;#26me<#A2Dz2H{4NGi=Rda8?%ny{TX2 zZo|<1@!+s2=p>uT%>E@}?Y`WOxG+R@5%Fbyo({d16u2V>KM%RZYWG?A=@I#(i4u(? zw*e6?1(bc9ntfZ))U$+uEx{~mA8q@-4z5D5eoO3tLX1BT8H%5ZsR5~&0gDvIogSN?o+9+g2ZU@8&KSb!49cS&3X#6ddArz45$wb-XAXF zNXI_yR~6;8UbVEVZ7+q|7;(%{)Rlo$P1sy3e~`p_x8GbV-`^59x3_YdVwweTB>-v+ z073%bWi?>-1hB4ULd@M6qh8lvs!5@CN3a74!r1q)1I!w8WZka$-_&i6sau>&7ji21 zrT7(8rrv z4?C$cH~@*Z^#@9-aEA^0hFh>hW|;2f0og~{T2C7Vz~Ub;djjZbIa)F|mBcNLb{=4S zaR@uY-@fo`>i&H2`ctWUy!J1_8`e=fc?nn$PhO)N0fSle;+I7A?@;~6>i#i{A8t7v z2pkI>QU6m%)&K7z;4{&VH=xU1VtwOLlb9p$LGBLYOwgNrcs+RiX;fdsWPz&QOhejE zQnQw7RqWIbs@V}bLs^|foCY%GJ!LwsVyge=z(1EEkXkcB zklXh92>#H&kn7(HKd+OXoT6>xM;lM5fqe2NxIMFwTe*qSa8;5<-T6pgb>F#r)iF1H}mN#p!-w7}9#na%qPQ-F|ydhu;!X-Q*<(*4pBX#n?LJAFkgIUzm{o~iX>!6v*T zZyAE;Z9%_^m+xpgR?AVR;EYZ20$_B@G#TnyH=PcBLIhj;=rzBS#BwmEoNYL5d5(0X zsDEzAB#Cn*FgYy^9UI9bTKo33|7rxMUc8oT9Wlo(+!C!}X;P+JBDav+1Kp;Nz|fA(l4b z^?$}mmq(k)V}ne9NF-0D@B`1--O7IHNlqoM`kqc*Wnx~?lDYClr}8mlUbQiGKG8-B zf+*3wdkc()?6RHSs*oMj_8N^6|K(U-{JzuEsIE`9XN(wYj`|DlMvIyKwQ9oGm)DJN zYxUucU=C!mkPRl;_<&N+DQ{Lw>(spJS;=GPiT|)~@&+=~8^gJxb6eq>{K4Z846KMy zcwE+NZDiPRw2Ix#*MD94u4| z0d?v~gvh6j$DpAOl z{*<|Nn7yJAtEw12R2T?fr6PwiXiC@enVmLv ze>Fbi?rt~$2A9lO26riRg2CjE4H@Z%T`0&IDep0Py+M;c0=tN28SGNMJ=add3y*+> zyn)F70)}jwx!mV{CPd*cUvsBcB>GC*AR8TGqp4;XN!p`<5XU*|qKQc_3_b^!bP|Qn zUUE4y z%rRH@zY}ci z-=ij%(jU*;M!_e=R{slOrQ&bmDTTZ|&I?&-hO$)WWZ7b~lw4%$LDFuUujDI}lo>5> z1#P#P2~(0vE4lnI6`6y)#=ZV*b3_IsI4hrUDv zD*VdO`Pn+C${(tU$EqTrW#Uydw&Y6gbEQhA6(s`tyv^hjLk>Dr$evBI`EBxls7V== zd98t@Y-3NUP}3j;+pOw%@=#T9Cx~GK(jWkQX6W3uPE>UjO)R;xYIMWugB6hP|FU@& zN>1~JBf*O4y+ASD_Ba(F;?&iu&LE+)SMtg4yR8(-DB;t3KApczp_d5FUhCI_E;UH= zx>C&}595mj!UG*@2<=0NbYV(82bLyYHI>(^5$s;hE(ag6{6OkAT7L+CM=e4hX~+IC zTrx}HB{huW>An1TWBZ+Y{96^85w74_MP{6v0bIoIIe#=mzC;lfs@r+$1THZMzG?McY<@(VI^}5-wB@EOA8;SxHGpA*rdQS z`q@I>x0UEj6@BJ9K?IWN=xro8r%SKT1S@*EgMotEnDHznAxl3FEKxPQ4HGDj0hzwd*8V1rsM2fJf$-}%1fab~5 zK{K>T{8RKzl&(8dTx-LWuAcDJyu9|41eE1Y#UqI@eAQ6TnOhTK@fs>U@p7@U4Js#U zTE0CIK9(htPbZMUnHx#5P$fSi)Vv%#LkttFPMljsS_?o z{;OAxhPvp0G=Ei^Gb>0V)-K94MIT8r}&Z%t+68E(es9V<79K1hpA%gVI{`jGot7Ao(44tkG z=VZJ&wPO~Cln{F{8Ea0cp_Zax9 zIu-BYv9VKip=G8`rr-usaJ+d`sb3#e>RF>oot0baiGHcCoMB4!5$2(5W2BizUfajn zSe;Vl4Cew{r?w_s&7czxl;4uYBwlVqlXwS{4XOW%W-!?RCRb4T)ZKH1yN<=BB`xTf zX^g>XLuzIz_nL7UfeWfogVs6cJFHy@;BBv`y{V^8S7SQ`-|K@_1UGi70ID5XklO(bql(;4 z?Q*;Q8S|bka+7)Yiu|oj+?zUIMJDHgE2u0Tn$V(x*gGmHZM}*m87r1lRpEzexGjiE zM4EyVO>N}eMkm5MSE**nTMY^Co#xM7@LR~Tmvg;uu)wiE`v<@gB(jzc8wf$=c{_~H6!oK@m<-}p_ zHbKOOv-2I(^w4eohqeBPXYGg8{)apL58d{|i~ffj{11krlA-->_R6@_e&A$dGIXRL z*=Qr%zWM8SX?HqQ(yY|I9r!yvpd($>I{v{T;15oCkbUR8!N7wYZ-%RV_A;e94JB`~ z_;5hMWm4bF`!MYR*o~!fL9>;T!~CULwvdf`2MSH=h07 z;$M=dwz!uFSTlQ<^=N}gOr{d0{HdRkuWS^PBu6^&*0b|;^k@z2nxi7LIq(HlV+t3f zM_Bz7-*&jB4=lHsJ_31wJDetK9;ni!!o-1drsTZUJlE=;jYuDV{rYaF6NMc8H0WnB zAKUp`#@`D5R`Rz>mF9z{B0|jymGwkct2xO(}rE~`;?jv%@a#t!_Vd;$t|jk8SQRy zzo}TH4p3D-C<61V)1k@aORdD7EyhSf+aUJf7GH5_eqQQ-=vu`#D^?|aMX|S=4v#zH zp?z~y=g_89%ER!{;0~aN$0*N&~sGjzKf!@t)3h3Kf52ieE-h z_c5f7Jx9!`r}Gvf_-_#0WDPUG`vOsQlkm{Ye5L#fFXpr?l-bT-AAkM)4e*!ZZ-~EP za7Vm1qMsy9ujb0!K|> zVFRIA3Is1ZVVoE7xu}geG2e>_^m}6IfsP5VN5O9}0 z2IwAME_(vlyP`hd0pNPYVHFcBml{DW77!X)?x2?5B<6COZ-1uyIMU@Ur! z<=1s{2Z*xJs3hemr`iHEA^>e_?SUMWs_eSCL*XUwwS(B;^*Sx?6x^x~NOv#UZ6p84 zp1lZtfLM>vQv?m6Pd`%~EHbEoMpwhyo}|wSuEBC{HP5wBxH1y#;;+|91}aSoL%x6I z3;!X8zwuCUJ)L<_nGF>ZvR`$psSQoYgocR_Aw#qhq4XKjsv<~CDGyA(6*I(_o`?oO zUTR>S;fGPxAUSWD=89#O8%9?4L{=&dm0ZE!D(L}91e+$?7Bq!4p)!{_+KYC_|#Cvu4@%T&@Vaf;@u;&8$37bDV}TyY&MNb!*{-Ma62k3j z65NcSo@J^ApOj2dbesX&(>c|P;bScO@ic6cK|55$7T`q4-w@ z_Qj?kZG5?tXHKFY(5>V0cxM|&Pt&b)V>S24g+uEh{()p;IcrWOb2KCMZaaj=k+g$# z3$YIUW6jDc=PB!sVf4A=EB@UKoPLxHjPYMlnx?l^v?OR+)7hm*^43-MEyY92*@Rg@ z@>pZzd+no756ad;(<$JJOot9O7)WX+mNfA2 zOei3Ed%5OxyqaJSv9J`cCU{&}SRuvI5IWxHi^oI z|N0-;t_G?}3LdXiB9;}1Sb@OXV3;udews`fi*M(n!6AYwS*yf)nFp1Hn;x=D4)mc~ zEjC4aruW0C)7HA^V(eQv&glm3%O`^b7F^XCMdfu2I?}2S%)t95PYiF@Mw7F(tq3x8MauD_myC@5BGK9ntk9kh+dEZG?HjGtqwJU5T8TTxq`P?aM4{*q(6Ot& zJSI+$r~Qs~g%0MsPvgUy`!qNmPJ?5W8?atw1MSJwC0}9*jDZWXy*l~%h{|MjqJkGY z+*Q)9N_DJn@|J+ed-led@NIDNX> zB)p1*f>iH8st3~VY=Rq2f{By{r21492!`|i%!8zax7vi~nuMvZ2^qy#@*-msck;g} zlsSWx-xjK`sckFKR8)Pe&;;VDwIjUFsePvP;Ew8J+xDujSzL%&bw?M*kKKc>Ij*_k z`A{cu#|k)=lbHN-13e&T36yn|9{BLzkNd4nHaf4n?d=HOxNEz^!CN-sw<)!FNbF)5 zN>@GI6PncT8{NP;JURIis2vK<9O~(8phr@V{zt93j;sj*TI@XaW`imkxJv~k3@1~m-CJPfM3A_s%30QFz;N+>)`nM#6wDfGtZ$| zGCrh1o4mDw7|`_-7XI8De!gBpzY5YMjsS6#wu^7yGT&amK;PyKIf>AB_%6bz@(81f zL|-$BUIYLXQI$IUi-scpjabnZDMDups#`qOB)s30H2HhH(6(iIRAQ#{KKC>{XYXm? z$fYTMQa_o-n@lg6Oo6kMnq!vF({HB}X2{UiAk8(byq6c++-D#C_Ay`|Df<|*k74^T zT82btQGr+YuR_gX_r>}eTFAxiz!j^IW-stP)C-O~q0dPq1sHbEFBqgBkEJGw&R2Mk zB_2{aJWM$~ohnC(7~CpL!5&kbiF5-Apkn^gXM04Ap~;hoOov{e$=p1L1|fL;RWOCr zeeepmKM@Gnh67Fv*SlXYnd~JZHSc5<@CUCbw|EJl63IXE&KNY|0m409+2bC!fXk@aewOh+_r?3@7N;O%8hCz}Z) zWr+H^$e((f>r4=i31<4Rlo$H2%s#5@quM^|?4wZ+H*l>1+*!dJQW?K0)eZca=tSr7 zO4HNn@Bpa_Hh3uz$-EF1`WcBW?o|nJzj9yU{;OZrOK6HpU^EDuEr*kEEs-rx{At6fwG>4|9UGhRu+`|6&E z20HHs1Tkw>Pl`76oVS8XzQYnf9oo<&=%F|GlvBy4&9`p8QOU>oW-7T#{fDe$A6@p* zYaatV+`zw)G|~AZ!mP^qRULXd<$_ZomS}-@+x5JzbB#$O7_Y$@-9VekDPTRFkBpJN zk2nZM#Xxxcg6S;k_##xHiQw0CmBAFqFg2(zpQ_}bL4UcTXa(;^46RRGTi6pR9f`Cu zAD>msEDfLWN<_j?M>OfDnf&eiEhB-^8-Vdu)Ir}C5Z7!0s$5hb6w*#VECIWwA6R10 z5ntb19dXxr(64q_uj^b%L5%*@s&X>Ld=l6aG=GgA$KPcM+a*P3?9?j1O)_3k}4{AOtRU$ne9TpwVSud3W6(4 zFusa3t8DBVV%HEf6>ip8?&)mhQ3fPPOfftb>jV5AF^Wg2q$yanIvw7|18H5sJU*y# zuxdhz?CG39$SS8hO9)aFk%EAzMCdR=r6x4MFRL0Akg|^<`w+;U&cE_v5SQ#U)wDV# z$|RE7(Tkm?Le$cvFX49*@up!Sa^RNPM} zJrULoB;occlQL4;6H#pyWAjETG$3)#Z*t}i$ORDRG7;tlk}xho@sY=ty(B}?0X=n% z-AO<*Axx8+!5nGgZvlFV3sefq0_8!@Y36S+HC<^dyxKn2sNS9T@~im8b zR%zW?|WoHF%psPxe zZlExrd9JhCzMXF0>Ug8(j!oIXg9(^lBpNm#0}Un>3qEx(B0^G)GcpZEeZ~6}iru|( zmSus6m(@hRTcF6B{K#8ZsAg{Q3nqFo(Q>fR+6eD6ToI;cxBO46+u39G{9k7q7p415 z_WXOa=l@6ZUdfq~wAu52#JqQ;&7S}7ZQ@~V?w((5fIruII>1*8op5a5o?xAUy}~A+ z4(xS;-ELv`J00&E*v%I9`v$i5lc@+|?v~VeV6!}0Nh?5QLXWIi{OJ)Q@n6RY0dJt# zB;|QZ>50T=o6Mo9CUfX>dUU^6kXrs}X&YW7@w4P~Dg#^ekijxLvrE^i9d014ICnxn zyxR;^H;`0*Z>&QVN0Cm0J$J}v=1>4!Xp(z#Cmn1EkfC6vPdBg&_)NHN;B6CF%J8PA zRq`D~AOrX3%&Yqog`P_G4r%PR1E1oZdLN*N>RsC(ydJOc%x|GDsMxHG){inF9lE|z zba~h$*Z+Lx;;WX9i0nWkp9R%xiX#fwb#VY-g5FOB}% zXudwnSL{u+T$e^*%Lz(+d%2eR_GY|=7>{3Fg(^q3rK8k2kN6IaLZ=$&?76TAkUed6 z@c!+_KGrmty1WL5p>pZacR*Dvae~2<8=5UVslg=a@soVRCi#NCxs$7?Cx|^#k1U-X zW6q1RnbkjR{N5BN{M_y83pel<@1iz5f`l$r-bCQfCdw9emY@8KLQmp^S*DZ|pD;<% zct>?7pQJDf<~qkc;W@&-643MP>R7j~oWo7MdkG!PNFJsCdpcMDw^P10G2z~ly5f3~ z18^z!e3jvb{%RgF=k;*$o3vph>CiMR8%SsB<%aH3IxopeKgo4A$qEbVn4BbSevz3}2&Iq|Eu$T|6pQaH<f>h*Q&{V@_j@a9TOWBSt7oJ;S`nO5_gB%vvo^UBtXXYS3nNk%cXuI5RJEV@WgzEcG@lIpO=BS$FH4vA00uoF7 zqEwG$;QkLZA=B|H7QbwCF6(N(RqQ}7)YaN&OhbkD z=|$e9j5kcTSbpE|&q!j&#pA%_KfJFNgx6MUHcBSzkQY+No~ZDx3LDFvwWml-VNv82 zDna9R@})yRu!UBWbD0YLWyp~JLwYf;v8+^bnMr&(iBTZ2Cr4v6Fsc_VT+y*R-MV+j ziFrh&+b(c1T%1(bam~|(d5h-YP^@@o;nVrS$d*KL>NQ)3DlN>aFi@ENH`1pw8x6r*{U z4q9O!EA3;oeXOyMwf51)Bays?UkH(3{)03il6{qlRW<>31D~e&MCWf!tn%?cleU)^ zb-*7@oRKe5Kd%*3EiIH1O@kNm#+@!8TID1ItIQzDj@p05H5u~3HZK%oR(3>p{7|)1sM_E$9AZ|Q-BV4PL*#%v zC(Y+cvoY}gFb%iry_P%8(9{t0EBo4H!-IP|SJ4I#V>`}BoSB`fE~4p&J|4)|dP5TL zRiqnP8FQ~)5k>MfV_ivNwb!l;-qJN?)6%YJupW;a$$ny|Y;prFsl2?Jzy^?(8a1qKy3 zp#Y77mL;12WoU{#!V2^D<9Bu=oz)H>9$2N$=+jp*(`zX8wo4fX7s#UcGfv)BaKW`` zB0H=nd_KsA(lbE&jMrr~*x@~o7etKqr7sPa6hCM*9c z7Y3=F)={A8D8k6L4qDKTFM?XyaRr7eiIZj6%Vk)RX@uq2It^C)5q$A#$5a=*z<#n~ zVttApT`MTsA3vdJ&y}!1Wy9O1nW!Eo_JT%p-< z&0%bYBVSfoS7Y>!Dy!%S#`7nTG@|300A4>Bh1dN>FKe>I`HHOcI4tOdEXsmC)c;@9 z|4F29;L$%~%6#k_JeOOJKp;XJew^)VB_lwt%z!U5I=~h?fgD^r@P-U{7#_@r0GPPM zUlOwZV_P181Mv^S0LpazKtO~mqzIPFRLaZIG)7FE^FZgHu~v`^V5NpDC6wC=rmA@J zfI$pS1zZ4VaPgPvZ4A)MB(@NL@t=acfp~Qq0s}!_IZj&$=Jrl&F~o%SWokddGz5_) zt%{@_at8v4?gk6E>VORGN^r@|B62YcN&hZMy1q}@5yd(gMJaZX%Uq76a$&^7_-08~ z*dl;@t6uZ4o&djdjHr?E7@1v3;7}k_^0i*C1fgo^vDyIeA2M-nNxd-`-yj`&%X;$` zKFycQQZgcW-%#R#f6@r!m?3U^Z)m8SR}WzQwrG1PPCh{8$d79m>}4-Px9*N~*}BWQ zL|4{6Iof=Na}{`|VEvGv0X&9zmX~pW-ESd)v6mf_3;W~*z@5^w6Q>U0&?pe!EZtey zM|x6D05nN=`$;(guv!rh;v3XJEJ{4&`@3&kY1Y4rcb$yJY4B)m0zcXV@k(z!f}7*f zvUy6g_A0?&4{V_i1ot3g1kQuts1ezF=a7?i4~LxW_3Qgn@-XCd>LDi&um}!2Ile?K zgKwSWGCdOZu8(Q9<4c72r6sWbBaW<6H^7u!09PYD4=9k`uBo;;TIo#kfTa%1KALs#~0uvu({k&k3;_x=95XnS%t9TQSegvwF zA4#g`6_cAZ6H={~0TcI->g4^BYL((8G!=)+_rY80^emDE(H$Y9uCu0-q-)L?0ONc< zb^Zut7>97RM!4_6U_AN@dj;Nd&0R1V;eE59&-)-i5lxVSCC-5>zn6u1DD@~pugEJ7HlND&cr&&^?)yl-+UZXcB_zH)_+Nv{A`?4XKDFpF3LmsQ2Q@Pm$Id< z*4|8o#o9adtRfi^-9TR`NpwUx%4?GJk~=MYsFaNz7BW9$2km1SgK7({YUZsae+DNx z#vOnSJFR}o*_HaW7ydoe*~fWi2TOVTyB2m1>xT3O;|}XKsn3=dU9+O0x!hN=WQ{^rg-aHApbxW5-# zk#8Uw8;d)wQ=l2s+tZn7rzyA=XivNqoDn~h#mBW3KVEL&3%(=#WRIf1@#KNxI^bPPqUyh|d(~ytx{f%zc3+*@P$fdXap^$D2^qlXOMm zhY$-Jr6~b$JYu0sZoh?)dk~tFDIRv(wn2rvKntnc;J&e&dQ;_&Ywqs2<~}MCIOLa2 z$y*;_%+ghW8Ga=h;T4rvumlg~go|4^zMOgd91fQMN&Xpo5? z!u<dd2Y46|>B`zgPe>J&9mV24c8;o8JOtQqwN*xTy_2~` z70jeeRc>W<(}H7f5?B0<0{t2S%{~yw#$-+){!J!(*8rHXZ}QyPfcPeY13-Ke!P+_4 z7m@Rq?H{0)v)=zE$>qL4G(d?@Qwfw>q{KXotdVS)D5x{CN-`W-g&7&Q9=K^P&Y_+& zFOlI|Z1b!gfWGM+iCyH=nfRNdD6>`{9N6Edg!^6q!ZjYIaBSot+Ej|xzSkAnyvXk} za8}yp&jqAU`ZhWNI6ZGC{&wze4_JH`^UvAsC!&+&w*keU`qAVJwWJ3w27tZWaivS( z|KQ`1*$0MFvoPnOaV)CYu8zb37%Onu-yi~2lZ_B~i-7GmN`-uf>CH5myI%_&Kk#qh z68{~5jA+Pb;AsCep6j2V#+Dk#<&T!_wEW`Ch=yOKJ*T1tH-3nghm%H6O!AG#--@JL zc^G5h`uk5BvZ6$wNx&{&`%voLmjuP#{5j`8ylI<1$9f&^KE4RSHh-QIMPnr6oqTbN zO*ujY&r7)>!!AEE`za=$X*?w9i*EohNsE@y9_#WD0d__Fx(q6#_cIR0*cp8@V`XJl z^elew<9BR-(X;WA1`Ph^p;QYHc*E6e zXPtYc6yb4X1I*R%9;;VTeL64~j@K7ZKYTir`ovtMt92!%>?*yuqi7+_(P?$U4WsN7 z;#GS`QGu)D#)~lz3Q||6wF>MoF1nJ;F@0E>{|u>E)1VAOK&(Dc7R+X%97A2v2AHj3 zriKne+@AuL+;4_RLF9nR7%e2hNvT7r=fW4sJhA=8q_MsXVxcc^>P_%%?t(Y}iah^9 z;B#cSW4#F5S-OPT`qD|1^1civgW7j|J@y?N`-qQT596}n>4$mK790;a$OadNqe^?z z)sYX6d^@|E*SmVvZ2p(nZP}0c|E|P#6sw`su8TQ5sz1pbb$aF^>Vz}PgXcAkpxARH z3Pg>l*azufsn}0bzd|vY9{LoW&zTDH@aNCf;iNc&7$=D*efvR5-&t%g&ic%?YY0oE z2V1V?j@vAC5?cESOY>F5SqF=;<4!(PxHPQJl*NbAmS(2k9`p}}QhzrGY7}Q3a19~o zXVJ2a+{MefvT#M~u9A7XFnB0QD(J%<95|lqV4y33G2;rTTMs2|c|eMRFUJ#3;8-(P zdw~s!6-*=wNXgbmNOlbl-3>`bF+2V85)Vjk2zRa&F8hBd+#s@o{B8iVgg9XdUrOUX z$#9`wK6-34G!8vyod0hc7{&jg{Ev1C`9GI=rT(N1wd08SpZYZ<-n;_%qu9VXW`^OdkKyS#D4X_3or;uG>$gyC_q^*u%qh6RW2dzb^Se3 ze)~BR*T`=wp^f|&6`tRsWU7Vbw}nVg9E<2@hBT6NIT|K>e?Hrs-=a>N-)@vis`K0b zQTeSip;zU$n6J!jO0KZnwlJ96qRgW8UP94!=lst^doPa3hgu7{Ev9pn%4=qkq?uLv zU`%)YyCnZ(NoQ(3-f&zaudmtl8I9ST;>yDLc`-OZ&RRh-nM$sCFD11I2iZ@xi-SA6?gV2{aJY#`lDxRogWM;_3sf@ zgVqh2-ve)@`M3E-X{6?8F|9WeV=iQd{)s=Ap{PH@2h-oIKN)NZGBBtA24YR=uk|<5 zA7`G0r9T>7=zpQ1KPv;N|0&b|4=VjfAsPsH;+@F!N8_Q>KRWQv6sn*27KoG;4)86^ z3NH{7p8kp7ll0VoGu}?Ae?u^+S^sYmd*=G1ZHE3c&s_g~O8Txp+IJ{u#lbX8p$#d*=G1-Gu&MJ9GV?R{B4RMOz5C z4d5&EKZ}YdUJoLre?8yA^j}4c(!UTNkvUxWV_S}cgLoSTySs8zioB-|t~!))G>_KN zNH#Xvn|HC)52b!K1C!+Rw@CkHe5b=IE$&X9bTrdQ-af zc{eXB;=Vw0(ujS9VeNWTGAV2d5I(oZhM=#vo9nvdg z?@iJj62D%$wfHL)abF;MWbq^VqvrdneOF;HkmhRt-0p>>xX4g`I5+4mkSMrDyP`34 zn)eFs-ql3WiYRNa1iK)xX@(n2UQ)d#IjTY#U3GNygE$_jedwed*6l^FVJP+K^QrCJ z>(iPYyhc}M_?H6x{mk$~z7|*e^(YF37^NLs1Vqa}B!ZW^dRhLQTk|NI90~-ZfqTzB z%&8BZ4y3(6?9=yOa`cK5ig1r!%k*98u}!y6-U;REs>GP}{Q*}; z!HHwGKuQ8DhQsRO=aSAPHuST(RTG~=KLx%L--KHuQFd{88am^45b4f%bYSag194F*Sq}oz%@Nd8{@=rw7UTsLY@i3rpttsl|i~AmpW8F}F%BjNG_e zu1S6&HSAo3!nB)flEusVs4`UvG1M;0LfvL54li3H~DG#M? zfK@O}y-a$~DI>_jZGtDImRuPvQsi<}ca#fVT``z2JL_yxc396N(5h#%gkhcB8b@SA zZc^jDuh19E-csK`O;`GY2e^!oj?YN~=sRR%++pS91EQq9jj!_!HBRbEyoL0=4b}p3 ziS(Z1B9QvZQbTl6ZAl!E$35&6H>_wN~}(f4IZ0DY5& z>-*&>#4PpQ_Zr`*?~`~7>H7m%3&;ng_nhqn;rf1GkxPB=`EgL+MpAZIHxfkZdlixY zTYdk1&S~_0TM|IuDO4^zeG`KE{^k%orM{2hEu`-wuom?Fq4b`!jUZg#E=4Z&{pmwN zebH9R;1G!3i z:^*LQ^?m-;@#_O>gA`FSQOJFFKIMCv<{$Px8LW1;tNWY7Kt166rV!e_ua^7FD= ztc9KXjwKgkrI_=}#Y6r@dAXwHQylJyK}X=edlC8mC28pGQ72c&M!R$C#qtbD+{)7F z#EpY~3qK}6oA4^u&tR5gSZaWjm`aZ*iov(f&eP@*|s ziN!56+Ae5;Yvi0;5quaH{r)Tag$T|`!96R&T_54jk8mG-)f}Gl*}?i;&52Fp3JNai z&F_`HWqd#|F&Wx#-Xopd@P^~SgY%XV?H`(%$(-UzX1U7v`!qI z@Dc9G#*FMJoLh+$dnxc7%5683^)>wEO|HI)@~C?B*cyR5ES|3 zl>k37L@0D2WvX%~ERwcNC9S!i{R~3{j&eQ}3nO`q1)mF~9@$R>M8gr99o9e4g}iU1YYKdj3{voCz(nzXS-pCJQ2n*Qg?N>8nHT_mkQw+V+Kj+^pOGyB zXC55~X0Ta!Uq4$Z{Bsx@w+X~6re~3c&O2G>K<0XU5O6^1lD4#0fpmwhamIO5AX+DVQx_b^GM(X!G3EALL=S8 zF`M&gi;VI1mjbWG!2dSHz}licPxz*>mOKks*aatH;x3by0;{cZ`HEv-to6MX_@;>- zYn6S0ze}xr$c^!V(TFVYaUwaM2po~{aG)1ZUaU6(6py~o(+>hX_z0#Lg$RBQZ|4>q zCKPx9Z?g|!tJ+l%p>xl%sUTda2yGzT3h$TOZpd&5(B1=w8i+Ve6_LD?JMe1+mx-|f zMJK$>zx=Z4KGk`NmS=_dxMi$s*Dm{-Pq8243>k`9*x7q6o-F0*5TMSe|&y?-C!mLS~6xzE{$V4oK!J&v2X zKf`&0u>T%@?1f6wLm?&xFs(I(Zz_%hlns8S#M}@tuHfI=;$*t(Aw1@gdO^#`H&4J5-@K zTIz)(nX!NjnU1}!R6^|qU3teB+9ZCD$9QEKeF-*(MW(u4oiWq1rM+D?etp#EOPTE- zbS3>VF>enx8+@3<3LtY2ZZ`Qem~J?Dc{x09J@70`A3o98YAt{}@Cz3FYcQxnHs(@`MR+~-x7&?W)%5}I7K1>}&NorQlfL~3lm9{`TNXYZI}c^LcQEuVnV_5*+7z9HX}6R!khbHGNxdCi_@PzQ)WLzK+Uv?;n1yLwBn!cNM$ z)~9~>@1d4Wt{Q6qFQ|NeB0++E^W3r`7j^JqM_UkgljeLtk zLd60`$#hF&bFJotZhtfI29js?A*6czmbY3)xxSrx`7Wp=3z?#6TW~!v;`TRvFMClb_6wkD`pbK0Q3i|LI{|rNtcgGB#b@$Oe)* z8mETJeByAmyVHd0*_?;|{Iz?s7`^oylw#hVmL}{LNE=H1*?3}$P@Iae&torKSlXDy z_igGOU$-fDFWuagRBFd8ox?R7@3V{SuGza>dZhuJ&VggL5!t*wkbD;!^glhtNU$H# z^=UubcRHcWHG8%TFn9U#fkqtZFd+H0|?0~_-O zq^~5xZN97yvrAB|c69~l0cR$8^2Nc8FL9Fft?~c??4K)hb;z?KkXO4TGQW2lW95lg z9rCNezzwqXR3XrZs=^$28JKWaiTt2oDD@X9$Xq9Iu{yV&fW(P?U4uz?qpriYE=GDI z2)J9%Ih1-GUeO%Iee%{M|5WSizkvt#WA(#K*pEeP$^BT&a}2nb0ihK_u=HH43$Uf- zV^>Z7eC!F>oF_}{`_HHn>q=`m+}^td8jRWSYJ2Yqyx`j+zf3r!HS#yp`onQDtq}(% zE;r}ZG_N0MnT&JiV_L>{x*T1b2R?83D{dIxkc62A=RFqRDBXJs+z9z1`biG#?>Tz~ z*mljC0B{U_ka?U!L%ZWqeMp@T0zXZom;K1WJR79RZdgEm%^@Fi=_U#-UBWMiA)v!W8aa;<4vGY_8oeTYtxkf#M z>QDUcd|2$Q=aDggMzdkZTOP{NZl)L#8dHPJBGz)5g4UlCbXX^YoRm(yJAm46{p2|z z80&uPAl_K~_FF&KZ@sASglnB23==;F&b2?r!k9yNe1G@^@Es*F4g<;kjaCDRfCsVZ zCBQQspjV!%Pvz2%6ICDfy-CG?#f&gpULt_KZdGWT@x2~>pzK_uB$cQWma9Ug2%mLg zhp8+>+)0IMZYcFdjMawrdz-qf1=0tjx-M&`e1*EZLUlUUX~_Ry+|n}=|0iHR1Wi39 z|M(F9LCHk%N039f1MCCV-#Gy8EI zcdv)k+00|QFXN&S9f65Vp{~?MmZ|{qlKDclwsSxY!;JCmaYrz*33#wwH zgm3Bs55Yuh&u3o_wV!Zooa92EjQ?PUq#y5Tjz@3{LKYxo5js@qo8J{lgRFRxs)U!m zG4P!Z-(Iv1qGQDX1UWMA#0?p<4@l7{QRxW?#a-q*-ybbGMj^~cD1P}#JKs%dZrpD@ z{0n8P{nkTxLxnEACN(c7#pLD^FthZ#cD?6t#rL~9R@hygMfQx?`LQiIoz{mDqP@?y z-}*hgApeR_iUN)fYZqL&(4fz;-+CQhQFCCAYb)MZ%jF?M67K(bLHPO`S+0gJ0FF2X!;oWw@qT6kbhGo=4tuf zEC244f1i_oeoQe`Wceo+95r#N@%y^niRnB;Q|`f8UXRTO~e6 z{#9~*M-HBPEUM2`+L@1qCtR6&FaW%exe4|_4}yKZqE0N-{xQG(eT*IsQJOeft@$qMqg(8BtJUI7l%9PqT2@sOQO+% zMkEKWKKf^TEd|~fhpBlDHT8mVmO~xuQQUbY1j0>Zujj!5uP27MqVpP*csXJr`UQy#t>iu~ z-C5{{}JI+3Y(cXWdH_0WLPh1L8e&0b8iXt$K-$&lN_aD|G;HooWKowR%m`L=)_?}M~YTDX57!{mNT`uFfX zQ@uO+UZUh2;Cpvdm@QX0U`T$U6!_s&CPTK8^ta#FA{hEt_#}U6D;M6Cbf{O1w81xj z{a};dYe#rBZHlio$$iC#$R9F$f>|B8Pq;J4z|imE7KMVliX?YmA>;G&J?(y(AJ~Q+ zO;GOx#0&NAApZD$l1AD(fOkVaL;fv_-wB56Q?$K&e@?w`<9m;KKg{=s)q5-7x8dEy zFOxVQ%bbz^>RqPmm){MlCWC-R(ifWOD^2wCw+*AuBmcYa4ceoU?kyp=OaouY?WBKg z$Q_`&0&c^WF4Oyb^}d5N*(Q7j-YGD`)X^`+aChF`1~)b(NwXwE^k9-B`)EI&n~x0e z57bHdy$;atq>zEDKM1CLkWCvJ!=w9Hc?D4%7aaZ7TGf_ld)<9cP#&2ETS9IpSReJl z4Ox5mUZ?%E{LFMmkK8NMr-$!{)w>LraF4{75${*{ZoWT>cO5>HJRg2dibH#BqkoE` zZR5L5(;D%ii*oegLr_)c-PVm#HkmIQ)cbbQbg!3qE+rf-?3eW2jLW5H{KT((RmDr* ze1gWtC133W77QO!+ue01@v%c$aHCichw-X_b;{a|AIqSI^~iDb?aV)L-J)Df82Hm;55sz*uVq$bYmEUU}PRdhE|NnMd1Fn>%${s<7SzC zU~?Y2f&QescXjSkpH<10uYJMmPa+cEC4uLAu%OY4KP8>kgTP^0a3=uf*yK{=%vSEu+LAX#VX695LkceWV(b}#@Q^6MsHIKR8; z4Sr{lUz+0gU6OUCUI$?C`%aLbhXL@AUnL2{`F;5-_<`T=9tXcZ{DHoo0SA8X0T}$g z9^`kJ0q~HYpM>H3_R<^tZX~}}#cvnMI#YiJp!h`vzWFvjq`txr`)y|cJmj~PgeHE+ zumoaJOKTn0w;n^qut=WQn)oW-v9{MTAgL9D|CjxpKMJK1&!JcaztnKN#;5U|7d$Z< z1|}%?iyHny!+SNnO~VEaD>YoCVWx&-HJqX2Db%n`!*6Qn)395^pK0jV@O2G8&~i&* z@h6X8BI0pq7^UHRNm_mlpVRP18hSOHtf7C5qTit5d<~B$D*R&_uF>!U4GVRA+Y=OB ziG~C5%KbA9uhsBu4d01V{IxrR0k|2kUv|3poS+kOh{Jo9M}5FM8Kbqbviwv(-x zds@@orQJ_z=-2QS4ga9wk97DCwEJ@nb2J>Q{U>Ya(fH#!eusA3w7(&*N4wwA{*yKR z-?V(MXz10@so_N$W@!2u+C5FfG!4I_{ZATvHS}tDxrQYgIyJmV!zVN}d<|z9^{R#^ttub8|W;bR(Z)38m$Lt3v&?RIK7L&FpeV>LW7gxOge zSfydNhJFo=@CNj)R(!HGKLdZJaGLdd>6EYHpwf@wn9|!mjuH+!@Nnkz@RKT@I=6y7 z7eq^W%d!;oJgwyztMssISoV~{H#8||;8$1II;tBQnrfMr@Ry3|_Y}(l9aJSaDJ&Fa<`Nc-DnkccsUE^!= z&ae2o!`Ia0XlQC&E779a=dEvC8$#AMZm4dkuXSj=r>VJK#^UMKdQMY%4muR<*Rbj= z7)*&60)=+v-G!zvBG{Jx?;{Gfu4hZOW^=+dxX`wtj! zr>0M~$b7V2%bBg+mCq^vo;c;dSS%Ce;tG+EzlG3q5#9>KWhM+QAmE!XmV#oLD3$IC z;YJSdAureBZH@4XYTQb&9`B97R^wMMn(*!bRXy+?(EvZ@81*>dQwEGz`je9i*&sH- zXFWK0@XrnI4)F1q_$>ftx@gh77J+6BD7=6(MGd%ZkP#G0{Hdin$HapHO2K1;j;|WB zHo#qD(xX|&IsI-Qtx41;cH&f(5JKE5& zTt3{4+bdQgz3GgWz8D5*U?zL-tAa9?Y#M4KX zufwW(v);eUn)v)Or{8J^vA+o@i<%gw9 z>etSwX!2Dz zIEZm%wltqVBR|-ZDP(DV&RSm{*1+5x7$%(-MH56SYCgf+acMFfS0@$ozGk~y~b=ExSMCF|BKvf>l+ts z^0}KUnv7T$OfSADx5a_rBYWi6*2-43x?!2e?Ug-&f~J;6--78aj*2?3ySg^$q^Qfh zwQlc%>9sSa&&*kKx<8D0hKyhR)kWg}`{`NI)a*0T5nU&TG|`MmtV(s6sxqak>l={h zrhKY`Egpss4;ZRQbg~E2J0g9`nwlCI+k)v04K0qvUexQsw3950eeU%^%!+1rtr;=~7!}h{z;25`J-0lbE&2Co`0hduLsw5zSfI z(^u6rtzXYxs1PRn9J7ZBKP%hhkux$I5yFSDG2z452#0DAn+^FK#p|orx>3_>e3s!} zz8>|15RQ^+?^-vYbZ2Q7+?Q8-H=!rHsu_s^Zpa>u&A#eJpGFv2z`cs`!`~>Ys~~4q zuo#Bq2>FM3hr-RB6Xu~6nBMFtcYEB`KF9PHW}=Y&6hY8Fn%=RzWp#6nw_f(;S6y@w zM3YZAe%9>S+9Lulbzy*Te2_c>KPwnPg(wos(Yq){uW<=lT*|r;zjAkTOM|a@W=*rv zPh>uy!}5{!{R8-`xdj4TK5MHrT?(>WkiVEmeC) z?UhneX2S@E4zj+14O+ZHD++NHvIKq1YHxnS+9q$kuWr34^SaR=^+I$iuNZAa8eZPg zu&%n%5#D?@d1n+rs%D259ae8Gikp}sR#mgnIq%X1fsE`$r1~Q9&1wW-X+q6(HrD?eQrPZGBXd~oW(YQW>B0Ri`M^&*Q z^9EHmzl5u=tr?k~il3~ijRZ6EIFt8?c*5n?>1qfJQ;+Dc1z<{q`!6x4LviDpCL{Yq z(6I?-<_TH9nTZjkX1`c=gk`UfS)#0NQ*(VybwiwV2L)>pTx zY!w3+hC3x+ z%IWboVO-$bv^XS3QLE40DD&0jm#wXK!Mvv}Fk<-zEN(H9m~6ctpgDp*!tz5GO(8ACz9`y zl4XnXOIDSxs3@uoaT^|gY4Orks&BDMbuE^Z=PzARl3!k20j1HJ=!_WRzZ@Nll2xvv z{IXT0`IW1dLv#~ih_0|`asG;uidEo+9*d-31O+b(525RcqQ+Vk(S~)XgH-*cGH<#2 zTP^io_j=^@<{Tubu306WEID6ECJK$@4;4+>FJYE*fLEb=O*J}ltjr|dc#-`K__OGc zPG&LE7hPUlP_&9E#qO0@?yL4OpljLkidB~vFE3tHBKvL9U9ikmRF1r1_!s0CxQaMC zi5(C5R~46BX{5PE0zatdH3yt}-ZNjp9u2dLG=7|d9_{YY?zY7W-+ifqWu*#Ml`GhP zm4g1O70kXyL8pfPZz}hohW_W(JZt|23co!|)6Y<_O~>1|7ke$LYbUm=FAIM*Ee1$F2|fy3Fe(S z%e)G+u|@cCX8KCF^5IhgR{`ECBz6_X7Wtqk172nufm*91|H`8BRV6EyqTsCJ*sQkN zTf0gwH^{EKjPMlwPRH`<^_~W|BWGr|lIwCgpS4cT(m4>5d2WsKj-mNs&bT=s59i_3 zEVl!*t1~e-%g0gQjP;oMYINQm8_^?npbh#OCt|C;juHJ$e)Kn<)h#~9SC;Y2^Br~7 z&0z0ytaiH_9q2P~W&#r#zB-H+Rg-CO!+MUv?-X^xY1q|s6isgEDSEiTgTB1PDJ2`K zH#N1$wIUK}TRarOU2Dv* z>ejTm#k;1u1{TxqjrEPQb5;eh5)ZM^>OzfGUa1G!i~QQ^^~jtX>uc84JBk`H7bwkc z@M+#y?Ww8LvXEZ$U+ea*QkDcWQ$t|*kJP#|xG>f&KiC^3-TcFK%M)-r=)hyak90AB z$M-}DQl`Kkn@@#j1UmY6gU)yuZadt08e#AwtTfRv-l`Gkbg&>`I0xt)8e;`C^w_dH z6tC0+?n+H(jyIC7djvWiEC?80fX<;YjQ547GA_Zv+6&+K)t?puI`PXF8NTA7p_^}p zw;w;#(*~N*qr!yTlw8IULW%f8FXKzQa6D;@pE(S1bW?ApjrnJX+x$y|+x#1X+x!c2 zz+))l7f#KP=7)N=eO>(o-|>FY~pDj&hLSNOblQ=rsEvVEAtEcW8_eFY%G`RgFN$c;D`4 z%BqL*2@{G0z0XmNkY49W6>%R-prNV7TIMvdH>IKrL zndq4R9B_97@-g_4(_=4(`0+b8KimU=e9Uw<6CLf92DhCAcnmtqSMpMbA3wfogL`WP z9Vy;^F+@jxgK!5%psR|Y^S~X+kHJz*bW&fqD)flFX{T>-mX#fH}YvD-B!@K zBKR@xV**Wq;JRLP@1^h16-&XBI|6m)688xQ$qo`Np#FX43C;2ud|Gabu| z3v^B$f!QvVN5c8l!5x_oBJIL-Jp6vhE_569WFGLw!+0y=%qc}slzW2_IU-uc( zRh}W819Xx3KGH7KW8mFWluJ`PL%QvtI}^LqogrQJ8Pc5?K>b2{<1zBVNP6^w?o8xs z1Kn1|?ko8@&yb%DbX8}-ukX)iCSUg%&@un+cxQNiW=^gI-gsCp>)@`^2*ZAoE}}lL zoA{aOL_|H*54U3kevx!r&ydbO0^NA<_q`R;gYN9NM{a*7o9L)-Hrx(CK87BI!|F-s z?;sH#qdpkPE;~$g2EUQ$>P|ry6Jh7H5$GuQ!*7Q4rMnOAGs(|e-xyhsmA?spi zNx2HY+}}ir+#VIq@Jj$b1;4xDcXxyz`*-}TVh8+oMEH$?pZU4l?D)U&-R4mHbR${X zw?;W>k2DP<{iU&hB#rTGKL^S@h4$phknIdV14^EtF`i7*J8_bS_Qa?CLwt4ePXwJg zUO(b(g+KNblXAp|KHVn&9i;#DaC-7Gc$?!VfAeF|5Enq06Y$4gWiv_GJM)_T>5zXm z-cKb11CpmfZI*-ajP$TfZ8zy-md|!>w5Y`8so0A>ENI*GXgR*7hdG|g2--@}89dE& z{qj=QVQgjHk+(AxpC=<)P)0uH@PUYU20+KhCja7lI6x@)E&$ICp! z{DHmclGgl(G8weycqpg&=>rclhIT8%Z_pH{nVz&eOgyp?cbm!nW?H_JzSTr;$9tqr z%wZ_=Hj_R(Oz)&KryJcy{*6pGvtDNXhv@}9&opl`KI5UBhfR7J>B@3tjxRD@?XcBX z+06AU@+*FkkCB$dmz@_aJfImE!R}knAGtiHJvW>udCBxN`H{B~mstxC7Gxfta7iN;MchYY&@$hGk98Y!x4|6;WLw|fD z5yso>pwCtlkI1ylM%YMPWcbSGhw~@KX~LW3It={}Q~wdW^-D`_f-TZ7JRyms(5zf=gC%yYfo47O2+2^)h#pcNG_rR$@(3 zd#i;(U>n?6#9OtYIVXG8oSB~5)r?JuujwD(&&O{`K}Ct2SVu%NeePCYMr}R)L^AsP zHBlP3UalQw`&6G@(XDUH$j%mJz*V6Gt#GZP=lUqY!(5enQ?2CHhPh8vL^JDau_G~K z4sqyj%4lb1XVShL$}2J|3-U>m8G$1*I~*t1=JSBd42506uvOu7R}ohi zhAWnFcp`8_wnpFzfNKlKNq$?yaAlHTcQ{VcZ41LyNV@I7*}+%oTOsLsfXkc#oR@Zz z-`a_lqYZb47vyrxaRaijqRF`Ky6b9J!+fG=m$skRSCdhkFXQM9!(Aog@SAZJimopV zr|9~D%g#YOtx6VgJW7;U40lE=Whwa=$}Vv#T|q5G?I%%k{G)NRiE9PUl>WKI@%zX= zte9xJi-;4BXtABT=_oli=*D1Cx}knH%5Ej|(duToheJ4{g^fHsIzUylaDvUuwQK5Y zn={m>Iv(_ei1RjkYcjHD;Z;-t*G9P&PRdRES3Q!B%MN;i3kpw#?<-~i^XV}EqwJBm^?l>{#kkX1e4g4fZz`Qa{6?Af*qu%!b@>wul zxs%glMS5+5NP)bu-Y5^_>qS_`Yw}Eq>tLe|v;Q^5)L^U93ENg9IZ3ShsK{m(- z*(|vCAZC#e=~JyDZFS1PSWj}5JzJ%Dw~m9{7@m;)_!v;R)EI%d`Fqt`G*bxKUec8T z=fu15v_a-PIGhFo@NwW=CO2OPPysUDrL5HPd>4>zhUe1-$T+_bNX&y8|2!b$PLGRX zJ}#rZs7pG|&4^z#sw~!|pR#QtY?~0YZJpM;tVPA+xj@BxFCd@lEakS0wTiK;$9NK} z63XJi(=t9rj4xLfOuv4DxOT?`OZB!iOXb#d%d*yqmL+wQEDL@4Q5V-bqb|%{7&XO` z7Zo>X8;I?X@ki-=e)#9e6KJQXXO)h>0g!sO0#c$afYk31?T=UaWTPQuxSRCGhwlP1 zoCo`S7_JyHGTcf)_UY@i`vySz??f#|`drzXVXr(O`RTf$EPYO#m~6RNBo4+8jP92u zr7Ai*s*HKL2Xa4zz@+y>E<)x>K7P&T;knW&F1Qw2ymOY$g@?6ZpG+8B|X8!eIt zP?t=N6X}+$VR11nw<0cv>(}}l@$}DE{8AUFc;fO^JPtrUw-xF5Wcg$nQ03E-G79N` zo=CAcgv~RmDz;3BsWz19xa-{F0zLx?*vUAmuGBE$#8fiL~dZgUumBVIUs z(}klpLpXBJ5f01QB4u#wfE{tDdNfJI-4*MRbyn>d;kXR;$FFQZcw14oRwPPy2XHHi zLwxi%yUk(JGe(KAwc}B*rph`r-h;lL7xg#7B|G9EM~awSF;1lAjui=qY=f}_F|t0X z(l#B?HvIr_pj6pw{89wQ^9?`?;Jw;Cewp%5|3<2CZo5#-+nOydYMmu=>ah39*C;Ni zT`$hfZ4eVJYed3e+<>h=)^F73d5G_C$R12bu|egNGR<$wH)M3;vr5f>Niz6e1ilx8 zZ!7q21m6wd>lwz^4!-5cQ`Nqf0u1$@}%1w!xWUw<>1 zHnCz9+Syn!F=zZhy0k|t?6DE{7&*VV5chRDzqIMN@6fR4yNZ9q_f&j;y-&q|$^FXx zukBjD;$)GxHA~EHogpr)`{Tfh)17x_*9#QqX{ZY04F&U72W&zUw zmp@bVuAi&$&ja$A`7#r zSAcL|4-4mEIA@}`(6>Nj)y_vfH4pVbZm8UWKkJk>$bA=RnJyk}pBn&akF9`A-Mayq zU$z4>|2zh0e!5>(;fDbEEa+GEEC=LMq1`P9Ret*mAo<;TNae@t|Df{cgKsEz^kL;+ z1IUZnZ1FU@}K*@qW>`<<4-(-KX{q|8LktMbdLg(?;b$X{TYzY7l5+7$+6rhIhIrX zi&)|LScv=Z>v>i6CCqN~crP3F=>A0WNwJ9()U7F~TT@WCrsUW?YD|am9L71Rz{xS) zJvMQV4}GeN;P|4-f^Q>gflIB$_z(Et`0rMmxb;ZHScJG*jM02HVb2{U>^TVomRoG% z7Q}7$#Q`^3*vo<2Y!jO+BEo(_h!J~my3?X9$?5R(CEzf28{{$-WTBJocE@_7f4Z=Yy<@@R`pj~YORZv|vJJ+9s9 zDKW60jbldH=Ni}%-cQbsQ{k=#g z%wfe_#1{Ov9aL#;cAv@?-C8E6wnd4yL1lz35WOhF`~dZQ2$0V!fPDL_##_cHdn^Ye zT^%6p@ql)pN`^YEUv`oTcMBk&UjXuLpT@rnNK4GND>@e-`QNPFk#=)xzT36`OMrZA z$tt{}j|+|i(=@#^%s-TVgO93o7&QE$w=rMR`vm3##wosXKEPBajQNRv_@Bft?=_V^ zW_LPvw#MZqia2j<75kOhkW+lL} z&2BX>6Rexmm<#nt*+h%@IevZ}#_Ud?6e|*PlSG17jSQIipw9zY(_pW(9NpiEs0X@{S6Dubdd7hEER}y|0rIH<aoET!@KBEMlw=^LVvZ*d<0dXIR8F_}%w=Wf!yC!f|FUcu$QLQ!VIQ436th_9vB1 z&9I4#BNIiMZ<4UrP8PP@X#(FqiQV`ONIo_+iW4!fHVKoYlQA$2_gDS$MD)uO&@Yci zza0H^@Av^r3fq$;v?pUwA7bw1F1t#LBMBlKa@6DJ`h$|w?6yov5L0|(#gy7)F$MMa zl)FNsFFB8O3);bHap)J~H}>qo$paJn)BNK-sZ}Xuma{PjR~Lge#4hZ2jUMDEe=4pG zyvjF5SdS!$!TA<3W`RXKC*wsP1U?MMb_Ds1dCdQH=#QVw%kij(n9b<^%HS)@}>y{&G3aR^<-!Zus_2j9X+8 z+uwjaaTU<$;|LRpPeGfGHY2ej2KrgWji7%FKR?5Ozu9e>oPaho5pxKDn2(&C6HEh- zB^v!BjOt{WJdV6~gn1A1-IpSM{2YhDPud_xx{m{wNgU#!f5=Vy52p#^BT#oB4uLpC zL>!iM%pK(>i*zq?@<4Kbl0VUtP!*3nFj|~FZQ@|s!1(?YznrHlOG3qJSsE{v9*+^l zN8&`GZ?u?SVH2YkK@<8(eW5Gz?_}hwFnwh{#=I>2Qs9^3MZZYanGtmi`ZV|^QJh_6 z5gBiyPnp0vB~B#ds5(Wq*~d|*9=<@_7lEe@Ma{`518)#^`C7 zvqbyAQMsSaEGS zHs&eWUrE#a+ame`Wjfr?0r~U;^6i+xk7W{h<}BoyG~^ld#VE7nIrvLe_jQ|j)+sqw z0`j>TkZ<>DeDYWD>jOW(rkh-^!Y>5m+f{(nFI|RDfIRV#N3LT8<=Kw#Ro5!IE7qy- z9e~mwx%rbhxt`6rTz}zBK_#ezW4mtlV?Obj%Bd z=SnsIUZ(l>fG?ltG&I@=1JaND%#iPW{{wm>Jj1>IzX-?hW_ie`TkHAl^-BNhMy)>} zHM$*;&m9`S8<73CKWTi{w=4oRtC$Oj+Er8l^0{2&?*PP7qIgu}-)=@74)@;xlK`zg zML!CV_yvG=z)FpOti>X*Y$f&sQhqrGv)mX)FmbBH;-+lb1c!j<&})PS`O1aJE`RH-6cFQf1ui9zh1bA%lPn ze70{={J#X`(|MDM_YpwGn|y8p%5M_N?--QdM3!G;l|h!d!+M{l|7Jx$r%mx$x>@n5 z2b6riqMY@BN4KW?2$0X5TU7WmK!#7hC|+ddW?(&aVrabWVGE(=Ahv?XZiFN4pvkTt z9sWx|J}I{l+}MCI!# zqHJ%faQVgyXYB+rry@n|w~3I0t=}vyqnR zNK5vWV((J*W*&4M=u&#z`5mR_&j4wkuL@^JxPDD{^PMW(y?`TZKOeD8o;Q6H_4Xqzb#3fIf5lT_{1G7+HS&4EtS0_avZ9 z7wt~Zi5HGKw4=x;a&5^!$`e~~ zCP0Qu&qf=E>ssuVaYB}M$QoT1mCgAv7s7d^l&Hrv{XO4T;hzLNUAzv2AJBAn|3HO% z9Po7EPW%Gtqv^K%P=$K{@O0t&5zepaeA`sGI{_I^*%$q6%j6O5y8~fa4tosxdsX;c zz)<*PgwGf;d>g_?AxM>`pL(APKOgXPa#td(N7H58ufi<^JYBeKgsakYDL+!-E(APX zI2*#1X}XvPRJbXCrwcdmU+7dK&tFgV@Q?ALm#7I<_0V2uI#x zlPWvo02%8NKw_>U{>O^Xoq&9L0r~ddfTS7ouxb~de?-yAabGR!8tl=K{pnncDd0Em zQH*=vS8a;foqme?vKzAT8ALeB_^TZ%{&xWBUi?$#ZUv;1j`zqm z?Fibmim@X5F^eeuFVG)R@tfVrv`=lin3|g=Qa=vuvY>tHw0(>|EBg?RU#R$o9#`?( zdz3x00HqvbV#S!dLT#E{m&LfQ4F1pJx0N#3uvUw82D<0`5_4b%4(%`9b@)A|WsP)) zg<;)|Fao?pL>z42GIAYQ*Bd`#`{smh55c~a*@JW@q<#axQhJ>Kl+y2NK)!7NBu&S! zmHS~px|7er+SWtk2eCV&-|i3XH=zB}n1;_Ny1M}ROngp-p99G7a%>E;sWF(h#u|Sa z@OAk0{zau#xEq8KG-};A8Dk)<+2oE933r9sgu%y@UTr4*oWD`=9(qB^^En{jQvF&V z&cmXun~)nXMweR$jQ&j@c=2&+y6*w0)=Q@cDpzTL~z` zvt31d8_|bQeR+iM(R7#XQQ_ABGMpN_+Hj5n#>V)my-tuBV}CPlgg#ubA6W+e1A7&} zy5B1Pp8)bLd7qZ=JgmQA57dN1!|Rhwgr_XsCVSI;AKcwK{IvawulIoB+XYCPo|ja8 zGUs;+Uj|5gdRd}a=u5=DlmzTciNn5>m|^=;1~B#;YiDX;KZy?a63(sp!9RWaXsnx06ydu} z)Vf$7{MFu!X~Ov@i})3Obpe%+&2Eb=M%Z#<*xSsO=~fXZ90L~N!|yRoZ+4s4J_Kyc z$MLSx!w6pm_mlV;;ltdV*FYSe_bg&Beg;jLThp|?ZxOHKXV8SXHBC3>?~dSS(1f{3 zgE@umf3b+?@#CDrRJIhua2P9?Xe>!Fp*=65wIMZE?A7xtavdJ?eCV4`MBjWO<_;#7W4@1Zp}cc0Ta}@Bv3(F_ z2*13)Auj3zo^<~nze3_*|46qvjAdMm7*`uF#+4^|rY6QQ`UP41}$;eONKS1scEqy?MS>BqvyZ}|7j74$I;K% zyv=S4?uANP6z_?{ct`388QVa6Cw|$-pd;tOG4EvXBK}eQ=F<=QN4m{nEv&=P$37=# z%HUZ8lls&Bavx_|qoc}7^G zFb9Lanv?~GH@Pi#tFYgq_iy$qM&gsB(ANH4r6cLg zZgW^U7Y@72`NJGp-}|S^xn{_QKAr<@b|rolnpe0x2D)SH7(r)gA1~TZ2yqk2a_eD? zWBQ`S`ra5(x5FxGw#ABNTStlfRvY%JU|rObDlFHgh-(8vT#b7B(!MCM5Fe-J?uZc= zZnKIDw#Eua>nJe^2MxpPs2z{UVLe{d#;~Y2b_*IlRgl2&Ce-GLu zKF z3&tq-j{(X52T6)gx&!+)b7x>*+yyc}vb}MEVfxBR;kI|10}a0`gHD3JO};nY3OG;1otO09}j|e@qz83s z)3*HuAZ_T^@D)I3g{A@I^8g@e4gwMrw?f%%KA^M@_jy!A?BCcDE4HBj5|^v?tEjcw z(Xb)bQp#aNxpzaYrE<>CR<8DL@cWYcsukbo0r`BU-J@kXgm`=ieJIx*H5iwn4SEN% z6B3u47QX-8pncUHCBDG#OVIKeorZN>mT$DZX?LmhT!VJT8r0MHW#W4wLbbmEYh}4u zKPs2&WFCX|C-8p-zk1M0zA;GSX+jwO3+tl9i}-yX{)Cn&^xct87Pg^igA={6W$Jyj zNW>m@ODgl&PUNv(na3=nF*m*%^y~+FRXS`E-== zbkqsy$V2HlsRNWv<~Iz1`BXM3c^$_UR9N+wm^|P!{}t|M2cF-g^I`fErs21(F`SNg zgMZry{K;WE@FT^+uzdKEP+g+=#4&ViM&7w_lR2P(hM-6VD^I~kgVVyoxLEm75 z{OC^WvjUl`<}jELC;V6L<0cMy5lA#a&@G-Oof_I__BJBn`h+z?>ohH_kb?8Mt_<;C>~3VfsSIJl80%=U*DILTxOE2y zegsqXDfoP&#ogk*qPpH!T#L;15HQ2Elq8F**Vi{}GMQU!RgT%LLoxg zUR(;SDt8fygB@@|D;Zv*C`i4;iyU5G(`@)0z*i+?Fe_mGQgT7zF_|2*L9ls^JgXB zz(XTRha26HEt*9r;hlyyb3CRBVss!~EohRmhZI zMlWmed0Kq(VvXhU`uBXa6pTT&N-9ps^%-#$xYj@@})&3vvZVvTB6EuHQF~? z>V5j!4ty6(DI4k`WI6N~)1yk7aFuO~x4FrCg*Z>Tiffu0uMkV3D{wDZa|62(>dp_W z%0-pA-Rrayu|FjW>zh5*zM8rat}Ch>8MWEx_7*gtnJ8)n-M6$gk+W-?HWKBNl7&J= z==@Nz;!MT4vHV z*|f9}+av}o+)O4j&9uqPY-Z9VZowePDTolXxa@B0PM~Fz35!ucKyV4#YEjUr1*_I2 zV8N<%-1EUUs;POKaJ3qqcDJG?Out;#%uO!p6$7#Y=U( zFLAuZrAsd=FSqwAHkU6pj7KH};|-zU^az7xWJ4yjbZND@vYtUV6U~G|cF5T^0ViRz zL#0L^@s!6|bS&W>pLAB6@nn5#EELkwDy;n7pcFMl6KgY#CaXp!TioCyDf+5i0+Dd$ z2nkqSYF@xWh`UBBuVmbEnytgt%$Xfh{${DHgVXE7tg*+EL5z#dO3~74GS?VxF*7Z6 z@N{UYoecZ`GN+yfCIr_;Gh~v4ysHR4UbeV2SXwhuJk-Q%aw)votIfr$V%bswBN)Ln$P$Jwb&`B#Y(C0XsX%WJ)Ry~8%`?)x}(-UKL%x_Ro%rW*$q?4ZS@#WRGGK3WAwvQ`4UjH^b2Q(Nc|D^Mm20ws1$`Er>!Ar*0*pm5n=geGn{c2A2*}zTs|5roG^CWoX73&ZJZm7s}9C# z+*_awkkL%IC1c#}e2a4>)io;5kz{LvOAMoPRGL|X*^XD~GwyNrUYFE5W7patTN743 zZqE-!P@`-)*p}YK@NBGHU9+UJXimVs$~wa^NpAx=jYRD=!kr@Ij*A7vnv)x%+^dWY z6XC|pFwS)RD;OuYAGt5+LOOy+eCS$QdRq)3%=K+H6EYq5wfehuF6vT_zrIxE$i8)9v#mUto)Gq|%K zyDe>RB8GM&#`qRtW{-wstuSb&!kU7jDCjP6RsiO%W9CFs#z9MT+4!i8Ze@28SeLH z+FCMVgbXtsZB!Ae3Zm^%+8-V|GlNNnMN4^GkNO~Qtdh;F9;*P;)}|R$ycS%h(V7YF zsu_L4Eo(QR>!31XhJ?d>YGhM1u_4})OlVD26us>7aYDI!dxOv$v{gl5%sWi54BaPS-HqOC;ieLn=V_g-YM$e(@N;tpI^-KZEC z%oN;Im(-$&R=>0JoN3g#%?cS+yeh3kw9T{bN~{nbvR!Q$34o6|Gf;)$7fXEH3slQWx& zW){t~2dgpLTNp)YZEq7o-7DR}?Od}r(AeXQjb!Zh){N%LI=oSgNeK;``)_Z6v2kdH z=C|9f7WsDD?OGZY!soeX*9|83x!kn>Qr^{8o~tOF(sVgxAZAoeST@q2SFhkoh9!C% zj5{Z;$}}+{NS|k3n!er6xktVg$>jRhRJl`AWr<7+BmRThQWI~8Ox8)OEV^;!f1^_&8hL8^!Tgqe{%`wStdO*Hv85mXZ{n< zJUw=)|HrZ2wogy)%<_*;y}}N%ArE3b2UXxcH>LMbCg4MFf+sEC2b;vX;b>+m-+F5@ zXIh?{g!wo}9cLU5o639ICm1KfMjxT`As&))EjW6czInGDT<;RU6V}=echMVzwI-(G z)^^9-?e2GP80&b1^L|FQ6OTUnJ{H}$yndIv+mt@n=c0F;yWQPp1?HcXFZLa~-J1?8 z9w&b9{;uD}kvPeNiF8ue%|d~?xt~+&pFPhba>*hBIBRCuZ!L|yxfh)^||QoH+H-Iz4aD_JNOye zlpj)`b3f;BYpKgFt_xzxP{`t-(n9A|oL?1p}aJsoHnXV62jLj^&j2Ew)K09^q zEXU);qk7Seu=lXzg>#Shj%G#dY`$%}k2uA3n0$}iyLu8letOLnVJ>JIE1A+^;-fVY z=hqA~m1)tpOe?>vh?x4}KH9=eOEqnmpVJ!bcr~Z9u!$MshOjwjL993un?G+}xS=6j z9E+BO=SE8E=f#Rj=EoW$(Yfb`Ld)#E?ElQ2vf_D|-*1?rXyP{UJwKx6hO9nf|96Ui z1m5HF2}I^L6wi2uQOgyzqUESMLK&56#A%rB}hE-Ic)`7k$DtzRCim{#E)pNmm` zE2DrYIeT8U)&Xxe{`=+lUZw24DsQ*N70s!iJ!fvTsH7-5d-j~@g2?Rp;sx^q;RW+c z!o_oniz(3~H??;i*&(?auicpHPtJHe-u9a2@M3OY`0Xtd*2)t6gq4W4#3P&*#vKn1 z3BQ8pJeckx_&W-31srMVe9$D}##GJ$f zU?-+{gx_}j-Ze?IR!I(<`mKBm?}g8R4}jZFu0QE&5FHntzq$L?%8(lAPhKwsjLRUvp>f1 zc9I6L74yRf!DlhWqXm{uz^r-)ADGM9qJ+`94uSJ%+23Jb@UewFs|D9O4Bso`nHzEN zhH~DgfqU0Q@UEGlwG&>%c<92yvn!|vyrDwblcezAQ(e^K_ju0^yzM65e}eae=XayM z0}p|}x`Q-)-_}$=`y1>F4}f<)U>KoCi68viZ&A+h0C>($+PXMci{AJFcpZ4_qa5oo z;s<>{q)gz#a!mDJ30~uH>3CoNbJ{Rrq+wnD3>q{)APo2>X0?H5`OO8>zIEWSzqYv_ zd;(J&9s)P)xBZ>qM}K3t;Q(0mJEt#zcVddCAN=6+w%-RX#jLb}UwOgy_k(X>@*6Mm zGz2D|1uq!3!&HJ_dC3Vc#1v*2Y-btd#+JoqGL^=t6{w-}4GFL(}f@mAji3z%%l;P-)_!Q}4(^Lc+z z+y`Fa@J{gh3Fzezz6;Ff9YiY(cv?Z8bzDDqBWA?|#*fOgj@t#k@ILfp*>@PcY!ccS zaA{=k~H6(K@d1W?!CB`D5A!c>J;G^1`Lvybl|K4}+gMF3&K2LVm&DVjA~` z!703Bt2W~U&&T|PsRaMQ@%Mu}F_pt0=;}5L3#X#jNqFIWYz;gFx*E;G2eD54!rjvoF$>pV_u&_2ut(vY;4vqVHnhLT$ydGK>_-=Q5WjE^ zwh!*rMwVuB&8d0DKK0t49`HrgK;tip8XM<2f*)Q z3O@)={HWdDq@A3_D%e+8^f5FS;a=Ti=^?)~llF#R_C6rt_!aYvhQgHuV%6lH|MEin! zur2T*uw=f?L!fktx8e8d6HB|e@O;V~zfiiv`{7>wVQChh7DNjKzwoEnR5X5tzr+G? zukNrkinm}P{K7q06J=Reul}!BFIc+4Yc9w$mJ&wzDXbPQ+<`U1g{3QOzpxkEh+p_R zc0F8JzS8zrg4L_?j2`@T;NT^cDO~u_CrLAW06hIt(hLuQ4_rpQ|BQM8U9Df?zl7-n z_`TY`()i6rNRRyBsSUINxF5VH%CYD#17KOq?&H$zy=X0MgD{n#(MbKmh4*45KPNoc z7H2$%cY;;x7$X?3q-T4_dfGOAY0`eSiT!^;d4kWhGG@Z1WqNKqASJ3X^1K@+7cG3)9dnM(DU;3c&&(L1s()6tPENu)uOzCqrTthnH zDRAm_lrP+?ed*Q6lvZZZ^?Am$C-W`zeV+Wng%@E}@Jdj6m^#-8cicdE;+NLt-8YgR zxO6LD=^`B5xQX!}^AC}CaPt?b3%Il*uiMPn0+;UN&u%7daA`Sy?@QG0Q^W~=;~yDY z;L=>o-9o*8(cbs=l&~cgG*cRBU#c0 z4}h=TN?9?UNE>j^?d1Jeqycm_{)F4G&G>~+V?A(b`R(i_4e&v5)*W`4c=hL)> z^0E(I3-Bat23)#(%lb(t+^ePM)x+~@1FzKi3+g&)9Ph5Nw0 z@;^*okg$aOtF7yNCLdAN=vpsMG!A9o+JByBxCMU3*C<{sC~&FYP|-)iRUDS?WpB zPZ;41*cNyf$O!u{Z-*j{)Y=xUh>Yo4au@C)m)$-m)P zpsQsjT>mSsCGZQ|umHRhbhXTcm+#9nLimN(VpVXjc9}HIu6c(3k6-u&tR3#vDU+tz zbJ!;Q!Z)xjaH0QM>Jl#Oz;v(L36}iYz6J<^U&00yCm7sspSwe#^vrZl_v)HS@9gy7 z;3vFr9`+nO1m242daoaR|L-_&;`f1To}=CVmNEqIc;4pyVD<&lgI~CH*iMgEv&^e! zCY`fWUt-K5yzmUH2`-$8ZG?Mu$(G=YVA3!XzJT?^h50X2PjKM`Y&YBoKJ^N14DQt= zlUCWMu!(4Q2|F=ATjTv+!e z>4Xc{{mJfkUX3v6hu!!V=_icvVQe#8s9&dh;6CtmOnug?2_~(ud$Apau@Jrw?$!13 zYJN%ks}Gy@yp8w3=fH(eVN2n|;Bga)hx+nrd`bK3X{?bj!rx<^aN(O+7hHH$f$jH! zC%m8Z@8@6#>5$S~sH)8>~ z^qU@8#Btz*;MNfOB`@IzU9BYHDdnUWzpxr>gx7)luy(jt`$#%TeOMQM;Q+P`Ef=hKtdMd71VB zzKm&1GcL(Dp1|ZE0(X3pYiIZ%c=sBc4}jY)wRu0d`7)bl!K+Q1cY$?v`OLFWKH$rk z+EO5l?i?mAeWtVO^No#!nE}>eDxVBED`Io4%O1j%HsQq%UjzQ6fwH-UFzs+WFn5Ys)T zaMlbV7;sZMgw}CldOx)j=Z)mMXap`1BBU@bBW6~WHmu8o=xWt9h+7cH^OG{iR-7In8 zzhfIoPe1r3X0@M<919b#1h+ao3qJ4gVQ|sqc9;;j-r>v!|G)eck?R0qb*Tx)<=kqG z{VWW&H#hk=@Mt^FB`+#0Iww%*=SA0K0}u8tDy*(5pIK7q=f#dhL%4~jm=_gxMAL=A z^QRoWkoQER&Gk(kej-Su7ZvjG{JH5!BkvBUXEw(pEy;8;mYK$4;Es&NxDB#b;Riv1d9OY2~$n4(B@_>&+S-er~92 ziErQ$D8xEOHhMRgwX>@p$6gWL5N+}|>Hng_a2hF& zKWikOg|iNSl!dcKrD@@;k<79C!dY&)@nvjfjQ72h^BI0$&$ONyJ%OH*o={IkPi0SS zPpYTAXHU;i&;Fj_o&!Bb@1$N|@3dZjZ%J>cx1x8bcYp72?}1(;H!0`KP0RUnGjf4k zNiLMD$W`WQb9K4KTq@U@+m!3d-IB}ZdUE}_`*H)hUAe*Bp4?Dwe{MK;AZPSV>htwY z>+|=`=nM3f^o9B=`YQX3Ea!95B?2C^mDP_`mlnXS#%WgD}pYF)2ouX~_-SGUoxdjo#I+t$DBzHK@wt?IB-{#+z!aKCh jx4m|I=k{IOC*2;nz4G4%dN%d;^zP}Mlq2B((a(PY!yy8Z delta 83688 zcmb@vdtB7T_dmX`7a*&My9%p|a#KMtP*J>qcwv>r*Xp8pLCq`H+q>lryi~dxpscS} zv8lF;+5K*5nWk0-sepGYEzL~JvKOJLT@)?d-}B6Cm)(WlpZE8VACCvTUNdv%oHJ+6 zoH=u5Uc;(u%~x#+*`^gbZ0xaOqQ?DmP18XWBDTe$}Q|G`dfHdpz?0~1wPo_qb+4Vou3Z@}X+X{cmQQi#CZ6cd5 zM(8*v;7HRNd3!|W6r-DI0yG-JazbREiP1g4H&1@~FPIlipEv)04d3WoX$!Sxk3f;#;uA6F&|7vyiHq6(Cj#EP05`-FnfJx1bQKCYw& zu=|^|*Ab*$ARx&VKRA}LCi8@Z$AeS#P!vf%49Ue#L`hf-8yx&VaJ;B-8E!_gkAeq@ z2`$*)!8v_msL`^iPRgiQ6znqO@pMwPRCG(b7-=(47u`x&f<&4MT&?H?X44Ms1I72P zdy=QUb@%hMyY5FSaJ8~t^LW$YLOV91X;&eVt!&ywh+$irP7tD5aI-k%CpSyN`}k(b z0g1aswzgS`5XC}6BHN%rCFO;glDuIR@4zu}DK5j3D3%%0Q)tTyLtbfvegH(CB+DL% zOqXG8B8+{t=zg9`YR!sti54T5$RJ4bbCTxo zZ77{|LMpn=rTHUI7v1L4JjE082}wSrymd>^$onKYr;n6+M9H`%us?PG2yNJP{WIdr zB73am4lp{X)vH1)_Diebf`Ro8J7xfjF2gG=RW@GXsV7^qi12noIO`RjYzgycrXNpx znMvkpH#0FRFlXkEKfz2Tm`NaJO8yks+u^-I`G@cr{1puGf{`T{?iF-wwV?;zKQ#31 zpyTT1GEC8nn#>zjLP5(tdYC-PA?LbWE@kZ>0@Jr16fg+Syw*j6o}FoJ7dEl`+iVg- zA2*Ih#u(#qLpZglxVhS*VxC&koTWv?4r{^NQm>|Fp7yF*FP?U*TA~VEt+=jY9Piv$ zD{d3enrqaW$F2(Om56&|0a8*f>C87VNGbiB#>1+G@IfMQtWq+qgu%Bz>z`M+8RlbV%Y7TDp)*P>voh}|=_ zM1KM3UjqH?x>~JLSt~F{RNsK5w*_`6s(l)_pf=Yu4%}4jpdkWtB zk!s;RQF0h^=i}XBjZ0LN8v@B?5|G-XX1|JTIe_GwtHocX?vj8G0bnlFsR^4Cb5C?A zVQFY|Gr$ci)m?#L_f9+rq#PoMruec5!dwrlG zEx`~wt~{f@=e#ME94rXglM9QcMExwVhvJfg(K#-ID6lPY#(OXk)z+qdY1);mqrv2$ zF{KQ-PTlq-A765);RR8noan)Ri|Zk}1lA$G`%u&>8mK7@*-cJ|%dieCW*TvSV=HHcw<*AVM5?$DDD@|IXHmZ!6)^=csM~qI zS07*DX|Fyqo_6cw8Wp%&nGEj>>fk8n&3fT)fn|114%fV<;WpQi4+>-(ww@Q*s?M{7 zb=zw?hYBGu$-G-T0bWSTiu&kj2>OGug2p0`Fxmx|VY3I~00v7%&uF$PDJh`vSAo5k zbTXvQUHe(sav=#*TyjU_(Q)3&5gvd{0H~FNS@SN%Ep7_b2U;>AmYV|G+$B%E*UA3s zk|HKJSwh#&Zh)o)V22(~={ic>^c4G`YfA8(r!-(Lnj+kO-%~9=5F90Kv>Z;ijC|U6}El@MD*4OC~Y%I9uurp9vH<|_q?a|?|>~D ztW{iwfu3qJc(oC3)_7|U!Zt_uvWZjIvln_ri3^&uy}cqt^F8cLucVOQh#Zv7cnvzz zmUVrm3G3Ed-|ABbD4xy%#io)xTB|&t%SQH&Y6~isqN_%$9jvq zuEmMzJqgp4^fBU|Cz$@dNOmIKn3|PNP>;Zt6&-d-u6?We#b^p|S{*^`4-{0v$U&}v z_?m1~n#*t!#M||a2;+S6oY(nl=tp)oq;F3VW_v~7Apw)l3aqMcMO;BC8a+Ro_<0}v z+p~?5Z2nBLn`fGfU4Lpy%aZ!V28?uR z*zkU%#6uePaz9hsh_#}|oa(aL(Z0mv zu5$QU$#8FA=kxmCBjT~T{~WPp7Q51apqSE`bsZ2HaNj=~mNQ_icTmWnZfXlENxmFm;6st^=77(K730|}kMYW@m1<}r#sA7HCi&f9D z4pp)02`d^KI)PXHPeZJF6bCSKt&wRJFKv#SJFY3gUYl_S?#<=-W_1}lterFJQ% zpVYG?y@I&hoRS*_F;@A&3|&kyq(XKB^JCyK$yJf#$XsU0IJW4D#jT%8>FqRT& zK*e&+w#vt$R=n#%%T_}l=e89zgf$i@voxBb{{3Fi6rk-#l~#9MGzg@k1Hk+JqDxYV zt6-S~=}1yil~hqDO1t4B($7GwLTI6J?I(8PQ-iIF+x)M7;#2()IMr3;RPW>Iq&lg{ z#hvQyJYD4CPW9_5FkGDRf1SRJ+T-!vBkWKBk%x;!9Cyf&%99? zlSwnqSQTM5s8A%m4h3)ziQgu9ef|!OCNm2Sd#MYW8cG|N7opGwaY7xh2m1i@dy^e7Ua^xy& z(&rZWn9H#Ag21%d4;X|01`7FZ&fL4ptO0i|88-`B63rCm$WCwd^&`M@y~yS}S`D#???1vO51Aoa9$^QDeA=n)!vO7-L7*0StX8=V>K4b?1y(o7<1WKD=LCiZ z2^QAV5-ZGNd6pin3=g4NO5b`4fAwR}ShB-%uJMNDa!FCcjuf)rEuVDyZ5hCu!m^U7 zAE#H2fgl>T3~GpmfpM4NowLIBlS5|;f{FFB8i|Y*WR?<{K3+2E>=|nZ@rNbsQ)@!2 zr+-mrxD@y7wyQO}VNGf^*GBf)!Mh7%YSYI8OU;=eyvWw)v>W;NbWeBY!c#)11A&b! z*ew`#^5irhmx1>g@HleHm9@W7Yx2sek1~}r3xT$%Rw%qkgGd6?G`nW9+c^aROTHD@ z!r{@qLkq!gVlS27(R$?ySeG25m#NZ(`EwZ#eS@r`tIMRK--JRwG9`9kcqiKzS3%5G zDHZL*NV~DMemJixzv< zQ4LIiAAy? zI`jsMQ}`w(1o$Tw`4kM_2e9diqd850FYx-9kSkANLhhEEFOP{^01^d6V*ZCN7n$K% z$SgqS8VX3Fd!Y%qmifdN-hc{)ay> z@H^_;yiB(rP>Oz~xg#wH%YsO&yl)S1Ya}D-b`JG9mAMQP_-p9gMJ!=teB6hp+-43l zi+Pu7<@`?Q(ao-X)Y(OU3yTwL z-nc`Zv>GIiV0o-Q7ORhW>SH<{WjQ5R3YLKU)0U(I|7v6ho)T>E=o?sB(~Se%#CIT6 z7TH{9$v8kpwiTtVNs6WD4mYwQFE6^ojqG3O@gr6l?&wkF;7)bY8Elzbxy<62s+9sx z*DJEcbxd+hES2Oit30?)VlktFJ2qJe&1v7=J-D>WrBsff@QWNwN?l35xy&*uzSVon z`S_>tO-qCGJFqkuH9?$^&%PVAP&_}MjTqgr>7FjsCH=S0XAh5#@AkJJP)Rik;;0;sdx z|Mv%3zcEPxhFT4qGbSl%Eqs^VouC^^=N;t51Wh>$O{5I0>#gzym%;HB(T#fW!R_T^ z<_G~j?r2z-vB?4V|Eyt?#tsVDyiLPi8XM6{D`-seU$s5lY>kFncw}r!)9Uwm?auSq z>T&Jw>HRC`E5!}e29<#N+FWW z3~k*oKCbCoTS+vV;x4ezogLZv@$JMObJ#!Q?~6K?K*P*D6H&eSuqL0)hE9CjnHUM( zKASy!Z?C9F{2?A7h*A||+H7|8-c(z%KSUHkqyd7WW)5=#H(*vr&it<~A_+>Nf6pRu znX53|uc}6^%6ym>+V&M+*(ZOaobBYt)DeIyza*+HC0E_&Ons;0od|AV`Onz2>iOi}a?=-iTigpScJ0(p-u)S@5VR5G&fscE!IIDU|Nwj`hIi<}1=75XX9-++AidAN(}aNs4c z@RBDFvW081n@vZ83`kBCt5`Eg#CQ!_+d&I25GbzZvud*eOyS78fJ!3>dT2B|xi(RZ&1ZMlMu&e-L$oA?nxI-M9oBLwr6s{L zr;70LuQ1cPjscFd8unvp|M2U8a*WpdcqfM|SgL)!apDvp9vew)Jh=)oR5>Xe$IjTN zi9b$e{nm9Bztymr>-vXZYlI`aob6hdV{9`SI9j#iILhjAbRWy2Xyl`l*d{sMIAjvg4Z=bNEK#?rEKSQ}SLC*V1{AV^ z>)Um3W8__^E=>c0QIh*u<)v6yM14Jx9Vtu*xbqE`vxP|k^WmGn_}D-(YA^fYvG~B& z6IC>ZgfpD6axO-`!VS8qVSOF(;-?eXG)JQNwT8)#?&1{<+v`XZpL&VaInsk`?~uadJNUVnd(SA6Ae* zw0AdVN4Pbw^VGK8EV?X8{9zpHUzR9hh|DbO9ZuF<-UpqfG2j?}4Zg)5_I_E4n32OS zmgU$kj|GXUtF-3Bq_|v?OElC5zE*3ze0mT*N~4J#TL64wAIS^l`n8*hr@d>p?mX>Y zyTz%%J^A9T?m6^Q$yr@-3H6iwhg5MX&=PQiuPr6lVac)NnpIw+waNov#WKdSqKygS zeP3dHCmTSxWP&*T`QsqCJ&C5tUO}VT?k5uZeX@`8c@~Dt zG3C48c~74$;I=H^nzT=~WnU?yCJ>|p>n?7~zH*WiWMGjybu9DA4AC-(t$1>sc;*$> z^r=VsC5$Ay)*k8yQ7z!28bj_*XbWC&0h56I=_=&&MyW-Az`t@Ce%{3@o(k{(4Ntre zI3Or>BY@8lx|bJJqaaJ(@w68MmA%Z%ig&SA&JF!8Zi5qq4S6t);HUv9>>0HFlpdiQ zz?jl(qsz7XFIt8eJ|fs)4{Xg=cFuVr(lwln>v83mX2e7mEhNYSA6J@Ske+!uuFX{J z^JhV!=D7@8;9c-Y;8)w+5y@$~E(m;-$zubt{Mn|&X0grBv=^Tr#44W2Xr1{U*2MO} z^XAlp5J2jesRt1fe~-0#wptuDfqnOEw~pki%HbM z=XDt(fbWP&K0wQ(tn>%p690999yt6km21K zSb?<5@W#7>COx2jXJ8gGb^5{B8Q@bNZ4NldSR&vJK%f1AfdI7H=hkJ}lSt7qK{nU9 z3kUc68)x4`pgx}KR|h|LS;#ZB4pnafgDA@^W`5sGJwd*Qht2aIifuB#P_aVY=_y+8(45pDp(( zJnh}X7|zq~Jq(iyT&)-#e3|7cn1^N7X^Yz!XY8YJStJK)w6;=Bu8)!)JZ3 zQie6D&f<=ro#y~<8~i0Nlz?CXzFPdZ+tp$J!|&tuy^dU25`4$fdnudEQ0UuAE< zvfZHTi&DqvM4$OT`E|DF)k}u(^hPD!Ut8cZ?_c9{n_%jY)iKI z^-*?uOR9MNBNqB@|G4{({yULTG-VRU%QvY+9Pf@3m4WQiyNTl7f!LgQIHdElZjjvP z*i-Kf(*Bf&+3-Ak;0y1?hJ5&pyYTF@EM#kDSj$_Osjz1=&68jHESs{`Za9|!7Ac}` zMwl{m>T58pUoK<+Y>gM6>%%&3>l{#aO2bBMix-vG*^+I~w|=X)y7nCpYr2gueUzvE z)UeL)w+mZ_BGt#Fpns-PeMZB6ct5h+tu53KbZ)e>10ar-H?I*u_g4bKolbN=+7jM7AkxfA09qa`3WFK#D zZ`AwOxXu+O^iV4Mb9+L^m&mP@}rb1gQO%rbVr;#1bM^*d9-?xv{vfSvxlUF5Kb9%o`=Jaf;8$js-4oUIyH~Kc;tD9tma0mg zMk)Q|kQ3aim0E=|-Bt2=6>;M$?A)Go@%3b8{5VQXc$M|}cvRn_`2^M*THZMs9r)jE zZeQpcl&0EkbK6z3eIHxJcV*V{lX>F6Bxd_0N-TewZT=*$dBlT+6TC!yb`NuX5*2p> zmX=o47?b8YxAttQ!_R^9kyq0iJ0SHrq9BYyMYC(Bu`}a*3Q%AF;{jvA!KY~UC zRJbmfc4`7_r3B)DEqu_>8AqiRaa+xXQex2SLTv z6O}zspZh`m2dnmB2jVTw<1oT4autP^6Qv?WCd==s_ zm?;@rFPBa#a*bk+XS)w1h?_dHK_|NOW-3q&@D4#Q%#VyrSWj(u1;O>7gx+EjHBm1vfY z$L?rWh{w8U_G(p*m=VqXK&p8(>rmZAJcn~2c)S8fEIWRvb@Q3^8KJT4`k}tfTh?b(#jrky?};0X3}wb-?Dc$^ z`bG-9!9lh0F4m$QSGcfWjOMtriQDJA?hPjElXEZJ=8~% z`iN7@qSRDt^`ToK3?mX+wNOxVZ$HB6{-HiDn9Qwgi*sx>qZU2Nn)aG4{rZ0Z)$M2)~eHp*i@3M8r(!`gwZ2z%!!YU#lZ{0SQud1)lBV;Y)^TKyPS`>=z4X2-R`bGy&ZezcpuA_i2r+V`~!HO!SORC zy@R7t#kNcNxh;!0ks$VqU_Ty6?z7nq!k7V^SFV^2_m}820!{12cWqT0GT?-X6r;oBZuLygl5CZU1VR*cBN` zqJY1x+2E4~@nlPSTY$GWTC%k#w~E=}EbUZR+p!XLtSnO=gGJnp16P{bM`UZ9we}&? zu9j*SJzftpi(u}j4jgaV)k-a$R9`xjN|Omg-}=%{^`*&FigCny$Bp?~NU?`}3j@1$ zYGC(5WGGLMa*s54{yDKq|N8|^WzcW|nnMFFg=ZG;b-5~o(gl|P^(--NEIah|z2d<< z7XQr@adsXn`37Tg47>WxEZdV~8V);G&D(s~J-3Lf{}bxNp+45&fnjG;Q_IxHLiI66 zExTV$O;R6u>SMTCHbhNjs*m(VLJkeP9%^Bdnj5D+qSQxgD#NKa1RF8#TJmjY&%hf> z`X@V9k)$#3=8k3^&-AqYwHh+gkHGQhVXxuOns@ML^&9we`4s+)`wo8|hcB$W>%=a$ zBj{Uz(|pMvI2YkE^aDv+20)UA>W6{s;%u6m>CD8L8uuHDR=`Rq)%8Vt>EH-}jKDCL zFgQY=AWqf}M}E=%#C`1UnP~CVQ7rn~4r1<9Ht5?fV!^{KQ;8T2Z*~Lb{eL@|xs>O7 zPiD^KX|KC9j;G!3(ohw+T4_IuP5Ul}MQWR|Cf~LS4ulW?>PUed{V~GuQzV47dazc5 zObKV>i|{4GJ~sK=VPf%o_V%}J(`L;lPP`Py^R$$z4R{f zw42`7JOPLB-4URdtDU=;Xcf+7yT6OkJ~EeR{XU%2`ffP8@m-84Ph@S*?b8lIhqw&Z zIqb)CgG8kT>-7Cr?U03(H*z*R_x&L8N+@f8eo4Sz%{6S5wl&*!e!4ao(sUVG&SHBn zU_x!daxTo)&Om!zh6OX(lpnj{?D}sP1_oT^RT7!@V!XI@7VCDgZ5o^9-=rd*_BLrT zPrI8mLj|r@`VVERFXrn0MiX6zpc#b1(eFuC{+&_VoKni87T&UX7T&UUW6`oGR&Z&o z_EbFmt()HdA_%+7xJ{1&uKjE=5ve+ zT&*Mxf#{>y38hcK4o<5zi>;ZiojHg2UU(mK)N~agn@?*NX+ND!c~#SR-niy$=C3B> zn9PK$(QT*m#JI`4%EQLRk?s{%;TV>Fb%-#6 zt-d->`^JB$##@uvUsq$a3wYwbNi6bObeo|((QlH6Vn?Djc{Ce&ZImz^b(7hNYw;7; z68d3|z*Xp$fcn$epYwp|TuxdZN*~u?RP`v z=6foTyxbh)X)iZ>dD_j*`zmlfH>>-zbHAsGPu$C@ZbwHK-0SA%mwt`7IoXd5{{zR8 zC$QOnOk#!``f+a%@0Pjc%NmGhgW9Hg@z{9U%g}tDb~ALJ3S7@nkACct8>ymsJbV8} zyO_lBZib%AXv9!yhTzoYh(__&couZ?#fb^TJbEniUwSM=ZCE`^e{;NEmTGz0&C>TO z(R!BN>-%`!Kyk-dW~z&ceqyYfqu{=cIQlc4ci8lC?3ubvGc!^Ju4krwI&<7gZQe7F<2Wu={1C*xxE&rHu4Ys#YttI>RglKJ?&mxf zaeKPhJB_WnZ5F$Yp)T*GgL}ah@XdmJt`81tv#Kx&b#@9R)tjr2zZT}uJZZ1)D z86M+=8wV1sDnlj=68?oCaUsD)YraCk56mngL(<(MCZLDqaz!aP#KY|C~ zLev19sTUE#F>Q&$hq>wpI)vA^e)v2fK->@KHGvg-I4;AYk!;JIIHtcF%TC;hYnN<7 ze$l>s9&y7_NgPL``F|>fQM%nt7&xV;+McmcXUUEae;(FK@0O2)zPP_t^RWZ`lVwLB7u7+!7>2Mt%4a^p$^ehF(!m0u=4f zoLUpu6^4xMxZ87Ph%dSp{-oOYmi;c(lN#bn>RZZ}$H595S9XqRKu#SKH8Z+6S2Yp3 z2RxVPa(>uEXfF=5I=^WmbQWKgoMNyL-{t{dN*Ydy95CmxU|~RbF5d#lA%iRhpcP*$ znha;GR_GG%{2D^04GH>wP`-|p(_5HZv*30hmK3`ugf#da=OTb6e)<-zErx4pIT{Nz zK&H&4`3%M@UtL*Yxi%!pS)~;^#os(tk2p<5916r%4`Qr~HA;wzIO<;zA5Ql{oJQ-w zH|`S|Dq3y?&g%t>v9vKtj_^!8^{XB!`$tmdQ>sn&%)7cXxSY|J!Sn z#by5)aLusWApD}0MvbocD2L8Qe>cy>jtBa?^dF6`WGBz|=|DkwnwPQfcA znS{cpaE}sAanFa&{#qf}Ii`h>Ek2s%+|)uC8zyGah&sSeVOHZ@x>DQI8Ke^i+KeRJ z*sv`3lb7?gHRw`HiX+&hGyP5|zeOl4dv}%B+ zAcQ`=z$T}$F)f3KvwD^^%ErdHe|uS@vAJ6i)}RIZZZ~439N-+zz+t6rW4cQ{h#ZDjZKM4qXKhT^;sN*4yF2+1xuGYlRymdW*(&Gg zmAQx&(zhkBu*Y08oG!i4&h{uOTm0;QrIONvTI#106Wui2jFJi|-!v9LPd6A8Z6R3k zRkWrgaDSvtUeeea_7k@L6&pt!f?hg69D~#YjY%DPv0kTR8jyOSt`VtCZfFnx@(Ls2 zHF!x)1gWAGS(<`5RB)#;w-dYs_i?A)JBYe9CYbDj_Ns!sB1?IV=U$>A9Mtkm0l7k)5?_i%iciBDMZ zg<@7IzE#A8_^>2TflVDPDEm1_*{}%BC;qO}Lk)h{wG1qU%`a8YsvWZr(kj34rud;B zM(Axa;WvB&_aEyfAns6^4f7FulUm^i9_8Ui1W9tLshrXje@ZrGC)Wi(QrkV?zNDrEDoORU7P@Igcxb_8fGw#ah&OPszf|r!VCQs2Tc`7YT z4N6}(u|bWAz2K?tFjyN9D`-q?p$FQB7*s~7;WB7Jw2uAZa@jD)2MW41cLd_?m~yzW zh!Q=R9EP8`EN06#O*%8Yld}G`nqM z088`PNC3!dr@LjhlVdWUR5V`2+E(0&a|QEIpl zc*AVRuB$DZ0iw_aEp*|oTNm~%r(S+~G?>Fa5tb*M7S2`?LOyMO&2z{gK*e z$R75<{dZkZIN?URV08p~tenZ7sa0C~V<+Sf)+0A$G+=q~?=D-z4$Aa^`^hX#*~P`n zW%fLM#*6NKt+B*KPi=?cD=c$;mEjvMXg~R!e1=byu#0}gG8L7A>2s5_LMJ3dZD`{z!R(Jp7qC8tb_f z`r0!VAHbu=8A%?ADb6y>+3g{rU7IQdC=tCF7OULBq)~?JE8r#%OW}php5%py9Zht$ z&l8QkGwE_1d?aiz9;)CxL!oeLM6e@ubZDY@D+xb*xS zrF~9fYzS^dco=i>PdG%3iZ+^$|3mX}lZP!b7Q3Wx-{P2>g*)ZVZT~tSAEx;@urY~g zZW5S|ha*0KX1To@LFvu=#gCy>K-+wr;jGg8VMqojTX@%TUwGpc>YKyLGd#<$_6^*p z;`X~=LHNO!)?xZ0*x#Y6_CTvFsiaow6y?vSoV@dQVo?xMkl6pcVqcV%K@+`E`f1@{b7 z1S|?#W~EKX%2=Y0`41~=8q=V2RSi^4as7(~JYWvPY6K2^2WaGvB8=0kgE?T|@Yl$G|M(tipkH5mS1JYL^UTz)i{ z0y70^6m78;^zi*M$48c;@;t9Jg2Hjbu@Xo4q`MlRX@I=YjT}k`Ij*qa{tL-lH5Anqt|6-zL}m>Eo4R7`Noe3zR1ML>J8@YYOO1Hf(>qqpX>@_B-AuH;!S8we-jwWxFG%*u zZ{W5Q^DoW;orHUA$S>SV#Pf+q2f*6*jJ+9N|AR`#BS9L8%=yffJU_ejAZb9BSk^?ov*)c)L5*9dTB?ytu zo{2(O5STL=oz_I5K(q!sKTi|}I@@;^Ecy?c()3~0;EqPjJmy^IL!AZFAftV9on+5B zA<4N~e>#(9TI`4EAI`#W+ zH7kPg`OR<|+YXaUXRfekwvyy}z(HaT&;AG&l zMct#%Ky0*uu}pIGnsoCnx^$kiO|sBgxZ^Y>3z6Zuy5enIiTmLbyra_1Q&#ye=fY$m zS~%q_OBP}h*TjaxpavXs{oojM=PkMgq!8{Y#s&XY!et4-W_g8l^DpN&$wET2TwTe0 zt~m3Y+AabNgL7CH;W6VKtdnpFGB_Ej#i=-5F>Y%^Xg$YywTm!HyCzK~;k0xWiUWFW z{?mD)tI#@rHjvMOoDWDhuS-e4TI|Vj7FfF#xct^WtmYd+96k+%jxv8KTjb96Xm*RIdekm^|M9|!A zCTPy$SHbTe_@%r-%>_+M{08Hf8%zw#%G*|n$eWD7fw1%3l5%u-gA zIKSvAMA#73a}2_QB;6vH+m}ORXCz!tmznmEUeaY%Np|d$*-f!hl__x*wcKvPPR#N& z{AS`;!f!5q$KyAj>Nv)UctvZMFSDY$)oz+4*>_XTs@0|VOBT?+`!SAbHwLFBSz28| zU)A7S(in0w_SrwC2|%)ce2pG_Yoka#op9uc(>y`Qw(X<>$7-sTdPP2$)>gt93w{C_ zAEp)U*3AHr^RN8rhHu0b)YdIl%liDn+;|+QzQ=dp1B9@|q z_`a%KlAXjUZl=_lTux)}Fqn>NT+$at2!btgV4P%cl4bvh2;e@rdLDY9L7wg5$|z^j zL?J%hXAx^IDThalI?id!5rs9$ibNO6-SRZ;thZq}_3CL@pC~WV47|G=hO2!Nk`Act zDvXk8x!#&f z0PJbnLsU_>b9iJ?jZlbdI#wc{jO**xkj$J5CJ7OS*L-+RCuq5r@Bq2oOXywouqlN6o0!AvFbiwV5MzT_ucuy-?Dt6aIa>TeWOIyG$D&w^y{jY(a5*r5O=KGi6t&sj$1V(2m{;2@65Ex?>!U5Dw^ zSc_9naIwaaM+LXalHS^)8;lR7amhdvldIe$S4$b|iK#`U-ZAmH%6e0|+J>7Aja=Gv zg)*d-ugb`??ZS)os9(G^HnDR_xP4Fk@#jS!PO!qn9n=L&LaM{Nj zbL8u0`2rn|$gt>_G}}e_HJ`wJlH?L@d(g-VMT`?{aP@K_xt2aZiKyfmQB#nj&WA z3-~2}>zaB?AK^0tzuATYT-k?kdYVQqxdu6@zGuk=yi!pjI6m!ukG0Cf6SL&(6q9_C zzUQx0T(08sLL*&jnZb!KD#c~F1YYw!B!ZJhx!AxwNY&-u7RMUmFZFC%v^on+hFXvG z7`T0{y$!0udzndILz04qbwW4VdBWR+f#@`kRePGiYx9Ah_;;4wC$ygOlsdS`k{W3e zxQ0m}#8kzxkPY!-dfP{m0|*nXA%B6-Dakv^0cj~(nJ*N!;L+~F;L0^*Eu1&*6S~{X z;DJ6-?<`em2q&x#v;)8TEPK9jMpb5{Mktk{vr1>hAS2T*Wl8e$bpWKJPY^A{`5MV8 zM?R7zze~KEF@$g%5y@81*X<4xL6+RqR8bRPf_~r*x5`10c%XkMo?sWP_WQJ^@?V0p zN*@BHIZ-I8<|GEeE-pgvQP&Sj$toS12Hz%DlA%8#%Ed47MaqquuknG9ReqO$_C59AFeziCVhzgHl7WWMk z9G1SQ!CAp2ZE%vex0F%62%+aQ__Wwnj?JjLfJNU^ylWOH!=L6C#eRjQmU2o0niC4%V1$&GHb>V1ME#8tl5_jlgMx0^0@j2?l{=U!*nNIM=p3ZdK_Z41@drWf-9GGz$97!C-b z{~y}_bxSY!90ISs-)+SC%tJdypbA4oM!_Px2g3+0JbTrq+y0wTU*UtfC1Iw~@C=MJ zwZ|k+#WCs?FKU)7g_mW1x@cLVrcjF&Dzuiwk{c@zGU7im6r?$t8%-HU7N=ax(!D>& zSXc|umEj)+&}aN9(9s1WNlAwyYO2E#qLyZS81ul3LP3Yxw`Hz3F1P*^)xm-$6Lko< zW)Ff0>ZGDmCl#%!a_yVc(E4HZG}!US@85;Y{a_?g7$Kcsot%5QAZllP5suW zXA5*3U3#-A?gWH~>jcQn@?IxEw+ZkcXZi<1G@TxuWt8j-U`l9hhBGie{%h)4kN=9L z#R9L!zKNiPM`O3(9p^+JOxz1Hm_AlUuUgb1ix_~hII8?bn>qZ_9Vw0UGllF{8kxRS zYx=@iTY(;XnouD%v+#0llr9la2!G)(O*v4RG31gTRnF2^iRp#;x)SUFYtjp~x)MZT zHLE`3YEXc62A_JRj3LG)i>$CK2QgB0#UBF3y-0*pm!AB2K`U1UqE8tFZPP5eQo{ft ze_>UtTO6}oQpTyp5V&fG9L=loCFXEkkc5LUXv;_(@qZseCJ)-Np%itM#~0#**4)g} zkBF1nAs&tf;N~H+5FSkO7sg6-=|`s55tt9Bh)E)EG(DDFrzgR#m+D$U#<&#lc(RGVDfG@)b;{NEXB3 zj2^^w0CdjYip4SzclV&aiTTtAv{pZ*JKmFAwM6h0?4C*~O_^v6R3Ux+x|{reNoJ*fW*v>KF0yQF{rAFUwv+!5`(7laP#zsgvI77%NS2t+HnvUqSb zT*D5(;g~XO9%0Rf$;B=_1^N=P^hrV2&c}{IA&86>=sxAu4^U&9FF~{$tEBK!zBQkS zqjDln$)#YQ6zI8ssVkm|iC%f>06!bzN9yieFR2cdLlSYLx1kxrIKeG$o{nu=N0gy6OAW}t`4stpB$+xlrI8luy*#2Is9YB?`?op}r_f_+?4eArQ&)noD zHYPWIw)5(Ip{?x*Z$EF-`=6=UEudm5vZcVK{g#hN+=tg#N&){uZ=jprH=Iwu=AE47 zFAdW56u7g;m0`fGYTw4a*nys_^lePc*`PKx?jk~FB#gO@iOp&NPO{{xex3`0w{|~> zar#FZ3#W5~`b2Ic0!T#e4wsT&+iuTn&^CfAY>f6xA2_$FPa;#~tLs;Z^uOr3kE(x0 zRu-Lu1vx%_$mCpU8&jJ;)A{~Fp|wqt!xi~)9@K|HiTSkANKN)(i-rgIaW9B5A2zR1 znq$c%;ntUotweayDv1VL!9o~H7*E<>H*3{R>?sW_)K@dm7Mt7jHp3OdMw%zmaXCFb zyZeL(``~*HAj=cUU_VdQ5D}-70f^LKAG@=~{hf}V*b6zFYX7i_ok!;a$dW%#?cqE= z2;Mwr06a`iY`|oS50-i!d%Jn;>NyYN$Kajm^$fm;@R2Wr13f1$NCi$2oDIh&)PoE4 z4BiDvpFsc8K(9j@62K=inh<68gh^zlxSvv!e?B>d)Vvsjqij-V(?oJIlSOw7>uu+$=-knC)mbHidGLfGbfYq2me z?mCA06C6)mnGw}uXC(elMeVh$8U+` zBI3Vzs1EHqbYYIXzKpQSO-%ArrJSHnX+Ma(%rzaUgnpn^76*?yy9_fB8>Ge7ZwE1O z9K)Sag@UnHE9BscSfH;BR$D7jGfSG1UtaI%uJ#e9K_fV477Cr6a-onnyV%7eqX=N* zzb@jvOup)ly)WqkWh-+#W@@aK25z+y0XI%Amjdcs2XJMYJ0|`OjlBSieSnwfGvL?b zNp*2KbCw7lo%5EUb6=!}Rhd8ID>geWI^zuuy5)c=q|Xe^Z4d?7YL$PGX=_rDKdhUD zP?2W?!Dj(B`6HsHZX(Ecc|)|+kPUdRGQR<|h`4+b=QeCsL?N*Lk!_Q=S{=!sSt@=G zv0#!xMO>0IoHK!?Isf|M(j!d;u^I8&0%o>xhBG4=pYAr$`K90usF`#L|rA5{Z zTLX&7O=(09nQ7LH2DFeVS<`(8xkTZiz${^rRuUHJt-?c`isTpyDh-Gr(?bMPm%FiO z6oT0esUXvzifn!Z0T}Zeavj&s|INYv8UEcHj!Tr3JLa(1rdctRaLSiEK;K53H2>iMpaRZd3yK3fYS zDSW%#f=DS2Kj43$VHIo3k~33q5UM!tP4K65Jj|n!xIz6B6=-m9P~GduBzo!)`SX7s6=lHO{hGjIuG1E{T|GbZ8 zMR~Paxx6q6m2oMdfE^j-)*0B#YC1Ada!^+?3$=$jnqx-)gmZ*zo?(6x72D6X#3o&_ z=@wuX+C)M{$PHGIb4P+wemzRQ7P`wSW2tG$wcv~u^%hq6?(5C_N$Ioo-YC*C1SAke z>O@hb1y)CK+$u0)pYgD?lu0br4$3vD7!zJ{>&`VN-Y%1Eq<& z*JTk_8u4jsEJh{vm96rSST)-87xq~ojeKiTCC;Q7`JtC01>W39TpMxkEs_;tO(XP5 z&Sbn}^1=V>K2~#qKDxx55QZ^OvkwWmqPFR-Ahfeqqysc=?U&WjK&+{g&xdg_e7FFB zMi6P;X%doma5)Vth3AA1o$s#{?iGKW;B0Oavcx0PoTF^QYhutu=PjF%(DJ(pRQEqL zaPH85aQct4`zj%9B;7?ylThg<{;-|M7z4ytqneFLZYXuNLCP%&(nY zR|}c#hyMY19QwiKSM-K|n`m_eIbbW0iEv%;P3MNyLR9=Lo@4myZw$I8exq}3!w7J> zn*BXxulUXR#cCnF#oNDQyeaf8XuiwvuG9Fa&}ZCh$U|rHY4(}s^O@%tGT-O4kx zkB<*hf@y^_LaTgv<*w_-kU)WsDI@f5Sa-wEUp)*iJ@#{|^YurCG0-v}f}dV@Mm#F? z9=8LT9*Vp1-k4&5hvJWy{V6{CmmkHmD(~F)neTk6NSNB{GpIurwlmOSFQUV6A(b<@ zSm+zN9r+%jA328?3qyNd4sy5aomzh?_y6ffyq?zUDtDYjU+3WxK@wZ8bQ;zO9a^pU zjYwUf2Cd=^dc$IOTL6X^-j$)du5AkVge))f!<= z_B&Wg&@xJCRRf-~)$YXZxZx1m>BCcq+8oZ)3s3%R3uqW?crwd9D<_Z~>Vf>OVQ#+l9z>ZQvJEU*#(1(e#GH z2qcpJzu(n3$J>RDgAbDANG8>n{AIEVLh#|PLTwr6E)R4d6KDVlWP@|BT^Oc+=T~2W zy!)&3mJF7V>EYKkrS z+rxP}b&2=qV|-GjE^U6;IcU9L5??BGmaG>#hBXI(o68e?B!*mae!N~t&(fg4!)74f z5mm-H0Gw92(ImDo(Tb&(!_bc=Jn(&i#^%f%m2XOLzFXdMrfm=o59tiIlgl#mg1;>5 zz^IQb(^ZO`@1sAt`KEb~_zpbi99kxf(6>W@w+Ewsa@rq94HPW8}5XCxDb>n{4z=yK7yoPh>1Jv2IBRB7y_ej_c(`QD#KkUx!V zl|N47+aFa`%=polNdF&Ih3|!250&&ERVtUbTLCJQ&-qg^Kwv&C8LM*0sr>qbN@ds& zzEpC4P^p-a>!D)#LDl$Z50wYL^QRII0r^mwp>p{DQT8ryQ5En1_&EcM+*TH36}+wr zisA+DC|*E87lkB4#Y{yd!!kt$Gs~7l3tG1*vr)66GP6FW^eGih6Vt@X63vX%lFDk$ zE@WnQbN}yWW|m!6`~E)v^Lp(*Gta!AXP&v7IWu!+=7?DS?l8h&T+GQgPmr72vby~ zWTBfKb%qaIt9P7qHyEk(rnx55?>+mu@%Ggpp+BNIKR+!EvN}eETm9(9xJw{w((Iv! zo%<2$s!2(Do766PGtC|Vz#_Nq=TGz3)?#Y92n_qdRQtg+d&c3wbjQ%c&P5PRwU4KK zdsx8{?wm^c4yGivUv!9a0io*=nzquF9JcmdbZ(qO8qG0pr;vWt&aNKTQLs{~V2b7p z_i$~e6!O|3M9`{d@}7`^10$In_{yJwK4j&F~_rp)pB@!`Nyq zJ~EMoNO2B$t!$#}FV96TCQ?l?QFm zyN4&2Zr=jDwQM`}HOjVE@nIYEVfuIcz76_ty_oOWfLW;e4f^e@!&yG&X}v28IL!;5 z);|f|e^#W8PNmz*Z{DbPVdgU&AAZTz-{7a=HLpHR0Gv*N_CJ9Z)lS8vH~)O2-dWG) zgP+ld^o$k=$7972Wbu8U3WbkRW+8*NetgXkslzJ#2d_fb zYB!vt^evY@TF3&q=cJDoeZyBKeR0w^Rr(U7?C2Kn6=9C_-66wrr7uPL@})0H`U>Eq9K)Ihhxsvdx?ScA$ZQ&E zfUy~jU@WFy3Y5!yeAf@p>J^M7zqRYp=k>0-Uc-03ppVrza{Wbp2xcvZzNnY7tB>-- zFY0lF1CDzhDZy}~cutMAul_y187kA4iNt7pSdTs7!G5u;>n7cxvx;53*Go7ru(CU! z{*u0zUFyc$Zq|3P!QJ>LoAv2~5^45>W<0`5{?LoM)9m|0;dtm;u%ZCZ==X{uu!K0y zURp|3&c6B>Rc7at`}rL&V+!XIU-z<}?Em^PEb~0~GRAA|ZsS2)5L?jOeDD^%FE$3v z+oF%q*Yd4f^km({f83(S8_#~=F|(e>zM^+$uN~z#zoJJPT}M4ZYxts9^hw4Z5-o;b z3A;XdMSoj29)I5hyM0&1fAj&meh)vjRo`S>RwIH0`~m*(YuKH1f^U9J?>i#QKFV?t zdjdzr*^4PP#eeEWdG^{XD^KO&W@Uvr_EBObQx-~1q#R?{)u$-EbW&UZkKKkFW`7T! z393)#h1>KgsN1#M^zM3J{?|5r7`uFs57~|}GToQ76LOIFu*rwcxA>y%dQ4!z5zK{` zZL8$ZZr2kP=n8>G(94_A{5ycUJP!La<*mKy$D+-i!p%DDN%MK$r&8}0f}Vg@yetvR zhEMa`EAismWvpCj>OXAa!MqxcwLW*2q%u@TkXZ=#BcYy-n!KVS{0vaU%wQ0 zWU75^6uvW;^g&@f&w5=?`TwhofB8CIgS^fEdL6qyD)3mLIIZ^52FeV-u?gw4D%?KO zlwOvM4_h1xbT3H4>ISSC7t6$Hk$A}uS=O55Epp;K2d`@Ku|T}uYQGR>Y8@7yQyFHO zfSq|+)mUXPlD0@E*J7K|X|c_y){2cS=vG?8B5CgemI)&0j0ma&n}76Qy{|39Q6yfb z;mA2G<3PuyAfx^@$ZLMIPBq ziNt4Tt@gjt%dja9Uz{8~BWlS^u@xMTHu~f|-B2kzDVML(vTB=Hx^qty*6k#v+aZHD zF5L)o4a(>RLkrj?SJmLqEolp8-LR_WpM#jmD{C8*f~_C-SRJdV1x}oXeHye$Xk$z< zEcH~D!~>qnG8FD|yQ2YkN6KWqY!$g^uCA=KnuHvVTERH!{41_WbCUOtN6A${%Q3w~ z8wjzi2g`ZxN>)pH=xa3x`Or7@(Z&@(A(upd?jiozoBG2;t{`@#M8)F#G^R-u$zFDz z+%JNz6ymMSN6ua^@vI$sXV&cYb=!{c`Ay^XKF<%xJ3 z&)W@|HoSbdKFJnK`t7Ia&1tZuNwOC2^~f}XJT~`?Z)-nZ@;zpTO3>`!eq12#$1$i& zTj+kI7C3}jV9ZC3Nw{b(wzby^JHZV=vAl)GBv1`(j><>Jk73K!j6HfcV>`?MiiLgj z$5-#c;=XEm^K6W%f?}~O=t5fGE2*VLS>mf&-=&r<&q*D;o4P1r4_!gGRn*rM`(>-c ze<#+(`lmY_qWj~ADcy4^#mk{3EDcZYX;t_g@nq+hyFJPL25u|}T=`qR4NoS#;i|Nj zETzNZZbbxXSQWc!7pys)mW8E7$DH*=bn7&EDLt$$8QXMG&frtTJLU{8ybj*n2QF}B zq}J9{@3(C{dhf$KxmuV3JL({S#VNGZ$${aJ)($AF9zc#`d>|GvrPYcxOlfsxVhNM8 z=k+!};-%ZYKIT~)UbB+X(-+e`{YhGkG-}~>_GkHs3Xu=4PiLGLZYs>L*M<4crsj{N z+)(D*-1gT?`<<{q)BYV+@2Vns2pjkyUN;AQiDS;3>*L(fn!?=+ziQH`{vpe*qUgM` z>=;?*lVuHv(#Gs*S4E!D@jIW6wTIR_3zr}R#L9v@k;BM#I_r220WPDmLrDPu=&;o>FSkkznB*l&$ zA2IJzfseDV?2&Jy?oUb8I%0&aq_<5t24EE}u05QV3`37OLl#kfb+FnO9RZF)!L&^G0m%<)xY7XXXoKA z&$Pc(k6r@~7YJKPO%j#uugEE%TAU?pY1*jYj~1BVG4ly-bKf4Y=_6-HAG`o=yk74g zX3}we3x!eWJ$R2>guclVQ5|VDNhsx>SQ&;MpK9A1>x2!BBPF7k3ItXqTpyXkD>6}^ z)9tJ3kod9o$gB1zs)b;fbIe`Wr9Rt75WOx33CdRS4`=okoV1hfs>w|Hd(ld=Ts4vC z0>XG&AU0Yp5c6#!|BCdazp;tzo9;SBP33is&W8dm8l4AlYl~LW-RO)Q;l1!`lR(OK zS9mL$j3r0n(3qo0iPu0g&8s?m_BzizfE|9dIG(C1!GzJ_rcE#|z8?2zV^1U(e0pJD zAn4v0^bC*-h9{EMi;Goap@#h{tWBxPu=iY)rE8gv;|mYzEo?1C43w{aov>2CipI)3R?H5OQO-N5>dZiyqZLCPbc#A6(aQFbw7Vg}A5jX3*Js^t$H(gSRdd#QCB@gWA>Qe-xa zpl0rvGd;hlIa7S(l)wp5;eE2Qr@)G%rqM_ve{gqbh%8sLCW_^Ajro~MS@Q4sDYBE2 zJMS)$h4X7LD}h$MNfv&M5sI{a{oJNDulGS?czX@IQ~ava?Hk|CN0v!3CerXRy`-`@ zzqtcCL>6DXQx@B3A1zdDk6=DRqN04_jPgM%*yEyfee&rCaRZ2aLa#10CSP>xuy_F# zfL0lbY#K)O&X01`;g2U8?UZnYt`O_%^HJ|~tY9c8OTh&1n8`?4%9Yf2HJz4UQDWMv zB-|q+YMN~a(escf-=a?UK_kZz>qUd8OLRt5B8^d}j&t}gDZYONj+f8!uD2l`>0rY3 zOLy3oU__2X<;4&~Veyp02p|kL;B^Pqsy>b;6i=yWE3H*MRbI9N``Ao?vSa96a^z{KVNo84^CL%(bIVQeej+;6H=($y>7ph zFuwhLJ*xj0JPP_U>uRlck@^>nlP z)YZ$FOOWRMca9nu9CPx#rcuA_Bd8=S;*RvGLiY)*=T=)PZnd;x%{s_ag{TItq10b@ zr!AGeqTpkR0kjH zJNTeARn?)H72Yak>{4^)x1mi_N}9(uYOeAr$toW-8SC#>8Lw16(zn`Kqh%wkfPeA&Bq-g-$nC5&t&oNOn4`_X--ilw13xKR8P)ony*g=izdXjLA3o? zFw~d{xjvbI26Vc5@PFhSCn6b5&p3AjI$bRGa<`uk0E(lC(5bZusOZxj-y>3w&oo>V z?Waed?JsH@(Tjw_<(>vK7MViL7!D|K9cmmZ?iWYKBiW6%-tJ>7M!ud-N`+7@+Gb&E zqPxI)dyXWqd6!p#5D?|nQCfV17Jr;atCzt=NG$C}b{(cpPfC*P^sJbVeX-O{#Ibr+ zV*4O%GcGJpOunRu0k{b3n6jK@B)g65yA*P(0W-vP72H20jjjy7dtzoXogiI)I`qFtGA zW>&b&`=P~*xE<8jrezQd%IK}##+U?t_ffnueC2b!<*w(B>iF2|_L2PO5A^}8`X%1w zBmE9`Y!`p*BfTec_Idg3k93_iKhCR<>G3ROH~;Y%UVN-r$$Ql5ox|@wizjCFMVz_U ziu%>PKIXZ#dJ^liia%4UPh*$f;^?RfNY-Na^};I5B#GfzIQw>FK%!GdB1QyP|$ zx9dM**PX}p2%Tkp#2+}J_h5ai_|_9xXkuHze?EbDUd75w#B-C1=dN1b`D1+|%U;2k zeT=tIe-7oFKE_+6=6fi6jdb~@Em{rBt9AxTF%>_!h6y1=lDM-^~ph1bl7|e zeJ1GcQ~G-L(=$BqQ+!is-=h-t&NBq-%x#|{@a#t3x(q z^u@YmE=m<8U4*UFIQ$H|AWS$&K|Ij$&U*TVJ-hmSuJ6!U|L|Skf1yv%S$PB>Ua$Yi zUJv1)pVMdHtfFD{dK0H=EK2XPI!Al>L3O0_V5TcVNa|%`C&# z?lQ~`RD<{KqLt}MS8QPqJp^kq9Mi8Vi!$xF%ah!(1Q%n6bKxXJ5yh*%JFG4s=>W%-yn-7(g(8Q zXk)n5YX4q*4uMXZa@>nL<;qCE{plDZEGL8JRu&cF3Sj?KoLVlyGn{ zRngWWL#c+w(gq7EUs*#Jn=>8$*!PcjL3mU5*u$i;{aqJRu&!N|PdYwCn_hAuu$W3e zP-u&~7(f(9LfNs=3NI051y~%IUWQZLj$b^QUe?-LQiCk~Cu|kQ^OysMfg%&AX|QI{ zLk?LHhn?cP!hY6o4(PrWJ!+VMjaehC@yD<&-?>3>zQ8n?HU6A6{v(`FoM}JoEa8v* zrgsaUf>6>xOJOe1a;G)tS6WcHKTCHc<01Nfh8;~0?)}fK>(lZ4iGRm>95?w9RL$kH z+F)yeYdrK?*CWq`N*g5+I`7{B zJ(S(3tMg(Yf=z>>V?kzkY97@coCk0}A@Bh)2nQkCkcVMAM|rh`Z9MH$OL4@Lr}6}3 zmIbGmwaFyQ(5%^RYl~!%Q>%f8AyILvP`)Z`jrUAAOFlZ{SZ~)Bg^O#ePkf`P_Y|Fm_{dOTO8q5ARfm zBaJY5VMWbBjYp3Xuc_}9h;I?%GF$Fj8^HRoY5r>mvz|j|J)vn=>N49@rekJ{(1Rrs3%Hhj&Hn4Zw z$34blWwbdIei)zS{5rkS_$(M-PR1{oz|ZNdCHpj=|E{x+%`Y#;ikeWX-;sVNT;^T8 z3qxXlC~9iXS>gAXb5!`1Ir}B}i1Vq*yp*w|srULw&-Ia>;w_!wmhL>6jGhQP#d`3K zi>Rz}0qu1v;U%RAtZ!J;^WlvCg)) zXl*VXUxA~viw2ea!EpGj!~Dc!gvHLYnIuFzlYS*JjOg6~R(HT@8L-72(AynwKn9$1 z2efntP>qMG<|r|1cZ%d6N+EjUW^g+amzYk;EP7z~Z)k#D<_f;UpT*ktETC|l(}mA) zw+9wkgH$Fh?$nm8IaK?f5Sa6BiCuZR;|unDGL zb&bJf;+G6;eRCZL6p3Mm!|)Kc1*{KX)67p9$p}MN97RO!U>p_!#)t8cW-KOX+6M$T zPBQYuW^6;`p7-hGC&OQVQ^sVj1F+P{ziq}kWu65qEIi$W&@Y8E1?zfD#&6)pluvqs z-S8EhWhs~xHB?i4cH_;V6klqLr6x@qi7a*|Lp%6$jeKGt>&{j!y?yiAyxjc$g+(LmMD zZrl~bYl7MMDFa1?T1(s%JgogD;`GTM=cI-$F#f$7V@)_>M>(X!JeC|`3kER7=$S4V z!*sH)pT5m<5-_|G~sYgufyr9m(jsP=T$m@}c2u>ZEp9SB?8<3HtgjbFPdkNb**IXDu_d zfe+^}%b=fVr;x*J6YxIZ^?Q$wE4q4lLlU4?KoNDBAOA6&wTapaFf9s0y-frv0h652 zw9}7UB3N&~v`ZSF5W(X0=lBB=tYuVbZ*1^}t_LX-#iwS`kj6O|@~sgpp;;BSi6k}X?R zNlzT(EkC1e$#mKc!+++bW;WWk4K1O|oR~+V<^$B^t~ya@607-)Pl&MlZ>O+{4c0RA z^9ZLDC#bwjfX#Oc&;lqxrX@J1_Yo<*2VBZDAk24>R=Y7(07r|M+Pw@9Y+c9Dw2p7d z+P4i8p$kQ*;Z6$0q`>E>R&#(TYsqfbU*Kn2vexZi{)Z$(YjGx39jbH9s(Sp>4P!S9 z1++ygmS|fKOok)0mXhIETTQjzZfpyt%RC=3xDF=a*%P*68J=*?9he8S*OeK}h_hx@ zwV$R@P3?CAtKO-Da|8^hc|3)kgi}Ohd>pFE9B1K2nF)72+%>DtO9X$v70YWMfB?~~ z;-)x_CLJdO3TuFIezlipM6smS4)9SnaY8ylN+Tgx{zgGZvF^imBU~oJgg_*s;x9_X zrCH=Lk0NOlS1VauD@E9Jgk_Ws>k1Wt(Jamn&l}z^nk5g}jbNc`VmTE3^(QHsI8!LP zh9ECR=S0}e2!o=p{?hnsr05Y|yY&vI&xZL2E%MTX)3`wurH*$u0pq-@m@!99w9 z$|^M5hToR+ZmrpggvX&(#E;ez2GBMRmw9L|o`#dFu}ypxCV~*(KGjbdwT#ENVObq% z>x#>~*aIt;uyPQzMPJ>^H?~2W{?b)~waVq^+OQG&y}LTJW$X1ebT)|n2S@_CR>#Z_r-0DpAxN!<^qV z-p#@~`{lwh-oo1SkGM?H6;72YLdC~HDaL=$TXUH^xS4CqkqKabE7*Zk`Lh<*Dd~Rn zNzthvgHC_TmslQG5OI%%-z%yQguToF7^I-m$g>|%9BySWH93%xM zsi4kU@|RGcNda;hlwDB28_) zF#81N=Ny4afPu$4vG$|SLAfwtZ2~Gd^6CKrD}@ob&wud^%@v_{l8Km!rZ=Z}WZ4Ju zex2Dszu9;Q&g;zDr(S9hN?qoTBDMh{A$IdrN(dQ^XpXrd*C?j40V62P-YyyXk@7yj zX#AtjtliAV-7ph0AwZ{a6y90n!E$n!8Lj~0);h}#KY)q?t+xtw!@d>}SK(cVVLZ9f zk4m=M2V88>ct#9s!*0JDU;1G^Y~QV*+(jT{7ZcEK_7^H>0!mqa5RV+>n9F=(qKHE% zx10BhIAr1~+>kJ7Muk*0*bB1U4at^}b`%z3Z+8lOq_gH{-XWH?jYx6BDi9g?>wo5% zv8+{GDfp=5c5`npoqlfc_tLT!NZ#sLX3eht0>Yv%QH&uC)+5qGZzv|F(J2{<;~}%r zh-AkM#p9s>Erc_=jTpQ?L!Q*ZKrX?u<&PD3HNeHgUigB?bzvi#y@r0Jvv4Gz--WeJ zdxScf4x`@!@aFWs<~nTXfrwRmh(#nce^+327#INbBuSv25&kezac+(xS;O!1`LQmn zYugi_qrolejTW(x3LeL@IA=@5(fK^0D;v=IH|lAYttv(YHR2FQO-m*XYv=QsU75pl z_(yu`AxRCW*DkY*U+c;&w&_0!sa)9*nxrouK2-KcAR%uBs|YsE1;b_j_-0D*!n0(M z!P)ZyHUl6X$?CpEH8Q3M++V;U7n2cX1>~K6*qVzc(+=`vVDSo_h1Mj=MTfky_Vc!ylK5fc2p8otkgVjg`CUWYXWbGW3e$d;nOG3 z_~RS`&&Z-nT(Y>lXZYZ5tV7mP3ZhO=1~NVYW2mV=z;MhS7&{3aKTtZ1 z6M({D;gRTtz*Pdc9+rBr%+L!y(ceg1^e85r3~WB%(v98Cwx8y$y0c#FuAzKPch*I} zi_h=Qx;ArNAS2iFjon$_fro`BcRbR2W-xjzOO|CgdZye(nIC&A$eeOaBF(*uem9Om z!D%Mgrg6;qIrooeo!RbTyjwhLV;KP*y6o^QdX7%A$i>=P3-Ya+^1OR{o)^#V3}m0- ztXV@t9X}P%dSothx(l`fwFL=%5Db_3MKZ+QFQWd4bJY+EHq1gWs+QOiiu6k#ofBU? zY@m~m@4*(>T8;w*lMOgsXsztZ9C3=b#bt&!zC)0Qn}>Chs9e8E8|>z~Pbm)La-dx1 zu$x59OLPZrmw~w=@KPqKc3MtX*i7S&v|Rl1?FJ4^r?RwpHEC3Dsp6gnTvUvSInW#< zgumr8da_nMw!s52Q%xG~9#Yp<${60I0LMf!<*RSWl#ERNLQmFhOge0FEM*yX@%(hG zO@IJE8wBWV20SGPF@=vc+Gn8+MFk5Cw833zTPT2MctkIJ>tZgS+KYARh@llaE18#J zYZj2w)=G*Rll(4o>Nx&FFLq~?<_=q13%v-QA%Y+0F}>MB%XjbyN%xcL)OivvA4^cf zlyCWt-l!*=zTu~PvyKCP0UD(scKh0miv{%abd{4>+h{ewa>%6+t_+g~cEh%BctQf^ zJMG`_*$K?r_xRVO|3&C`KJ*p7PD1Z{aokT@MP_^r4m|Or33_8&YAPL$QmTEq{PP4% zh7BD}@=Lm|319YmmgG`c&7sk;*<{P7Lel53*7;~GuECD%lE z=aTTy;db}}MI2(HBU@Tz#{ZUn;F=#^1mG5N&Tk|5=drnoCN(nSyB?CpopfeX5R4Oz z`W?}fUI#q@=$RcA$2a$3Lv0(rqC~E=YTE0%^KS9+Rr5ngIF6K7&%32SAm3f)JCJ?W zvSIlM#e?S;A__9m4e^(dPb8#^7o@Kn@;IV`mS737jZ&BvXo$80;|zWYkKnRl@iP1e zuqYNBxNMLS8~^wcsK^%t>O%^)n;%t52(?HE8b2n~PtOa~0;$WSP}Kd{gkVaP%{&eT zfcP$xCP%vggmDaL58^%o@fP%D@g@vLw1D6|C1~y}(vf6M`T~pI1`q22`NBT1*NDWWiRBkJ*KF4j(en4jv$;%wJOQH;YL1{j(Ph+wrr7|mzjC)0nM z2r?Kb2DED(U`~pPhLTNcDZemG9o9`mVP2$yDLW^rVDoEfA|mpGJwib6I!VOzjtqUp0=~cfDxUNdL2(3LoHe0Fz*YyhQp}DA;~mjTk@q&G~nmacsN zNG(H}#DvGn_e+m6Jot_y&b!Z%1tei!K@r-GQ^65B)Ow7k1)C2Vzy{s&R4T&k#>paX zyb3|G+rW+Of_CF2bY9YNWFBe5tlMod7W}L(^B%|yi|l**p|F*A`NQ%ITAjSM7>Njr zGyF4)9~i(!n#zP3tIEMe({k|(-eMq&VTN;j@Icm0pUG##+kP#I1y!~^u;{8lQL@X7 z<2L9`jHF? zc~~4y*J4E9jM4m5p%z6$VA^TQoiR|%P%x_r(Maax#zswy>~_jlkUhu}p*&-clX zSvoN_WIRj}(R827{J~IBHR8}H_lU=m-uToQ9Iy7k;8oQR3`jDPTv1?>AsVzWw#?X$ z^w^Du(Qhd;_``$g!4V?%HBQ7nzmCQw=&n)RU$zIfh(q9!Ss@= zZ3 z#`h0pZIVVJ#BTVZ224lhpt7_Ph#k!Y{IFqzxLd-;@M*welffoRfNmn;v9D2#zFJa%6t0pThq*Wh7aoip;kC&uz1hboW~@cymK4?@=$rl&$73SNZZu~{L!?I_lB#JJ;B1gA6X=D~2`sJ`_0_wmtN zhOZwjj`;4y?Y$_le9h1tey$9QQfnTB*N{S-bE~J*nG|#V_)8e*8fGA1mvaU=eJGA$I_Y1sooIzjk=^Ld1gO znJ@PMujZj*=%EXZF#CZ2ZDk$&7IwVG+o!W=90HP<&T_N!dQpQl_Z=FftFg_{D+k2T zCS3%1)^`jRe$SGKZo;pYM6{LQBhFvLsF8|%YAl9}Z}Rl9tj*wcu+)2@NU8996fG5g zrRXjRKH?lH6j}Jbv20v6C5X%n#N*w^%+ExS$ISPH-=lrE@GI?GCHRQ5q`5GY7mQc$wdX3!mtrsaU)h9^IAr=I2E4fC?h2 zHobiiIAPOPxI8vZ>Oy=o7r_J85fL`wC4jWaA>lOKcCP|6iNf zG=Z2mvT0w8v}q#`x*0aLB%H9xA1;qgb7F~a{`nmu0~&}lwdotM02emBLR$hH{$$hJ zge)`E5b?F?Rbt-ArVl$yo3`>l$fo9m6E^*Qz}u!JF~m222PSNC5@~AF=MpY#+WkkP zO*;u$X4p@}*QOVUdA&{XSduNO^j>~p0!%tp4VW6HzLg9w20y;b9`aCf!L)ACnxnuvGmZKXD7|slWDAHk&>2mHnF5xVjMItp|Pl zyM6qR`}ha%-}Mso=sJ#Kn#4wQ?Z1mGGFi*8&a2GOg?zAK>OoiRfAOSn#cmS+Pw@ql z@Kk@J84kVCHH;zroCp1}*}uAQp7T@?CC1n$7!x8p1Ms&TDC|GDDU7C?rbwB5;y#Z*Xf`+_$TtEZVsv1cI`H6bcr>B#sB|s? z11AJw5wEkK8yI&`_9eMF&X72I(4$L;=AeCS#|2GWADOO`^45tx5 zNy8Mu8me#(VMMM3R)dJ4Nq*N^e< zZ)KfW>3@0isjN-Q7ynD?dW#Z-Y4uXW8u*?6URS6gsePM)Xe&@ zc{6$&GHEgwiidqr$=Ng(-~8pxr2aEJjry5qSMsE37)Ok%Ij=SA7t3u*Ox}0dU2hGK*^HFpytXzo}RFg6S;UoC+1lL#$pk z48Yw*GlBE! zbqGYOBxW}{a5jS_vr~~gbp{5@OL@@@)=^)^`3zuX@YO8J2a5PBLnrzBwlD^cu(C^I|@0Q#A~ z6Dc#~Zj)YnZ6fy-G7D;;x?5g^n;wrtj{We-DSe{f(ty1ykr)O8PgGZuo(iIz61N=yT~{ya4c48S_7TU{Se?VM2m2&*;XyW zQRpg5F(D*Ogy2(m4#VYW{?jZLZ95SCFUqzdeZYMw!f&MPQxPJR?G;X>_W<1lhP#!w z)>gnchXZrvpk8Pc%^B68L;RW9EGB;Yy<`LJ zPwy(!l>C61Px9VMx#%$e+!)@B|1g_b!cRVjqxq15VK1cG57ZbAJjdJRVUb^(?R%pb7^S;n&x-#q#DBf`|6cslF#xXED>gx#;{OHI z8dq#6-8`{h2zQqFUm^a_i+?(y(G^=N{vQ$ltHuAL;=e)sKOz2~6#s|C|2px%LHwT= zalI}6tHRM2cg4OSJe$P-Ht}y2P$vGRrPFskH-{DJElQh{1O`m_&k+ARdDnc_r&DCZ zKjJtttK*Smz%GZ}?#VRF<|BmFq44Yy9PW)_3HKPQ#0d>wfz zWk4OdH%a$Wa<7BiOJN-G3jKwFiR3PoZY#OZRy8J0h(Nc%r+VRYyzoElyJl-tP7)9h zN>g!&j`HV_;{G1LD25o1KbOKI;fHJ#I)6hT|oHra7+CeGRsf8 z8TLnt0l@4~9pT!+*v4YUjiSQT(Z4R-&nZ(~Q0cB9j0LqA zn9}y-Idn`2l#^A?+`i)V2cdHV%e_IZv^Uw|d$Z}CSK;C*S{CI}Y}aek;_ zJaz2e647ZwIY4hGY`Jg~h0bI09mhh#qa!iL{msyN!aOZ#n5( zD)mhvnB6=Q#I3KF(r2qNB$5O!QfMJ$Q-v`U423;OVVqRhkzjUnFo;{>7iZiCY9xS* z^f_O=PTy*aYuhEi zQqe~houg=fMcXU-H}%AEeWmC&MW0Y~siN9WCgRs(;GnC&qREP8D0-`+a}>Q_(bbBs zQ*^VUZz{S`rCcb--v(7ww<-ftw=qgTB36t}iZ)mDPbKiZqNfynU(vmaZdFt$!{14F z<=Z6_)>_dRMTaVS(=M9?+^QIPiY`-hqoPkJ>W*NgSTTcpRZ%y8rx;&qTHISIM->fJ zbhx6M6;9?qq%ZA}0&YcLxNjQ;{MMZ!1-Pste!#6*`DZErH6=G)(IHCzZHneAYFnZl z4=6fXDR@%(Ur_Y4QnW|;4=Ea@=tgD05rwN%e0PGPl-`Hj@-jzl+7n7(iK49)bt;8l zD0);C!Aa$RU(sqs2P(Q*(I*sbt>`r+e_2tdps0UeD8{V{=(hMLmF0ITgWUW?<=?32 zSfB7oK7LI#Oy~b2i}<9XZxf~ZzgaOJRdlJMxr&Zfw6CJUiZ;9|6`ob}ZACXL`lzBy z70pw0w4!}M#r+qh7!k_SE{YCO^oLhuM28jKqUckKW+>ckiq3KK6&`>Oa}*t^Xct9W zD%!9~Dy&zuTG2{HpH;L}(Ituw$y1IjMH3Wt*Y@HbS~9w-T0smG)Xgtw!MiVGqm7#) zBtN$W=$ejx{dnav7C39=GHDON{qRd%$21FhaM3i!mRu=TVU{%EeMxH-O`fIr5i)v{ zqL!ITt^;pZyM z6|GcssG?Rke~!e}ek%0;slB-_8Hztjok=1Xu)D)*3Q^1kTU%C?lNyn%JK-N&>9sX)|EcTB$Z_y}+o9~>p^ zyjOjvEGh89xh=X8&ZKZptz(vn)llNAoP>RiD8B)Yl>7S_kw7?+TwC(J*Q>^ePwT(b zlam-dYX$HZfYRl*n{YArcyM@?pyk5v&M!BPX#Nrpj!LEg{&M-4^C%?9-$foE2B?|| zzVqSGs27eB5(j?;C|zy?b$ZOcxu~haYWS}Tm$;59NAC>ISVLVZexl1R`nc-6Wx?XAgQV=ZmKZq1`2P4t6P12#Dpsa z-+hTpYJ&5sKU?fgfq?+}Hfn_M#!&^Wf?TqS!JT5l9eJdwzB>4Qi(Awm@2sKXvI1yP z3d!$oowt;E6jH=T;J;2G_}==cD0BV~oXmfK2x$2~fCv^B;hd%aTf~+BZ=6j;yc9s+ z8h)cnlL*{8Wr#bgZ-l%0@c(9D^~2~U{x4Zr`aden0?ztB7-(|qqx}EBQd0L2TD$*2 z;X2^{mxSa3m+Pg^_HWNa%m1j*x$6HNv1r7Av#5yOl^O}RT~}L=ph3COZRguA6a(j0 z=*~g9KYSacN&fN`*FVt6&?dJh z#a{tI3n*Rglw1cySiTp|orM&oZ_dPd;oSOegma>$wOOEmE^pvUpC))6{1$=o;3y|z zd~jCyeJdeFYGLDeLM%WW^|TgN^6MT-o- zg+RF6IC_SKKIOsT-J_NTe-#=uQ3>2Yf+Vc<04d^R_^TDmjU#&gNe_-1#s>J4LFsbi zDB{{R9vqg=XeQviGU_(MYmfI`K9wjzXW+LwpJ}Uam4fatyr9&uTtoW-VL$kUQRgU- zte6X#sf}_xelwfzU(J&B$2+z#niAmvCW`3e9V2=DR@RKyZDz*cgG{TZI`nfZ^(h2MQGrLfr^ zznfnTIWmwgZ?&Wg7eDtZGlw_Me@ktgD7Kzz2qIUfB5 zhnQ$seDzlO=&c8iE*gf=MO8l;zeF!x@M?ibgZpY?C(@-Z?8v}nf}!rq8(4n3hfm@8 zFeDJXD*Ud_qAYJy4%Ur|4?N^4l?YophhKYzwHgSkDJ2+iyW?u4RS3ILPUYS%O9pk2 zRnF#LyvkaRu>hL@1Cl+dXw+l%(e2JD0(d73IYlK-m65FzdRt9@7u5(VF<;QA&h4Nm zZ)6^s;e8!}4sQ+>Ocs9CA~QsEEtY!AAg(MjC0d4Ay~64dhUB(6u)=& zk>6b*kbUHByu9u6&JNhuh#^NUipi&}G4T^C+JX^Vyp+20mujN7Lf@KLPcrVrnwv%4 zfod>^@#@ViivRp7GYrbAL4kr>=~IQtn$q_?)_F(byrZ!YyDmjL=)OqTjYSjyjhK3ID{Z6{b|dYC$0ge(G8yJ+m`rjPV3&L^6RVHhSc!&3p44U{ zSc5pGdfas*Eyv3iH*aR) z%$Nbh!*hjpn`^xi5@yf2L(?qqQ|;(AY68wW9F+p6O^fqk5;vhKF9R$s5xh81B|MRM zxlM7m5--0AFP+qB1wK4tE)_gH<)y;(lu=+7Hw6lXrA>HQLScDRp1`eZ!pj!83h+b@ zh!kfFTqSr{yCU92$SK=QF*~ZAr|c!h%$P-FF+Jz+$8%fpmMvSBHw!ta1qB=3HVFJJQ2I~@$kCb-=s1}OD(KpYZ#|OU2Mn+_JhQ$+@ zwgO?)#1zNyN8V(;S`HO>+-!vb8YY^xQTPp=$3J?LwGSYfiV6JEo2-k7W9bCmeh2F% zbY(%;ETyYFhELzYy0;`d;=m{U6}Y>IhIZo5?qD|wc@4UDu)ae!Vh$BCiXSpeG&Yss zCByB$Dj`o}qhvP&^c>>Zi52)1M%O-26rlDlC~=M}{*R!fH#(SU!F>a=$Sw_bMThul zAtRe*1tg$XsCePg0a|pPQH$&!ss$`!1;nqrS=wG4E8{8XF7s>yC@$V?7n{RZJoh)l#HeZkYZ zNCObIp^D8Kn271Ij)AyW&c|vmkJhyRj=_Qo;z7T!zZ~JjOI*nN?q)-7swA9&X@+d0 z)_)upY2vq6$@=;&2qPZJM8`DKBJx^l5&g|tgdU-V6kr(EiSOCX+6!BAl&vOZYuhgT zr`@cR-in9sVS~C;auWdg9x0(@CMZ}X)=87!b~9|pHDwB4w1@S&mGFVUPe7hgc(TH0 zfKs;RD1Yq~8Qx7a45+>sP*C{ipphAmuN*8@=mU8CggTkwQ2}*LmgA&4j zg`DCVTq3(kS)eqsyG{CS+Cn(|!1(1MDNy>bRPYZdUH6vC2*P^@YhCqttxZF8eXCk? zb&GIcA&*EsEkVgJ*IMcC_7sKl;cv4cR1C}BW?e*P zBtzFCs3Cn8r7u%aT69eOUC&7!tt+I?I8eGGpO@i-KKsI2}+q;0ZOH|71aAm*ed0p1EuS8P?Eo-{FjyAyiFF)Dp2UNX|da7akhCw7H7(v z(*GhTMd-If3fv7!3O@yqk3mVk zs7l)Q4JgU4+AVEAvrnY?7OmZW$+#Dk0H1-9>QU8F!E2y&g&mN%cu>NP0VUjQP#x3; zO4ri|_^St4>;9IbXh`wX@QSG;h_^o)ha&sWl242ZBtE%T8qY)DVRL!eJ8Ybd9vVv# zL}@a`$+WfwC4Zcvp}=@w$>*e+CqU_X8YYOO3uF|N&8yH zI02Nd2S9burxgEnP>Rd%yu?L;`oTY5`NP`=YHjl(w5aTG{=qveNElI#PRZB90<^H~KrO6)GcBxdNUnG&qxaQ-@b(w*z{V6|M;OxJ zw0RiOcxfHn->3yI3dpTPO+pVwKFte5wOj~o!*8Av^!EG8QI|?a7>XlUHSH$+HYqu8 zzmATpsdr(ZW`dQbzM=I(C@(;(fWXK2omE2Ke%|mdv&<>DsA(_ZXId`P;qBK0m=@5N z$uyr2(o8?%Wg~tWgpCZ;A`xR)b~7Y30Ii5mI8688`P+wCYg@%nn)Vld>r`Cc{^*W@ zT10jjIxfw%h(*ooLkmJg)=?JdQNdbNUMOl>h!%x1j+zkJ5K(U`hz^5&6IcS>(77lx zs#xy(iDyw9goA3~SrwoMQ#l1wITh#;WPlL{glZ8Jnm3RE9gY5)@qCD8 zI25cE|D|b9;Fp0pVk&SiKR;K)VuQ<>u6>1H=>xp^5!N!)<7XPT9${@;SKx^6&+w~L zDm;Gvz!BDgzjTBRvK6BuK8s%}-T-x^d!(?r)`UkkBjXLo^emiGe>Z-jKtx4t27Dm8 zgFa!fw+WVdmZ-!9c(45oeqO_hHa<)YTpL0Bfe$bzPPv_^a=SpaEzsYX`&0_#rz;@> z=U3yZ0HrIig)VOU&Y;8(?-HnW$qUvxW(R3)`v+>xCj^VT)1>ql`?OtIE%a7Ga2piR zbq*YkT7~P4uYJ%p5tLj7 zpk#1#=U}aKVTjfqMn}VF)uz`|yz3yJMIRNx71c`0j|C;aXe_fswO0K@w4e!wdcOje zOELwJ!8KDWP{5@$$i=k=R0#GB(faluqIJ;|wAKw#^(|{7t0M}+b4^+D8D~|Atw3Vw z+N!AceL#F(`yq>psJbqc|M?;7)RGW~ZWz|Z=GIiHEV@N3h{4<6)-r1bf>PW!gVJ@2 z;+KO`ll~vYx3tqWR8OrxD5|J79+a+|75`DtAoyQV{49%3=XBe&JUHmBhz*pk2W5b^ z57Y#Io#Jokq-$pQcYunl!xsfDdEr{yY!jM|FfH_nz=nW&e^J3KD%)z2Xu?t=>y_NF z&N7~lV<>@W)_;P7Ksxqb1}#G2<>J(8aT|u=Hf*jH#Om5X{K`vI8|vi`rwRiD+Q0zg zqM(Lm^@dt^;a|N?mRUs?ssA5Px(@b~Ir{~ui@$V?br?u>r5129qu$GAfybxjh+hOs z*X{%fCN+zlg2L?E{%T#lv2$Lj!N`03-UQ+o=pSMRff$-MOm)?VnS0;V2(3V$+1hHp>d z6HY*f7>XkNzsR~;gb}9qD2W+6%A+G6m}2lLynm_;AD_zKM;x64|21RwMiAe?ab zrb*nBX&yaMz*U1!;f14R_+z6z;SG4tR|!6a=Z%r!wlSXYBM7empTe`%LZ$^){@;&P zS}G9!4s0k#7-9ZMmpUTH@=I>aQef~*)sn|bj85}{xY}lT;>iLg3w+YypDDvTWO~Bm z5S|P^gu2TX2{qUh79Ja{#rBWT zA{Ut&&~>Q|t`00{mMf>SDj`iVH>l>-ubUiw4FaXxe=KPI1gU+&EmHeBP;%9P66PmR zYDz06Nu21n<4`FSn*v3ghUioh%iv$&m)U4N69L znIio;Q}~~sU{RLoHeKsEP1g#r5~w5Oy!|2*@}jkl{i8JVrI3c8dNGjCqN<;(GQr(n z5YLq9GLCcj!XsS`pyUe4Rf%jKpf!IYxFN7Upw_>d6?6>5+U>ypCM^Mgr}4W=Mg-!% z%MKTQ^GrNgu;xkdFfbv%6~DH`gJNGl>1dRp76LLFbZI_uP`en@-=ek9dFxNrxH=ID zdI35qlX6jlL@80#v!(Gtx64E&fs$(yC}HN_A^j^s$sgW1NNfE<%ZAAMh+0#1^8#aT zXjVvqe6*VoM}5A;Et)I!eSa5LW|0%01Q7~!e~fogh#m3gkJ)Yu_=)(%t(J*y>IcvV z6W$?M>yRCR=To>A_C#Z;iOz{LM<%SmD}k2zs%pHpKw5MPlw7}niWJeX@CH4dYAB=2 zR3LG)LFxJvlw9V8o~PA1U~?Br+$m7H!WT(g4^WwN|K1ZTfPAsSwOK6X2ZEBE98U#m zx$u|cC#T?H8~V-Q1vN5x{YSX9srFi-uCh>yClyI`HK64B3RIZW71MA1TWPIMT;Bv( zAWxWLuY4r`YWNG5NO}Kzq~4-sQg1mZAU2m_xoiV|dq4)5Y*HXP%czYg3}b`xnzP>h zLvfx|0BgKi)DycB-RMpxk$qZ%k_)xVhz5a@5sxVUf0bWv8Kku=G-xgHJ@S^>eE4bH z_H|UVp(Z*uNNc6H)|$@J(F`+QS0G&6d6zqB73I3t;&BXgur8(}_}+d!&|eGeEBX}) zqAap8wpycWC-G~GHJ^m{_G3cF);Obc6?4sW6+bDBcPprdKXR={K@&d>b`VJ|!egHD zV4C<9rXJz_xd$Wt$T_miXDWbNUd@0(b^`>A>yQFrJq)~#hsMndqmvx^Omg&z;d9>r zImE_@ta5{{9mX%FT;_!E8w7*@d4^3GUjBlvRpM7p0HG*Y_}>L@qgx1WZ@))|KXR6} z9&ULN-?YN-yjuqbGUUGnzXqkl*YAizBR`h>rXDU||UHT==ymKB|?k-&;-(^3#I)`)dKXw}kFO zKa@W1i~s1_Px#FfI;b7-125Z~iIAmSut8Gw>7H>;Hz<#MH+G4;IOb@)g>MpVxGy5m%@!1~GmS3_?@zjCMg7LRP2Sr@3 zjH&og%AW#mP?z!jty=T~WbRe>yY599Bl=O|~xv+A){5eO0oV4uM%V%xG z`ud7v18O<>ssn}&%;dX^8WZT|OjkBkGjpLly`hFYA8N>lK~2{f=zRsga5_~2_3BXT zbb35g$unO%Q#b9PJu?O7(?ibObvx{{r)~#S#TOrOrmg_$jYmLL@BviSUqZcpIpT|T z{*p)2>0_dX5*<6Fm<}a9P2-6{6G(8b`WGJ+OPta1F1K#Z8_aY*UbrSQ_O;IKY;T|U z8k(XLKC=)wzWvnU7jnW;gFE%9oziz+~OdwWDqb zF+$0*|7mHNi!;f@q^w@&Wi{pos(lwEmUhTGFEd$iEQLz3EV5}zGh;mtS($y4>psSR z2D_Pcb|t#f>{S0I#aWg)jXlo#(^;{H4q4YUQjwY5Y^pe96{ODX;B5H*GAmZR)@nJl zW*IjzmOD>ty<5z}cD6seoczfObC+AOIx0*ZLS=5HWKs=PKBZpXdh*;x z*em#>66@IP*X_{y2-p95g_bKZ%TKywtkG9Qq2IM)J-@Qrg-f2PZ_}g=q=)q348Ee? zH}^$!ki7U+^}b0jk-FB4Y#UFxA6{kmqFlmL?v--dM7c=3e%ra5xeV@&|7qLDohV)# zaF$)dNzifHzTNnO_=b2IcpP6Y4V>ulwb()#d_j+|FTQ}vdFmUZa@1Gm@s(_8O(Yy= z+~ZtI!N?^Vt#glcIQLkGbB}c}=N{{D?y(N%9_w)Ku@3)tk9EixzuLKQsei0P`9lfo zK#YO`t^Wasy5*7nJkr;?(dGiCEZ@bhHYaj)9u3O zfvjngp*7{bGW;EUaAb2oKgs6}Tln7tb6Wb_Kh@cPvz}ks6YEW)YY>ieHIXT^z|~Q> z)zuZS&eZ{KAy2@R)*OQGxVj29w6`6 z!sL&`E<`%*ieLm&MdI78zdobbvNMZcAjUhLGGaM43ta(Mw;~?{T?W^1;k`jRi~A?~ zH*OUy)TbF+cDnI6{Ea7DM*IVojgG)u+c}LWg74+fd;EcpJcJYrp`$Rj6K{+T!tt0I z9fE<*j?Ra#b)i9%!?X@Du&c8{J}kMAFKc=;I6c>x9Hu4$t1(Se4ZNotqk>iraQI^U z#gu_OOv4_nD%drT2H_9F`>_bR3LeH(kLhkqW|%+r+{4X%`07ES3K+bCuaqG+hP(Q6 zuLNBMcMReyi!_`+n4V6h$I!{?wa&`-RUU91Rzw-G>kvAG7GK3m(cub2i=oUp3*H1@ zCHf|lfyM1u6=g&-j0h)MT!SpB}EYhaJ$vu9MNLvDt`U!^$D*|vPrXwN>SN(%g(FQPWuj3EEZ?F~k6Y$ng9e)v= z^cho#KLl$r)sui{{^`_X_R&yG{t&!&Kl87OLi_320frA<1qXc2-i%f{=9(IQ8ZAF; z_Jz~q0DKG!Jw)SR-&%GwbQGSzw5{?UCmiG{1^7cysfzN8gAQ{V{v-Qb81Xh%#)47^ z7kx!Rw9*B8ea+tfFpYrAF`Z1}@D0qK8h8}Tp^P_8Q0ao#U_3&KBd>Gq=UtEOkM0>OQ_TcOO9k-tFixV;P2t9_$d_VEuM`$R1G4*>o zfF^H`-$b#XyfSeDHVQ2+JW6F~F^0`TdlUJT;1{Z6*zt>{*cNmYERUY1Pn(0#+ zSc{n%j22X0UH%Nt|DFUpJMk``3y2~LiKUj$l%TyicFMggZQwIY@ryBR6S_Xj&a$)b zMmO}CBlyM1nBl!b@cl*v2+&n95^m-*d1?eqX5)$F=ljeU{Gu}ILTGPhowD)PW2N}T z2?4?~Xz^Zb6S_X3&a$)b^3r*8{NhAxA37{C1xsiH*fE1h0mrm5>I%3oY9hQbOl!eL zXeGGq!Zx9m#5S)b!SF{JA$WbJv#pZTidzvB$FF>|=d@|- z7=^mjCpIa;X+~FigI`>NZA2@1>121GsYII|KJzoC1W(h`XO0JHL@D|=geFqOd`TKCcurK$~(E<45W!%ht9RKBn+>*&X-eesm^rZJ^*eN6?uqayWd8N;6KnJ0c zbu?Py>H!Qdex>Ft8py~!L8IZGd?MxO8o1*s_9wK`YNlUJge}a1a%k=;;7!oVlX-0r zt@{&|!WRZJ>}ch`EWd`{pyO~3xh#9p%4r#J9Xr%4c33!g82bxac`BC=r_u7mZr9T| zwDM2d+(6@ITXwEVxJeUnn(iLo86(TkJ6hM$~o-$a2u9|R#wLv zGB|S4WiV~5Gb-M!4P|p2#3GatPhm^v=qY)(`piW|KC_M$-i!-x)`T~sLYWpjZ}*uX z*%o3oR)7}2#YUmUqu6+K0&cy-ohDdKrb7h3G9Fq_V*bBo!JECHtcE36aIR%@ExG_L zW=!S{&|>Bk$Dae6hL|S&N;_saAU;AeczxnhY_L|3jgU*LPW7=WOd?M#j zLZ_Qpk%L%?>5|zSLRVPdP)8SyCAbODvZsZTlOy&-gk)+a5ZA(RnIu>R->d;=SX z_D0N!q=(Js#S9%Q#P_g=(cX}`Lg~A(x%kCTu{hcrLRTpL6RZrsSc_Gnz0q(*&ii8r z@ry&TlM6WONrW&GscmjY+t=!F)-r~X&+I7dNpxHvyFsXMc{b&}VRVJl)BnPLgJ0~5 z6`+IgNlX_Vad>J4`w)J!k`W27qW3Hm!WBkL; zyqJMiqQ!PtHM%~mZk-i^4=Wab0BgL^viSo#8!aBg^3Z0r&wRLs-lOZ|@0J~xzvp?M z8H-<3JiP?%jjMZO?uy0tUdy3Q8FAoR=D(5!gvrOM(PA%R_GDp^^-$i>Eix16GLZUt}84V&Emt?`W~j z-<-qA8$VZUeP9_MRFn~?Vmr~|%UBJ%0>1FFv+e(4>0wqV$bQ!=9GhryI+pzuhYc+M zJ5zwJfRT-ilKk+AP4osGg(qI;s6p#@>W^USR0bArWuHKY;JOOuxDUQXgDUxiTtuBP zd~BzVQ#Rhkl?=88tr++E-RwkYg|4?9VE+8a&wMw%6YZdt?5OnI>@R)!Wo z!z$2X$`?%GV$S~x6JPc{6|*8AHcRkHwS=LD$$+gm_eVY<@r&nzfE)jItb@uW6)7}#VO{0A`6A52sNK}&S*LCfU9fZz@MF=%ZJl3odcq9 z&@WE;LRgB~e)!!Pr+fm|{m1bq;5UX758EbSozK-_M4ja>z&gLH;ZG@i%+fg1fN8+A zv3OfUM~7feBS(wF8xs=b6kZ7R-xqblse;d+$8Qy=rwn##;`oEGVJg>uS`k1T$9~qE zHRbaelTN_uW{$3bmFGLU3T{kubOnqCEYox;=LXm*-LgMmir}-ZRtWmW49jFwW(?#J zrgmpmXE6WcnmY@6I70}tznj6?uAU41E$I~-r@%~1WrA=7R)T*NjA9xgp1W#3aP>~u zCzCHPl<5bTW2@1tVMZ%PXDR3Z7Kq`PDlLS|FjceyPRO#%1OxqG zsS)Ch?Rk^|^^Aeru~BG+wo^MeWfY#i*VS=Yi|INbd=Qb6!{NgQ;svf2zwJoR@hf%V zM@)@72}g8hNbxIUWDiz~PQa@!VEEBIQP$ z=AkQlJ|>-WA=80LD~DP6%hJk^P;P{@aubwuAT27bKw4B9fwZWU0cmk1mQ927K#%V+ zbtZA~rIxue&soTa^IRQ;`(0fFyLWfW1mW+cV%?bNPrS{(ipy0+YCSp#T5? diff --git a/docs/html/custom_memory_pools.html b/docs/html/custom_memory_pools.html index 9f72b3f..8b7e428 100644 --- a/docs/html/custom_memory_pools.html +++ b/docs/html/custom_memory_pools.html @@ -132,7 +132,7 @@ Ring buffer
Ring buffer
-

Pools with linear algorithm support lost allocations when used as ring buffer. If there is not enough free space for a new allocation, but existing allocations from the front of the queue can become lost, they become lost and the allocation succeeds.

+
Ring buffer with lost allocations
diff --git a/docs/html/index.html b/docs/html/index.html index e231206..336fb86 100644 --- a/docs/html/index.html +++ b/docs/html/index.html @@ -62,7 +62,7 @@ $(function() {
Vulkan Memory Allocator
-

Version 2.1.0-alpha.4 (2018-08-22)

+

Version 2.1.0-beta.1 (2018-08-24)

Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
License: MIT

Documentation of all members: vk_mem_alloc.h

diff --git a/docs/html/vk__mem__alloc_8h_source.html b/docs/html/vk__mem__alloc_8h_source.html index 4cf5c94..88e2ea2 100644 --- a/docs/html/vk__mem__alloc_8h_source.html +++ b/docs/html/vk__mem__alloc_8h_source.html @@ -62,7 +62,7 @@ $(function() {
vk_mem_alloc.h
-Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
1342 #include <vulkan/vulkan.h>
1343 
1344 #if !defined(VMA_DEDICATED_ALLOCATION)
1345  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
1346  #define VMA_DEDICATED_ALLOCATION 1
1347  #else
1348  #define VMA_DEDICATED_ALLOCATION 0
1349  #endif
1350 #endif
1351 
1361 VK_DEFINE_HANDLE(VmaAllocator)
1362 
1363 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
1365  VmaAllocator allocator,
1366  uint32_t memoryType,
1367  VkDeviceMemory memory,
1368  VkDeviceSize size);
1370 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
1371  VmaAllocator allocator,
1372  uint32_t memoryType,
1373  VkDeviceMemory memory,
1374  VkDeviceSize size);
1375 
1383 typedef struct VmaDeviceMemoryCallbacks {
1385  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
1387  PFN_vmaFreeDeviceMemoryFunction pfnFree;
1388 } VmaDeviceMemoryCallbacks;
1389 
1391 typedef enum VmaAllocatorCreateFlagBits {
1396  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
1418  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
1419 
1420  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1421 } VmaAllocatorCreateFlagBits;
1422 typedef VkFlags VmaAllocatorCreateFlags;
1423 
1428 typedef struct VmaVulkanFunctions {
1429  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
1430  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
1431  PFN_vkAllocateMemory vkAllocateMemory;
1432  PFN_vkFreeMemory vkFreeMemory;
1433  PFN_vkMapMemory vkMapMemory;
1434  PFN_vkUnmapMemory vkUnmapMemory;
1435  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
1436  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
1437  PFN_vkBindBufferMemory vkBindBufferMemory;
1438  PFN_vkBindImageMemory vkBindImageMemory;
1439  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
1440  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
1441  PFN_vkCreateBuffer vkCreateBuffer;
1442  PFN_vkDestroyBuffer vkDestroyBuffer;
1443  PFN_vkCreateImage vkCreateImage;
1444  PFN_vkDestroyImage vkDestroyImage;
1445 #if VMA_DEDICATED_ALLOCATION
1446  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
1447  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
1448 #endif
1449 } VmaVulkanFunctions;
1450 
1452 typedef enum VmaRecordFlagBits {
1458  VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
1459 
1460  VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1461 } VmaRecordFlagBits;
1462 typedef VkFlags VmaRecordFlags;
1463 
1464 /*
1465 Define this macro to 0/1 to disable/enable support for recording functionality,
1466 available through VmaAllocatorCreateInfo::pRecordSettings.
1467 */
1468 #ifndef VMA_RECORDING_ENABLED
1469  #ifdef _WIN32
1470  #define VMA_RECORDING_ENABLED 1
1471  #else
1472  #define VMA_RECORDING_ENABLED 0
1473  #endif
1474 #endif
1475 
1477 typedef struct VmaRecordSettings
1478 {
1480  VmaRecordFlags flags;
1488  const char* pFilePath;
1489 } VmaRecordSettings;
1490 
1492 typedef struct VmaAllocatorCreateInfo
1493 {
1495  VmaAllocatorCreateFlags flags;
1497 
1498  VkPhysicalDevice physicalDevice;
1500 
1501  VkDevice device;
1503 
1504  VkDeviceSize preferredLargeHeapBlockSize;
1506 
1507  const VkAllocationCallbacks* pAllocationCallbacks;
1509 
1510  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
1524  uint32_t frameInUseCount;
1548  const VkDeviceSize* pHeapSizeLimit;
1560  const VmaVulkanFunctions* pVulkanFunctions;
1567  const VmaRecordSettings* pRecordSettings;
1568 } VmaAllocatorCreateInfo;
1569 
1571 VkResult vmaCreateAllocator(
1572  const VmaAllocatorCreateInfo* pCreateInfo,
1573  VmaAllocator* pAllocator);
1574 
1576 void vmaDestroyAllocator(
1577  VmaAllocator allocator);
1578 
1583 void vmaGetPhysicalDeviceProperties(
1584  VmaAllocator allocator,
1585  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
1586 
1591 void vmaGetMemoryProperties(
1592  VmaAllocator allocator,
1593  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1594 
1601 void vmaGetMemoryTypeProperties(
1602  VmaAllocator allocator,
1603  uint32_t memoryTypeIndex,
1604  VkMemoryPropertyFlags* pFlags);
1605 
1614 void vmaSetCurrentFrameIndex(
1615  VmaAllocator allocator,
1616  uint32_t frameIndex);
1617 
1620 typedef struct VmaStatInfo
1621 {
1623  uint32_t blockCount;
1625  uint32_t allocationCount;
1627  uint32_t unusedRangeCount;
1629  VkDeviceSize usedBytes;
1631  VkDeviceSize unusedBytes;
1632  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1633  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1634 } VmaStatInfo;
1635 
1637 typedef struct VmaStats
1638 {
1639  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1640  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1641  VmaStatInfo total;
1642 } VmaStats;
1643 
1645 void vmaCalculateStats(
1646  VmaAllocator allocator,
1647  VmaStats* pStats);
1648 
1649 #define VMA_STATS_STRING_ENABLED 1
1650 
1651 #if VMA_STATS_STRING_ENABLED
1652 
1654 
1656 void vmaBuildStatsString(
1657  VmaAllocator allocator,
1658  char** ppStatsString,
1659  VkBool32 detailedMap);
1660 
1661 void vmaFreeStatsString(
1662  VmaAllocator allocator,
1663  char* pStatsString);
1664 
1665 #endif // #if VMA_STATS_STRING_ENABLED
1666 
1675 VK_DEFINE_HANDLE(VmaPool)
1676 
1677 typedef enum VmaMemoryUsage
1678 {
1682  VMA_MEMORY_USAGE_UNKNOWN = 0,
1699  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1709  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1716  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1725  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1726  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1727 } VmaMemoryUsage;
1728 
1730 typedef enum VmaAllocationCreateFlagBits {
1742  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1743 
1753  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1766  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1779  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1786  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1792  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1797  VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT = 0x00000040,
1798 
1799  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1800 } VmaAllocationCreateFlagBits;
1801 typedef VkFlags VmaAllocationCreateFlags;
1802 
1803 typedef struct VmaAllocationCreateInfo
1804 {
1806  VmaAllocationCreateFlags flags;
1812  VmaMemoryUsage usage;
1817  VkMemoryPropertyFlags requiredFlags;
1822  VkMemoryPropertyFlags preferredFlags;
1830  uint32_t memoryTypeBits;
1836  VmaPool pool;
1843  void* pUserData;
1844 } VmaAllocationCreateInfo;
1845 
1862 VkResult vmaFindMemoryTypeIndex(
1863  VmaAllocator allocator,
1864  uint32_t memoryTypeBits,
1865  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1866  uint32_t* pMemoryTypeIndex);
1867 
1880 VkResult vmaFindMemoryTypeIndexForBufferInfo(
1881  VmaAllocator allocator,
1882  const VkBufferCreateInfo* pBufferCreateInfo,
1883  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1884  uint32_t* pMemoryTypeIndex);
1885 
1898 VkResult vmaFindMemoryTypeIndexForImageInfo(
1899  VmaAllocator allocator,
1900  const VkImageCreateInfo* pImageCreateInfo,
1901  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1902  uint32_t* pMemoryTypeIndex);
1903 
1905 typedef enum VmaPoolCreateFlagBits {
1923  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1924 
1938  VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT = 0x00000004,
1939 
1940  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1941 } VmaPoolCreateFlagBits;
1942 typedef VkFlags VmaPoolCreateFlags;
1943 
1946 typedef struct VmaPoolCreateInfo {
1949  uint32_t memoryTypeIndex;
1952  VmaPoolCreateFlags flags;
1957  VkDeviceSize blockSize;
1962  size_t minBlockCount;
1971  size_t maxBlockCount;
1985  uint32_t frameInUseCount;
1986 } VmaPoolCreateInfo;
1987 
1990 typedef struct VmaPoolStats {
1993  VkDeviceSize size;
1996  VkDeviceSize unusedSize;
1999  size_t allocationCount;
2002  size_t unusedRangeCount;
2009  VkDeviceSize unusedRangeSizeMax;
2010 } VmaPoolStats;
2011 
2018 VkResult vmaCreatePool(
2019  VmaAllocator allocator,
2020  const VmaPoolCreateInfo* pCreateInfo,
2021  VmaPool* pPool);
2022 
2025 void vmaDestroyPool(
2026  VmaAllocator allocator,
2027  VmaPool pool);
2028 
2035 void vmaGetPoolStats(
2036  VmaAllocator allocator,
2037  VmaPool pool,
2038  VmaPoolStats* pPoolStats);
2039 
2046 void vmaMakePoolAllocationsLost(
2047  VmaAllocator allocator,
2048  VmaPool pool,
2049  size_t* pLostAllocationCount);
2050 
2065 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
2066 
2091 VK_DEFINE_HANDLE(VmaAllocation)
2092 
2093 
2095 typedef struct VmaAllocationInfo {
2100  uint32_t memoryType;
2109  VkDeviceMemory deviceMemory;
2114  VkDeviceSize offset;
2119  VkDeviceSize size;
2128  void* pMappedData;
2133  void* pUserData;
2134 } VmaAllocationInfo;
2135 
2146 VkResult vmaAllocateMemory(
2147  VmaAllocator allocator,
2148  const VkMemoryRequirements* pVkMemoryRequirements,
2149  const VmaAllocationCreateInfo* pCreateInfo,
2150  VmaAllocation* pAllocation,
2151  VmaAllocationInfo* pAllocationInfo);
2152 
2159 VkResult vmaAllocateMemoryForBuffer(
2160  VmaAllocator allocator,
2161  VkBuffer buffer,
2162  const VmaAllocationCreateInfo* pCreateInfo,
2163  VmaAllocation* pAllocation,
2164  VmaAllocationInfo* pAllocationInfo);
2165 
2167 VkResult vmaAllocateMemoryForImage(
2168  VmaAllocator allocator,
2169  VkImage image,
2170  const VmaAllocationCreateInfo* pCreateInfo,
2171  VmaAllocation* pAllocation,
2172  VmaAllocationInfo* pAllocationInfo);
2173 
2175 void vmaFreeMemory(
2176  VmaAllocator allocator,
2177  VmaAllocation allocation);
2178 
2195 void vmaGetAllocationInfo(
2196  VmaAllocator allocator,
2197  VmaAllocation allocation,
2198  VmaAllocationInfo* pAllocationInfo);
2199 
2214 VkBool32 vmaTouchAllocation(
2215  VmaAllocator allocator,
2216  VmaAllocation allocation);
2217 
2231 void vmaSetAllocationUserData(
2232  VmaAllocator allocator,
2233  VmaAllocation allocation,
2234  void* pUserData);
2235 
2246 void vmaCreateLostAllocation(
2247  VmaAllocator allocator,
2248  VmaAllocation* pAllocation);
2249 
2284 VkResult vmaMapMemory(
2285  VmaAllocator allocator,
2286  VmaAllocation allocation,
2287  void** ppData);
2288 
2293 void vmaUnmapMemory(
2294  VmaAllocator allocator,
2295  VmaAllocation allocation);
2296 
2309 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
2310 
2323 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
2324 
2341 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
2342 
2344 typedef struct VmaDefragmentationInfo {
2349  VkDeviceSize maxBytesToMove;
2354  uint32_t maxAllocationsToMove;
2355 } VmaDefragmentationInfo;
2356 
2358 typedef struct VmaDefragmentationStats {
2360  VkDeviceSize bytesMoved;
2362  VkDeviceSize bytesFreed;
2364  uint32_t allocationsMoved;
2366  uint32_t deviceMemoryBlocksFreed;
2367 } VmaDefragmentationStats;
2368 
2455 VkResult vmaDefragment(
2456  VmaAllocator allocator,
2457  VmaAllocation* pAllocations,
2458  size_t allocationCount,
2459  VkBool32* pAllocationsChanged,
2460  const VmaDefragmentationInfo *pDefragmentationInfo,
2461  VmaDefragmentationStats* pDefragmentationStats);
2462 
2475 VkResult vmaBindBufferMemory(
2476  VmaAllocator allocator,
2477  VmaAllocation allocation,
2478  VkBuffer buffer);
2479 
2492 VkResult vmaBindImageMemory(
2493  VmaAllocator allocator,
2494  VmaAllocation allocation,
2495  VkImage image);
2496 
2523 VkResult vmaCreateBuffer(
2524  VmaAllocator allocator,
2525  const VkBufferCreateInfo* pBufferCreateInfo,
2526  const VmaAllocationCreateInfo* pAllocationCreateInfo,
2527  VkBuffer* pBuffer,
2528  VmaAllocation* pAllocation,
2529  VmaAllocationInfo* pAllocationInfo);
2530 
2542 void vmaDestroyBuffer(
2543  VmaAllocator allocator,
2544  VkBuffer buffer,
2545  VmaAllocation allocation);
2546 
2548 VkResult vmaCreateImage(
2549  VmaAllocator allocator,
2550  const VkImageCreateInfo* pImageCreateInfo,
2551  const VmaAllocationCreateInfo* pAllocationCreateInfo,
2552  VkImage* pImage,
2553  VmaAllocation* pAllocation,
2554  VmaAllocationInfo* pAllocationInfo);
2555 
2567 void vmaDestroyImage(
2568  VmaAllocator allocator,
2569  VkImage image,
2570  VmaAllocation allocation);
2571 
2572 #ifdef __cplusplus
2573 }
2574 #endif
2575 
2576 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
2577 
2578 // For Visual Studio IntelliSense.
2579 #if defined(__cplusplus) && defined(__INTELLISENSE__)
2580 #define VMA_IMPLEMENTATION
2581 #endif
2582 
2583 #ifdef VMA_IMPLEMENTATION
2584 #undef VMA_IMPLEMENTATION
2585 
2586 #include <cstdint>
2587 #include <cstdlib>
2588 #include <cstring>
2589 
2590 /*******************************************************************************
2591 CONFIGURATION SECTION
2592 
2593 Define some of these macros before each #include of this header or change them
2594 here if you need other then default behavior depending on your environment.
2595 */
2596 
2597 /*
2598 Define this macro to 1 to make the library fetch pointers to Vulkan functions
2599 internally, like:
2600 
2601  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
2602 
2603 Define to 0 if you are going to provide you own pointers to Vulkan functions via
2604 VmaAllocatorCreateInfo::pVulkanFunctions.
2605 */
2606 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
2607 #define VMA_STATIC_VULKAN_FUNCTIONS 1
2608 #endif
2609 
2610 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
2611 //#define VMA_USE_STL_CONTAINERS 1
2612 
2613 /* Set this macro to 1 to make the library including and using STL containers:
2614 std::pair, std::vector, std::list, std::unordered_map.
2615 
2616 Set it to 0 or undefined to make the library using its own implementation of
2617 the containers.
2618 */
2619 #if VMA_USE_STL_CONTAINERS
2620  #define VMA_USE_STL_VECTOR 1
2621  #define VMA_USE_STL_UNORDERED_MAP 1
2622  #define VMA_USE_STL_LIST 1
2623 #endif
2624 
2625 #if VMA_USE_STL_VECTOR
2626  #include <vector>
2627 #endif
2628 
2629 #if VMA_USE_STL_UNORDERED_MAP
2630  #include <unordered_map>
2631 #endif
2632 
2633 #if VMA_USE_STL_LIST
2634  #include <list>
2635 #endif
2636 
2637 /*
2638 Following headers are used in this CONFIGURATION section only, so feel free to
2639 remove them if not needed.
2640 */
2641 #include <cassert> // for assert
2642 #include <algorithm> // for min, max
2643 #include <mutex> // for std::mutex
2644 #include <atomic> // for std::atomic
2645 
2646 #ifndef VMA_NULL
2647  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
2648  #define VMA_NULL nullptr
2649 #endif
2650 
2651 #if defined(__APPLE__) || defined(__ANDROID__)
2652 #include <cstdlib>
2653 void *aligned_alloc(size_t alignment, size_t size)
2654 {
2655  // alignment must be >= sizeof(void*)
2656  if(alignment < sizeof(void*))
2657  {
2658  alignment = sizeof(void*);
2659  }
2660 
2661  void *pointer;
2662  if(posix_memalign(&pointer, alignment, size) == 0)
2663  return pointer;
2664  return VMA_NULL;
2665 }
2666 #endif
2667 
2668 // If your compiler is not compatible with C++11 and definition of
2669 // aligned_alloc() function is missing, uncommeting following line may help:
2670 
2671 //#include <malloc.h>
2672 
2673 // Normal assert to check for programmer's errors, especially in Debug configuration.
2674 #ifndef VMA_ASSERT
2675  #ifdef _DEBUG
2676  #define VMA_ASSERT(expr) assert(expr)
2677  #else
2678  #define VMA_ASSERT(expr)
2679  #endif
2680 #endif
2681 
2682 // Assert that will be called very often, like inside data structures e.g. operator[].
2683 // Making it non-empty can make program slow.
2684 #ifndef VMA_HEAVY_ASSERT
2685  #ifdef _DEBUG
2686  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
2687  #else
2688  #define VMA_HEAVY_ASSERT(expr)
2689  #endif
2690 #endif
2691 
2692 #ifndef VMA_ALIGN_OF
2693  #define VMA_ALIGN_OF(type) (__alignof(type))
2694 #endif
2695 
2696 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
2697  #if defined(_WIN32)
2698  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
2699  #else
2700  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
2701  #endif
2702 #endif
2703 
2704 #ifndef VMA_SYSTEM_FREE
2705  #if defined(_WIN32)
2706  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
2707  #else
2708  #define VMA_SYSTEM_FREE(ptr) free(ptr)
2709  #endif
2710 #endif
2711 
2712 #ifndef VMA_MIN
2713  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
2714 #endif
2715 
2716 #ifndef VMA_MAX
2717  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
2718 #endif
2719 
2720 #ifndef VMA_SWAP
2721  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
2722 #endif
2723 
2724 #ifndef VMA_SORT
2725  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
2726 #endif
2727 
2728 #ifndef VMA_DEBUG_LOG
2729  #define VMA_DEBUG_LOG(format, ...)
2730  /*
2731  #define VMA_DEBUG_LOG(format, ...) do { \
2732  printf(format, __VA_ARGS__); \
2733  printf("\n"); \
2734  } while(false)
2735  */
2736 #endif
2737 
2738 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
2739 #if VMA_STATS_STRING_ENABLED
2740  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
2741  {
2742  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
2743  }
2744  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
2745  {
2746  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
2747  }
2748  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
2749  {
2750  snprintf(outStr, strLen, "%p", ptr);
2751  }
2752 #endif
2753 
2754 #ifndef VMA_MUTEX
2755  class VmaMutex
2756  {
2757  public:
2758  VmaMutex() { }
2759  ~VmaMutex() { }
2760  void Lock() { m_Mutex.lock(); }
2761  void Unlock() { m_Mutex.unlock(); }
2762  private:
2763  std::mutex m_Mutex;
2764  };
2765  #define VMA_MUTEX VmaMutex
2766 #endif
2767 
2768 /*
2769 If providing your own implementation, you need to implement a subset of std::atomic:
2770 
2771 - Constructor(uint32_t desired)
2772 - uint32_t load() const
2773 - void store(uint32_t desired)
2774 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
2775 */
2776 #ifndef VMA_ATOMIC_UINT32
2777  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
2778 #endif
2779 
2780 #ifndef VMA_BEST_FIT
2781 
2793  #define VMA_BEST_FIT (1)
2794 #endif
2795 
2796 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
2797 
2801  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
2802 #endif
2803 
2804 #ifndef VMA_DEBUG_ALIGNMENT
2805 
2809  #define VMA_DEBUG_ALIGNMENT (1)
2810 #endif
2811 
2812 #ifndef VMA_DEBUG_MARGIN
2813 
2817  #define VMA_DEBUG_MARGIN (0)
2818 #endif
2819 
2820 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
2821 
2825  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
2826 #endif
2827 
2828 #ifndef VMA_DEBUG_DETECT_CORRUPTION
2829 
2834  #define VMA_DEBUG_DETECT_CORRUPTION (0)
2835 #endif
2836 
2837 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2838 
2842  #define VMA_DEBUG_GLOBAL_MUTEX (0)
2843 #endif
2844 
2845 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2846 
2850  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2851 #endif
2852 
2853 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2854  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2856 #endif
2857 
2858 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2859  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2861 #endif
2862 
2863 #ifndef VMA_CLASS_NO_COPY
2864  #define VMA_CLASS_NO_COPY(className) \
2865  private: \
2866  className(const className&) = delete; \
2867  className& operator=(const className&) = delete;
2868 #endif
2869 
2870 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2871 
2872 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
2873 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
2874 
2875 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
2876 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
2877 
2878 /*******************************************************************************
2879 END OF CONFIGURATION
2880 */
2881 
2882 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2883  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2884 
2885 // Returns number of bits set to 1 in (v).
2886 static inline uint32_t VmaCountBitsSet(uint32_t v)
2887 {
2888  uint32_t c = v - ((v >> 1) & 0x55555555);
2889  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2890  c = ((c >> 4) + c) & 0x0F0F0F0F;
2891  c = ((c >> 8) + c) & 0x00FF00FF;
2892  c = ((c >> 16) + c) & 0x0000FFFF;
2893  return c;
2894 }
2895 
2896 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2897 // Use types like uint32_t, uint64_t as T.
2898 template <typename T>
2899 static inline T VmaAlignUp(T val, T align)
2900 {
2901  return (val + align - 1) / align * align;
2902 }
2903 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
2904 // Use types like uint32_t, uint64_t as T.
2905 template <typename T>
2906 static inline T VmaAlignDown(T val, T align)
2907 {
2908  return val / align * align;
2909 }
2910 
2911 // Division with mathematical rounding to nearest number.
2912 template <typename T>
2913 inline T VmaRoundDiv(T x, T y)
2914 {
2915  return (x + (y / (T)2)) / y;
2916 }
2917 
2918 static inline bool VmaStrIsEmpty(const char* pStr)
2919 {
2920  return pStr == VMA_NULL || *pStr == '\0';
2921 }
2922 
2923 #ifndef VMA_SORT
2924 
2925 template<typename Iterator, typename Compare>
2926 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2927 {
2928  Iterator centerValue = end; --centerValue;
2929  Iterator insertIndex = beg;
2930  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2931  {
2932  if(cmp(*memTypeIndex, *centerValue))
2933  {
2934  if(insertIndex != memTypeIndex)
2935  {
2936  VMA_SWAP(*memTypeIndex, *insertIndex);
2937  }
2938  ++insertIndex;
2939  }
2940  }
2941  if(insertIndex != centerValue)
2942  {
2943  VMA_SWAP(*insertIndex, *centerValue);
2944  }
2945  return insertIndex;
2946 }
2947 
2948 template<typename Iterator, typename Compare>
2949 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2950 {
2951  if(beg < end)
2952  {
2953  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2954  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2955  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2956  }
2957 }
2958 
2959 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2960 
2961 #endif // #ifndef VMA_SORT
2962 
2963 /*
2964 Returns true if two memory blocks occupy overlapping pages.
2965 ResourceA must be in less memory offset than ResourceB.
2966 
2967 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2968 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2969 */
2970 static inline bool VmaBlocksOnSamePage(
2971  VkDeviceSize resourceAOffset,
2972  VkDeviceSize resourceASize,
2973  VkDeviceSize resourceBOffset,
2974  VkDeviceSize pageSize)
2975 {
2976  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2977  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2978  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2979  VkDeviceSize resourceBStart = resourceBOffset;
2980  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2981  return resourceAEndPage == resourceBStartPage;
2982 }
2983 
2984 enum VmaSuballocationType
2985 {
2986  VMA_SUBALLOCATION_TYPE_FREE = 0,
2987  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2988  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2989  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2990  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2991  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2992  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2993 };
2994 
2995 /*
2996 Returns true if given suballocation types could conflict and must respect
2997 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2998 or linear image and another one is optimal image. If type is unknown, behave
2999 conservatively.
3000 */
3001 static inline bool VmaIsBufferImageGranularityConflict(
3002  VmaSuballocationType suballocType1,
3003  VmaSuballocationType suballocType2)
3004 {
3005  if(suballocType1 > suballocType2)
3006  {
3007  VMA_SWAP(suballocType1, suballocType2);
3008  }
3009 
3010  switch(suballocType1)
3011  {
3012  case VMA_SUBALLOCATION_TYPE_FREE:
3013  return false;
3014  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
3015  return true;
3016  case VMA_SUBALLOCATION_TYPE_BUFFER:
3017  return
3018  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
3019  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
3020  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
3021  return
3022  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
3023  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
3024  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
3025  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
3026  return
3027  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
3028  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
3029  return false;
3030  default:
3031  VMA_ASSERT(0);
3032  return true;
3033  }
3034 }
3035 
3036 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
3037 {
3038  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
3039  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
3040  for(size_t i = 0; i < numberCount; ++i, ++pDst)
3041  {
3042  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
3043  }
3044 }
3045 
3046 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
3047 {
3048  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
3049  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
3050  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
3051  {
3052  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
3053  {
3054  return false;
3055  }
3056  }
3057  return true;
3058 }
3059 
3060 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
3061 struct VmaMutexLock
3062 {
3063  VMA_CLASS_NO_COPY(VmaMutexLock)
3064 public:
3065  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
3066  m_pMutex(useMutex ? &mutex : VMA_NULL)
3067  {
3068  if(m_pMutex)
3069  {
3070  m_pMutex->Lock();
3071  }
3072  }
3073 
3074  ~VmaMutexLock()
3075  {
3076  if(m_pMutex)
3077  {
3078  m_pMutex->Unlock();
3079  }
3080  }
3081 
3082 private:
3083  VMA_MUTEX* m_pMutex;
3084 };
3085 
3086 #if VMA_DEBUG_GLOBAL_MUTEX
3087  static VMA_MUTEX gDebugGlobalMutex;
3088  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
3089 #else
3090  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
3091 #endif
3092 
3093 // Minimum size of a free suballocation to register it in the free suballocation collection.
3094 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
3095 
3096 /*
3097 Performs binary search and returns iterator to first element that is greater or
3098 equal to (key), according to comparison (cmp).
3099 
3100 Cmp should return true if first argument is less than second argument.
3101 
3102 Returned value is the found element, if present in the collection or place where
3103 new element with value (key) should be inserted.
3104 */
3105 template <typename CmpLess, typename IterT, typename KeyT>
3106 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpLess cmp)
3107 {
3108  size_t down = 0, up = (end - beg);
3109  while(down < up)
3110  {
3111  const size_t mid = (down + up) / 2;
3112  if(cmp(*(beg+mid), key))
3113  {
3114  down = mid + 1;
3115  }
3116  else
3117  {
3118  up = mid;
3119  }
3120  }
3121  return beg + down;
3122 }
3123 
3125 // Memory allocation
3126 
3127 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
3128 {
3129  if((pAllocationCallbacks != VMA_NULL) &&
3130  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
3131  {
3132  return (*pAllocationCallbacks->pfnAllocation)(
3133  pAllocationCallbacks->pUserData,
3134  size,
3135  alignment,
3136  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
3137  }
3138  else
3139  {
3140  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
3141  }
3142 }
3143 
3144 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
3145 {
3146  if((pAllocationCallbacks != VMA_NULL) &&
3147  (pAllocationCallbacks->pfnFree != VMA_NULL))
3148  {
3149  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
3150  }
3151  else
3152  {
3153  VMA_SYSTEM_FREE(ptr);
3154  }
3155 }
3156 
3157 template<typename T>
3158 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
3159 {
3160  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
3161 }
3162 
3163 template<typename T>
3164 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
3165 {
3166  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
3167 }
3168 
3169 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
3170 
3171 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
3172 
3173 template<typename T>
3174 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
3175 {
3176  ptr->~T();
3177  VmaFree(pAllocationCallbacks, ptr);
3178 }
3179 
3180 template<typename T>
3181 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
3182 {
3183  if(ptr != VMA_NULL)
3184  {
3185  for(size_t i = count; i--; )
3186  {
3187  ptr[i].~T();
3188  }
3189  VmaFree(pAllocationCallbacks, ptr);
3190  }
3191 }
3192 
3193 // STL-compatible allocator.
3194 template<typename T>
3195 class VmaStlAllocator
3196 {
3197 public:
3198  const VkAllocationCallbacks* const m_pCallbacks;
3199  typedef T value_type;
3200 
3201  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
3202  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
3203 
3204  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
3205  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
3206 
3207  template<typename U>
3208  bool operator==(const VmaStlAllocator<U>& rhs) const
3209  {
3210  return m_pCallbacks == rhs.m_pCallbacks;
3211  }
3212  template<typename U>
3213  bool operator!=(const VmaStlAllocator<U>& rhs) const
3214  {
3215  return m_pCallbacks != rhs.m_pCallbacks;
3216  }
3217 
3218  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
3219 };
3220 
3221 #if VMA_USE_STL_VECTOR
3222 
3223 #define VmaVector std::vector
3224 
3225 template<typename T, typename allocatorT>
3226 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
3227 {
3228  vec.insert(vec.begin() + index, item);
3229 }
3230 
3231 template<typename T, typename allocatorT>
3232 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
3233 {
3234  vec.erase(vec.begin() + index);
3235 }
3236 
3237 #else // #if VMA_USE_STL_VECTOR
3238 
3239 /* Class with interface compatible with subset of std::vector.
3240 T must be POD because constructors and destructors are not called and memcpy is
3241 used for these objects. */
3242 template<typename T, typename AllocatorT>
3243 class VmaVector
3244 {
3245 public:
3246  typedef T value_type;
3247 
3248  VmaVector(const AllocatorT& allocator) :
3249  m_Allocator(allocator),
3250  m_pArray(VMA_NULL),
3251  m_Count(0),
3252  m_Capacity(0)
3253  {
3254  }
3255 
3256  VmaVector(size_t count, const AllocatorT& allocator) :
3257  m_Allocator(allocator),
3258  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
3259  m_Count(count),
3260  m_Capacity(count)
3261  {
3262  }
3263 
3264  VmaVector(const VmaVector<T, AllocatorT>& src) :
3265  m_Allocator(src.m_Allocator),
3266  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
3267  m_Count(src.m_Count),
3268  m_Capacity(src.m_Count)
3269  {
3270  if(m_Count != 0)
3271  {
3272  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
3273  }
3274  }
3275 
3276  ~VmaVector()
3277  {
3278  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3279  }
3280 
3281  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
3282  {
3283  if(&rhs != this)
3284  {
3285  resize(rhs.m_Count);
3286  if(m_Count != 0)
3287  {
3288  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
3289  }
3290  }
3291  return *this;
3292  }
3293 
3294  bool empty() const { return m_Count == 0; }
3295  size_t size() const { return m_Count; }
3296  T* data() { return m_pArray; }
3297  const T* data() const { return m_pArray; }
3298 
3299  T& operator[](size_t index)
3300  {
3301  VMA_HEAVY_ASSERT(index < m_Count);
3302  return m_pArray[index];
3303  }
3304  const T& operator[](size_t index) const
3305  {
3306  VMA_HEAVY_ASSERT(index < m_Count);
3307  return m_pArray[index];
3308  }
3309 
3310  T& front()
3311  {
3312  VMA_HEAVY_ASSERT(m_Count > 0);
3313  return m_pArray[0];
3314  }
3315  const T& front() const
3316  {
3317  VMA_HEAVY_ASSERT(m_Count > 0);
3318  return m_pArray[0];
3319  }
3320  T& back()
3321  {
3322  VMA_HEAVY_ASSERT(m_Count > 0);
3323  return m_pArray[m_Count - 1];
3324  }
3325  const T& back() const
3326  {
3327  VMA_HEAVY_ASSERT(m_Count > 0);
3328  return m_pArray[m_Count - 1];
3329  }
3330 
3331  void reserve(size_t newCapacity, bool freeMemory = false)
3332  {
3333  newCapacity = VMA_MAX(newCapacity, m_Count);
3334 
3335  if((newCapacity < m_Capacity) && !freeMemory)
3336  {
3337  newCapacity = m_Capacity;
3338  }
3339 
3340  if(newCapacity != m_Capacity)
3341  {
3342  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
3343  if(m_Count != 0)
3344  {
3345  memcpy(newArray, m_pArray, m_Count * sizeof(T));
3346  }
3347  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3348  m_Capacity = newCapacity;
3349  m_pArray = newArray;
3350  }
3351  }
3352 
3353  void resize(size_t newCount, bool freeMemory = false)
3354  {
3355  size_t newCapacity = m_Capacity;
3356  if(newCount > m_Capacity)
3357  {
3358  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
3359  }
3360  else if(freeMemory)
3361  {
3362  newCapacity = newCount;
3363  }
3364 
3365  if(newCapacity != m_Capacity)
3366  {
3367  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
3368  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
3369  if(elementsToCopy != 0)
3370  {
3371  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
3372  }
3373  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3374  m_Capacity = newCapacity;
3375  m_pArray = newArray;
3376  }
3377 
3378  m_Count = newCount;
3379  }
3380 
3381  void clear(bool freeMemory = false)
3382  {
3383  resize(0, freeMemory);
3384  }
3385 
3386  void insert(size_t index, const T& src)
3387  {
3388  VMA_HEAVY_ASSERT(index <= m_Count);
3389  const size_t oldCount = size();
3390  resize(oldCount + 1);
3391  if(index < oldCount)
3392  {
3393  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
3394  }
3395  m_pArray[index] = src;
3396  }
3397 
3398  void remove(size_t index)
3399  {
3400  VMA_HEAVY_ASSERT(index < m_Count);
3401  const size_t oldCount = size();
3402  if(index < oldCount - 1)
3403  {
3404  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
3405  }
3406  resize(oldCount - 1);
3407  }
3408 
3409  void push_back(const T& src)
3410  {
3411  const size_t newIndex = size();
3412  resize(newIndex + 1);
3413  m_pArray[newIndex] = src;
3414  }
3415 
3416  void pop_back()
3417  {
3418  VMA_HEAVY_ASSERT(m_Count > 0);
3419  resize(size() - 1);
3420  }
3421 
3422  void push_front(const T& src)
3423  {
3424  insert(0, src);
3425  }
3426 
3427  void pop_front()
3428  {
3429  VMA_HEAVY_ASSERT(m_Count > 0);
3430  remove(0);
3431  }
3432 
3433  typedef T* iterator;
3434 
3435  iterator begin() { return m_pArray; }
3436  iterator end() { return m_pArray + m_Count; }
3437 
3438 private:
3439  AllocatorT m_Allocator;
3440  T* m_pArray;
3441  size_t m_Count;
3442  size_t m_Capacity;
3443 };
3444 
3445 template<typename T, typename allocatorT>
3446 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
3447 {
3448  vec.insert(index, item);
3449 }
3450 
3451 template<typename T, typename allocatorT>
3452 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
3453 {
3454  vec.remove(index);
3455 }
3456 
3457 #endif // #if VMA_USE_STL_VECTOR
3458 
3459 template<typename CmpLess, typename VectorT>
3460 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
3461 {
3462  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3463  vector.data(),
3464  vector.data() + vector.size(),
3465  value,
3466  CmpLess()) - vector.data();
3467  VmaVectorInsert(vector, indexToInsert, value);
3468  return indexToInsert;
3469 }
3470 
3471 template<typename CmpLess, typename VectorT>
3472 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
3473 {
3474  CmpLess comparator;
3475  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
3476  vector.begin(),
3477  vector.end(),
3478  value,
3479  comparator);
3480  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
3481  {
3482  size_t indexToRemove = it - vector.begin();
3483  VmaVectorRemove(vector, indexToRemove);
3484  return true;
3485  }
3486  return false;
3487 }
3488 
3489 template<typename CmpLess, typename IterT, typename KeyT>
3490 IterT VmaVectorFindSorted(const IterT& beg, const IterT& end, const KeyT& value)
3491 {
3492  CmpLess comparator;
3493  typename IterT it = VmaBinaryFindFirstNotLess<CmpLess, IterT, KeyT>(
3494  beg, end, value, comparator);
3495  if(it == end ||
3496  !comparator(*it, value) && !comparator(value, *it))
3497  {
3498  return it;
3499  }
3500  return end;
3501 }
3502 
3504 // class VmaPoolAllocator
3505 
3506 /*
3507 Allocator for objects of type T using a list of arrays (pools) to speed up
3508 allocation. Number of elements that can be allocated is not bounded because
3509 allocator can create multiple blocks.
3510 */
3511 template<typename T>
3512 class VmaPoolAllocator
3513 {
3514  VMA_CLASS_NO_COPY(VmaPoolAllocator)
3515 public:
3516  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
3517  ~VmaPoolAllocator();
3518  void Clear();
3519  T* Alloc();
3520  void Free(T* ptr);
3521 
3522 private:
3523  union Item
3524  {
3525  uint32_t NextFreeIndex;
3526  T Value;
3527  };
3528 
3529  struct ItemBlock
3530  {
3531  Item* pItems;
3532  uint32_t FirstFreeIndex;
3533  };
3534 
3535  const VkAllocationCallbacks* m_pAllocationCallbacks;
3536  size_t m_ItemsPerBlock;
3537  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
3538 
3539  ItemBlock& CreateNewBlock();
3540 };
3541 
3542 template<typename T>
3543 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
3544  m_pAllocationCallbacks(pAllocationCallbacks),
3545  m_ItemsPerBlock(itemsPerBlock),
3546  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
3547 {
3548  VMA_ASSERT(itemsPerBlock > 0);
3549 }
3550 
3551 template<typename T>
3552 VmaPoolAllocator<T>::~VmaPoolAllocator()
3553 {
3554  Clear();
3555 }
3556 
3557 template<typename T>
3558 void VmaPoolAllocator<T>::Clear()
3559 {
3560  for(size_t i = m_ItemBlocks.size(); i--; )
3561  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
3562  m_ItemBlocks.clear();
3563 }
3564 
3565 template<typename T>
3566 T* VmaPoolAllocator<T>::Alloc()
3567 {
3568  for(size_t i = m_ItemBlocks.size(); i--; )
3569  {
3570  ItemBlock& block = m_ItemBlocks[i];
3571  // This block has some free items: Use first one.
3572  if(block.FirstFreeIndex != UINT32_MAX)
3573  {
3574  Item* const pItem = &block.pItems[block.FirstFreeIndex];
3575  block.FirstFreeIndex = pItem->NextFreeIndex;
3576  return &pItem->Value;
3577  }
3578  }
3579 
3580  // No block has free item: Create new one and use it.
3581  ItemBlock& newBlock = CreateNewBlock();
3582  Item* const pItem = &newBlock.pItems[0];
3583  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
3584  return &pItem->Value;
3585 }
3586 
3587 template<typename T>
3588 void VmaPoolAllocator<T>::Free(T* ptr)
3589 {
3590  // Search all memory blocks to find ptr.
3591  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
3592  {
3593  ItemBlock& block = m_ItemBlocks[i];
3594 
3595  // Casting to union.
3596  Item* pItemPtr;
3597  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
3598 
3599  // Check if pItemPtr is in address range of this block.
3600  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
3601  {
3602  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
3603  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
3604  block.FirstFreeIndex = index;
3605  return;
3606  }
3607  }
3608  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
3609 }
3610 
3611 template<typename T>
3612 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
3613 {
3614  ItemBlock newBlock = {
3615  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
3616 
3617  m_ItemBlocks.push_back(newBlock);
3618 
3619  // Setup singly-linked list of all free items in this block.
3620  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
3621  newBlock.pItems[i].NextFreeIndex = i + 1;
3622  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
3623  return m_ItemBlocks.back();
3624 }
3625 
3627 // class VmaRawList, VmaList
3628 
3629 #if VMA_USE_STL_LIST
3630 
3631 #define VmaList std::list
3632 
3633 #else // #if VMA_USE_STL_LIST
3634 
3635 template<typename T>
3636 struct VmaListItem
3637 {
3638  VmaListItem* pPrev;
3639  VmaListItem* pNext;
3640  T Value;
3641 };
3642 
3643 // Doubly linked list.
3644 template<typename T>
3645 class VmaRawList
3646 {
3647  VMA_CLASS_NO_COPY(VmaRawList)
3648 public:
3649  typedef VmaListItem<T> ItemType;
3650 
3651  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
3652  ~VmaRawList();
3653  void Clear();
3654 
3655  size_t GetCount() const { return m_Count; }
3656  bool IsEmpty() const { return m_Count == 0; }
3657 
3658  ItemType* Front() { return m_pFront; }
3659  const ItemType* Front() const { return m_pFront; }
3660  ItemType* Back() { return m_pBack; }
3661  const ItemType* Back() const { return m_pBack; }
3662 
3663  ItemType* PushBack();
3664  ItemType* PushFront();
3665  ItemType* PushBack(const T& value);
3666  ItemType* PushFront(const T& value);
3667  void PopBack();
3668  void PopFront();
3669 
3670  // Item can be null - it means PushBack.
3671  ItemType* InsertBefore(ItemType* pItem);
3672  // Item can be null - it means PushFront.
3673  ItemType* InsertAfter(ItemType* pItem);
3674 
3675  ItemType* InsertBefore(ItemType* pItem, const T& value);
3676  ItemType* InsertAfter(ItemType* pItem, const T& value);
3677 
3678  void Remove(ItemType* pItem);
3679 
3680 private:
3681  const VkAllocationCallbacks* const m_pAllocationCallbacks;
3682  VmaPoolAllocator<ItemType> m_ItemAllocator;
3683  ItemType* m_pFront;
3684  ItemType* m_pBack;
3685  size_t m_Count;
3686 };
3687 
3688 template<typename T>
3689 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
3690  m_pAllocationCallbacks(pAllocationCallbacks),
3691  m_ItemAllocator(pAllocationCallbacks, 128),
3692  m_pFront(VMA_NULL),
3693  m_pBack(VMA_NULL),
3694  m_Count(0)
3695 {
3696 }
3697 
3698 template<typename T>
3699 VmaRawList<T>::~VmaRawList()
3700 {
3701  // Intentionally not calling Clear, because that would be unnecessary
3702  // computations to return all items to m_ItemAllocator as free.
3703 }
3704 
3705 template<typename T>
3706 void VmaRawList<T>::Clear()
3707 {
3708  if(IsEmpty() == false)
3709  {
3710  ItemType* pItem = m_pBack;
3711  while(pItem != VMA_NULL)
3712  {
3713  ItemType* const pPrevItem = pItem->pPrev;
3714  m_ItemAllocator.Free(pItem);
3715  pItem = pPrevItem;
3716  }
3717  m_pFront = VMA_NULL;
3718  m_pBack = VMA_NULL;
3719  m_Count = 0;
3720  }
3721 }
3722 
3723 template<typename T>
3724 VmaListItem<T>* VmaRawList<T>::PushBack()
3725 {
3726  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3727  pNewItem->pNext = VMA_NULL;
3728  if(IsEmpty())
3729  {
3730  pNewItem->pPrev = VMA_NULL;
3731  m_pFront = pNewItem;
3732  m_pBack = pNewItem;
3733  m_Count = 1;
3734  }
3735  else
3736  {
3737  pNewItem->pPrev = m_pBack;
3738  m_pBack->pNext = pNewItem;
3739  m_pBack = pNewItem;
3740  ++m_Count;
3741  }
3742  return pNewItem;
3743 }
3744 
3745 template<typename T>
3746 VmaListItem<T>* VmaRawList<T>::PushFront()
3747 {
3748  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3749  pNewItem->pPrev = VMA_NULL;
3750  if(IsEmpty())
3751  {
3752  pNewItem->pNext = VMA_NULL;
3753  m_pFront = pNewItem;
3754  m_pBack = pNewItem;
3755  m_Count = 1;
3756  }
3757  else
3758  {
3759  pNewItem->pNext = m_pFront;
3760  m_pFront->pPrev = pNewItem;
3761  m_pFront = pNewItem;
3762  ++m_Count;
3763  }
3764  return pNewItem;
3765 }
3766 
3767 template<typename T>
3768 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
3769 {
3770  ItemType* const pNewItem = PushBack();
3771  pNewItem->Value = value;
3772  return pNewItem;
3773 }
3774 
3775 template<typename T>
3776 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
3777 {
3778  ItemType* const pNewItem = PushFront();
3779  pNewItem->Value = value;
3780  return pNewItem;
3781 }
3782 
3783 template<typename T>
3784 void VmaRawList<T>::PopBack()
3785 {
3786  VMA_HEAVY_ASSERT(m_Count > 0);
3787  ItemType* const pBackItem = m_pBack;
3788  ItemType* const pPrevItem = pBackItem->pPrev;
3789  if(pPrevItem != VMA_NULL)
3790  {
3791  pPrevItem->pNext = VMA_NULL;
3792  }
3793  m_pBack = pPrevItem;
3794  m_ItemAllocator.Free(pBackItem);
3795  --m_Count;
3796 }
3797 
3798 template<typename T>
3799 void VmaRawList<T>::PopFront()
3800 {
3801  VMA_HEAVY_ASSERT(m_Count > 0);
3802  ItemType* const pFrontItem = m_pFront;
3803  ItemType* const pNextItem = pFrontItem->pNext;
3804  if(pNextItem != VMA_NULL)
3805  {
3806  pNextItem->pPrev = VMA_NULL;
3807  }
3808  m_pFront = pNextItem;
3809  m_ItemAllocator.Free(pFrontItem);
3810  --m_Count;
3811 }
3812 
3813 template<typename T>
3814 void VmaRawList<T>::Remove(ItemType* pItem)
3815 {
3816  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
3817  VMA_HEAVY_ASSERT(m_Count > 0);
3818 
3819  if(pItem->pPrev != VMA_NULL)
3820  {
3821  pItem->pPrev->pNext = pItem->pNext;
3822  }
3823  else
3824  {
3825  VMA_HEAVY_ASSERT(m_pFront == pItem);
3826  m_pFront = pItem->pNext;
3827  }
3828 
3829  if(pItem->pNext != VMA_NULL)
3830  {
3831  pItem->pNext->pPrev = pItem->pPrev;
3832  }
3833  else
3834  {
3835  VMA_HEAVY_ASSERT(m_pBack == pItem);
3836  m_pBack = pItem->pPrev;
3837  }
3838 
3839  m_ItemAllocator.Free(pItem);
3840  --m_Count;
3841 }
3842 
3843 template<typename T>
3844 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
3845 {
3846  if(pItem != VMA_NULL)
3847  {
3848  ItemType* const prevItem = pItem->pPrev;
3849  ItemType* const newItem = m_ItemAllocator.Alloc();
3850  newItem->pPrev = prevItem;
3851  newItem->pNext = pItem;
3852  pItem->pPrev = newItem;
3853  if(prevItem != VMA_NULL)
3854  {
3855  prevItem->pNext = newItem;
3856  }
3857  else
3858  {
3859  VMA_HEAVY_ASSERT(m_pFront == pItem);
3860  m_pFront = newItem;
3861  }
3862  ++m_Count;
3863  return newItem;
3864  }
3865  else
3866  return PushBack();
3867 }
3868 
3869 template<typename T>
3870 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
3871 {
3872  if(pItem != VMA_NULL)
3873  {
3874  ItemType* const nextItem = pItem->pNext;
3875  ItemType* const newItem = m_ItemAllocator.Alloc();
3876  newItem->pNext = nextItem;
3877  newItem->pPrev = pItem;
3878  pItem->pNext = newItem;
3879  if(nextItem != VMA_NULL)
3880  {
3881  nextItem->pPrev = newItem;
3882  }
3883  else
3884  {
3885  VMA_HEAVY_ASSERT(m_pBack == pItem);
3886  m_pBack = newItem;
3887  }
3888  ++m_Count;
3889  return newItem;
3890  }
3891  else
3892  return PushFront();
3893 }
3894 
3895 template<typename T>
3896 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3897 {
3898  ItemType* const newItem = InsertBefore(pItem);
3899  newItem->Value = value;
3900  return newItem;
3901 }
3902 
3903 template<typename T>
3904 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3905 {
3906  ItemType* const newItem = InsertAfter(pItem);
3907  newItem->Value = value;
3908  return newItem;
3909 }
3910 
3911 template<typename T, typename AllocatorT>
3912 class VmaList
3913 {
3914  VMA_CLASS_NO_COPY(VmaList)
3915 public:
3916  class iterator
3917  {
3918  public:
3919  iterator() :
3920  m_pList(VMA_NULL),
3921  m_pItem(VMA_NULL)
3922  {
3923  }
3924 
3925  T& operator*() const
3926  {
3927  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3928  return m_pItem->Value;
3929  }
3930  T* operator->() const
3931  {
3932  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3933  return &m_pItem->Value;
3934  }
3935 
3936  iterator& operator++()
3937  {
3938  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3939  m_pItem = m_pItem->pNext;
3940  return *this;
3941  }
3942  iterator& operator--()
3943  {
3944  if(m_pItem != VMA_NULL)
3945  {
3946  m_pItem = m_pItem->pPrev;
3947  }
3948  else
3949  {
3950  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3951  m_pItem = m_pList->Back();
3952  }
3953  return *this;
3954  }
3955 
3956  iterator operator++(int)
3957  {
3958  iterator result = *this;
3959  ++*this;
3960  return result;
3961  }
3962  iterator operator--(int)
3963  {
3964  iterator result = *this;
3965  --*this;
3966  return result;
3967  }
3968 
3969  bool operator==(const iterator& rhs) const
3970  {
3971  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3972  return m_pItem == rhs.m_pItem;
3973  }
3974  bool operator!=(const iterator& rhs) const
3975  {
3976  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3977  return m_pItem != rhs.m_pItem;
3978  }
3979 
3980  private:
3981  VmaRawList<T>* m_pList;
3982  VmaListItem<T>* m_pItem;
3983 
3984  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3985  m_pList(pList),
3986  m_pItem(pItem)
3987  {
3988  }
3989 
3990  friend class VmaList<T, AllocatorT>;
3991  };
3992 
3993  class const_iterator
3994  {
3995  public:
3996  const_iterator() :
3997  m_pList(VMA_NULL),
3998  m_pItem(VMA_NULL)
3999  {
4000  }
4001 
4002  const_iterator(const iterator& src) :
4003  m_pList(src.m_pList),
4004  m_pItem(src.m_pItem)
4005  {
4006  }
4007 
4008  const T& operator*() const
4009  {
4010  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
4011  return m_pItem->Value;
4012  }
4013  const T* operator->() const
4014  {
4015  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
4016  return &m_pItem->Value;
4017  }
4018 
4019  const_iterator& operator++()
4020  {
4021  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
4022  m_pItem = m_pItem->pNext;
4023  return *this;
4024  }
4025  const_iterator& operator--()
4026  {
4027  if(m_pItem != VMA_NULL)
4028  {
4029  m_pItem = m_pItem->pPrev;
4030  }
4031  else
4032  {
4033  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
4034  m_pItem = m_pList->Back();
4035  }
4036  return *this;
4037  }
4038 
4039  const_iterator operator++(int)
4040  {
4041  const_iterator result = *this;
4042  ++*this;
4043  return result;
4044  }
4045  const_iterator operator--(int)
4046  {
4047  const_iterator result = *this;
4048  --*this;
4049  return result;
4050  }
4051 
4052  bool operator==(const const_iterator& rhs) const
4053  {
4054  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
4055  return m_pItem == rhs.m_pItem;
4056  }
4057  bool operator!=(const const_iterator& rhs) const
4058  {
4059  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
4060  return m_pItem != rhs.m_pItem;
4061  }
4062 
4063  private:
4064  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
4065  m_pList(pList),
4066  m_pItem(pItem)
4067  {
4068  }
4069 
4070  const VmaRawList<T>* m_pList;
4071  const VmaListItem<T>* m_pItem;
4072 
4073  friend class VmaList<T, AllocatorT>;
4074  };
4075 
4076  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
4077 
4078  bool empty() const { return m_RawList.IsEmpty(); }
4079  size_t size() const { return m_RawList.GetCount(); }
4080 
4081  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
4082  iterator end() { return iterator(&m_RawList, VMA_NULL); }
4083 
4084  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
4085  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
4086 
4087  void clear() { m_RawList.Clear(); }
4088  void push_back(const T& value) { m_RawList.PushBack(value); }
4089  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
4090  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
4091 
4092 private:
4093  VmaRawList<T> m_RawList;
4094 };
4095 
4096 #endif // #if VMA_USE_STL_LIST
4097 
4099 // class VmaMap
4100 
4101 // Unused in this version.
4102 #if 0
4103 
4104 #if VMA_USE_STL_UNORDERED_MAP
4105 
4106 #define VmaPair std::pair
4107 
4108 #define VMA_MAP_TYPE(KeyT, ValueT) \
4109  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
4110 
4111 #else // #if VMA_USE_STL_UNORDERED_MAP
4112 
4113 template<typename T1, typename T2>
4114 struct VmaPair
4115 {
4116  T1 first;
4117  T2 second;
4118 
4119  VmaPair() : first(), second() { }
4120  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
4121 };
4122 
4123 /* Class compatible with subset of interface of std::unordered_map.
4124 KeyT, ValueT must be POD because they will be stored in VmaVector.
4125 */
4126 template<typename KeyT, typename ValueT>
4127 class VmaMap
4128 {
4129 public:
4130  typedef VmaPair<KeyT, ValueT> PairType;
4131  typedef PairType* iterator;
4132 
4133  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
4134 
4135  iterator begin() { return m_Vector.begin(); }
4136  iterator end() { return m_Vector.end(); }
4137 
4138  void insert(const PairType& pair);
4139  iterator find(const KeyT& key);
4140  void erase(iterator it);
4141 
4142 private:
4143  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
4144 };
4145 
4146 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
4147 
4148 template<typename FirstT, typename SecondT>
4149 struct VmaPairFirstLess
4150 {
4151  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
4152  {
4153  return lhs.first < rhs.first;
4154  }
4155  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
4156  {
4157  return lhs.first < rhsFirst;
4158  }
4159 };
4160 
4161 template<typename KeyT, typename ValueT>
4162 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
4163 {
4164  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
4165  m_Vector.data(),
4166  m_Vector.data() + m_Vector.size(),
4167  pair,
4168  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
4169  VmaVectorInsert(m_Vector, indexToInsert, pair);
4170 }
4171 
4172 template<typename KeyT, typename ValueT>
4173 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
4174 {
4175  PairType* it = VmaBinaryFindFirstNotLess(
4176  m_Vector.data(),
4177  m_Vector.data() + m_Vector.size(),
4178  key,
4179  VmaPairFirstLess<KeyT, ValueT>());
4180  if((it != m_Vector.end()) && (it->first == key))
4181  {
4182  return it;
4183  }
4184  else
4185  {
4186  return m_Vector.end();
4187  }
4188 }
4189 
4190 template<typename KeyT, typename ValueT>
4191 void VmaMap<KeyT, ValueT>::erase(iterator it)
4192 {
4193  VmaVectorRemove(m_Vector, it - m_Vector.begin());
4194 }
4195 
4196 #endif // #if VMA_USE_STL_UNORDERED_MAP
4197 
4198 #endif // #if 0
4199 
4201 
4202 class VmaDeviceMemoryBlock;
4203 
4204 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
4205 
4206 struct VmaAllocation_T
4207 {
4208  VMA_CLASS_NO_COPY(VmaAllocation_T)
4209 private:
4210  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
4211 
4212  enum FLAGS
4213  {
4214  FLAG_USER_DATA_STRING = 0x01,
4215  };
4216 
4217 public:
4218  enum ALLOCATION_TYPE
4219  {
4220  ALLOCATION_TYPE_NONE,
4221  ALLOCATION_TYPE_BLOCK,
4222  ALLOCATION_TYPE_DEDICATED,
4223  };
4224 
4225  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
4226  m_Alignment(1),
4227  m_Size(0),
4228  m_pUserData(VMA_NULL),
4229  m_LastUseFrameIndex(currentFrameIndex),
4230  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
4231  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
4232  m_MapCount(0),
4233  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
4234  {
4235 #if VMA_STATS_STRING_ENABLED
4236  m_CreationFrameIndex = currentFrameIndex;
4237  m_BufferImageUsage = 0;
4238 #endif
4239  }
4240 
4241  ~VmaAllocation_T()
4242  {
4243  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
4244 
4245  // Check if owned string was freed.
4246  VMA_ASSERT(m_pUserData == VMA_NULL);
4247  }
4248 
4249  void InitBlockAllocation(
4250  VmaPool hPool,
4251  VmaDeviceMemoryBlock* block,
4252  VkDeviceSize offset,
4253  VkDeviceSize alignment,
4254  VkDeviceSize size,
4255  VmaSuballocationType suballocationType,
4256  bool mapped,
4257  bool canBecomeLost)
4258  {
4259  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4260  VMA_ASSERT(block != VMA_NULL);
4261  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
4262  m_Alignment = alignment;
4263  m_Size = size;
4264  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
4265  m_SuballocationType = (uint8_t)suballocationType;
4266  m_BlockAllocation.m_hPool = hPool;
4267  m_BlockAllocation.m_Block = block;
4268  m_BlockAllocation.m_Offset = offset;
4269  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
4270  }
4271 
4272  void InitLost()
4273  {
4274  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4275  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
4276  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
4277  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
4278  m_BlockAllocation.m_Block = VMA_NULL;
4279  m_BlockAllocation.m_Offset = 0;
4280  m_BlockAllocation.m_CanBecomeLost = true;
4281  }
4282 
4283  void ChangeBlockAllocation(
4284  VmaAllocator hAllocator,
4285  VmaDeviceMemoryBlock* block,
4286  VkDeviceSize offset);
4287 
4288  // pMappedData not null means allocation is created with MAPPED flag.
4289  void InitDedicatedAllocation(
4290  uint32_t memoryTypeIndex,
4291  VkDeviceMemory hMemory,
4292  VmaSuballocationType suballocationType,
4293  void* pMappedData,
4294  VkDeviceSize size)
4295  {
4296  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4297  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
4298  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
4299  m_Alignment = 0;
4300  m_Size = size;
4301  m_SuballocationType = (uint8_t)suballocationType;
4302  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
4303  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
4304  m_DedicatedAllocation.m_hMemory = hMemory;
4305  m_DedicatedAllocation.m_pMappedData = pMappedData;
4306  }
4307 
4308  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
4309  VkDeviceSize GetAlignment() const { return m_Alignment; }
4310  VkDeviceSize GetSize() const { return m_Size; }
4311  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
4312  void* GetUserData() const { return m_pUserData; }
4313  void SetUserData(VmaAllocator hAllocator, void* pUserData);
4314  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
4315 
4316  VmaDeviceMemoryBlock* GetBlock() const
4317  {
4318  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4319  return m_BlockAllocation.m_Block;
4320  }
4321  VkDeviceSize GetOffset() const;
4322  VkDeviceMemory GetMemory() const;
4323  uint32_t GetMemoryTypeIndex() const;
4324  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
4325  void* GetMappedData() const;
4326  bool CanBecomeLost() const;
4327  VmaPool GetPool() const;
4328 
4329  uint32_t GetLastUseFrameIndex() const
4330  {
4331  return m_LastUseFrameIndex.load();
4332  }
4333  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
4334  {
4335  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
4336  }
4337  /*
4338  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
4339  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
4340  - Else, returns false.
4341 
4342  If hAllocation is already lost, assert - you should not call it then.
4343  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
4344  */
4345  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4346 
4347  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
4348  {
4349  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
4350  outInfo.blockCount = 1;
4351  outInfo.allocationCount = 1;
4352  outInfo.unusedRangeCount = 0;
4353  outInfo.usedBytes = m_Size;
4354  outInfo.unusedBytes = 0;
4355  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
4356  outInfo.unusedRangeSizeMin = UINT64_MAX;
4357  outInfo.unusedRangeSizeMax = 0;
4358  }
4359 
4360  void BlockAllocMap();
4361  void BlockAllocUnmap();
4362  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
4363  void DedicatedAllocUnmap(VmaAllocator hAllocator);
4364 
4365 #if VMA_STATS_STRING_ENABLED
4366  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
4367  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
4368 
4369  void InitBufferImageUsage(uint32_t bufferImageUsage)
4370  {
4371  VMA_ASSERT(m_BufferImageUsage == 0);
4372  m_BufferImageUsage = bufferImageUsage;
4373  }
4374 
4375  void PrintParameters(class VmaJsonWriter& json) const;
4376 #endif
4377 
4378 private:
4379  VkDeviceSize m_Alignment;
4380  VkDeviceSize m_Size;
4381  void* m_pUserData;
4382  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
4383  uint8_t m_Type; // ALLOCATION_TYPE
4384  uint8_t m_SuballocationType; // VmaSuballocationType
4385  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
4386  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
4387  uint8_t m_MapCount;
4388  uint8_t m_Flags; // enum FLAGS
4389 
4390  // Allocation out of VmaDeviceMemoryBlock.
4391  struct BlockAllocation
4392  {
4393  VmaPool m_hPool; // Null if belongs to general memory.
4394  VmaDeviceMemoryBlock* m_Block;
4395  VkDeviceSize m_Offset;
4396  bool m_CanBecomeLost;
4397  };
4398 
4399  // Allocation for an object that has its own private VkDeviceMemory.
4400  struct DedicatedAllocation
4401  {
4402  uint32_t m_MemoryTypeIndex;
4403  VkDeviceMemory m_hMemory;
4404  void* m_pMappedData; // Not null means memory is mapped.
4405  };
4406 
4407  union
4408  {
4409  // Allocation out of VmaDeviceMemoryBlock.
4410  BlockAllocation m_BlockAllocation;
4411  // Allocation for an object that has its own private VkDeviceMemory.
4412  DedicatedAllocation m_DedicatedAllocation;
4413  };
4414 
4415 #if VMA_STATS_STRING_ENABLED
4416  uint32_t m_CreationFrameIndex;
4417  uint32_t m_BufferImageUsage; // 0 if unknown.
4418 #endif
4419 
4420  void FreeUserDataString(VmaAllocator hAllocator);
4421 };
4422 
4423 /*
4424 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
4425 allocated memory block or free.
4426 */
4427 struct VmaSuballocation
4428 {
4429  VkDeviceSize offset;
4430  VkDeviceSize size;
4431  VmaAllocation hAllocation;
4432  VmaSuballocationType type;
4433 };
4434 
4435 // Comparator for offsets.
4436 struct VmaSuballocationOffsetLess
4437 {
4438  bool operator()(const VmaSuballocation& lhs, const VmaSuballocation& rhs) const
4439  {
4440  return lhs.offset < rhs.offset;
4441  }
4442 };
4443 struct VmaSuballocationOffsetGreater
4444 {
4445  bool operator()(const VmaSuballocation& lhs, const VmaSuballocation& rhs) const
4446  {
4447  return lhs.offset > rhs.offset;
4448  }
4449 };
4450 
4451 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
4452 
4453 // Cost of one additional allocation lost, as equivalent in bytes.
4454 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
4455 
4456 /*
4457 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
4458 
4459 If canMakeOtherLost was false:
4460 - item points to a FREE suballocation.
4461 - itemsToMakeLostCount is 0.
4462 
4463 If canMakeOtherLost was true:
4464 - item points to first of sequence of suballocations, which are either FREE,
4465  or point to VmaAllocations that can become lost.
4466 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
4467  the requested allocation to succeed.
4468 */
4469 struct VmaAllocationRequest
4470 {
4471  VkDeviceSize offset;
4472  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
4473  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
4474  VmaSuballocationList::iterator item;
4475  size_t itemsToMakeLostCount;
4476 
4477  VkDeviceSize CalcCost() const
4478  {
4479  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
4480  }
4481 };
4482 
4483 /*
4484 Data structure used for bookkeeping of allocations and unused ranges of memory
4485 in a single VkDeviceMemory block.
4486 */
4487 class VmaBlockMetadata
4488 {
4489 public:
4490  VmaBlockMetadata() : m_Size(0) { }
4491  virtual ~VmaBlockMetadata() { }
4492  virtual void Init(VkDeviceSize size) { m_Size = size; }
4493 
4494  // Validates all data structures inside this object. If not valid, returns false.
4495  virtual bool Validate() const = 0;
4496  VkDeviceSize GetSize() const { return m_Size; }
4497  virtual size_t GetAllocationCount() const = 0;
4498  virtual VkDeviceSize GetSumFreeSize() const = 0;
4499  virtual VkDeviceSize GetUnusedRangeSizeMax() const = 0;
4500  // Returns true if this block is empty - contains only single free suballocation.
4501  virtual bool IsEmpty() const = 0;
4502 
4503  virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const = 0;
4504  virtual void AddPoolStats(VmaPoolStats& inoutStats) const = 0;
4505 
4506 #if VMA_STATS_STRING_ENABLED
4507  virtual void PrintDetailedMap(class VmaJsonWriter& json) const = 0;
4508 #endif
4509 
4510  // Tries to find a place for suballocation with given parameters inside this block.
4511  // If succeeded, fills pAllocationRequest and returns true.
4512  // If failed, returns false.
4513  virtual bool CreateAllocationRequest(
4514  uint32_t currentFrameIndex,
4515  uint32_t frameInUseCount,
4516  VkDeviceSize bufferImageGranularity,
4517  VkDeviceSize allocSize,
4518  VkDeviceSize allocAlignment,
4519  bool upperAddress,
4520  VmaSuballocationType allocType,
4521  bool canMakeOtherLost,
4522  VmaAllocationRequest* pAllocationRequest) = 0;
4523 
4524  virtual bool MakeRequestedAllocationsLost(
4525  uint32_t currentFrameIndex,
4526  uint32_t frameInUseCount,
4527  VmaAllocationRequest* pAllocationRequest) = 0;
4528 
4529  virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) = 0;
4530 
4531  virtual VkResult CheckCorruption(const void* pBlockData) = 0;
4532 
4533  // Makes actual allocation based on request. Request must already be checked and valid.
4534  virtual void Alloc(
4535  const VmaAllocationRequest& request,
4536  VmaSuballocationType type,
4537  VkDeviceSize allocSize,
4538  bool upperAddress,
4539  VmaAllocation hAllocation) = 0;
4540 
4541  // Frees suballocation assigned to given memory region.
4542  virtual void Free(const VmaAllocation allocation) = 0;
4543  virtual void FreeAtOffset(VkDeviceSize offset) = 0;
4544 
4545 protected:
4546 #if VMA_STATS_STRING_ENABLED
4547  void PrintDetailedMap_Begin(class VmaJsonWriter& json,
4548  VkDeviceSize unusedBytes,
4549  size_t allocationCount,
4550  size_t unusedRangeCount) const;
4551  void PrintDetailedMap_Allocation(class VmaJsonWriter& json,
4552  VkDeviceSize offset,
4553  VmaAllocation hAllocation) const;
4554  void PrintDetailedMap_UnusedRange(class VmaJsonWriter& json,
4555  VkDeviceSize offset,
4556  VkDeviceSize size) const;
4557  void PrintDetailedMap_End(class VmaJsonWriter& json) const;
4558 #endif
4559 
4560 private:
4561  VkDeviceSize m_Size;
4562 };
4563 
4564 class VmaBlockMetadata_Generic : public VmaBlockMetadata
4565 {
4566  VMA_CLASS_NO_COPY(VmaBlockMetadata_Generic)
4567 public:
4568  VmaBlockMetadata_Generic(VmaAllocator hAllocator);
4569  virtual ~VmaBlockMetadata_Generic();
4570  virtual void Init(VkDeviceSize size);
4571 
4572  virtual bool Validate() const;
4573  virtual size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
4574  virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
4575  virtual VkDeviceSize GetUnusedRangeSizeMax() const;
4576  virtual bool IsEmpty() const;
4577 
4578  virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
4579  virtual void AddPoolStats(VmaPoolStats& inoutStats) const;
4580 
4581 #if VMA_STATS_STRING_ENABLED
4582  virtual void PrintDetailedMap(class VmaJsonWriter& json) const;
4583 #endif
4584 
4585  virtual bool CreateAllocationRequest(
4586  uint32_t currentFrameIndex,
4587  uint32_t frameInUseCount,
4588  VkDeviceSize bufferImageGranularity,
4589  VkDeviceSize allocSize,
4590  VkDeviceSize allocAlignment,
4591  bool upperAddress,
4592  VmaSuballocationType allocType,
4593  bool canMakeOtherLost,
4594  VmaAllocationRequest* pAllocationRequest);
4595 
4596  virtual bool MakeRequestedAllocationsLost(
4597  uint32_t currentFrameIndex,
4598  uint32_t frameInUseCount,
4599  VmaAllocationRequest* pAllocationRequest);
4600 
4601  virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4602 
4603  virtual VkResult CheckCorruption(const void* pBlockData);
4604 
4605  virtual void Alloc(
4606  const VmaAllocationRequest& request,
4607  VmaSuballocationType type,
4608  VkDeviceSize allocSize,
4609  bool upperAddress,
4610  VmaAllocation hAllocation);
4611 
4612  virtual void Free(const VmaAllocation allocation);
4613  virtual void FreeAtOffset(VkDeviceSize offset);
4614 
4615 private:
4616  uint32_t m_FreeCount;
4617  VkDeviceSize m_SumFreeSize;
4618  VmaSuballocationList m_Suballocations;
4619  // Suballocations that are free and have size greater than certain threshold.
4620  // Sorted by size, ascending.
4621  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
4622 
4623  bool ValidateFreeSuballocationList() const;
4624 
4625  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
4626  // If yes, fills pOffset and returns true. If no, returns false.
4627  bool CheckAllocation(
4628  uint32_t currentFrameIndex,
4629  uint32_t frameInUseCount,
4630  VkDeviceSize bufferImageGranularity,
4631  VkDeviceSize allocSize,
4632  VkDeviceSize allocAlignment,
4633  VmaSuballocationType allocType,
4634  VmaSuballocationList::const_iterator suballocItem,
4635  bool canMakeOtherLost,
4636  VkDeviceSize* pOffset,
4637  size_t* itemsToMakeLostCount,
4638  VkDeviceSize* pSumFreeSize,
4639  VkDeviceSize* pSumItemSize) const;
4640  // Given free suballocation, it merges it with following one, which must also be free.
4641  void MergeFreeWithNext(VmaSuballocationList::iterator item);
4642  // Releases given suballocation, making it free.
4643  // Merges it with adjacent free suballocations if applicable.
4644  // Returns iterator to new free suballocation at this place.
4645  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
4646  // Given free suballocation, it inserts it into sorted list of
4647  // m_FreeSuballocationsBySize if it's suitable.
4648  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
4649  // Given free suballocation, it removes it from sorted list of
4650  // m_FreeSuballocationsBySize if it's suitable.
4651  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
4652 };
4653 
4654 /*
4655 Allocations and their references in internal data structure look like this:
4656 
4657 if(m_2ndVectorMode == SECOND_VECTOR_EMPTY):
4658 
4659  0 +-------+
4660  | |
4661  | |
4662  | |
4663  +-------+
4664  | Alloc | 1st[m_1stNullItemsBeginCount]
4665  +-------+
4666  | Alloc | 1st[m_1stNullItemsBeginCount + 1]
4667  +-------+
4668  | ... |
4669  +-------+
4670  | Alloc | 1st[1st.size() - 1]
4671  +-------+
4672  | |
4673  | |
4674  | |
4675 GetSize() +-------+
4676 
4677 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER):
4678 
4679  0 +-------+
4680  | Alloc | 2nd[0]
4681  +-------+
4682  | Alloc | 2nd[1]
4683  +-------+
4684  | ... |
4685  +-------+
4686  | Alloc | 2nd[2nd.size() - 1]
4687  +-------+
4688  | |
4689  | |
4690  | |
4691  +-------+
4692  | Alloc | 1st[m_1stNullItemsBeginCount]
4693  +-------+
4694  | Alloc | 1st[m_1stNullItemsBeginCount + 1]
4695  +-------+
4696  | ... |
4697  +-------+
4698  | Alloc | 1st[1st.size() - 1]
4699  +-------+
4700  | |
4701 GetSize() +-------+
4702 
4703 if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK):
4704 
4705  0 +-------+
4706  | |
4707  | |
4708  | |
4709  +-------+
4710  | Alloc | 1st[m_1stNullItemsBeginCount]
4711  +-------+
4712  | Alloc | 1st[m_1stNullItemsBeginCount + 1]
4713  +-------+
4714  | ... |
4715  +-------+
4716  | Alloc | 1st[1st.size() - 1]
4717  +-------+
4718  | |
4719  | |
4720  | |
4721  +-------+
4722  | Alloc | 2nd[2nd.size() - 1]
4723  +-------+
4724  | ... |
4725  +-------+
4726  | Alloc | 2nd[1]
4727  +-------+
4728  | Alloc | 2nd[0]
4729 GetSize() +-------+
4730 
4731 */
4732 class VmaBlockMetadata_Linear : public VmaBlockMetadata
4733 {
4734  VMA_CLASS_NO_COPY(VmaBlockMetadata_Linear)
4735 public:
4736  VmaBlockMetadata_Linear(VmaAllocator hAllocator);
4737  virtual ~VmaBlockMetadata_Linear();
4738  virtual void Init(VkDeviceSize size);
4739 
4740  virtual bool Validate() const;
4741  virtual size_t GetAllocationCount() const;
4742  virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
4743  virtual VkDeviceSize GetUnusedRangeSizeMax() const;
4744  virtual bool IsEmpty() const { return GetAllocationCount() == 0; }
4745 
4746  virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
4747  virtual void AddPoolStats(VmaPoolStats& inoutStats) const;
4748 
4749 #if VMA_STATS_STRING_ENABLED
4750  virtual void PrintDetailedMap(class VmaJsonWriter& json) const;
4751 #endif
4752 
4753  virtual bool CreateAllocationRequest(
4754  uint32_t currentFrameIndex,
4755  uint32_t frameInUseCount,
4756  VkDeviceSize bufferImageGranularity,
4757  VkDeviceSize allocSize,
4758  VkDeviceSize allocAlignment,
4759  bool upperAddress,
4760  VmaSuballocationType allocType,
4761  bool canMakeOtherLost,
4762  VmaAllocationRequest* pAllocationRequest);
4763 
4764  virtual bool MakeRequestedAllocationsLost(
4765  uint32_t currentFrameIndex,
4766  uint32_t frameInUseCount,
4767  VmaAllocationRequest* pAllocationRequest);
4768 
4769  virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4770 
4771  virtual VkResult CheckCorruption(const void* pBlockData);
4772 
4773  virtual void Alloc(
4774  const VmaAllocationRequest& request,
4775  VmaSuballocationType type,
4776  VkDeviceSize allocSize,
4777  bool upperAddress,
4778  VmaAllocation hAllocation);
4779 
4780  virtual void Free(const VmaAllocation allocation);
4781  virtual void FreeAtOffset(VkDeviceSize offset);
4782 
4783 private:
4784  /*
4785  There are two suballocation vectors, used in ping-pong way.
4786  The one with index m_1stVectorIndex is called 1st.
4787  The one with index (m_1stVectorIndex ^ 1) is called 2nd.
4788  2nd can be non-empty only when 1st is not empty.
4789  When 2nd is not empty, m_2ndVectorMode indicates its mode of operation.
4790  */
4791  typedef VmaVector< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > SuballocationVectorType;
4792 
4793  enum SECOND_VECTOR_MODE
4794  {
4795  SECOND_VECTOR_EMPTY,
4796  /*
4797  Suballocations in 2nd vector are created later than the ones in 1st, but they
4798  all have smaller offset.
4799  */
4800  SECOND_VECTOR_RING_BUFFER,
4801  /*
4802  Suballocations in 2nd vector are upper side of double stack.
4803  They all have offsets higher than those in 1st vector.
4804  Top of this stack means smaller offsets, but higher indices in this vector.
4805  */
4806  SECOND_VECTOR_DOUBLE_STACK,
4807  };
4808 
4809  VkDeviceSize m_SumFreeSize;
4810  SuballocationVectorType m_Suballocations0, m_Suballocations1;
4811  uint32_t m_1stVectorIndex;
4812  SECOND_VECTOR_MODE m_2ndVectorMode;
4813 
4814  SuballocationVectorType& AccessSuballocations1st() { return m_1stVectorIndex ? m_Suballocations1 : m_Suballocations0; }
4815  SuballocationVectorType& AccessSuballocations2nd() { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; }
4816  const SuballocationVectorType& AccessSuballocations1st() const { return m_1stVectorIndex ? m_Suballocations1 : m_Suballocations0; }
4817  const SuballocationVectorType& AccessSuballocations2nd() const { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; }
4818 
4819  // Number of items in 1st vector with hAllocation = null at the beginning.
4820  size_t m_1stNullItemsBeginCount;
4821  // Number of other items in 1st vector with hAllocation = null somewhere in the middle.
4822  size_t m_1stNullItemsMiddleCount;
4823  // Number of items in 2nd vector with hAllocation = null.
4824  size_t m_2ndNullItemsCount;
4825 
4826  bool ShouldCompact1st() const;
4827  void CleanupAfterFree();
4828 };
4829 
4830 /*
4831 Represents a single block of device memory (`VkDeviceMemory`) with all the
4832 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
4833 
4834 Thread-safety: This class must be externally synchronized.
4835 */
4836 class VmaDeviceMemoryBlock
4837 {
4838  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
4839 public:
4840  VmaBlockMetadata* m_pMetadata;
4841 
4842  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
4843 
4844  ~VmaDeviceMemoryBlock()
4845  {
4846  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
4847  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
4848  }
4849 
4850  // Always call after construction.
4851  void Init(
4852  VmaAllocator hAllocator,
4853  uint32_t newMemoryTypeIndex,
4854  VkDeviceMemory newMemory,
4855  VkDeviceSize newSize,
4856  uint32_t id,
4857  bool linearAlgorithm);
4858  // Always call before destruction.
4859  void Destroy(VmaAllocator allocator);
4860 
4861  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
4862  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
4863  uint32_t GetId() const { return m_Id; }
4864  void* GetMappedData() const { return m_pMappedData; }
4865 
4866  // Validates all data structures inside this object. If not valid, returns false.
4867  bool Validate() const;
4868 
4869  VkResult CheckCorruption(VmaAllocator hAllocator);
4870 
4871  // ppData can be null.
4872  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
4873  void Unmap(VmaAllocator hAllocator, uint32_t count);
4874 
4875  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
4876  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
4877 
4878  VkResult BindBufferMemory(
4879  const VmaAllocator hAllocator,
4880  const VmaAllocation hAllocation,
4881  VkBuffer hBuffer);
4882  VkResult BindImageMemory(
4883  const VmaAllocator hAllocator,
4884  const VmaAllocation hAllocation,
4885  VkImage hImage);
4886 
4887 private:
4888  uint32_t m_MemoryTypeIndex;
4889  uint32_t m_Id;
4890  VkDeviceMemory m_hMemory;
4891 
4892  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
4893  // Also protects m_MapCount, m_pMappedData.
4894  VMA_MUTEX m_Mutex;
4895  uint32_t m_MapCount;
4896  void* m_pMappedData;
4897 };
4898 
4899 struct VmaPointerLess
4900 {
4901  bool operator()(const void* lhs, const void* rhs) const
4902  {
4903  return lhs < rhs;
4904  }
4905 };
4906 
4907 class VmaDefragmentator;
4908 
4909 /*
4910 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
4911 Vulkan memory type.
4912 
4913 Synchronized internally with a mutex.
4914 */
4915 struct VmaBlockVector
4916 {
4917  VMA_CLASS_NO_COPY(VmaBlockVector)
4918 public:
4919  VmaBlockVector(
4920  VmaAllocator hAllocator,
4921  uint32_t memoryTypeIndex,
4922  VkDeviceSize preferredBlockSize,
4923  size_t minBlockCount,
4924  size_t maxBlockCount,
4925  VkDeviceSize bufferImageGranularity,
4926  uint32_t frameInUseCount,
4927  bool isCustomPool,
4928  bool linearAlgorithm);
4929  ~VmaBlockVector();
4930 
4931  VkResult CreateMinBlocks();
4932 
4933  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
4934  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
4935  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
4936  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
4937  bool UsesLinearAlgorithm() const { return m_LinearAlgorithm; }
4938 
4939  void GetPoolStats(VmaPoolStats* pStats);
4940 
4941  bool IsEmpty() const { return m_Blocks.empty(); }
4942  bool IsCorruptionDetectionEnabled() const;
4943 
4944  VkResult Allocate(
4945  VmaPool hCurrentPool,
4946  uint32_t currentFrameIndex,
4947  VkDeviceSize size,
4948  VkDeviceSize alignment,
4949  const VmaAllocationCreateInfo& createInfo,
4950  VmaSuballocationType suballocType,
4951  VmaAllocation* pAllocation);
4952 
4953  void Free(
4954  VmaAllocation hAllocation);
4955 
4956  // Adds statistics of this BlockVector to pStats.
4957  void AddStats(VmaStats* pStats);
4958 
4959 #if VMA_STATS_STRING_ENABLED
4960  void PrintDetailedMap(class VmaJsonWriter& json);
4961 #endif
4962 
4963  void MakePoolAllocationsLost(
4964  uint32_t currentFrameIndex,
4965  size_t* pLostAllocationCount);
4966  VkResult CheckCorruption();
4967 
4968  VmaDefragmentator* EnsureDefragmentator(
4969  VmaAllocator hAllocator,
4970  uint32_t currentFrameIndex);
4971 
4972  VkResult Defragment(
4973  VmaDefragmentationStats* pDefragmentationStats,
4974  VkDeviceSize& maxBytesToMove,
4975  uint32_t& maxAllocationsToMove);
4976 
4977  void DestroyDefragmentator();
4978 
4979 private:
4980  friend class VmaDefragmentator;
4981 
4982  const VmaAllocator m_hAllocator;
4983  const uint32_t m_MemoryTypeIndex;
4984  const VkDeviceSize m_PreferredBlockSize;
4985  const size_t m_MinBlockCount;
4986  const size_t m_MaxBlockCount;
4987  const VkDeviceSize m_BufferImageGranularity;
4988  const uint32_t m_FrameInUseCount;
4989  const bool m_IsCustomPool;
4990  const bool m_LinearAlgorithm;
4991  bool m_HasEmptyBlock;
4992  VMA_MUTEX m_Mutex;
4993  // Incrementally sorted by sumFreeSize, ascending.
4994  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
4995  /* There can be at most one allocation that is completely empty - a
4996  hysteresis to avoid pessimistic case of alternating creation and destruction
4997  of a VkDeviceMemory. */
4998  VmaDefragmentator* m_pDefragmentator;
4999  uint32_t m_NextBlockId;
5000 
5001  VkDeviceSize CalcMaxBlockSize() const;
5002 
5003  // Finds and removes given block from vector.
5004  void Remove(VmaDeviceMemoryBlock* pBlock);
5005 
5006  // Performs single step in sorting m_Blocks. They may not be fully sorted
5007  // after this call.
5008  void IncrementallySortBlocks();
5009 
5010  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
5011 };
5012 
5013 struct VmaPool_T
5014 {
5015  VMA_CLASS_NO_COPY(VmaPool_T)
5016 public:
5017  VmaBlockVector m_BlockVector;
5018 
5019  VmaPool_T(
5020  VmaAllocator hAllocator,
5021  const VmaPoolCreateInfo& createInfo);
5022  ~VmaPool_T();
5023 
5024  uint32_t GetId() const { return m_Id; }
5025  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
5026 
5027 #if VMA_STATS_STRING_ENABLED
5028  //void PrintDetailedMap(class VmaStringBuilder& sb);
5029 #endif
5030 
5031 private:
5032  uint32_t m_Id;
5033 };
5034 
5035 class VmaDefragmentator
5036 {
5037  VMA_CLASS_NO_COPY(VmaDefragmentator)
5038 private:
5039  const VmaAllocator m_hAllocator;
5040  VmaBlockVector* const m_pBlockVector;
5041  uint32_t m_CurrentFrameIndex;
5042  VkDeviceSize m_BytesMoved;
5043  uint32_t m_AllocationsMoved;
5044 
5045  struct AllocationInfo
5046  {
5047  VmaAllocation m_hAllocation;
5048  VkBool32* m_pChanged;
5049 
5050  AllocationInfo() :
5051  m_hAllocation(VK_NULL_HANDLE),
5052  m_pChanged(VMA_NULL)
5053  {
5054  }
5055  };
5056 
5057  struct AllocationInfoSizeGreater
5058  {
5059  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
5060  {
5061  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
5062  }
5063  };
5064 
5065  // Used between AddAllocation and Defragment.
5066  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
5067 
5068  struct BlockInfo
5069  {
5070  VmaDeviceMemoryBlock* m_pBlock;
5071  bool m_HasNonMovableAllocations;
5072  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
5073 
5074  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
5075  m_pBlock(VMA_NULL),
5076  m_HasNonMovableAllocations(true),
5077  m_Allocations(pAllocationCallbacks),
5078  m_pMappedDataForDefragmentation(VMA_NULL)
5079  {
5080  }
5081 
5082  void CalcHasNonMovableAllocations()
5083  {
5084  const size_t blockAllocCount = m_pBlock->m_pMetadata->GetAllocationCount();
5085  const size_t defragmentAllocCount = m_Allocations.size();
5086  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
5087  }
5088 
5089  void SortAllocationsBySizeDescecnding()
5090  {
5091  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
5092  }
5093 
5094  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
5095  void Unmap(VmaAllocator hAllocator);
5096 
5097  private:
5098  // Not null if mapped for defragmentation only, not originally mapped.
5099  void* m_pMappedDataForDefragmentation;
5100  };
5101 
5102  struct BlockPointerLess
5103  {
5104  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
5105  {
5106  return pLhsBlockInfo->m_pBlock < pRhsBlock;
5107  }
5108  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
5109  {
5110  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
5111  }
5112  };
5113 
5114  // 1. Blocks with some non-movable allocations go first.
5115  // 2. Blocks with smaller sumFreeSize go first.
5116  struct BlockInfoCompareMoveDestination
5117  {
5118  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
5119  {
5120  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
5121  {
5122  return true;
5123  }
5124  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
5125  {
5126  return false;
5127  }
5128  if(pLhsBlockInfo->m_pBlock->m_pMetadata->GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_pMetadata->GetSumFreeSize())
5129  {
5130  return true;
5131  }
5132  return false;
5133  }
5134  };
5135 
5136  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
5137  BlockInfoVector m_Blocks;
5138 
5139  VkResult DefragmentRound(
5140  VkDeviceSize maxBytesToMove,
5141  uint32_t maxAllocationsToMove);
5142 
5143  static bool MoveMakesSense(
5144  size_t dstBlockIndex, VkDeviceSize dstOffset,
5145  size_t srcBlockIndex, VkDeviceSize srcOffset);
5146 
5147 public:
5148  VmaDefragmentator(
5149  VmaAllocator hAllocator,
5150  VmaBlockVector* pBlockVector,
5151  uint32_t currentFrameIndex);
5152 
5153  ~VmaDefragmentator();
5154 
5155  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
5156  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
5157 
5158  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
5159 
5160  VkResult Defragment(
5161  VkDeviceSize maxBytesToMove,
5162  uint32_t maxAllocationsToMove);
5163 };
5164 
5165 #if VMA_RECORDING_ENABLED
5166 
5167 class VmaRecorder
5168 {
5169 public:
5170  VmaRecorder();
5171  VkResult Init(const VmaRecordSettings& settings, bool useMutex);
5172  void WriteConfiguration(
5173  const VkPhysicalDeviceProperties& devProps,
5174  const VkPhysicalDeviceMemoryProperties& memProps,
5175  bool dedicatedAllocationExtensionEnabled);
5176  ~VmaRecorder();
5177 
5178  void RecordCreateAllocator(uint32_t frameIndex);
5179  void RecordDestroyAllocator(uint32_t frameIndex);
5180  void RecordCreatePool(uint32_t frameIndex,
5181  const VmaPoolCreateInfo& createInfo,
5182  VmaPool pool);
5183  void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
5184  void RecordAllocateMemory(uint32_t frameIndex,
5185  const VkMemoryRequirements& vkMemReq,
5186  const VmaAllocationCreateInfo& createInfo,
5187  VmaAllocation allocation);
5188  void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
5189  const VkMemoryRequirements& vkMemReq,
5190  bool requiresDedicatedAllocation,
5191  bool prefersDedicatedAllocation,
5192  const VmaAllocationCreateInfo& createInfo,
5193  VmaAllocation allocation);
5194  void RecordAllocateMemoryForImage(uint32_t frameIndex,
5195  const VkMemoryRequirements& vkMemReq,
5196  bool requiresDedicatedAllocation,
5197  bool prefersDedicatedAllocation,
5198  const VmaAllocationCreateInfo& createInfo,
5199  VmaAllocation allocation);
5200  void RecordFreeMemory(uint32_t frameIndex,
5201  VmaAllocation allocation);
5202  void RecordSetAllocationUserData(uint32_t frameIndex,
5203  VmaAllocation allocation,
5204  const void* pUserData);
5205  void RecordCreateLostAllocation(uint32_t frameIndex,
5206  VmaAllocation allocation);
5207  void RecordMapMemory(uint32_t frameIndex,
5208  VmaAllocation allocation);
5209  void RecordUnmapMemory(uint32_t frameIndex,
5210  VmaAllocation allocation);
5211  void RecordFlushAllocation(uint32_t frameIndex,
5212  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
5213  void RecordInvalidateAllocation(uint32_t frameIndex,
5214  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
5215  void RecordCreateBuffer(uint32_t frameIndex,
5216  const VkBufferCreateInfo& bufCreateInfo,
5217  const VmaAllocationCreateInfo& allocCreateInfo,
5218  VmaAllocation allocation);
5219  void RecordCreateImage(uint32_t frameIndex,
5220  const VkImageCreateInfo& imageCreateInfo,
5221  const VmaAllocationCreateInfo& allocCreateInfo,
5222  VmaAllocation allocation);
5223  void RecordDestroyBuffer(uint32_t frameIndex,
5224  VmaAllocation allocation);
5225  void RecordDestroyImage(uint32_t frameIndex,
5226  VmaAllocation allocation);
5227  void RecordTouchAllocation(uint32_t frameIndex,
5228  VmaAllocation allocation);
5229  void RecordGetAllocationInfo(uint32_t frameIndex,
5230  VmaAllocation allocation);
5231  void RecordMakePoolAllocationsLost(uint32_t frameIndex,
5232  VmaPool pool);
5233 
5234 private:
5235  struct CallParams
5236  {
5237  uint32_t threadId;
5238  double time;
5239  };
5240 
5241  class UserDataString
5242  {
5243  public:
5244  UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
5245  const char* GetString() const { return m_Str; }
5246 
5247  private:
5248  char m_PtrStr[17];
5249  const char* m_Str;
5250  };
5251 
5252  bool m_UseMutex;
5253  VmaRecordFlags m_Flags;
5254  FILE* m_File;
5255  VMA_MUTEX m_FileMutex;
5256  int64_t m_Freq;
5257  int64_t m_StartCounter;
5258 
5259  void GetBasicParams(CallParams& outParams);
5260  void Flush();
5261 };
5262 
5263 #endif // #if VMA_RECORDING_ENABLED
5264 
5265 // Main allocator object.
5266 struct VmaAllocator_T
5267 {
5268  VMA_CLASS_NO_COPY(VmaAllocator_T)
5269 public:
5270  bool m_UseMutex;
5271  bool m_UseKhrDedicatedAllocation;
5272  VkDevice m_hDevice;
5273  bool m_AllocationCallbacksSpecified;
5274  VkAllocationCallbacks m_AllocationCallbacks;
5275  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
5276 
5277  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
5278  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
5279  VMA_MUTEX m_HeapSizeLimitMutex;
5280 
5281  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
5282  VkPhysicalDeviceMemoryProperties m_MemProps;
5283 
5284  // Default pools.
5285  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
5286 
5287  // Each vector is sorted by memory (handle value).
5288  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
5289  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
5290  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
5291 
5292  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
5293  VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
5294  ~VmaAllocator_T();
5295 
5296  const VkAllocationCallbacks* GetAllocationCallbacks() const
5297  {
5298  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
5299  }
5300  const VmaVulkanFunctions& GetVulkanFunctions() const
5301  {
5302  return m_VulkanFunctions;
5303  }
5304 
5305  VkDeviceSize GetBufferImageGranularity() const
5306  {
5307  return VMA_MAX(
5308  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
5309  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
5310  }
5311 
5312  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
5313  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
5314 
5315  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
5316  {
5317  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
5318  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
5319  }
5320  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
5321  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
5322  {
5323  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
5324  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
5325  }
5326  // Minimum alignment for all allocations in specific memory type.
5327  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
5328  {
5329  return IsMemoryTypeNonCoherent(memTypeIndex) ?
5330  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
5331  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
5332  }
5333 
5334  bool IsIntegratedGpu() const
5335  {
5336  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
5337  }
5338 
5339 #if VMA_RECORDING_ENABLED
5340  VmaRecorder* GetRecorder() const { return m_pRecorder; }
5341 #endif
5342 
5343  void GetBufferMemoryRequirements(
5344  VkBuffer hBuffer,
5345  VkMemoryRequirements& memReq,
5346  bool& requiresDedicatedAllocation,
5347  bool& prefersDedicatedAllocation) const;
5348  void GetImageMemoryRequirements(
5349  VkImage hImage,
5350  VkMemoryRequirements& memReq,
5351  bool& requiresDedicatedAllocation,
5352  bool& prefersDedicatedAllocation) const;
5353 
5354  // Main allocation function.
5355  VkResult AllocateMemory(
5356  const VkMemoryRequirements& vkMemReq,
5357  bool requiresDedicatedAllocation,
5358  bool prefersDedicatedAllocation,
5359  VkBuffer dedicatedBuffer,
5360  VkImage dedicatedImage,
5361  const VmaAllocationCreateInfo& createInfo,
5362  VmaSuballocationType suballocType,
5363  VmaAllocation* pAllocation);
5364 
5365  // Main deallocation function.
5366  void FreeMemory(const VmaAllocation allocation);
5367 
5368  void CalculateStats(VmaStats* pStats);
5369 
5370 #if VMA_STATS_STRING_ENABLED
5371  void PrintDetailedMap(class VmaJsonWriter& json);
5372 #endif
5373 
5374  VkResult Defragment(
5375  VmaAllocation* pAllocations,
5376  size_t allocationCount,
5377  VkBool32* pAllocationsChanged,
5378  const VmaDefragmentationInfo* pDefragmentationInfo,
5379  VmaDefragmentationStats* pDefragmentationStats);
5380 
5381  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
5382  bool TouchAllocation(VmaAllocation hAllocation);
5383 
5384  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
5385  void DestroyPool(VmaPool pool);
5386  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
5387 
5388  void SetCurrentFrameIndex(uint32_t frameIndex);
5389  uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
5390 
5391  void MakePoolAllocationsLost(
5392  VmaPool hPool,
5393  size_t* pLostAllocationCount);
5394  VkResult CheckPoolCorruption(VmaPool hPool);
5395  VkResult CheckCorruption(uint32_t memoryTypeBits);
5396 
5397  void CreateLostAllocation(VmaAllocation* pAllocation);
5398 
5399  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
5400  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
5401 
5402  VkResult Map(VmaAllocation hAllocation, void** ppData);
5403  void Unmap(VmaAllocation hAllocation);
5404 
5405  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
5406  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
5407 
5408  void FlushOrInvalidateAllocation(
5409  VmaAllocation hAllocation,
5410  VkDeviceSize offset, VkDeviceSize size,
5411  VMA_CACHE_OPERATION op);
5412 
5413  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
5414 
5415 private:
5416  VkDeviceSize m_PreferredLargeHeapBlockSize;
5417 
5418  VkPhysicalDevice m_PhysicalDevice;
5419  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
5420 
5421  VMA_MUTEX m_PoolsMutex;
5422  // Protected by m_PoolsMutex. Sorted by pointer value.
5423  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
5424  uint32_t m_NextPoolId;
5425 
5426  VmaVulkanFunctions m_VulkanFunctions;
5427 
5428 #if VMA_RECORDING_ENABLED
5429  VmaRecorder* m_pRecorder;
5430 #endif
5431 
5432  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
5433 
5434  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
5435 
5436  VkResult AllocateMemoryOfType(
5437  VkDeviceSize size,
5438  VkDeviceSize alignment,
5439  bool dedicatedAllocation,
5440  VkBuffer dedicatedBuffer,
5441  VkImage dedicatedImage,
5442  const VmaAllocationCreateInfo& createInfo,
5443  uint32_t memTypeIndex,
5444  VmaSuballocationType suballocType,
5445  VmaAllocation* pAllocation);
5446 
5447  // Allocates and registers new VkDeviceMemory specifically for single allocation.
5448  VkResult AllocateDedicatedMemory(
5449  VkDeviceSize size,
5450  VmaSuballocationType suballocType,
5451  uint32_t memTypeIndex,
5452  bool map,
5453  bool isUserDataString,
5454  void* pUserData,
5455  VkBuffer dedicatedBuffer,
5456  VkImage dedicatedImage,
5457  VmaAllocation* pAllocation);
5458 
5459  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
5460  void FreeDedicatedMemory(VmaAllocation allocation);
5461 };
5462 
5464 // Memory allocation #2 after VmaAllocator_T definition
5465 
5466 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
5467 {
5468  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
5469 }
5470 
5471 static void VmaFree(VmaAllocator hAllocator, void* ptr)
5472 {
5473  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
5474 }
5475 
5476 template<typename T>
5477 static T* VmaAllocate(VmaAllocator hAllocator)
5478 {
5479  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
5480 }
5481 
5482 template<typename T>
5483 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
5484 {
5485  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
5486 }
5487 
5488 template<typename T>
5489 static void vma_delete(VmaAllocator hAllocator, T* ptr)
5490 {
5491  if(ptr != VMA_NULL)
5492  {
5493  ptr->~T();
5494  VmaFree(hAllocator, ptr);
5495  }
5496 }
5497 
5498 template<typename T>
5499 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
5500 {
5501  if(ptr != VMA_NULL)
5502  {
5503  for(size_t i = count; i--; )
5504  ptr[i].~T();
5505  VmaFree(hAllocator, ptr);
5506  }
5507 }
5508 
5510 // VmaStringBuilder
5511 
5512 #if VMA_STATS_STRING_ENABLED
5513 
5514 class VmaStringBuilder
5515 {
5516 public:
5517  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
5518  size_t GetLength() const { return m_Data.size(); }
5519  const char* GetData() const { return m_Data.data(); }
5520 
5521  void Add(char ch) { m_Data.push_back(ch); }
5522  void Add(const char* pStr);
5523  void AddNewLine() { Add('\n'); }
5524  void AddNumber(uint32_t num);
5525  void AddNumber(uint64_t num);
5526  void AddPointer(const void* ptr);
5527 
5528 private:
5529  VmaVector< char, VmaStlAllocator<char> > m_Data;
5530 };
5531 
5532 void VmaStringBuilder::Add(const char* pStr)
5533 {
5534  const size_t strLen = strlen(pStr);
5535  if(strLen > 0)
5536  {
5537  const size_t oldCount = m_Data.size();
5538  m_Data.resize(oldCount + strLen);
5539  memcpy(m_Data.data() + oldCount, pStr, strLen);
5540  }
5541 }
5542 
5543 void VmaStringBuilder::AddNumber(uint32_t num)
5544 {
5545  char buf[11];
5546  VmaUint32ToStr(buf, sizeof(buf), num);
5547  Add(buf);
5548 }
5549 
5550 void VmaStringBuilder::AddNumber(uint64_t num)
5551 {
5552  char buf[21];
5553  VmaUint64ToStr(buf, sizeof(buf), num);
5554  Add(buf);
5555 }
5556 
5557 void VmaStringBuilder::AddPointer(const void* ptr)
5558 {
5559  char buf[21];
5560  VmaPtrToStr(buf, sizeof(buf), ptr);
5561  Add(buf);
5562 }
5563 
5564 #endif // #if VMA_STATS_STRING_ENABLED
5565 
5567 // VmaJsonWriter
5568 
5569 #if VMA_STATS_STRING_ENABLED
5570 
5571 class VmaJsonWriter
5572 {
5573  VMA_CLASS_NO_COPY(VmaJsonWriter)
5574 public:
5575  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
5576  ~VmaJsonWriter();
5577 
5578  void BeginObject(bool singleLine = false);
5579  void EndObject();
5580 
5581  void BeginArray(bool singleLine = false);
5582  void EndArray();
5583 
5584  void WriteString(const char* pStr);
5585  void BeginString(const char* pStr = VMA_NULL);
5586  void ContinueString(const char* pStr);
5587  void ContinueString(uint32_t n);
5588  void ContinueString(uint64_t n);
5589  void ContinueString_Pointer(const void* ptr);
5590  void EndString(const char* pStr = VMA_NULL);
5591 
5592  void WriteNumber(uint32_t n);
5593  void WriteNumber(uint64_t n);
5594  void WriteBool(bool b);
5595  void WriteNull();
5596 
5597 private:
5598  static const char* const INDENT;
5599 
5600  enum COLLECTION_TYPE
5601  {
5602  COLLECTION_TYPE_OBJECT,
5603  COLLECTION_TYPE_ARRAY,
5604  };
5605  struct StackItem
5606  {
5607  COLLECTION_TYPE type;
5608  uint32_t valueCount;
5609  bool singleLineMode;
5610  };
5611 
5612  VmaStringBuilder& m_SB;
5613  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
5614  bool m_InsideString;
5615 
5616  void BeginValue(bool isString);
5617  void WriteIndent(bool oneLess = false);
5618 };
5619 
5620 const char* const VmaJsonWriter::INDENT = " ";
5621 
5622 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
5623  m_SB(sb),
5624  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
5625  m_InsideString(false)
5626 {
5627 }
5628 
5629 VmaJsonWriter::~VmaJsonWriter()
5630 {
5631  VMA_ASSERT(!m_InsideString);
5632  VMA_ASSERT(m_Stack.empty());
5633 }
5634 
5635 void VmaJsonWriter::BeginObject(bool singleLine)
5636 {
5637  VMA_ASSERT(!m_InsideString);
5638 
5639  BeginValue(false);
5640  m_SB.Add('{');
5641 
5642  StackItem item;
5643  item.type = COLLECTION_TYPE_OBJECT;
5644  item.valueCount = 0;
5645  item.singleLineMode = singleLine;
5646  m_Stack.push_back(item);
5647 }
5648 
5649 void VmaJsonWriter::EndObject()
5650 {
5651  VMA_ASSERT(!m_InsideString);
5652 
5653  WriteIndent(true);
5654  m_SB.Add('}');
5655 
5656  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
5657  m_Stack.pop_back();
5658 }
5659 
5660 void VmaJsonWriter::BeginArray(bool singleLine)
5661 {
5662  VMA_ASSERT(!m_InsideString);
5663 
5664  BeginValue(false);
5665  m_SB.Add('[');
5666 
5667  StackItem item;
5668  item.type = COLLECTION_TYPE_ARRAY;
5669  item.valueCount = 0;
5670  item.singleLineMode = singleLine;
5671  m_Stack.push_back(item);
5672 }
5673 
5674 void VmaJsonWriter::EndArray()
5675 {
5676  VMA_ASSERT(!m_InsideString);
5677 
5678  WriteIndent(true);
5679  m_SB.Add(']');
5680 
5681  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
5682  m_Stack.pop_back();
5683 }
5684 
5685 void VmaJsonWriter::WriteString(const char* pStr)
5686 {
5687  BeginString(pStr);
5688  EndString();
5689 }
5690 
5691 void VmaJsonWriter::BeginString(const char* pStr)
5692 {
5693  VMA_ASSERT(!m_InsideString);
5694 
5695  BeginValue(true);
5696  m_SB.Add('"');
5697  m_InsideString = true;
5698  if(pStr != VMA_NULL && pStr[0] != '\0')
5699  {
5700  ContinueString(pStr);
5701  }
5702 }
5703 
5704 void VmaJsonWriter::ContinueString(const char* pStr)
5705 {
5706  VMA_ASSERT(m_InsideString);
5707 
5708  const size_t strLen = strlen(pStr);
5709  for(size_t i = 0; i < strLen; ++i)
5710  {
5711  char ch = pStr[i];
5712  if(ch == '\\')
5713  {
5714  m_SB.Add("\\\\");
5715  }
5716  else if(ch == '"')
5717  {
5718  m_SB.Add("\\\"");
5719  }
5720  else if(ch >= 32)
5721  {
5722  m_SB.Add(ch);
5723  }
5724  else switch(ch)
5725  {
5726  case '\b':
5727  m_SB.Add("\\b");
5728  break;
5729  case '\f':
5730  m_SB.Add("\\f");
5731  break;
5732  case '\n':
5733  m_SB.Add("\\n");
5734  break;
5735  case '\r':
5736  m_SB.Add("\\r");
5737  break;
5738  case '\t':
5739  m_SB.Add("\\t");
5740  break;
5741  default:
5742  VMA_ASSERT(0 && "Character not currently supported.");
5743  break;
5744  }
5745  }
5746 }
5747 
5748 void VmaJsonWriter::ContinueString(uint32_t n)
5749 {
5750  VMA_ASSERT(m_InsideString);
5751  m_SB.AddNumber(n);
5752 }
5753 
5754 void VmaJsonWriter::ContinueString(uint64_t n)
5755 {
5756  VMA_ASSERT(m_InsideString);
5757  m_SB.AddNumber(n);
5758 }
5759 
5760 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
5761 {
5762  VMA_ASSERT(m_InsideString);
5763  m_SB.AddPointer(ptr);
5764 }
5765 
5766 void VmaJsonWriter::EndString(const char* pStr)
5767 {
5768  VMA_ASSERT(m_InsideString);
5769  if(pStr != VMA_NULL && pStr[0] != '\0')
5770  {
5771  ContinueString(pStr);
5772  }
5773  m_SB.Add('"');
5774  m_InsideString = false;
5775 }
5776 
5777 void VmaJsonWriter::WriteNumber(uint32_t n)
5778 {
5779  VMA_ASSERT(!m_InsideString);
5780  BeginValue(false);
5781  m_SB.AddNumber(n);
5782 }
5783 
5784 void VmaJsonWriter::WriteNumber(uint64_t n)
5785 {
5786  VMA_ASSERT(!m_InsideString);
5787  BeginValue(false);
5788  m_SB.AddNumber(n);
5789 }
5790 
5791 void VmaJsonWriter::WriteBool(bool b)
5792 {
5793  VMA_ASSERT(!m_InsideString);
5794  BeginValue(false);
5795  m_SB.Add(b ? "true" : "false");
5796 }
5797 
5798 void VmaJsonWriter::WriteNull()
5799 {
5800  VMA_ASSERT(!m_InsideString);
5801  BeginValue(false);
5802  m_SB.Add("null");
5803 }
5804 
5805 void VmaJsonWriter::BeginValue(bool isString)
5806 {
5807  if(!m_Stack.empty())
5808  {
5809  StackItem& currItem = m_Stack.back();
5810  if(currItem.type == COLLECTION_TYPE_OBJECT &&
5811  currItem.valueCount % 2 == 0)
5812  {
5813  VMA_ASSERT(isString);
5814  }
5815 
5816  if(currItem.type == COLLECTION_TYPE_OBJECT &&
5817  currItem.valueCount % 2 != 0)
5818  {
5819  m_SB.Add(": ");
5820  }
5821  else if(currItem.valueCount > 0)
5822  {
5823  m_SB.Add(", ");
5824  WriteIndent();
5825  }
5826  else
5827  {
5828  WriteIndent();
5829  }
5830  ++currItem.valueCount;
5831  }
5832 }
5833 
5834 void VmaJsonWriter::WriteIndent(bool oneLess)
5835 {
5836  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
5837  {
5838  m_SB.AddNewLine();
5839 
5840  size_t count = m_Stack.size();
5841  if(count > 0 && oneLess)
5842  {
5843  --count;
5844  }
5845  for(size_t i = 0; i < count; ++i)
5846  {
5847  m_SB.Add(INDENT);
5848  }
5849  }
5850 }
5851 
5852 #endif // #if VMA_STATS_STRING_ENABLED
5853 
5855 
5856 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
5857 {
5858  if(IsUserDataString())
5859  {
5860  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
5861 
5862  FreeUserDataString(hAllocator);
5863 
5864  if(pUserData != VMA_NULL)
5865  {
5866  const char* const newStrSrc = (char*)pUserData;
5867  const size_t newStrLen = strlen(newStrSrc);
5868  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
5869  memcpy(newStrDst, newStrSrc, newStrLen + 1);
5870  m_pUserData = newStrDst;
5871  }
5872  }
5873  else
5874  {
5875  m_pUserData = pUserData;
5876  }
5877 }
5878 
5879 void VmaAllocation_T::ChangeBlockAllocation(
5880  VmaAllocator hAllocator,
5881  VmaDeviceMemoryBlock* block,
5882  VkDeviceSize offset)
5883 {
5884  VMA_ASSERT(block != VMA_NULL);
5885  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
5886 
5887  // Move mapping reference counter from old block to new block.
5888  if(block != m_BlockAllocation.m_Block)
5889  {
5890  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
5891  if(IsPersistentMap())
5892  ++mapRefCount;
5893  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
5894  block->Map(hAllocator, mapRefCount, VMA_NULL);
5895  }
5896 
5897  m_BlockAllocation.m_Block = block;
5898  m_BlockAllocation.m_Offset = offset;
5899 }
5900 
5901 VkDeviceSize VmaAllocation_T::GetOffset() const
5902 {
5903  switch(m_Type)
5904  {
5905  case ALLOCATION_TYPE_BLOCK:
5906  return m_BlockAllocation.m_Offset;
5907  case ALLOCATION_TYPE_DEDICATED:
5908  return 0;
5909  default:
5910  VMA_ASSERT(0);
5911  return 0;
5912  }
5913 }
5914 
5915 VkDeviceMemory VmaAllocation_T::GetMemory() const
5916 {
5917  switch(m_Type)
5918  {
5919  case ALLOCATION_TYPE_BLOCK:
5920  return m_BlockAllocation.m_Block->GetDeviceMemory();
5921  case ALLOCATION_TYPE_DEDICATED:
5922  return m_DedicatedAllocation.m_hMemory;
5923  default:
5924  VMA_ASSERT(0);
5925  return VK_NULL_HANDLE;
5926  }
5927 }
5928 
5929 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
5930 {
5931  switch(m_Type)
5932  {
5933  case ALLOCATION_TYPE_BLOCK:
5934  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
5935  case ALLOCATION_TYPE_DEDICATED:
5936  return m_DedicatedAllocation.m_MemoryTypeIndex;
5937  default:
5938  VMA_ASSERT(0);
5939  return UINT32_MAX;
5940  }
5941 }
5942 
5943 void* VmaAllocation_T::GetMappedData() const
5944 {
5945  switch(m_Type)
5946  {
5947  case ALLOCATION_TYPE_BLOCK:
5948  if(m_MapCount != 0)
5949  {
5950  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
5951  VMA_ASSERT(pBlockData != VMA_NULL);
5952  return (char*)pBlockData + m_BlockAllocation.m_Offset;
5953  }
5954  else
5955  {
5956  return VMA_NULL;
5957  }
5958  break;
5959  case ALLOCATION_TYPE_DEDICATED:
5960  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
5961  return m_DedicatedAllocation.m_pMappedData;
5962  default:
5963  VMA_ASSERT(0);
5964  return VMA_NULL;
5965  }
5966 }
5967 
5968 bool VmaAllocation_T::CanBecomeLost() const
5969 {
5970  switch(m_Type)
5971  {
5972  case ALLOCATION_TYPE_BLOCK:
5973  return m_BlockAllocation.m_CanBecomeLost;
5974  case ALLOCATION_TYPE_DEDICATED:
5975  return false;
5976  default:
5977  VMA_ASSERT(0);
5978  return false;
5979  }
5980 }
5981 
5982 VmaPool VmaAllocation_T::GetPool() const
5983 {
5984  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
5985  return m_BlockAllocation.m_hPool;
5986 }
5987 
5988 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5989 {
5990  VMA_ASSERT(CanBecomeLost());
5991 
5992  /*
5993  Warning: This is a carefully designed algorithm.
5994  Do not modify unless you really know what you're doing :)
5995  */
5996  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
5997  for(;;)
5998  {
5999  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
6000  {
6001  VMA_ASSERT(0);
6002  return false;
6003  }
6004  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
6005  {
6006  return false;
6007  }
6008  else // Last use time earlier than current time.
6009  {
6010  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
6011  {
6012  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
6013  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
6014  return true;
6015  }
6016  }
6017  }
6018 }
6019 
6020 #if VMA_STATS_STRING_ENABLED
6021 
6022 // Correspond to values of enum VmaSuballocationType.
6023 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
6024  "FREE",
6025  "UNKNOWN",
6026  "BUFFER",
6027  "IMAGE_UNKNOWN",
6028  "IMAGE_LINEAR",
6029  "IMAGE_OPTIMAL",
6030 };
6031 
6032 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
6033 {
6034  json.WriteString("Type");
6035  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
6036 
6037  json.WriteString("Size");
6038  json.WriteNumber(m_Size);
6039 
6040  if(m_pUserData != VMA_NULL)
6041  {
6042  json.WriteString("UserData");
6043  if(IsUserDataString())
6044  {
6045  json.WriteString((const char*)m_pUserData);
6046  }
6047  else
6048  {
6049  json.BeginString();
6050  json.ContinueString_Pointer(m_pUserData);
6051  json.EndString();
6052  }
6053  }
6054 
6055  json.WriteString("CreationFrameIndex");
6056  json.WriteNumber(m_CreationFrameIndex);
6057 
6058  json.WriteString("LastUseFrameIndex");
6059  json.WriteNumber(GetLastUseFrameIndex());
6060 
6061  if(m_BufferImageUsage != 0)
6062  {
6063  json.WriteString("Usage");
6064  json.WriteNumber(m_BufferImageUsage);
6065  }
6066 }
6067 
6068 #endif
6069 
6070 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
6071 {
6072  VMA_ASSERT(IsUserDataString());
6073  if(m_pUserData != VMA_NULL)
6074  {
6075  char* const oldStr = (char*)m_pUserData;
6076  const size_t oldStrLen = strlen(oldStr);
6077  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
6078  m_pUserData = VMA_NULL;
6079  }
6080 }
6081 
6082 void VmaAllocation_T::BlockAllocMap()
6083 {
6084  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
6085 
6086  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
6087  {
6088  ++m_MapCount;
6089  }
6090  else
6091  {
6092  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
6093  }
6094 }
6095 
6096 void VmaAllocation_T::BlockAllocUnmap()
6097 {
6098  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
6099 
6100  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
6101  {
6102  --m_MapCount;
6103  }
6104  else
6105  {
6106  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
6107  }
6108 }
6109 
6110 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
6111 {
6112  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
6113 
6114  if(m_MapCount != 0)
6115  {
6116  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
6117  {
6118  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
6119  *ppData = m_DedicatedAllocation.m_pMappedData;
6120  ++m_MapCount;
6121  return VK_SUCCESS;
6122  }
6123  else
6124  {
6125  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
6126  return VK_ERROR_MEMORY_MAP_FAILED;
6127  }
6128  }
6129  else
6130  {
6131  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
6132  hAllocator->m_hDevice,
6133  m_DedicatedAllocation.m_hMemory,
6134  0, // offset
6135  VK_WHOLE_SIZE,
6136  0, // flags
6137  ppData);
6138  if(result == VK_SUCCESS)
6139  {
6140  m_DedicatedAllocation.m_pMappedData = *ppData;
6141  m_MapCount = 1;
6142  }
6143  return result;
6144  }
6145 }
6146 
6147 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
6148 {
6149  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
6150 
6151  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
6152  {
6153  --m_MapCount;
6154  if(m_MapCount == 0)
6155  {
6156  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
6157  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
6158  hAllocator->m_hDevice,
6159  m_DedicatedAllocation.m_hMemory);
6160  }
6161  }
6162  else
6163  {
6164  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
6165  }
6166 }
6167 
6168 #if VMA_STATS_STRING_ENABLED
6169 
6170 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
6171 {
6172  json.BeginObject();
6173 
6174  json.WriteString("Blocks");
6175  json.WriteNumber(stat.blockCount);
6176 
6177  json.WriteString("Allocations");
6178  json.WriteNumber(stat.allocationCount);
6179 
6180  json.WriteString("UnusedRanges");
6181  json.WriteNumber(stat.unusedRangeCount);
6182 
6183  json.WriteString("UsedBytes");
6184  json.WriteNumber(stat.usedBytes);
6185 
6186  json.WriteString("UnusedBytes");
6187  json.WriteNumber(stat.unusedBytes);
6188 
6189  if(stat.allocationCount > 1)
6190  {
6191  json.WriteString("AllocationSize");
6192  json.BeginObject(true);
6193  json.WriteString("Min");
6194  json.WriteNumber(stat.allocationSizeMin);
6195  json.WriteString("Avg");
6196  json.WriteNumber(stat.allocationSizeAvg);
6197  json.WriteString("Max");
6198  json.WriteNumber(stat.allocationSizeMax);
6199  json.EndObject();
6200  }
6201 
6202  if(stat.unusedRangeCount > 1)
6203  {
6204  json.WriteString("UnusedRangeSize");
6205  json.BeginObject(true);
6206  json.WriteString("Min");
6207  json.WriteNumber(stat.unusedRangeSizeMin);
6208  json.WriteString("Avg");
6209  json.WriteNumber(stat.unusedRangeSizeAvg);
6210  json.WriteString("Max");
6211  json.WriteNumber(stat.unusedRangeSizeMax);
6212  json.EndObject();
6213  }
6214 
6215  json.EndObject();
6216 }
6217 
6218 #endif // #if VMA_STATS_STRING_ENABLED
6219 
6220 struct VmaSuballocationItemSizeLess
6221 {
6222  bool operator()(
6223  const VmaSuballocationList::iterator lhs,
6224  const VmaSuballocationList::iterator rhs) const
6225  {
6226  return lhs->size < rhs->size;
6227  }
6228  bool operator()(
6229  const VmaSuballocationList::iterator lhs,
6230  VkDeviceSize rhsSize) const
6231  {
6232  return lhs->size < rhsSize;
6233  }
6234 };
6235 
6236 
6238 // class VmaBlockMetadata
6239 
6240 #if VMA_STATS_STRING_ENABLED
6241 
6242 void VmaBlockMetadata::PrintDetailedMap_Begin(class VmaJsonWriter& json,
6243  VkDeviceSize unusedBytes,
6244  size_t allocationCount,
6245  size_t unusedRangeCount) const
6246 {
6247  json.BeginObject();
6248 
6249  json.WriteString("TotalBytes");
6250  json.WriteNumber(GetSize());
6251 
6252  json.WriteString("UnusedBytes");
6253  json.WriteNumber(unusedBytes);
6254 
6255  json.WriteString("Allocations");
6256  json.WriteNumber((uint64_t)allocationCount);
6257 
6258  json.WriteString("UnusedRanges");
6259  json.WriteNumber((uint64_t)unusedRangeCount);
6260 
6261  json.WriteString("Suballocations");
6262  json.BeginArray();
6263 }
6264 
6265 void VmaBlockMetadata::PrintDetailedMap_Allocation(class VmaJsonWriter& json,
6266  VkDeviceSize offset,
6267  VmaAllocation hAllocation) const
6268 {
6269  json.BeginObject(true);
6270 
6271  json.WriteString("Offset");
6272  json.WriteNumber(offset);
6273 
6274  hAllocation->PrintParameters(json);
6275 
6276  json.EndObject();
6277 }
6278 
6279 void VmaBlockMetadata::PrintDetailedMap_UnusedRange(class VmaJsonWriter& json,
6280  VkDeviceSize offset,
6281  VkDeviceSize size) const
6282 {
6283  json.BeginObject(true);
6284 
6285  json.WriteString("Offset");
6286  json.WriteNumber(offset);
6287 
6288  json.WriteString("Type");
6289  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
6290 
6291  json.WriteString("Size");
6292  json.WriteNumber(size);
6293 
6294  json.EndObject();
6295 }
6296 
6297 void VmaBlockMetadata::PrintDetailedMap_End(class VmaJsonWriter& json) const
6298 {
6299  json.EndArray();
6300  json.EndObject();
6301 }
6302 
6303 #endif // #if VMA_STATS_STRING_ENABLED
6304 
6306 // class VmaBlockMetadata_Generic
6307 
6308 VmaBlockMetadata_Generic::VmaBlockMetadata_Generic(VmaAllocator hAllocator) :
6309  m_FreeCount(0),
6310  m_SumFreeSize(0),
6311  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
6312  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
6313 {
6314 }
6315 
6316 VmaBlockMetadata_Generic::~VmaBlockMetadata_Generic()
6317 {
6318 }
6319 
6320 void VmaBlockMetadata_Generic::Init(VkDeviceSize size)
6321 {
6322  VmaBlockMetadata::Init(size);
6323  m_FreeCount = 1;
6324  m_SumFreeSize = size;
6325 
6326  VmaSuballocation suballoc = {};
6327  suballoc.offset = 0;
6328  suballoc.size = size;
6329  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6330  suballoc.hAllocation = VK_NULL_HANDLE;
6331 
6332  m_Suballocations.push_back(suballoc);
6333  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
6334  --suballocItem;
6335  m_FreeSuballocationsBySize.push_back(suballocItem);
6336 }
6337 
6338 bool VmaBlockMetadata_Generic::Validate() const
6339 {
6340  if(m_Suballocations.empty())
6341  {
6342  return false;
6343  }
6344 
6345  // Expected offset of new suballocation as calculated from previous ones.
6346  VkDeviceSize calculatedOffset = 0;
6347  // Expected number of free suballocations as calculated from traversing their list.
6348  uint32_t calculatedFreeCount = 0;
6349  // Expected sum size of free suballocations as calculated from traversing their list.
6350  VkDeviceSize calculatedSumFreeSize = 0;
6351  // Expected number of free suballocations that should be registered in
6352  // m_FreeSuballocationsBySize calculated from traversing their list.
6353  size_t freeSuballocationsToRegister = 0;
6354  // True if previous visited suballocation was free.
6355  bool prevFree = false;
6356 
6357  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6358  suballocItem != m_Suballocations.cend();
6359  ++suballocItem)
6360  {
6361  const VmaSuballocation& subAlloc = *suballocItem;
6362 
6363  // Actual offset of this suballocation doesn't match expected one.
6364  if(subAlloc.offset != calculatedOffset)
6365  {
6366  return false;
6367  }
6368 
6369  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
6370  // Two adjacent free suballocations are invalid. They should be merged.
6371  if(prevFree && currFree)
6372  {
6373  return false;
6374  }
6375 
6376  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
6377  {
6378  return false;
6379  }
6380 
6381  if(currFree)
6382  {
6383  calculatedSumFreeSize += subAlloc.size;
6384  ++calculatedFreeCount;
6385  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6386  {
6387  ++freeSuballocationsToRegister;
6388  }
6389 
6390  // Margin required between allocations - every free space must be at least that large.
6391  if(subAlloc.size < VMA_DEBUG_MARGIN)
6392  {
6393  return false;
6394  }
6395  }
6396  else
6397  {
6398  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
6399  {
6400  return false;
6401  }
6402  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
6403  {
6404  return false;
6405  }
6406 
6407  // Margin required between allocations - previous allocation must be free.
6408  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
6409  {
6410  return false;
6411  }
6412  }
6413 
6414  calculatedOffset += subAlloc.size;
6415  prevFree = currFree;
6416  }
6417 
6418  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
6419  // match expected one.
6420  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
6421  {
6422  return false;
6423  }
6424 
6425  VkDeviceSize lastSize = 0;
6426  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
6427  {
6428  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
6429 
6430  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
6431  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
6432  {
6433  return false;
6434  }
6435  // They must be sorted by size ascending.
6436  if(suballocItem->size < lastSize)
6437  {
6438  return false;
6439  }
6440 
6441  lastSize = suballocItem->size;
6442  }
6443 
6444  // Check if totals match calculacted values.
6445  if(!ValidateFreeSuballocationList() ||
6446  (calculatedOffset != GetSize()) ||
6447  (calculatedSumFreeSize != m_SumFreeSize) ||
6448  (calculatedFreeCount != m_FreeCount))
6449  {
6450  return false;
6451  }
6452 
6453  return true;
6454 }
6455 
6456 VkDeviceSize VmaBlockMetadata_Generic::GetUnusedRangeSizeMax() const
6457 {
6458  if(!m_FreeSuballocationsBySize.empty())
6459  {
6460  return m_FreeSuballocationsBySize.back()->size;
6461  }
6462  else
6463  {
6464  return 0;
6465  }
6466 }
6467 
6468 bool VmaBlockMetadata_Generic::IsEmpty() const
6469 {
6470  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
6471 }
6472 
6473 void VmaBlockMetadata_Generic::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
6474 {
6475  outInfo.blockCount = 1;
6476 
6477  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
6478  outInfo.allocationCount = rangeCount - m_FreeCount;
6479  outInfo.unusedRangeCount = m_FreeCount;
6480 
6481  outInfo.unusedBytes = m_SumFreeSize;
6482  outInfo.usedBytes = GetSize() - outInfo.unusedBytes;
6483 
6484  outInfo.allocationSizeMin = UINT64_MAX;
6485  outInfo.allocationSizeMax = 0;
6486  outInfo.unusedRangeSizeMin = UINT64_MAX;
6487  outInfo.unusedRangeSizeMax = 0;
6488 
6489  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6490  suballocItem != m_Suballocations.cend();
6491  ++suballocItem)
6492  {
6493  const VmaSuballocation& suballoc = *suballocItem;
6494  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
6495  {
6496  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
6497  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
6498  }
6499  else
6500  {
6501  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
6502  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
6503  }
6504  }
6505 }
6506 
6507 void VmaBlockMetadata_Generic::AddPoolStats(VmaPoolStats& inoutStats) const
6508 {
6509  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
6510 
6511  inoutStats.size += GetSize();
6512  inoutStats.unusedSize += m_SumFreeSize;
6513  inoutStats.allocationCount += rangeCount - m_FreeCount;
6514  inoutStats.unusedRangeCount += m_FreeCount;
6515  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
6516 }
6517 
6518 #if VMA_STATS_STRING_ENABLED
6519 
6520 void VmaBlockMetadata_Generic::PrintDetailedMap(class VmaJsonWriter& json) const
6521 {
6522  PrintDetailedMap_Begin(json,
6523  m_SumFreeSize, // unusedBytes
6524  m_Suballocations.size() - (size_t)m_FreeCount, // allocationCount
6525  m_FreeCount); // unusedRangeCount
6526 
6527  size_t i = 0;
6528  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6529  suballocItem != m_Suballocations.cend();
6530  ++suballocItem, ++i)
6531  {
6532  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6533  {
6534  PrintDetailedMap_UnusedRange(json, suballocItem->offset, suballocItem->size);
6535  }
6536  else
6537  {
6538  PrintDetailedMap_Allocation(json, suballocItem->offset, suballocItem->hAllocation);
6539  }
6540  }
6541 
6542  PrintDetailedMap_End(json);
6543 }
6544 
6545 #endif // #if VMA_STATS_STRING_ENABLED
6546 
6547 /*
6548 How many suitable free suballocations to analyze before choosing best one.
6549 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
6550  be chosen.
6551 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
6552  suballocations will be analized and best one will be chosen.
6553 - Any other value is also acceptable.
6554 */
6555 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
6556 
6557 bool VmaBlockMetadata_Generic::CreateAllocationRequest(
6558  uint32_t currentFrameIndex,
6559  uint32_t frameInUseCount,
6560  VkDeviceSize bufferImageGranularity,
6561  VkDeviceSize allocSize,
6562  VkDeviceSize allocAlignment,
6563  bool upperAddress,
6564  VmaSuballocationType allocType,
6565  bool canMakeOtherLost,
6566  VmaAllocationRequest* pAllocationRequest)
6567 {
6568  VMA_ASSERT(allocSize > 0);
6569  VMA_ASSERT(!upperAddress);
6570  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
6571  VMA_ASSERT(pAllocationRequest != VMA_NULL);
6572  VMA_HEAVY_ASSERT(Validate());
6573 
6574  // There is not enough total free space in this block to fullfill the request: Early return.
6575  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
6576  {
6577  return false;
6578  }
6579 
6580  // New algorithm, efficiently searching freeSuballocationsBySize.
6581  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
6582  if(freeSuballocCount > 0)
6583  {
6584  if(VMA_BEST_FIT)
6585  {
6586  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
6587  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
6588  m_FreeSuballocationsBySize.data(),
6589  m_FreeSuballocationsBySize.data() + freeSuballocCount,
6590  allocSize + 2 * VMA_DEBUG_MARGIN,
6591  VmaSuballocationItemSizeLess());
6592  size_t index = it - m_FreeSuballocationsBySize.data();
6593  for(; index < freeSuballocCount; ++index)
6594  {
6595  if(CheckAllocation(
6596  currentFrameIndex,
6597  frameInUseCount,
6598  bufferImageGranularity,
6599  allocSize,
6600  allocAlignment,
6601  allocType,
6602  m_FreeSuballocationsBySize[index],
6603  false, // canMakeOtherLost
6604  &pAllocationRequest->offset,
6605  &pAllocationRequest->itemsToMakeLostCount,
6606  &pAllocationRequest->sumFreeSize,
6607  &pAllocationRequest->sumItemSize))
6608  {
6609  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
6610  return true;
6611  }
6612  }
6613  }
6614  else
6615  {
6616  // Search staring from biggest suballocations.
6617  for(size_t index = freeSuballocCount; index--; )
6618  {
6619  if(CheckAllocation(
6620  currentFrameIndex,
6621  frameInUseCount,
6622  bufferImageGranularity,
6623  allocSize,
6624  allocAlignment,
6625  allocType,
6626  m_FreeSuballocationsBySize[index],
6627  false, // canMakeOtherLost
6628  &pAllocationRequest->offset,
6629  &pAllocationRequest->itemsToMakeLostCount,
6630  &pAllocationRequest->sumFreeSize,
6631  &pAllocationRequest->sumItemSize))
6632  {
6633  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
6634  return true;
6635  }
6636  }
6637  }
6638  }
6639 
6640  if(canMakeOtherLost)
6641  {
6642  // Brute-force algorithm. TODO: Come up with something better.
6643 
6644  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
6645  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
6646 
6647  VmaAllocationRequest tmpAllocRequest = {};
6648  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
6649  suballocIt != m_Suballocations.end();
6650  ++suballocIt)
6651  {
6652  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
6653  suballocIt->hAllocation->CanBecomeLost())
6654  {
6655  if(CheckAllocation(
6656  currentFrameIndex,
6657  frameInUseCount,
6658  bufferImageGranularity,
6659  allocSize,
6660  allocAlignment,
6661  allocType,
6662  suballocIt,
6663  canMakeOtherLost,
6664  &tmpAllocRequest.offset,
6665  &tmpAllocRequest.itemsToMakeLostCount,
6666  &tmpAllocRequest.sumFreeSize,
6667  &tmpAllocRequest.sumItemSize))
6668  {
6669  tmpAllocRequest.item = suballocIt;
6670 
6671  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
6672  {
6673  *pAllocationRequest = tmpAllocRequest;
6674  }
6675  }
6676  }
6677  }
6678 
6679  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
6680  {
6681  return true;
6682  }
6683  }
6684 
6685  return false;
6686 }
6687 
6688 bool VmaBlockMetadata_Generic::MakeRequestedAllocationsLost(
6689  uint32_t currentFrameIndex,
6690  uint32_t frameInUseCount,
6691  VmaAllocationRequest* pAllocationRequest)
6692 {
6693  while(pAllocationRequest->itemsToMakeLostCount > 0)
6694  {
6695  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
6696  {
6697  ++pAllocationRequest->item;
6698  }
6699  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
6700  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
6701  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
6702  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
6703  {
6704  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
6705  --pAllocationRequest->itemsToMakeLostCount;
6706  }
6707  else
6708  {
6709  return false;
6710  }
6711  }
6712 
6713  VMA_HEAVY_ASSERT(Validate());
6714  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
6715  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
6716 
6717  return true;
6718 }
6719 
6720 uint32_t VmaBlockMetadata_Generic::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
6721 {
6722  uint32_t lostAllocationCount = 0;
6723  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
6724  it != m_Suballocations.end();
6725  ++it)
6726  {
6727  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
6728  it->hAllocation->CanBecomeLost() &&
6729  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
6730  {
6731  it = FreeSuballocation(it);
6732  ++lostAllocationCount;
6733  }
6734  }
6735  return lostAllocationCount;
6736 }
6737 
6738 VkResult VmaBlockMetadata_Generic::CheckCorruption(const void* pBlockData)
6739 {
6740  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
6741  it != m_Suballocations.end();
6742  ++it)
6743  {
6744  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
6745  {
6746  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
6747  {
6748  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
6749  return VK_ERROR_VALIDATION_FAILED_EXT;
6750  }
6751  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
6752  {
6753  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
6754  return VK_ERROR_VALIDATION_FAILED_EXT;
6755  }
6756  }
6757  }
6758 
6759  return VK_SUCCESS;
6760 }
6761 
6762 void VmaBlockMetadata_Generic::Alloc(
6763  const VmaAllocationRequest& request,
6764  VmaSuballocationType type,
6765  VkDeviceSize allocSize,
6766  bool upperAddress,
6767  VmaAllocation hAllocation)
6768 {
6769  VMA_ASSERT(!upperAddress);
6770  VMA_ASSERT(request.item != m_Suballocations.end());
6771  VmaSuballocation& suballoc = *request.item;
6772  // Given suballocation is a free block.
6773  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
6774  // Given offset is inside this suballocation.
6775  VMA_ASSERT(request.offset >= suballoc.offset);
6776  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
6777  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
6778  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
6779 
6780  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
6781  // it to become used.
6782  UnregisterFreeSuballocation(request.item);
6783 
6784  suballoc.offset = request.offset;
6785  suballoc.size = allocSize;
6786  suballoc.type = type;
6787  suballoc.hAllocation = hAllocation;
6788 
6789  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
6790  if(paddingEnd)
6791  {
6792  VmaSuballocation paddingSuballoc = {};
6793  paddingSuballoc.offset = request.offset + allocSize;
6794  paddingSuballoc.size = paddingEnd;
6795  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6796  VmaSuballocationList::iterator next = request.item;
6797  ++next;
6798  const VmaSuballocationList::iterator paddingEndItem =
6799  m_Suballocations.insert(next, paddingSuballoc);
6800  RegisterFreeSuballocation(paddingEndItem);
6801  }
6802 
6803  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
6804  if(paddingBegin)
6805  {
6806  VmaSuballocation paddingSuballoc = {};
6807  paddingSuballoc.offset = request.offset - paddingBegin;
6808  paddingSuballoc.size = paddingBegin;
6809  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6810  const VmaSuballocationList::iterator paddingBeginItem =
6811  m_Suballocations.insert(request.item, paddingSuballoc);
6812  RegisterFreeSuballocation(paddingBeginItem);
6813  }
6814 
6815  // Update totals.
6816  m_FreeCount = m_FreeCount - 1;
6817  if(paddingBegin > 0)
6818  {
6819  ++m_FreeCount;
6820  }
6821  if(paddingEnd > 0)
6822  {
6823  ++m_FreeCount;
6824  }
6825  m_SumFreeSize -= allocSize;
6826 }
6827 
6828 void VmaBlockMetadata_Generic::Free(const VmaAllocation allocation)
6829 {
6830  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
6831  suballocItem != m_Suballocations.end();
6832  ++suballocItem)
6833  {
6834  VmaSuballocation& suballoc = *suballocItem;
6835  if(suballoc.hAllocation == allocation)
6836  {
6837  FreeSuballocation(suballocItem);
6838  VMA_HEAVY_ASSERT(Validate());
6839  return;
6840  }
6841  }
6842  VMA_ASSERT(0 && "Not found!");
6843 }
6844 
6845 void VmaBlockMetadata_Generic::FreeAtOffset(VkDeviceSize offset)
6846 {
6847  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
6848  suballocItem != m_Suballocations.end();
6849  ++suballocItem)
6850  {
6851  VmaSuballocation& suballoc = *suballocItem;
6852  if(suballoc.offset == offset)
6853  {
6854  FreeSuballocation(suballocItem);
6855  return;
6856  }
6857  }
6858  VMA_ASSERT(0 && "Not found!");
6859 }
6860 
6861 bool VmaBlockMetadata_Generic::ValidateFreeSuballocationList() const
6862 {
6863  VkDeviceSize lastSize = 0;
6864  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
6865  {
6866  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
6867 
6868  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
6869  {
6870  VMA_ASSERT(0);
6871  return false;
6872  }
6873  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6874  {
6875  VMA_ASSERT(0);
6876  return false;
6877  }
6878  if(it->size < lastSize)
6879  {
6880  VMA_ASSERT(0);
6881  return false;
6882  }
6883 
6884  lastSize = it->size;
6885  }
6886  return true;
6887 }
6888 
6889 bool VmaBlockMetadata_Generic::CheckAllocation(
6890  uint32_t currentFrameIndex,
6891  uint32_t frameInUseCount,
6892  VkDeviceSize bufferImageGranularity,
6893  VkDeviceSize allocSize,
6894  VkDeviceSize allocAlignment,
6895  VmaSuballocationType allocType,
6896  VmaSuballocationList::const_iterator suballocItem,
6897  bool canMakeOtherLost,
6898  VkDeviceSize* pOffset,
6899  size_t* itemsToMakeLostCount,
6900  VkDeviceSize* pSumFreeSize,
6901  VkDeviceSize* pSumItemSize) const
6902 {
6903  VMA_ASSERT(allocSize > 0);
6904  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
6905  VMA_ASSERT(suballocItem != m_Suballocations.cend());
6906  VMA_ASSERT(pOffset != VMA_NULL);
6907 
6908  *itemsToMakeLostCount = 0;
6909  *pSumFreeSize = 0;
6910  *pSumItemSize = 0;
6911 
6912  if(canMakeOtherLost)
6913  {
6914  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6915  {
6916  *pSumFreeSize = suballocItem->size;
6917  }
6918  else
6919  {
6920  if(suballocItem->hAllocation->CanBecomeLost() &&
6921  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
6922  {
6923  ++*itemsToMakeLostCount;
6924  *pSumItemSize = suballocItem->size;
6925  }
6926  else
6927  {
6928  return false;
6929  }
6930  }
6931 
6932  // Remaining size is too small for this request: Early return.
6933  if(GetSize() - suballocItem->offset < allocSize)
6934  {
6935  return false;
6936  }
6937 
6938  // Start from offset equal to beginning of this suballocation.
6939  *pOffset = suballocItem->offset;
6940 
6941  // Apply VMA_DEBUG_MARGIN at the beginning.
6942  if(VMA_DEBUG_MARGIN > 0)
6943  {
6944  *pOffset += VMA_DEBUG_MARGIN;
6945  }
6946 
6947  // Apply alignment.
6948  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
6949 
6950  // Check previous suballocations for BufferImageGranularity conflicts.
6951  // Make bigger alignment if necessary.
6952  if(bufferImageGranularity > 1)
6953  {
6954  bool bufferImageGranularityConflict = false;
6955  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
6956  while(prevSuballocItem != m_Suballocations.cbegin())
6957  {
6958  --prevSuballocItem;
6959  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
6960  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
6961  {
6962  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
6963  {
6964  bufferImageGranularityConflict = true;
6965  break;
6966  }
6967  }
6968  else
6969  // Already on previous page.
6970  break;
6971  }
6972  if(bufferImageGranularityConflict)
6973  {
6974  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
6975  }
6976  }
6977 
6978  // Now that we have final *pOffset, check if we are past suballocItem.
6979  // If yes, return false - this function should be called for another suballocItem as starting point.
6980  if(*pOffset >= suballocItem->offset + suballocItem->size)
6981  {
6982  return false;
6983  }
6984 
6985  // Calculate padding at the beginning based on current offset.
6986  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
6987 
6988  // Calculate required margin at the end.
6989  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
6990 
6991  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
6992  // Another early return check.
6993  if(suballocItem->offset + totalSize > GetSize())
6994  {
6995  return false;
6996  }
6997 
6998  // Advance lastSuballocItem until desired size is reached.
6999  // Update itemsToMakeLostCount.
7000  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
7001  if(totalSize > suballocItem->size)
7002  {
7003  VkDeviceSize remainingSize = totalSize - suballocItem->size;
7004  while(remainingSize > 0)
7005  {
7006  ++lastSuballocItem;
7007  if(lastSuballocItem == m_Suballocations.cend())
7008  {
7009  return false;
7010  }
7011  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
7012  {
7013  *pSumFreeSize += lastSuballocItem->size;
7014  }
7015  else
7016  {
7017  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
7018  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
7019  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
7020  {
7021  ++*itemsToMakeLostCount;
7022  *pSumItemSize += lastSuballocItem->size;
7023  }
7024  else
7025  {
7026  return false;
7027  }
7028  }
7029  remainingSize = (lastSuballocItem->size < remainingSize) ?
7030  remainingSize - lastSuballocItem->size : 0;
7031  }
7032  }
7033 
7034  // Check next suballocations for BufferImageGranularity conflicts.
7035  // If conflict exists, we must mark more allocations lost or fail.
7036  if(bufferImageGranularity > 1)
7037  {
7038  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
7039  ++nextSuballocItem;
7040  while(nextSuballocItem != m_Suballocations.cend())
7041  {
7042  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
7043  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
7044  {
7045  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
7046  {
7047  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
7048  if(nextSuballoc.hAllocation->CanBecomeLost() &&
7049  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
7050  {
7051  ++*itemsToMakeLostCount;
7052  }
7053  else
7054  {
7055  return false;
7056  }
7057  }
7058  }
7059  else
7060  {
7061  // Already on next page.
7062  break;
7063  }
7064  ++nextSuballocItem;
7065  }
7066  }
7067  }
7068  else
7069  {
7070  const VmaSuballocation& suballoc = *suballocItem;
7071  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
7072 
7073  *pSumFreeSize = suballoc.size;
7074 
7075  // Size of this suballocation is too small for this request: Early return.
7076  if(suballoc.size < allocSize)
7077  {
7078  return false;
7079  }
7080 
7081  // Start from offset equal to beginning of this suballocation.
7082  *pOffset = suballoc.offset;
7083 
7084  // Apply VMA_DEBUG_MARGIN at the beginning.
7085  if(VMA_DEBUG_MARGIN > 0)
7086  {
7087  *pOffset += VMA_DEBUG_MARGIN;
7088  }
7089 
7090  // Apply alignment.
7091  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
7092 
7093  // Check previous suballocations for BufferImageGranularity conflicts.
7094  // Make bigger alignment if necessary.
7095  if(bufferImageGranularity > 1)
7096  {
7097  bool bufferImageGranularityConflict = false;
7098  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
7099  while(prevSuballocItem != m_Suballocations.cbegin())
7100  {
7101  --prevSuballocItem;
7102  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
7103  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
7104  {
7105  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
7106  {
7107  bufferImageGranularityConflict = true;
7108  break;
7109  }
7110  }
7111  else
7112  // Already on previous page.
7113  break;
7114  }
7115  if(bufferImageGranularityConflict)
7116  {
7117  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
7118  }
7119  }
7120 
7121  // Calculate padding at the beginning based on current offset.
7122  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
7123 
7124  // Calculate required margin at the end.
7125  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
7126 
7127  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
7128  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
7129  {
7130  return false;
7131  }
7132 
7133  // Check next suballocations for BufferImageGranularity conflicts.
7134  // If conflict exists, allocation cannot be made here.
7135  if(bufferImageGranularity > 1)
7136  {
7137  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
7138  ++nextSuballocItem;
7139  while(nextSuballocItem != m_Suballocations.cend())
7140  {
7141  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
7142  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
7143  {
7144  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
7145  {
7146  return false;
7147  }
7148  }
7149  else
7150  {
7151  // Already on next page.
7152  break;
7153  }
7154  ++nextSuballocItem;
7155  }
7156  }
7157  }
7158 
7159  // All tests passed: Success. pOffset is already filled.
7160  return true;
7161 }
7162 
7163 void VmaBlockMetadata_Generic::MergeFreeWithNext(VmaSuballocationList::iterator item)
7164 {
7165  VMA_ASSERT(item != m_Suballocations.end());
7166  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
7167 
7168  VmaSuballocationList::iterator nextItem = item;
7169  ++nextItem;
7170  VMA_ASSERT(nextItem != m_Suballocations.end());
7171  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
7172 
7173  item->size += nextItem->size;
7174  --m_FreeCount;
7175  m_Suballocations.erase(nextItem);
7176 }
7177 
7178 VmaSuballocationList::iterator VmaBlockMetadata_Generic::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
7179 {
7180  // Change this suballocation to be marked as free.
7181  VmaSuballocation& suballoc = *suballocItem;
7182  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
7183  suballoc.hAllocation = VK_NULL_HANDLE;
7184 
7185  // Update totals.
7186  ++m_FreeCount;
7187  m_SumFreeSize += suballoc.size;
7188 
7189  // Merge with previous and/or next suballocation if it's also free.
7190  bool mergeWithNext = false;
7191  bool mergeWithPrev = false;
7192 
7193  VmaSuballocationList::iterator nextItem = suballocItem;
7194  ++nextItem;
7195  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
7196  {
7197  mergeWithNext = true;
7198  }
7199 
7200  VmaSuballocationList::iterator prevItem = suballocItem;
7201  if(suballocItem != m_Suballocations.begin())
7202  {
7203  --prevItem;
7204  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
7205  {
7206  mergeWithPrev = true;
7207  }
7208  }
7209 
7210  if(mergeWithNext)
7211  {
7212  UnregisterFreeSuballocation(nextItem);
7213  MergeFreeWithNext(suballocItem);
7214  }
7215 
7216  if(mergeWithPrev)
7217  {
7218  UnregisterFreeSuballocation(prevItem);
7219  MergeFreeWithNext(prevItem);
7220  RegisterFreeSuballocation(prevItem);
7221  return prevItem;
7222  }
7223  else
7224  {
7225  RegisterFreeSuballocation(suballocItem);
7226  return suballocItem;
7227  }
7228 }
7229 
7230 void VmaBlockMetadata_Generic::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
7231 {
7232  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
7233  VMA_ASSERT(item->size > 0);
7234 
7235  // You may want to enable this validation at the beginning or at the end of
7236  // this function, depending on what do you want to check.
7237  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
7238 
7239  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
7240  {
7241  if(m_FreeSuballocationsBySize.empty())
7242  {
7243  m_FreeSuballocationsBySize.push_back(item);
7244  }
7245  else
7246  {
7247  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
7248  }
7249  }
7250 
7251  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
7252 }
7253 
7254 
7255 void VmaBlockMetadata_Generic::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
7256 {
7257  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
7258  VMA_ASSERT(item->size > 0);
7259 
7260  // You may want to enable this validation at the beginning or at the end of
7261  // this function, depending on what do you want to check.
7262  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
7263 
7264  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
7265  {
7266  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
7267  m_FreeSuballocationsBySize.data(),
7268  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
7269  item,
7270  VmaSuballocationItemSizeLess());
7271  for(size_t index = it - m_FreeSuballocationsBySize.data();
7272  index < m_FreeSuballocationsBySize.size();
7273  ++index)
7274  {
7275  if(m_FreeSuballocationsBySize[index] == item)
7276  {
7277  VmaVectorRemove(m_FreeSuballocationsBySize, index);
7278  return;
7279  }
7280  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
7281  }
7282  VMA_ASSERT(0 && "Not found.");
7283  }
7284 
7285  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
7286 }
7287 
7289 // class VmaBlockMetadata_Linear
7290 
7291 VmaBlockMetadata_Linear::VmaBlockMetadata_Linear(VmaAllocator hAllocator) :
7292  m_SumFreeSize(0),
7293  m_Suballocations0(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
7294  m_Suballocations1(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
7295  m_1stVectorIndex(0),
7296  m_2ndVectorMode(SECOND_VECTOR_EMPTY),
7297  m_1stNullItemsBeginCount(0),
7298  m_1stNullItemsMiddleCount(0),
7299  m_2ndNullItemsCount(0)
7300 {
7301 }
7302 
7303 VmaBlockMetadata_Linear::~VmaBlockMetadata_Linear()
7304 {
7305 }
7306 
7307 void VmaBlockMetadata_Linear::Init(VkDeviceSize size)
7308 {
7309  VmaBlockMetadata::Init(size);
7310  m_SumFreeSize = size;
7311 }
7312 
7313 bool VmaBlockMetadata_Linear::Validate() const
7314 {
7315  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7316  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7317 
7318  if(suballocations2nd.empty() != (m_2ndVectorMode == SECOND_VECTOR_EMPTY))
7319  {
7320  return false;
7321  }
7322  if(suballocations1st.empty() && !suballocations2nd.empty() &&
7323  m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7324  {
7325  return false;
7326  }
7327  if(!suballocations1st.empty())
7328  {
7329  // Null item at the beginning should be accounted into m_1stNullItemsBeginCount.
7330  if(suballocations1st[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE)
7331  {
7332  return false;
7333  }
7334  // Null item at the end should be just pop_back().
7335  if(suballocations1st.back().hAllocation == VK_NULL_HANDLE)
7336  {
7337  return false;
7338  }
7339  }
7340  if(!suballocations2nd.empty())
7341  {
7342  // Null item at the end should be just pop_back().
7343  if(suballocations2nd.back().hAllocation == VK_NULL_HANDLE)
7344  {
7345  return false;
7346  }
7347  }
7348 
7349  if(m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount > suballocations1st.size())
7350  {
7351  return false;
7352  }
7353  if(m_2ndNullItemsCount > suballocations2nd.size())
7354  {
7355  return false;
7356  }
7357 
7358  VkDeviceSize sumUsedSize = 0;
7359  const size_t suballoc1stCount = suballocations1st.size();
7360  VkDeviceSize offset = VMA_DEBUG_MARGIN;
7361 
7362  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7363  {
7364  const size_t suballoc2ndCount = suballocations2nd.size();
7365  size_t nullItem2ndCount = 0;
7366  for(size_t i = 0; i < suballoc2ndCount; ++i)
7367  {
7368  const VmaSuballocation& suballoc = suballocations2nd[i];
7369  const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
7370 
7371  if(currFree != (suballoc.hAllocation == VK_NULL_HANDLE))
7372  {
7373  return false;
7374  }
7375  if(suballoc.offset < offset)
7376  {
7377  return false;
7378  }
7379 
7380  if(!currFree)
7381  {
7382  if(suballoc.hAllocation->GetOffset() != suballoc.offset)
7383  {
7384  return false;
7385  }
7386  if(suballoc.hAllocation->GetSize() != suballoc.size)
7387  {
7388  return false;
7389  }
7390  sumUsedSize += suballoc.size;
7391  }
7392  else
7393  {
7394  ++nullItem2ndCount;
7395  }
7396 
7397  offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN;
7398  }
7399 
7400  if(nullItem2ndCount != m_2ndNullItemsCount)
7401  {
7402  return false;
7403  }
7404  }
7405 
7406  for(size_t i = 0; i < m_1stNullItemsBeginCount; ++i)
7407  {
7408  const VmaSuballocation& suballoc = suballocations1st[i];
7409  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE ||
7410  suballoc.hAllocation != VK_NULL_HANDLE)
7411  {
7412  return false;
7413  }
7414  }
7415 
7416  size_t nullItem1stCount = m_1stNullItemsBeginCount;
7417 
7418  for(size_t i = m_1stNullItemsBeginCount; i < suballoc1stCount; ++i)
7419  {
7420  const VmaSuballocation& suballoc = suballocations1st[i];
7421  const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
7422 
7423  if(currFree != (suballoc.hAllocation == VK_NULL_HANDLE))
7424  {
7425  return false;
7426  }
7427  if(suballoc.offset < offset)
7428  {
7429  return false;
7430  }
7431  if(i < m_1stNullItemsBeginCount && !currFree)
7432  {
7433  return false;
7434  }
7435 
7436  if(!currFree)
7437  {
7438  if(suballoc.hAllocation->GetOffset() != suballoc.offset)
7439  {
7440  return false;
7441  }
7442  if(suballoc.hAllocation->GetSize() != suballoc.size)
7443  {
7444  return false;
7445  }
7446  sumUsedSize += suballoc.size;
7447  }
7448  else
7449  {
7450  ++nullItem1stCount;
7451  }
7452 
7453  offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN;
7454  }
7455  if(nullItem1stCount != m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount)
7456  {
7457  return false;
7458  }
7459 
7460  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
7461  {
7462  const size_t suballoc2ndCount = suballocations2nd.size();
7463  size_t nullItem2ndCount = 0;
7464  for(size_t i = suballoc2ndCount; i--; )
7465  {
7466  const VmaSuballocation& suballoc = suballocations2nd[i];
7467  const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
7468 
7469  if(currFree != (suballoc.hAllocation == VK_NULL_HANDLE))
7470  {
7471  return false;
7472  }
7473  if(suballoc.offset < offset)
7474  {
7475  return false;
7476  }
7477 
7478  if(!currFree)
7479  {
7480  if(suballoc.hAllocation->GetOffset() != suballoc.offset)
7481  {
7482  return false;
7483  }
7484  if(suballoc.hAllocation->GetSize() != suballoc.size)
7485  {
7486  return false;
7487  }
7488  sumUsedSize += suballoc.size;
7489  }
7490  else
7491  {
7492  ++nullItem2ndCount;
7493  }
7494 
7495  offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN;
7496  }
7497 
7498  if(nullItem2ndCount != m_2ndNullItemsCount)
7499  {
7500  return false;
7501  }
7502  }
7503 
7504  if(offset > GetSize())
7505  {
7506  return false;
7507  }
7508  if(m_SumFreeSize != GetSize() - sumUsedSize)
7509  {
7510  return false;
7511  }
7512 
7513  return true;
7514 }
7515 
7516 size_t VmaBlockMetadata_Linear::GetAllocationCount() const
7517 {
7518  return AccessSuballocations1st().size() - (m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount) +
7519  AccessSuballocations2nd().size() - m_2ndNullItemsCount;
7520 }
7521 
7522 VkDeviceSize VmaBlockMetadata_Linear::GetUnusedRangeSizeMax() const
7523 {
7524  const VkDeviceSize size = GetSize();
7525 
7526  /*
7527  We don't consider gaps inside allocation vectors with freed allocations because
7528  they are not suitable for reuse in linear allocator. We consider only space that
7529  is available for new allocations.
7530  */
7531  if(IsEmpty())
7532  {
7533  return size;
7534  }
7535 
7536  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7537 
7538  switch(m_2ndVectorMode)
7539  {
7540  case SECOND_VECTOR_EMPTY:
7541  /*
7542  Available space is after end of 1st, as well as before beginning of 1st (which
7543  whould make it a ring buffer).
7544  */
7545  {
7546  const size_t suballocations1stCount = suballocations1st.size();
7547  VMA_ASSERT(suballocations1stCount > m_1stNullItemsBeginCount);
7548  const VmaSuballocation& firstSuballoc = suballocations1st[m_1stNullItemsBeginCount];
7549  const VmaSuballocation& lastSuballoc = suballocations1st[suballocations1stCount - 1];
7550  return VMA_MAX(
7551  firstSuballoc.offset,
7552  size - (lastSuballoc.offset + lastSuballoc.size));
7553  }
7554  break;
7555 
7556  case SECOND_VECTOR_RING_BUFFER:
7557  /*
7558  Available space is only between end of 2nd and beginning of 1st.
7559  */
7560  {
7561  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7562  const VmaSuballocation& lastSuballoc2nd = suballocations2nd.back();
7563  const VmaSuballocation& firstSuballoc1st = suballocations1st[m_1stNullItemsBeginCount];
7564  return firstSuballoc1st.offset - (lastSuballoc2nd.offset + lastSuballoc2nd.size);
7565  }
7566  break;
7567 
7568  case SECOND_VECTOR_DOUBLE_STACK:
7569  /*
7570  Available space is only between end of 1st and top of 2nd.
7571  */
7572  {
7573  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7574  const VmaSuballocation& topSuballoc2nd = suballocations2nd.back();
7575  const VmaSuballocation& lastSuballoc1st = suballocations1st.back();
7576  return topSuballoc2nd.offset - (lastSuballoc1st.offset + lastSuballoc1st.size);
7577  }
7578  break;
7579 
7580  default:
7581  VMA_ASSERT(0);
7582  return 0;
7583  }
7584 }
7585 
7586 void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
7587 {
7588  const VkDeviceSize size = GetSize();
7589  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7590  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7591  const size_t suballoc1stCount = suballocations1st.size();
7592  const size_t suballoc2ndCount = suballocations2nd.size();
7593 
7594  outInfo.blockCount = 1;
7595  outInfo.allocationCount = (uint32_t)GetAllocationCount();
7596  outInfo.unusedRangeCount = 0;
7597  outInfo.usedBytes = 0;
7598  outInfo.allocationSizeMin = UINT64_MAX;
7599  outInfo.allocationSizeMax = 0;
7600  outInfo.unusedRangeSizeMin = UINT64_MAX;
7601  outInfo.unusedRangeSizeMax = 0;
7602 
7603  VkDeviceSize lastOffset = 0;
7604 
7605  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7606  {
7607  const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
7608  size_t nextAlloc2ndIndex = 0;
7609  while(lastOffset < freeSpace2ndTo1stEnd)
7610  {
7611  // Find next non-null allocation or move nextAllocIndex to the end.
7612  while(nextAlloc2ndIndex < suballoc2ndCount &&
7613  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7614  {
7615  ++nextAlloc2ndIndex;
7616  }
7617 
7618  // Found non-null allocation.
7619  if(nextAlloc2ndIndex < suballoc2ndCount)
7620  {
7621  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7622 
7623  // 1. Process free space before this allocation.
7624  if(lastOffset < suballoc.offset)
7625  {
7626  // There is free space from lastOffset to suballoc.offset.
7627  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7628  ++outInfo.unusedRangeCount;
7629  outInfo.unusedBytes += unusedRangeSize;
7630  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7631  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7632  }
7633 
7634  // 2. Process this allocation.
7635  // There is allocation with suballoc.offset, suballoc.size.
7636  outInfo.usedBytes += suballoc.size;
7637  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
7638  outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size);
7639 
7640  // 3. Prepare for next iteration.
7641  lastOffset = suballoc.offset + suballoc.size;
7642  ++nextAlloc2ndIndex;
7643  }
7644  // We are at the end.
7645  else
7646  {
7647  // There is free space from lastOffset to freeSpace2ndTo1stEnd.
7648  if(lastOffset < freeSpace2ndTo1stEnd)
7649  {
7650  const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset;
7651  ++outInfo.unusedRangeCount;
7652  outInfo.unusedBytes += unusedRangeSize;
7653  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7654  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7655  }
7656 
7657  // End of loop.
7658  lastOffset = freeSpace2ndTo1stEnd;
7659  }
7660  }
7661  }
7662 
7663  size_t nextAlloc1stIndex = m_1stNullItemsBeginCount;
7664  const VkDeviceSize freeSpace1stTo2ndEnd =
7665  m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size;
7666  while(lastOffset < freeSpace1stTo2ndEnd)
7667  {
7668  // Find next non-null allocation or move nextAllocIndex to the end.
7669  while(nextAlloc1stIndex < suballoc1stCount &&
7670  suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
7671  {
7672  ++nextAlloc1stIndex;
7673  }
7674 
7675  // Found non-null allocation.
7676  if(nextAlloc1stIndex < suballoc1stCount)
7677  {
7678  const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
7679 
7680  // 1. Process free space before this allocation.
7681  if(lastOffset < suballoc.offset)
7682  {
7683  // There is free space from lastOffset to suballoc.offset.
7684  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7685  ++outInfo.unusedRangeCount;
7686  outInfo.unusedBytes += unusedRangeSize;
7687  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7688  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7689  }
7690 
7691  // 2. Process this allocation.
7692  // There is allocation with suballoc.offset, suballoc.size.
7693  outInfo.usedBytes += suballoc.size;
7694  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
7695  outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size);
7696 
7697  // 3. Prepare for next iteration.
7698  lastOffset = suballoc.offset + suballoc.size;
7699  ++nextAlloc1stIndex;
7700  }
7701  // We are at the end.
7702  else
7703  {
7704  // There is free space from lastOffset to freeSpace1stTo2ndEnd.
7705  if(lastOffset < freeSpace1stTo2ndEnd)
7706  {
7707  const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset;
7708  ++outInfo.unusedRangeCount;
7709  outInfo.unusedBytes += unusedRangeSize;
7710  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7711  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7712  }
7713 
7714  // End of loop.
7715  lastOffset = freeSpace1stTo2ndEnd;
7716  }
7717  }
7718 
7719  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
7720  {
7721  size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
7722  while(lastOffset < size)
7723  {
7724  // Find next non-null allocation or move nextAllocIndex to the end.
7725  while(nextAlloc2ndIndex != SIZE_MAX &&
7726  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7727  {
7728  --nextAlloc2ndIndex;
7729  }
7730 
7731  // Found non-null allocation.
7732  if(nextAlloc2ndIndex != SIZE_MAX)
7733  {
7734  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7735 
7736  // 1. Process free space before this allocation.
7737  if(lastOffset < suballoc.offset)
7738  {
7739  // There is free space from lastOffset to suballoc.offset.
7740  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7741  ++outInfo.unusedRangeCount;
7742  outInfo.unusedBytes += unusedRangeSize;
7743  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7744  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7745  }
7746 
7747  // 2. Process this allocation.
7748  // There is allocation with suballoc.offset, suballoc.size.
7749  outInfo.usedBytes += suballoc.size;
7750  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
7751  outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size);
7752 
7753  // 3. Prepare for next iteration.
7754  lastOffset = suballoc.offset + suballoc.size;
7755  --nextAlloc2ndIndex;
7756  }
7757  // We are at the end.
7758  else
7759  {
7760  // There is free space from lastOffset to size.
7761  if(lastOffset < size)
7762  {
7763  const VkDeviceSize unusedRangeSize = size - lastOffset;
7764  ++outInfo.unusedRangeCount;
7765  outInfo.unusedBytes += unusedRangeSize;
7766  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7767  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7768  }
7769 
7770  // End of loop.
7771  lastOffset = size;
7772  }
7773  }
7774  }
7775 
7776  outInfo.unusedBytes = size - outInfo.usedBytes;
7777 }
7778 
7779 void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const
7780 {
7781  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7782  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7783  const VkDeviceSize size = GetSize();
7784  const size_t suballoc1stCount = suballocations1st.size();
7785  const size_t suballoc2ndCount = suballocations2nd.size();
7786 
7787  inoutStats.size += size;
7788 
7789  VkDeviceSize lastOffset = 0;
7790 
7791  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7792  {
7793  const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
7794  size_t nextAlloc2ndIndex = m_1stNullItemsBeginCount;
7795  while(lastOffset < freeSpace2ndTo1stEnd)
7796  {
7797  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
7798  while(nextAlloc2ndIndex < suballoc2ndCount &&
7799  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7800  {
7801  ++nextAlloc2ndIndex;
7802  }
7803 
7804  // Found non-null allocation.
7805  if(nextAlloc2ndIndex < suballoc2ndCount)
7806  {
7807  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7808 
7809  // 1. Process free space before this allocation.
7810  if(lastOffset < suballoc.offset)
7811  {
7812  // There is free space from lastOffset to suballoc.offset.
7813  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7814  inoutStats.unusedSize += unusedRangeSize;
7815  ++inoutStats.unusedRangeCount;
7816  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7817  }
7818 
7819  // 2. Process this allocation.
7820  // There is allocation with suballoc.offset, suballoc.size.
7821  ++inoutStats.allocationCount;
7822 
7823  // 3. Prepare for next iteration.
7824  lastOffset = suballoc.offset + suballoc.size;
7825  ++nextAlloc2ndIndex;
7826  }
7827  // We are at the end.
7828  else
7829  {
7830  if(lastOffset < freeSpace2ndTo1stEnd)
7831  {
7832  // There is free space from lastOffset to freeSpace2ndTo1stEnd.
7833  const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset;
7834  inoutStats.unusedSize += unusedRangeSize;
7835  ++inoutStats.unusedRangeCount;
7836  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7837  }
7838 
7839  // End of loop.
7840  lastOffset = freeSpace2ndTo1stEnd;
7841  }
7842  }
7843  }
7844 
7845  size_t nextAlloc1stIndex = m_1stNullItemsBeginCount;
7846  const VkDeviceSize freeSpace1stTo2ndEnd =
7847  m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size;
7848  while(lastOffset < freeSpace1stTo2ndEnd)
7849  {
7850  // Find next non-null allocation or move nextAllocIndex to the end.
7851  while(nextAlloc1stIndex < suballoc1stCount &&
7852  suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
7853  {
7854  ++nextAlloc1stIndex;
7855  }
7856 
7857  // Found non-null allocation.
7858  if(nextAlloc1stIndex < suballoc1stCount)
7859  {
7860  const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
7861 
7862  // 1. Process free space before this allocation.
7863  if(lastOffset < suballoc.offset)
7864  {
7865  // There is free space from lastOffset to suballoc.offset.
7866  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7867  inoutStats.unusedSize += unusedRangeSize;
7868  ++inoutStats.unusedRangeCount;
7869  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7870  }
7871 
7872  // 2. Process this allocation.
7873  // There is allocation with suballoc.offset, suballoc.size.
7874  ++inoutStats.allocationCount;
7875 
7876  // 3. Prepare for next iteration.
7877  lastOffset = suballoc.offset + suballoc.size;
7878  ++nextAlloc1stIndex;
7879  }
7880  // We are at the end.
7881  else
7882  {
7883  if(lastOffset < freeSpace1stTo2ndEnd)
7884  {
7885  // There is free space from lastOffset to freeSpace1stTo2ndEnd.
7886  const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset;
7887  inoutStats.unusedSize += unusedRangeSize;
7888  ++inoutStats.unusedRangeCount;
7889  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7890  }
7891 
7892  // End of loop.
7893  lastOffset = freeSpace1stTo2ndEnd;
7894  }
7895  }
7896 
7897  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
7898  {
7899  size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
7900  while(lastOffset < size)
7901  {
7902  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
7903  while(nextAlloc2ndIndex != SIZE_MAX &&
7904  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7905  {
7906  --nextAlloc2ndIndex;
7907  }
7908 
7909  // Found non-null allocation.
7910  if(nextAlloc2ndIndex != SIZE_MAX)
7911  {
7912  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7913 
7914  // 1. Process free space before this allocation.
7915  if(lastOffset < suballoc.offset)
7916  {
7917  // There is free space from lastOffset to suballoc.offset.
7918  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7919  inoutStats.unusedSize += unusedRangeSize;
7920  ++inoutStats.unusedRangeCount;
7921  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7922  }
7923 
7924  // 2. Process this allocation.
7925  // There is allocation with suballoc.offset, suballoc.size.
7926  ++inoutStats.allocationCount;
7927 
7928  // 3. Prepare for next iteration.
7929  lastOffset = suballoc.offset + suballoc.size;
7930  --nextAlloc2ndIndex;
7931  }
7932  // We are at the end.
7933  else
7934  {
7935  if(lastOffset < size)
7936  {
7937  // There is free space from lastOffset to size.
7938  const VkDeviceSize unusedRangeSize = size - lastOffset;
7939  inoutStats.unusedSize += unusedRangeSize;
7940  ++inoutStats.unusedRangeCount;
7941  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7942  }
7943 
7944  // End of loop.
7945  lastOffset = size;
7946  }
7947  }
7948  }
7949 }
7950 
7951 #if VMA_STATS_STRING_ENABLED
7952 void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const
7953 {
7954  const VkDeviceSize size = GetSize();
7955  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7956  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7957  const size_t suballoc1stCount = suballocations1st.size();
7958  const size_t suballoc2ndCount = suballocations2nd.size();
7959 
7960  // FIRST PASS
7961 
7962  size_t unusedRangeCount = 0;
7963  VkDeviceSize usedBytes = 0;
7964 
7965  VkDeviceSize lastOffset = 0;
7966 
7967  size_t alloc2ndCount = 0;
7968  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7969  {
7970  const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
7971  size_t nextAlloc2ndIndex = 0;
7972  while(lastOffset < freeSpace2ndTo1stEnd)
7973  {
7974  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
7975  while(nextAlloc2ndIndex < suballoc2ndCount &&
7976  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7977  {
7978  ++nextAlloc2ndIndex;
7979  }
7980 
7981  // Found non-null allocation.
7982  if(nextAlloc2ndIndex < suballoc2ndCount)
7983  {
7984  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7985 
7986  // 1. Process free space before this allocation.
7987  if(lastOffset < suballoc.offset)
7988  {
7989  // There is free space from lastOffset to suballoc.offset.
7990  ++unusedRangeCount;
7991  }
7992 
7993  // 2. Process this allocation.
7994  // There is allocation with suballoc.offset, suballoc.size.
7995  ++alloc2ndCount;
7996  usedBytes += suballoc.size;
7997 
7998  // 3. Prepare for next iteration.
7999  lastOffset = suballoc.offset + suballoc.size;
8000  ++nextAlloc2ndIndex;
8001  }
8002  // We are at the end.
8003  else
8004  {
8005  if(lastOffset < freeSpace2ndTo1stEnd)
8006  {
8007  // There is free space from lastOffset to freeSpace2ndTo1stEnd.
8008  ++unusedRangeCount;
8009  }
8010 
8011  // End of loop.
8012  lastOffset = freeSpace2ndTo1stEnd;
8013  }
8014  }
8015  }
8016 
8017  size_t nextAlloc1stIndex = m_1stNullItemsBeginCount;
8018  size_t alloc1stCount = 0;
8019  const VkDeviceSize freeSpace1stTo2ndEnd =
8020  m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size;
8021  while(lastOffset < freeSpace1stTo2ndEnd)
8022  {
8023  // Find next non-null allocation or move nextAllocIndex to the end.
8024  while(nextAlloc1stIndex < suballoc1stCount &&
8025  suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
8026  {
8027  ++nextAlloc1stIndex;
8028  }
8029 
8030  // Found non-null allocation.
8031  if(nextAlloc1stIndex < suballoc1stCount)
8032  {
8033  const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
8034 
8035  // 1. Process free space before this allocation.
8036  if(lastOffset < suballoc.offset)
8037  {
8038  // There is free space from lastOffset to suballoc.offset.
8039  ++unusedRangeCount;
8040  }
8041 
8042  // 2. Process this allocation.
8043  // There is allocation with suballoc.offset, suballoc.size.
8044  ++alloc1stCount;
8045  usedBytes += suballoc.size;
8046 
8047  // 3. Prepare for next iteration.
8048  lastOffset = suballoc.offset + suballoc.size;
8049  ++nextAlloc1stIndex;
8050  }
8051  // We are at the end.
8052  else
8053  {
8054  if(lastOffset < size)
8055  {
8056  // There is free space from lastOffset to freeSpace1stTo2ndEnd.
8057  ++unusedRangeCount;
8058  }
8059 
8060  // End of loop.
8061  lastOffset = freeSpace1stTo2ndEnd;
8062  }
8063  }
8064 
8065  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8066  {
8067  size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
8068  while(lastOffset < size)
8069  {
8070  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
8071  while(nextAlloc2ndIndex != SIZE_MAX &&
8072  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
8073  {
8074  --nextAlloc2ndIndex;
8075  }
8076 
8077  // Found non-null allocation.
8078  if(nextAlloc2ndIndex != SIZE_MAX)
8079  {
8080  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
8081 
8082  // 1. Process free space before this allocation.
8083  if(lastOffset < suballoc.offset)
8084  {
8085  // There is free space from lastOffset to suballoc.offset.
8086  ++unusedRangeCount;
8087  }
8088 
8089  // 2. Process this allocation.
8090  // There is allocation with suballoc.offset, suballoc.size.
8091  ++alloc2ndCount;
8092  usedBytes += suballoc.size;
8093 
8094  // 3. Prepare for next iteration.
8095  lastOffset = suballoc.offset + suballoc.size;
8096  --nextAlloc2ndIndex;
8097  }
8098  // We are at the end.
8099  else
8100  {
8101  if(lastOffset < size)
8102  {
8103  // There is free space from lastOffset to size.
8104  ++unusedRangeCount;
8105  }
8106 
8107  // End of loop.
8108  lastOffset = size;
8109  }
8110  }
8111  }
8112 
8113  const VkDeviceSize unusedBytes = size - usedBytes;
8114  PrintDetailedMap_Begin(json, unusedBytes, alloc1stCount + alloc2ndCount, unusedRangeCount);
8115 
8116  // SECOND PASS
8117  lastOffset = 0;
8118 
8119  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
8120  {
8121  const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
8122  size_t nextAlloc2ndIndex = 0;
8123  while(lastOffset < freeSpace2ndTo1stEnd)
8124  {
8125  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
8126  while(nextAlloc2ndIndex < suballoc2ndCount &&
8127  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
8128  {
8129  ++nextAlloc2ndIndex;
8130  }
8131 
8132  // Found non-null allocation.
8133  if(nextAlloc2ndIndex < suballoc2ndCount)
8134  {
8135  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
8136 
8137  // 1. Process free space before this allocation.
8138  if(lastOffset < suballoc.offset)
8139  {
8140  // There is free space from lastOffset to suballoc.offset.
8141  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
8142  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8143  }
8144 
8145  // 2. Process this allocation.
8146  // There is allocation with suballoc.offset, suballoc.size.
8147  PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation);
8148 
8149  // 3. Prepare for next iteration.
8150  lastOffset = suballoc.offset + suballoc.size;
8151  ++nextAlloc2ndIndex;
8152  }
8153  // We are at the end.
8154  else
8155  {
8156  if(lastOffset < freeSpace2ndTo1stEnd)
8157  {
8158  // There is free space from lastOffset to freeSpace2ndTo1stEnd.
8159  const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset;
8160  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8161  }
8162 
8163  // End of loop.
8164  lastOffset = freeSpace2ndTo1stEnd;
8165  }
8166  }
8167  }
8168 
8169  nextAlloc1stIndex = m_1stNullItemsBeginCount;
8170  while(lastOffset < freeSpace1stTo2ndEnd)
8171  {
8172  // Find next non-null allocation or move nextAllocIndex to the end.
8173  while(nextAlloc1stIndex < suballoc1stCount &&
8174  suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
8175  {
8176  ++nextAlloc1stIndex;
8177  }
8178 
8179  // Found non-null allocation.
8180  if(nextAlloc1stIndex < suballoc1stCount)
8181  {
8182  const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
8183 
8184  // 1. Process free space before this allocation.
8185  if(lastOffset < suballoc.offset)
8186  {
8187  // There is free space from lastOffset to suballoc.offset.
8188  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
8189  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8190  }
8191 
8192  // 2. Process this allocation.
8193  // There is allocation with suballoc.offset, suballoc.size.
8194  PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation);
8195 
8196  // 3. Prepare for next iteration.
8197  lastOffset = suballoc.offset + suballoc.size;
8198  ++nextAlloc1stIndex;
8199  }
8200  // We are at the end.
8201  else
8202  {
8203  if(lastOffset < freeSpace1stTo2ndEnd)
8204  {
8205  // There is free space from lastOffset to freeSpace1stTo2ndEnd.
8206  const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset;
8207  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8208  }
8209 
8210  // End of loop.
8211  lastOffset = freeSpace1stTo2ndEnd;
8212  }
8213  }
8214 
8215  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8216  {
8217  size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
8218  while(lastOffset < size)
8219  {
8220  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
8221  while(nextAlloc2ndIndex != SIZE_MAX &&
8222  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
8223  {
8224  --nextAlloc2ndIndex;
8225  }
8226 
8227  // Found non-null allocation.
8228  if(nextAlloc2ndIndex != SIZE_MAX)
8229  {
8230  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
8231 
8232  // 1. Process free space before this allocation.
8233  if(lastOffset < suballoc.offset)
8234  {
8235  // There is free space from lastOffset to suballoc.offset.
8236  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
8237  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8238  }
8239 
8240  // 2. Process this allocation.
8241  // There is allocation with suballoc.offset, suballoc.size.
8242  PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation);
8243 
8244  // 3. Prepare for next iteration.
8245  lastOffset = suballoc.offset + suballoc.size;
8246  --nextAlloc2ndIndex;
8247  }
8248  // We are at the end.
8249  else
8250  {
8251  if(lastOffset < size)
8252  {
8253  // There is free space from lastOffset to size.
8254  const VkDeviceSize unusedRangeSize = size - lastOffset;
8255  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8256  }
8257 
8258  // End of loop.
8259  lastOffset = size;
8260  }
8261  }
8262  }
8263 
8264  PrintDetailedMap_End(json);
8265 }
8266 #endif // #if VMA_STATS_STRING_ENABLED
8267 
8268 bool VmaBlockMetadata_Linear::CreateAllocationRequest(
8269  uint32_t currentFrameIndex,
8270  uint32_t frameInUseCount,
8271  VkDeviceSize bufferImageGranularity,
8272  VkDeviceSize allocSize,
8273  VkDeviceSize allocAlignment,
8274  bool upperAddress,
8275  VmaSuballocationType allocType,
8276  bool canMakeOtherLost,
8277  VmaAllocationRequest* pAllocationRequest)
8278 {
8279  VMA_ASSERT(allocSize > 0);
8280  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
8281  VMA_ASSERT(pAllocationRequest != VMA_NULL);
8282  VMA_HEAVY_ASSERT(Validate());
8283 
8284  const VkDeviceSize size = GetSize();
8285  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8286  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8287 
8288  if(upperAddress)
8289  {
8290  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
8291  {
8292  VMA_ASSERT(0 && "Trying to use pool with linear algorithm as double stack, while it is already being used as ring buffer.");
8293  return false;
8294  }
8295 
8296  // Try to allocate before 2nd.back(), or end of block if 2nd.empty().
8297  if(allocSize > size)
8298  {
8299  return false;
8300  }
8301  VkDeviceSize resultBaseOffset = size - allocSize;
8302  if(!suballocations2nd.empty())
8303  {
8304  const VmaSuballocation& lastSuballoc = suballocations2nd.back();
8305  resultBaseOffset = lastSuballoc.offset - allocSize;
8306  if(allocSize > lastSuballoc.offset)
8307  {
8308  return false;
8309  }
8310  }
8311 
8312  // Start from offset equal to end of free space.
8313  VkDeviceSize resultOffset = resultBaseOffset;
8314 
8315  // Apply VMA_DEBUG_MARGIN at the end.
8316  if(VMA_DEBUG_MARGIN > 0)
8317  {
8318  if(resultOffset < VMA_DEBUG_MARGIN)
8319  {
8320  return false;
8321  }
8322  resultOffset -= VMA_DEBUG_MARGIN;
8323  }
8324 
8325  // Apply alignment.
8326  resultOffset = VmaAlignDown(resultOffset, allocAlignment);
8327 
8328  // Check next suballocations from 2nd for BufferImageGranularity conflicts.
8329  // Make bigger alignment if necessary.
8330  if(bufferImageGranularity > 1 && !suballocations2nd.empty())
8331  {
8332  bool bufferImageGranularityConflict = false;
8333  for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; )
8334  {
8335  const VmaSuballocation& nextSuballoc = suballocations2nd[nextSuballocIndex];
8336  if(VmaBlocksOnSamePage(nextSuballoc.offset, nextSuballoc.size, resultOffset, bufferImageGranularity))
8337  {
8338  if(VmaIsBufferImageGranularityConflict(nextSuballoc.type, allocType))
8339  {
8340  bufferImageGranularityConflict = true;
8341  break;
8342  }
8343  }
8344  else
8345  // Already on previous page.
8346  break;
8347  }
8348  if(bufferImageGranularityConflict)
8349  {
8350  resultOffset = VmaAlignDown(resultOffset, bufferImageGranularity);
8351  }
8352  }
8353 
8354  // There is enough free space.
8355  const VkDeviceSize endOf1st = !suballocations1st.empty() ?
8356  suballocations1st.back().offset + suballocations1st.back().size :
8357  0;
8358  if(endOf1st + VMA_DEBUG_MARGIN <= resultOffset)
8359  {
8360  // Check previous suballocations for BufferImageGranularity conflicts.
8361  // If conflict exists, allocation cannot be made here.
8362  if(bufferImageGranularity > 1)
8363  {
8364  for(size_t prevSuballocIndex = suballocations1st.size(); prevSuballocIndex--; )
8365  {
8366  const VmaSuballocation& prevSuballoc = suballocations1st[prevSuballocIndex];
8367  if(VmaBlocksOnSamePage(resultOffset, allocSize, prevSuballoc.offset, bufferImageGranularity))
8368  {
8369  if(VmaIsBufferImageGranularityConflict(allocType, prevSuballoc.type))
8370  {
8371  return false;
8372  }
8373  }
8374  else
8375  {
8376  // Already on next page.
8377  break;
8378  }
8379  }
8380  }
8381 
8382  // All tests passed: Success.
8383  pAllocationRequest->offset = resultOffset;
8384  pAllocationRequest->sumFreeSize = resultBaseOffset + allocSize - endOf1st;
8385  pAllocationRequest->sumItemSize = 0;
8386  // pAllocationRequest->item unused.
8387  pAllocationRequest->itemsToMakeLostCount = 0;
8388  return true;
8389  }
8390  }
8391  else // !upperAddress
8392  {
8393  if(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8394  {
8395  // Try to allocate at the end of 1st vector.
8396 
8397  VkDeviceSize resultBaseOffset = 0;
8398  if(!suballocations1st.empty())
8399  {
8400  const VmaSuballocation& lastSuballoc = suballocations1st.back();
8401  resultBaseOffset = lastSuballoc.offset + lastSuballoc.size;
8402  }
8403 
8404  // Start from offset equal to beginning of free space.
8405  VkDeviceSize resultOffset = resultBaseOffset;
8406 
8407  // Apply VMA_DEBUG_MARGIN at the beginning.
8408  if(VMA_DEBUG_MARGIN > 0)
8409  {
8410  resultOffset += VMA_DEBUG_MARGIN;
8411  }
8412 
8413  // Apply alignment.
8414  resultOffset = VmaAlignUp(resultOffset, allocAlignment);
8415 
8416  // Check previous suballocations for BufferImageGranularity conflicts.
8417  // Make bigger alignment if necessary.
8418  if(bufferImageGranularity > 1 && !suballocations1st.empty())
8419  {
8420  bool bufferImageGranularityConflict = false;
8421  for(size_t prevSuballocIndex = suballocations1st.size(); prevSuballocIndex--; )
8422  {
8423  const VmaSuballocation& prevSuballoc = suballocations1st[prevSuballocIndex];
8424  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity))
8425  {
8426  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
8427  {
8428  bufferImageGranularityConflict = true;
8429  break;
8430  }
8431  }
8432  else
8433  // Already on previous page.
8434  break;
8435  }
8436  if(bufferImageGranularityConflict)
8437  {
8438  resultOffset = VmaAlignUp(resultOffset, bufferImageGranularity);
8439  }
8440  }
8441 
8442  const VkDeviceSize freeSpaceEnd = m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ?
8443  suballocations2nd.back().offset : size;
8444 
8445  // There is enough free space at the end after alignment.
8446  if(resultOffset + allocSize + VMA_DEBUG_MARGIN <= freeSpaceEnd)
8447  {
8448  // Check next suballocations for BufferImageGranularity conflicts.
8449  // If conflict exists, allocation cannot be made here.
8450  if(bufferImageGranularity > 1 && m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8451  {
8452  for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; )
8453  {
8454  const VmaSuballocation& nextSuballoc = suballocations2nd[nextSuballocIndex];
8455  if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
8456  {
8457  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
8458  {
8459  return false;
8460  }
8461  }
8462  else
8463  {
8464  // Already on previous page.
8465  break;
8466  }
8467  }
8468  }
8469 
8470  // All tests passed: Success.
8471  pAllocationRequest->offset = resultOffset;
8472  pAllocationRequest->sumFreeSize = freeSpaceEnd - resultBaseOffset;
8473  pAllocationRequest->sumItemSize = 0;
8474  // pAllocationRequest->item unused.
8475  pAllocationRequest->itemsToMakeLostCount = 0;
8476  return true;
8477  }
8478  }
8479 
8480  // Wrap-around to end of 2nd vector. Try to allocate there, watching for the
8481  // beginning of 1st vector as the end of free space.
8482  if(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
8483  {
8484  VMA_ASSERT(!suballocations1st.empty());
8485 
8486  VkDeviceSize resultBaseOffset = 0;
8487  if(!suballocations2nd.empty())
8488  {
8489  const VmaSuballocation& lastSuballoc = suballocations2nd.back();
8490  resultBaseOffset = lastSuballoc.offset + lastSuballoc.size;
8491  }
8492 
8493  // Start from offset equal to beginning of free space.
8494  VkDeviceSize resultOffset = resultBaseOffset;
8495 
8496  // Apply VMA_DEBUG_MARGIN at the beginning.
8497  if(VMA_DEBUG_MARGIN > 0)
8498  {
8499  resultOffset += VMA_DEBUG_MARGIN;
8500  }
8501 
8502  // Apply alignment.
8503  resultOffset = VmaAlignUp(resultOffset, allocAlignment);
8504 
8505  // Check previous suballocations for BufferImageGranularity conflicts.
8506  // Make bigger alignment if necessary.
8507  if(bufferImageGranularity > 1 && !suballocations2nd.empty())
8508  {
8509  bool bufferImageGranularityConflict = false;
8510  for(size_t prevSuballocIndex = suballocations2nd.size(); prevSuballocIndex--; )
8511  {
8512  const VmaSuballocation& prevSuballoc = suballocations2nd[prevSuballocIndex];
8513  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity))
8514  {
8515  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
8516  {
8517  bufferImageGranularityConflict = true;
8518  break;
8519  }
8520  }
8521  else
8522  // Already on previous page.
8523  break;
8524  }
8525  if(bufferImageGranularityConflict)
8526  {
8527  resultOffset = VmaAlignUp(resultOffset, bufferImageGranularity);
8528  }
8529  }
8530 
8531  pAllocationRequest->itemsToMakeLostCount = 0;
8532  pAllocationRequest->sumItemSize = 0;
8533  size_t index1st = m_1stNullItemsBeginCount;
8534 
8535  if(canMakeOtherLost)
8536  {
8537  while(index1st < suballocations1st.size() &&
8538  resultOffset + allocSize + VMA_DEBUG_MARGIN > suballocations1st[index1st].offset)
8539  {
8540  // Next colliding allocation at the beginning of 1st vector found. Try to make it lost.
8541  const VmaSuballocation& suballoc = suballocations1st[index1st];
8542  if(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE)
8543  {
8544  // No problem.
8545  }
8546  else
8547  {
8548  VMA_ASSERT(suballoc.hAllocation != VK_NULL_HANDLE);
8549  if(suballoc.hAllocation->CanBecomeLost() &&
8550  suballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
8551  {
8552  ++pAllocationRequest->itemsToMakeLostCount;
8553  pAllocationRequest->sumItemSize += suballoc.size;
8554  }
8555  else
8556  {
8557  return false;
8558  }
8559  }
8560  ++index1st;
8561  }
8562 
8563  // Check next suballocations for BufferImageGranularity conflicts.
8564  // If conflict exists, we must mark more allocations lost or fail.
8565  if(bufferImageGranularity > 1)
8566  {
8567  while(index1st < suballocations1st.size())
8568  {
8569  const VmaSuballocation& suballoc = suballocations1st[index1st];
8570  if(VmaBlocksOnSamePage(resultOffset, allocSize, suballoc.offset, bufferImageGranularity))
8571  {
8572  if(suballoc.hAllocation != VK_NULL_HANDLE)
8573  {
8574  // Not checking actual VmaIsBufferImageGranularityConflict(allocType, suballoc.type).
8575  if(suballoc.hAllocation->CanBecomeLost() &&
8576  suballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
8577  {
8578  ++pAllocationRequest->itemsToMakeLostCount;
8579  pAllocationRequest->sumItemSize += suballoc.size;
8580  }
8581  else
8582  {
8583  return false;
8584  }
8585  }
8586  }
8587  else
8588  {
8589  // Already on next page.
8590  break;
8591  }
8592  ++index1st;
8593  }
8594  }
8595  }
8596 
8597  // There is enough free space at the end after alignment.
8598  if(index1st == suballocations1st.size() && resultOffset + allocSize + VMA_DEBUG_MARGIN < size ||
8599  index1st < suballocations1st.size() && resultOffset + allocSize + VMA_DEBUG_MARGIN <= suballocations1st[index1st].offset)
8600  {
8601  // Check next suballocations for BufferImageGranularity conflicts.
8602  // If conflict exists, allocation cannot be made here.
8603  if(bufferImageGranularity > 1)
8604  {
8605  for(size_t nextSuballocIndex = index1st;
8606  nextSuballocIndex < suballocations1st.size();
8607  nextSuballocIndex++)
8608  {
8609  const VmaSuballocation& nextSuballoc = suballocations1st[nextSuballocIndex];
8610  if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
8611  {
8612  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
8613  {
8614  return false;
8615  }
8616  }
8617  else
8618  {
8619  // Already on next page.
8620  break;
8621  }
8622  }
8623  }
8624 
8625  // All tests passed: Success.
8626  pAllocationRequest->offset = resultOffset;
8627  pAllocationRequest->sumFreeSize =
8628  (index1st < suballocations1st.size() ? suballocations1st[index1st].offset : size)
8629  - resultBaseOffset
8630  - pAllocationRequest->sumItemSize;
8631  // pAllocationRequest->item unused.
8632  return true;
8633  }
8634  }
8635  }
8636 
8637  return false;
8638 }
8639 
8640 bool VmaBlockMetadata_Linear::MakeRequestedAllocationsLost(
8641  uint32_t currentFrameIndex,
8642  uint32_t frameInUseCount,
8643  VmaAllocationRequest* pAllocationRequest)
8644 {
8645  if(pAllocationRequest->itemsToMakeLostCount == 0)
8646  {
8647  return true;
8648  }
8649 
8650  VMA_ASSERT(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER);
8651 
8652  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8653  size_t index1st = m_1stNullItemsBeginCount;
8654  size_t madeLostCount = 0;
8655  while(madeLostCount < pAllocationRequest->itemsToMakeLostCount)
8656  {
8657  VMA_ASSERT(index1st < suballocations1st.size());
8658  VmaSuballocation& suballoc = suballocations1st[index1st];
8659  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
8660  {
8661  VMA_ASSERT(suballoc.hAllocation != VK_NULL_HANDLE);
8662  VMA_ASSERT(suballoc.hAllocation->CanBecomeLost());
8663  if(suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
8664  {
8665  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
8666  suballoc.hAllocation = VK_NULL_HANDLE;
8667  m_SumFreeSize += suballoc.size;
8668  ++m_1stNullItemsMiddleCount;
8669  ++madeLostCount;
8670  }
8671  else
8672  {
8673  return false;
8674  }
8675  }
8676  ++index1st;
8677  }
8678 
8679  CleanupAfterFree();
8680  //VMA_HEAVY_ASSERT(Validate()); // Already called by ClanupAfterFree().
8681 
8682  return true;
8683 }
8684 
8685 uint32_t VmaBlockMetadata_Linear::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
8686 {
8687  uint32_t lostAllocationCount = 0;
8688 
8689  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8690  for(size_t i = m_1stNullItemsBeginCount, count = suballocations1st.size(); i < count; ++i)
8691  {
8692  VmaSuballocation& suballoc = suballocations1st[i];
8693  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE &&
8694  suballoc.hAllocation->CanBecomeLost() &&
8695  suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
8696  {
8697  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
8698  suballoc.hAllocation = VK_NULL_HANDLE;
8699  ++m_1stNullItemsMiddleCount;
8700  m_SumFreeSize += suballoc.size;
8701  ++lostAllocationCount;
8702  }
8703  }
8704 
8705  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8706  for(size_t i = 0, count = suballocations2nd.size(); i < count; ++i)
8707  {
8708  VmaSuballocation& suballoc = suballocations2nd[i];
8709  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE &&
8710  suballoc.hAllocation->CanBecomeLost() &&
8711  suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
8712  {
8713  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
8714  suballoc.hAllocation = VK_NULL_HANDLE;
8715  ++m_2ndNullItemsCount;
8716  ++lostAllocationCount;
8717  }
8718  }
8719 
8720  if(lostAllocationCount)
8721  {
8722  CleanupAfterFree();
8723  }
8724 
8725  return lostAllocationCount;
8726 }
8727 
8728 VkResult VmaBlockMetadata_Linear::CheckCorruption(const void* pBlockData)
8729 {
8730  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8731  for(size_t i = m_1stNullItemsBeginCount, count = suballocations1st.size(); i < count; ++i)
8732  {
8733  const VmaSuballocation& suballoc = suballocations1st[i];
8734  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
8735  {
8736  if(!VmaValidateMagicValue(pBlockData, suballoc.offset - VMA_DEBUG_MARGIN))
8737  {
8738  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
8739  return VK_ERROR_VALIDATION_FAILED_EXT;
8740  }
8741  if(!VmaValidateMagicValue(pBlockData, suballoc.offset + suballoc.size))
8742  {
8743  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
8744  return VK_ERROR_VALIDATION_FAILED_EXT;
8745  }
8746  }
8747  }
8748 
8749  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8750  for(size_t i = 0, count = suballocations2nd.size(); i < count; ++i)
8751  {
8752  const VmaSuballocation& suballoc = suballocations2nd[i];
8753  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
8754  {
8755  if(!VmaValidateMagicValue(pBlockData, suballoc.offset - VMA_DEBUG_MARGIN))
8756  {
8757  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
8758  return VK_ERROR_VALIDATION_FAILED_EXT;
8759  }
8760  if(!VmaValidateMagicValue(pBlockData, suballoc.offset + suballoc.size))
8761  {
8762  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
8763  return VK_ERROR_VALIDATION_FAILED_EXT;
8764  }
8765  }
8766  }
8767 
8768  return VK_SUCCESS;
8769 }
8770 
8771 void VmaBlockMetadata_Linear::Alloc(
8772  const VmaAllocationRequest& request,
8773  VmaSuballocationType type,
8774  VkDeviceSize allocSize,
8775  bool upperAddress,
8776  VmaAllocation hAllocation)
8777 {
8778  const VmaSuballocation newSuballoc = { request.offset, allocSize, hAllocation, type };
8779 
8780  if(upperAddress)
8781  {
8782  VMA_ASSERT(m_2ndVectorMode != SECOND_VECTOR_RING_BUFFER &&
8783  "CRITICAL ERROR: Trying to use linear allocator as double stack while it was already used as ring buffer.");
8784  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8785  suballocations2nd.push_back(newSuballoc);
8786  m_2ndVectorMode = SECOND_VECTOR_DOUBLE_STACK;
8787  }
8788  else
8789  {
8790  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8791 
8792  // First allocation.
8793  if(suballocations1st.empty())
8794  {
8795  suballocations1st.push_back(newSuballoc);
8796  }
8797  else
8798  {
8799  // New allocation at the end of 1st vector.
8800  if(request.offset >= suballocations1st.back().offset + suballocations1st.back().size)
8801  {
8802  // Check if it fits before the end of the block.
8803  VMA_ASSERT(request.offset + allocSize <= GetSize());
8804  suballocations1st.push_back(newSuballoc);
8805  }
8806  // New allocation at the end of 2-part ring buffer, so before first allocation from 1st vector.
8807  else if(request.offset + allocSize <= suballocations1st[m_1stNullItemsBeginCount].offset)
8808  {
8809  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8810 
8811  switch(m_2ndVectorMode)
8812  {
8813  case SECOND_VECTOR_EMPTY:
8814  // First allocation from second part ring buffer.
8815  VMA_ASSERT(suballocations2nd.empty());
8816  m_2ndVectorMode = SECOND_VECTOR_RING_BUFFER;
8817  break;
8818  case SECOND_VECTOR_RING_BUFFER:
8819  // 2-part ring buffer is already started.
8820  VMA_ASSERT(!suballocations2nd.empty());
8821  break;
8822  case SECOND_VECTOR_DOUBLE_STACK:
8823  VMA_ASSERT(0 && "CRITICAL ERROR: Trying to use linear allocator as ring buffer while it was already used as double stack.");
8824  break;
8825  default:
8826  VMA_ASSERT(0);
8827  }
8828 
8829  suballocations2nd.push_back(newSuballoc);
8830  }
8831  else
8832  {
8833  VMA_ASSERT(0 && "CRITICAL INTERNAL ERROR.");
8834  }
8835  }
8836  }
8837 
8838  m_SumFreeSize -= newSuballoc.size;
8839 }
8840 
8841 void VmaBlockMetadata_Linear::Free(const VmaAllocation allocation)
8842 {
8843  FreeAtOffset(allocation->GetOffset());
8844 }
8845 
8846 void VmaBlockMetadata_Linear::FreeAtOffset(VkDeviceSize offset)
8847 {
8848  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8849  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8850 
8851  if(!suballocations1st.empty())
8852  {
8853  // First allocation: Mark it as next empty at the beginning.
8854  VmaSuballocation& firstSuballoc = suballocations1st[m_1stNullItemsBeginCount];
8855  if(firstSuballoc.offset == offset)
8856  {
8857  firstSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
8858  firstSuballoc.hAllocation = VK_NULL_HANDLE;
8859  m_SumFreeSize += firstSuballoc.size;
8860  ++m_1stNullItemsBeginCount;
8861  CleanupAfterFree();
8862  return;
8863  }
8864  }
8865 
8866  // Last allocation in 2-part ring buffer or top of upper stack (same logic).
8867  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER ||
8868  m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8869  {
8870  VmaSuballocation& lastSuballoc = suballocations2nd.back();
8871  if(lastSuballoc.offset == offset)
8872  {
8873  m_SumFreeSize += lastSuballoc.size;
8874  suballocations2nd.pop_back();
8875  CleanupAfterFree();
8876  return;
8877  }
8878  }
8879  // Last allocation in 1st vector.
8880  else if(m_2ndVectorMode == SECOND_VECTOR_EMPTY)
8881  {
8882  VmaSuballocation& lastSuballoc = suballocations1st.back();
8883  if(lastSuballoc.offset == offset)
8884  {
8885  m_SumFreeSize += lastSuballoc.size;
8886  suballocations1st.pop_back();
8887  CleanupAfterFree();
8888  return;
8889  }
8890  }
8891 
8892  // Item from the middle of 1st vector.
8893  {
8894  VmaSuballocation refSuballoc;
8895  refSuballoc.offset = offset;
8896  // Rest of members stays uninitialized intentionally for better performance.
8897  SuballocationVectorType::iterator it = VmaVectorFindSorted<VmaSuballocationOffsetLess>(
8898  suballocations1st.begin() + m_1stNullItemsBeginCount,
8899  suballocations1st.end(),
8900  refSuballoc);
8901  if(it != suballocations1st.end())
8902  {
8903  it->type = VMA_SUBALLOCATION_TYPE_FREE;
8904  it->hAllocation = VK_NULL_HANDLE;
8905  ++m_1stNullItemsMiddleCount;
8906  m_SumFreeSize += it->size;
8907  CleanupAfterFree();
8908  return;
8909  }
8910  }
8911 
8912  if(m_2ndVectorMode != SECOND_VECTOR_EMPTY)
8913  {
8914  // Item from the middle of 2nd vector.
8915  VmaSuballocation refSuballoc;
8916  refSuballoc.offset = offset;
8917  // Rest of members stays uninitialized intentionally for better performance.
8918  SuballocationVectorType::iterator it = m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER ?
8919  VmaVectorFindSorted<VmaSuballocationOffsetLess>(suballocations2nd.begin(), suballocations2nd.end(), refSuballoc) :
8920  VmaVectorFindSorted<VmaSuballocationOffsetGreater>(suballocations2nd.begin(), suballocations2nd.end(), refSuballoc);
8921  if(it != suballocations2nd.end())
8922  {
8923  it->type = VMA_SUBALLOCATION_TYPE_FREE;
8924  it->hAllocation = VK_NULL_HANDLE;
8925  ++m_2ndNullItemsCount;
8926  m_SumFreeSize += it->size;
8927  CleanupAfterFree();
8928  return;
8929  }
8930  }
8931 
8932  VMA_ASSERT(0 && "Allocation to free not found in linear allocator!");
8933 }
8934 
8935 bool VmaBlockMetadata_Linear::ShouldCompact1st() const
8936 {
8937  const size_t nullItemCount = m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount;
8938  const size_t suballocCount = AccessSuballocations1st().size();
8939  return suballocCount > 32 && nullItemCount * 2 >= (suballocCount - nullItemCount) * 3;
8940 }
8941 
8942 void VmaBlockMetadata_Linear::CleanupAfterFree()
8943 {
8944  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8945  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8946 
8947  if(IsEmpty())
8948  {
8949  suballocations1st.clear();
8950  suballocations2nd.clear();
8951  m_1stNullItemsBeginCount = 0;
8952  m_1stNullItemsMiddleCount = 0;
8953  m_2ndNullItemsCount = 0;
8954  m_2ndVectorMode = SECOND_VECTOR_EMPTY;
8955  }
8956  else
8957  {
8958  const size_t suballoc1stCount = suballocations1st.size();
8959  const size_t nullItem1stCount = m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount;
8960  VMA_ASSERT(nullItem1stCount <= suballoc1stCount);
8961 
8962  // Find more null items at the beginning of 1st vector.
8963  while(m_1stNullItemsBeginCount < suballoc1stCount &&
8964  suballocations1st[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE)
8965  {
8966  ++m_1stNullItemsBeginCount;
8967  --m_1stNullItemsMiddleCount;
8968  }
8969 
8970  // Find more null items at the end of 1st vector.
8971  while(m_1stNullItemsMiddleCount > 0 &&
8972  suballocations1st.back().hAllocation == VK_NULL_HANDLE)
8973  {
8974  --m_1stNullItemsMiddleCount;
8975  suballocations1st.pop_back();
8976  }
8977 
8978  // Find more null items at the end of 2nd vector.
8979  while(m_2ndNullItemsCount > 0 &&
8980  suballocations2nd.back().hAllocation == VK_NULL_HANDLE)
8981  {
8982  --m_2ndNullItemsCount;
8983  suballocations2nd.pop_back();
8984  }
8985 
8986  if(ShouldCompact1st())
8987  {
8988  const size_t nonNullItemCount = suballoc1stCount - nullItem1stCount;
8989  size_t srcIndex = m_1stNullItemsBeginCount;
8990  for(size_t dstIndex = 0; dstIndex < nonNullItemCount; ++dstIndex)
8991  {
8992  while(suballocations1st[srcIndex].hAllocation == VK_NULL_HANDLE)
8993  {
8994  ++srcIndex;
8995  }
8996  if(dstIndex != srcIndex)
8997  {
8998  suballocations1st[dstIndex] = suballocations1st[srcIndex];
8999  }
9000  ++srcIndex;
9001  }
9002  suballocations1st.resize(nonNullItemCount);
9003  m_1stNullItemsBeginCount = 0;
9004  m_1stNullItemsMiddleCount = 0;
9005  }
9006 
9007  // 2nd vector became empty.
9008  if(suballocations2nd.empty())
9009  {
9010  m_2ndVectorMode = SECOND_VECTOR_EMPTY;
9011  }
9012 
9013  // 1st vector became empty.
9014  if(suballocations1st.size() - m_1stNullItemsBeginCount == 0)
9015  {
9016  suballocations1st.clear();
9017  m_1stNullItemsBeginCount = 0;
9018 
9019  if(!suballocations2nd.empty() && m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
9020  {
9021  // Swap 1st with 2nd. Now 2nd is empty.
9022  m_2ndVectorMode = SECOND_VECTOR_EMPTY;
9023  m_1stNullItemsMiddleCount = m_2ndNullItemsCount;
9024  while(m_1stNullItemsBeginCount < suballocations2nd.size() &&
9025  suballocations2nd[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE)
9026  {
9027  ++m_1stNullItemsBeginCount;
9028  --m_1stNullItemsMiddleCount;
9029  }
9030  m_2ndNullItemsCount = 0;
9031  m_1stVectorIndex ^= 1;
9032  }
9033  }
9034  }
9035 
9036  VMA_HEAVY_ASSERT(Validate());
9037 }
9038 
9039 
9041 // class VmaDeviceMemoryBlock
9042 
9043 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
9044  m_pMetadata(VMA_NULL),
9045  m_MemoryTypeIndex(UINT32_MAX),
9046  m_Id(0),
9047  m_hMemory(VK_NULL_HANDLE),
9048  m_MapCount(0),
9049  m_pMappedData(VMA_NULL)
9050 {
9051 }
9052 
9053 void VmaDeviceMemoryBlock::Init(
9054  VmaAllocator hAllocator,
9055  uint32_t newMemoryTypeIndex,
9056  VkDeviceMemory newMemory,
9057  VkDeviceSize newSize,
9058  uint32_t id,
9059  bool linearAlgorithm)
9060 {
9061  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
9062 
9063  m_MemoryTypeIndex = newMemoryTypeIndex;
9064  m_Id = id;
9065  m_hMemory = newMemory;
9066 
9067  if(linearAlgorithm)
9068  {
9069  m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Linear)(hAllocator);
9070  }
9071  else
9072  {
9073  m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Generic)(hAllocator);
9074  }
9075  m_pMetadata->Init(newSize);
9076 }
9077 
9078 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
9079 {
9080  // This is the most important assert in the entire library.
9081  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
9082  VMA_ASSERT(m_pMetadata->IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
9083 
9084  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
9085  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_pMetadata->GetSize(), m_hMemory);
9086  m_hMemory = VK_NULL_HANDLE;
9087 
9088  vma_delete(allocator, m_pMetadata);
9089  m_pMetadata = VMA_NULL;
9090 }
9091 
9092 bool VmaDeviceMemoryBlock::Validate() const
9093 {
9094  if((m_hMemory == VK_NULL_HANDLE) ||
9095  (m_pMetadata->GetSize() == 0))
9096  {
9097  return false;
9098  }
9099 
9100  return m_pMetadata->Validate();
9101 }
9102 
9103 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
9104 {
9105  void* pData = nullptr;
9106  VkResult res = Map(hAllocator, 1, &pData);
9107  if(res != VK_SUCCESS)
9108  {
9109  return res;
9110  }
9111 
9112  res = m_pMetadata->CheckCorruption(pData);
9113 
9114  Unmap(hAllocator, 1);
9115 
9116  return res;
9117 }
9118 
9119 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
9120 {
9121  if(count == 0)
9122  {
9123  return VK_SUCCESS;
9124  }
9125 
9126  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
9127  if(m_MapCount != 0)
9128  {
9129  m_MapCount += count;
9130  VMA_ASSERT(m_pMappedData != VMA_NULL);
9131  if(ppData != VMA_NULL)
9132  {
9133  *ppData = m_pMappedData;
9134  }
9135  return VK_SUCCESS;
9136  }
9137  else
9138  {
9139  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
9140  hAllocator->m_hDevice,
9141  m_hMemory,
9142  0, // offset
9143  VK_WHOLE_SIZE,
9144  0, // flags
9145  &m_pMappedData);
9146  if(result == VK_SUCCESS)
9147  {
9148  if(ppData != VMA_NULL)
9149  {
9150  *ppData = m_pMappedData;
9151  }
9152  m_MapCount = count;
9153  }
9154  return result;
9155  }
9156 }
9157 
9158 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
9159 {
9160  if(count == 0)
9161  {
9162  return;
9163  }
9164 
9165  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
9166  if(m_MapCount >= count)
9167  {
9168  m_MapCount -= count;
9169  if(m_MapCount == 0)
9170  {
9171  m_pMappedData = VMA_NULL;
9172  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
9173  }
9174  }
9175  else
9176  {
9177  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
9178  }
9179 }
9180 
9181 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
9182 {
9183  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
9184  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
9185 
9186  void* pData;
9187  VkResult res = Map(hAllocator, 1, &pData);
9188  if(res != VK_SUCCESS)
9189  {
9190  return res;
9191  }
9192 
9193  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
9194  VmaWriteMagicValue(pData, allocOffset + allocSize);
9195 
9196  Unmap(hAllocator, 1);
9197 
9198  return VK_SUCCESS;
9199 }
9200 
9201 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
9202 {
9203  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
9204  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
9205 
9206  void* pData;
9207  VkResult res = Map(hAllocator, 1, &pData);
9208  if(res != VK_SUCCESS)
9209  {
9210  return res;
9211  }
9212 
9213  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
9214  {
9215  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
9216  }
9217  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
9218  {
9219  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
9220  }
9221 
9222  Unmap(hAllocator, 1);
9223 
9224  return VK_SUCCESS;
9225 }
9226 
9227 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
9228  const VmaAllocator hAllocator,
9229  const VmaAllocation hAllocation,
9230  VkBuffer hBuffer)
9231 {
9232  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
9233  hAllocation->GetBlock() == this);
9234  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
9235  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
9236  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
9237  hAllocator->m_hDevice,
9238  hBuffer,
9239  m_hMemory,
9240  hAllocation->GetOffset());
9241 }
9242 
9243 VkResult VmaDeviceMemoryBlock::BindImageMemory(
9244  const VmaAllocator hAllocator,
9245  const VmaAllocation hAllocation,
9246  VkImage hImage)
9247 {
9248  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
9249  hAllocation->GetBlock() == this);
9250  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
9251  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
9252  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
9253  hAllocator->m_hDevice,
9254  hImage,
9255  m_hMemory,
9256  hAllocation->GetOffset());
9257 }
9258 
9259 static void InitStatInfo(VmaStatInfo& outInfo)
9260 {
9261  memset(&outInfo, 0, sizeof(outInfo));
9262  outInfo.allocationSizeMin = UINT64_MAX;
9263  outInfo.unusedRangeSizeMin = UINT64_MAX;
9264 }
9265 
9266 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
9267 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
9268 {
9269  inoutInfo.blockCount += srcInfo.blockCount;
9270  inoutInfo.allocationCount += srcInfo.allocationCount;
9271  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
9272  inoutInfo.usedBytes += srcInfo.usedBytes;
9273  inoutInfo.unusedBytes += srcInfo.unusedBytes;
9274  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
9275  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
9276  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
9277  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
9278 }
9279 
9280 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
9281 {
9282  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
9283  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
9284  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
9285  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
9286 }
9287 
9288 VmaPool_T::VmaPool_T(
9289  VmaAllocator hAllocator,
9290  const VmaPoolCreateInfo& createInfo) :
9291  m_BlockVector(
9292  hAllocator,
9293  createInfo.memoryTypeIndex,
9294  createInfo.blockSize,
9295  createInfo.minBlockCount,
9296  createInfo.maxBlockCount,
9297  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
9298  createInfo.frameInUseCount,
9299  true, // isCustomPool
9300  (createInfo.flags & VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT) != 0), // linearAlgorithm
9301  m_Id(0)
9302 {
9303 }
9304 
9305 VmaPool_T::~VmaPool_T()
9306 {
9307 }
9308 
9309 #if VMA_STATS_STRING_ENABLED
9310 
9311 #endif // #if VMA_STATS_STRING_ENABLED
9312 
9313 VmaBlockVector::VmaBlockVector(
9314  VmaAllocator hAllocator,
9315  uint32_t memoryTypeIndex,
9316  VkDeviceSize preferredBlockSize,
9317  size_t minBlockCount,
9318  size_t maxBlockCount,
9319  VkDeviceSize bufferImageGranularity,
9320  uint32_t frameInUseCount,
9321  bool isCustomPool,
9322  bool linearAlgorithm) :
9323  m_hAllocator(hAllocator),
9324  m_MemoryTypeIndex(memoryTypeIndex),
9325  m_PreferredBlockSize(preferredBlockSize),
9326  m_MinBlockCount(minBlockCount),
9327  m_MaxBlockCount(maxBlockCount),
9328  m_BufferImageGranularity(bufferImageGranularity),
9329  m_FrameInUseCount(frameInUseCount),
9330  m_IsCustomPool(isCustomPool),
9331  m_LinearAlgorithm(linearAlgorithm),
9332  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
9333  m_HasEmptyBlock(false),
9334  m_pDefragmentator(VMA_NULL),
9335  m_NextBlockId(0)
9336 {
9337 }
9338 
9339 VmaBlockVector::~VmaBlockVector()
9340 {
9341  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
9342 
9343  for(size_t i = m_Blocks.size(); i--; )
9344  {
9345  m_Blocks[i]->Destroy(m_hAllocator);
9346  vma_delete(m_hAllocator, m_Blocks[i]);
9347  }
9348 }
9349 
9350 VkResult VmaBlockVector::CreateMinBlocks()
9351 {
9352  for(size_t i = 0; i < m_MinBlockCount; ++i)
9353  {
9354  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
9355  if(res != VK_SUCCESS)
9356  {
9357  return res;
9358  }
9359  }
9360  return VK_SUCCESS;
9361 }
9362 
9363 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
9364 {
9365  pStats->size = 0;
9366  pStats->unusedSize = 0;
9367  pStats->allocationCount = 0;
9368  pStats->unusedRangeCount = 0;
9369  pStats->unusedRangeSizeMax = 0;
9370 
9371  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9372 
9373  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
9374  {
9375  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
9376  VMA_ASSERT(pBlock);
9377  VMA_HEAVY_ASSERT(pBlock->Validate());
9378  pBlock->m_pMetadata->AddPoolStats(*pStats);
9379  }
9380 }
9381 
9382 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
9383 {
9384  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
9385  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
9386  (VMA_DEBUG_MARGIN > 0) &&
9387  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
9388 }
9389 
9390 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
9391 
9392 VkResult VmaBlockVector::Allocate(
9393  VmaPool hCurrentPool,
9394  uint32_t currentFrameIndex,
9395  VkDeviceSize size,
9396  VkDeviceSize alignment,
9397  const VmaAllocationCreateInfo& createInfo,
9398  VmaSuballocationType suballocType,
9399  VmaAllocation* pAllocation)
9400 {
9401  const bool isUpperAddress = (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0;
9402  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
9403  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
9404  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
9405  const bool canCreateNewBlock =
9406  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
9407  (m_Blocks.size() < m_MaxBlockCount);
9408 
9409  // Upper address can only be used with linear allocator.
9410  if(isUpperAddress && !m_LinearAlgorithm)
9411  {
9412  return VK_ERROR_FEATURE_NOT_PRESENT;
9413  }
9414 
9415  // Early reject: requested allocation size is larger that maximum block size for this block vector.
9416  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
9417  {
9418  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9419  }
9420 
9421  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9422 
9423  /*
9424  Under certain condition, this whole section can be skipped for optimization, so
9425  we move on directly to trying to allocate with canMakeOtherLost. That's the case
9426  e.g. for custom pools with linear algorithm.
9427  */
9428  if(!canMakeOtherLost || canCreateNewBlock)
9429  {
9430  // 1. Search existing allocations. Try to allocate without making other allocations lost.
9431  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
9432  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
9433  {
9434  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
9435  VMA_ASSERT(pCurrBlock);
9436  VmaAllocationRequest currRequest = {};
9437  if(pCurrBlock->m_pMetadata->CreateAllocationRequest(
9438  currentFrameIndex,
9439  m_FrameInUseCount,
9440  m_BufferImageGranularity,
9441  size,
9442  alignment,
9443  isUpperAddress,
9444  suballocType,
9445  false, // canMakeOtherLost
9446  &currRequest))
9447  {
9448  // Allocate from pCurrBlock.
9449  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
9450 
9451  if(mapped)
9452  {
9453  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
9454  if(res != VK_SUCCESS)
9455  {
9456  return res;
9457  }
9458  }
9459 
9460  // We no longer have an empty Allocation.
9461  if(pCurrBlock->m_pMetadata->IsEmpty())
9462  {
9463  m_HasEmptyBlock = false;
9464  }
9465 
9466  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
9467  pCurrBlock->m_pMetadata->Alloc(currRequest, suballocType, size, isUpperAddress, *pAllocation);
9468  (*pAllocation)->InitBlockAllocation(
9469  hCurrentPool,
9470  pCurrBlock,
9471  currRequest.offset,
9472  alignment,
9473  size,
9474  suballocType,
9475  mapped,
9476  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
9477  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
9478  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
9479  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
9480  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
9481  {
9482  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
9483  }
9484  if(IsCorruptionDetectionEnabled())
9485  {
9486  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
9487  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
9488  }
9489  return VK_SUCCESS;
9490  }
9491  }
9492 
9493  // 2. Try to create new block.
9494  if(canCreateNewBlock)
9495  {
9496  // Calculate optimal size for new block.
9497  VkDeviceSize newBlockSize = m_PreferredBlockSize;
9498  uint32_t newBlockSizeShift = 0;
9499  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
9500 
9501  // Allocating blocks of other sizes is allowed only in default pools.
9502  // In custom pools block size is fixed.
9503  if(m_IsCustomPool == false)
9504  {
9505  // Allocate 1/8, 1/4, 1/2 as first blocks.
9506  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
9507  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
9508  {
9509  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
9510  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
9511  {
9512  newBlockSize = smallerNewBlockSize;
9513  ++newBlockSizeShift;
9514  }
9515  else
9516  {
9517  break;
9518  }
9519  }
9520  }
9521 
9522  size_t newBlockIndex = 0;
9523  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
9524  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
9525  if(m_IsCustomPool == false)
9526  {
9527  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
9528  {
9529  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
9530  if(smallerNewBlockSize >= size)
9531  {
9532  newBlockSize = smallerNewBlockSize;
9533  ++newBlockSizeShift;
9534  res = CreateBlock(newBlockSize, &newBlockIndex);
9535  }
9536  else
9537  {
9538  break;
9539  }
9540  }
9541  }
9542 
9543  if(res == VK_SUCCESS)
9544  {
9545  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
9546  VMA_ASSERT(pBlock->m_pMetadata->GetSize() >= size);
9547 
9548  if(mapped)
9549  {
9550  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
9551  if(res != VK_SUCCESS)
9552  {
9553  return res;
9554  }
9555  }
9556 
9557  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
9558  VmaAllocationRequest allocRequest;
9559  if(pBlock->m_pMetadata->CreateAllocationRequest(
9560  currentFrameIndex,
9561  m_FrameInUseCount,
9562  m_BufferImageGranularity,
9563  size,
9564  alignment,
9565  isUpperAddress,
9566  suballocType,
9567  false, // canMakeOtherLost
9568  &allocRequest))
9569  {
9570  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
9571  pBlock->m_pMetadata->Alloc(allocRequest, suballocType, size, isUpperAddress, *pAllocation);
9572  (*pAllocation)->InitBlockAllocation(
9573  hCurrentPool,
9574  pBlock,
9575  allocRequest.offset,
9576  alignment,
9577  size,
9578  suballocType,
9579  mapped,
9580  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
9581  VMA_HEAVY_ASSERT(pBlock->Validate());
9582  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
9583  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
9584  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
9585  {
9586  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
9587  }
9588  if(IsCorruptionDetectionEnabled())
9589  {
9590  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
9591  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
9592  }
9593  return VK_SUCCESS;
9594  }
9595  else
9596  {
9597  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
9598  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9599  }
9600  }
9601  }
9602  }
9603 
9604  // 3. Try to allocate from existing blocks with making other allocations lost.
9605  if(canMakeOtherLost)
9606  {
9607  uint32_t tryIndex = 0;
9608  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
9609  {
9610  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
9611  VmaAllocationRequest bestRequest = {};
9612  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
9613 
9614  // 1. Search existing allocations.
9615  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
9616  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
9617  {
9618  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
9619  VMA_ASSERT(pCurrBlock);
9620  VmaAllocationRequest currRequest = {};
9621  if(pCurrBlock->m_pMetadata->CreateAllocationRequest(
9622  currentFrameIndex,
9623  m_FrameInUseCount,
9624  m_BufferImageGranularity,
9625  size,
9626  alignment,
9627  (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0,
9628  suballocType,
9629  canMakeOtherLost,
9630  &currRequest))
9631  {
9632  const VkDeviceSize currRequestCost = currRequest.CalcCost();
9633  if(pBestRequestBlock == VMA_NULL ||
9634  currRequestCost < bestRequestCost)
9635  {
9636  pBestRequestBlock = pCurrBlock;
9637  bestRequest = currRequest;
9638  bestRequestCost = currRequestCost;
9639 
9640  if(bestRequestCost == 0)
9641  {
9642  break;
9643  }
9644  }
9645  }
9646  }
9647 
9648  if(pBestRequestBlock != VMA_NULL)
9649  {
9650  if(mapped)
9651  {
9652  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
9653  if(res != VK_SUCCESS)
9654  {
9655  return res;
9656  }
9657  }
9658 
9659  if(pBestRequestBlock->m_pMetadata->MakeRequestedAllocationsLost(
9660  currentFrameIndex,
9661  m_FrameInUseCount,
9662  &bestRequest))
9663  {
9664  // We no longer have an empty Allocation.
9665  if(pBestRequestBlock->m_pMetadata->IsEmpty())
9666  {
9667  m_HasEmptyBlock = false;
9668  }
9669  // Allocate from this pBlock.
9670  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
9671  pBestRequestBlock->m_pMetadata->Alloc(bestRequest, suballocType, size, isUpperAddress, *pAllocation);
9672  (*pAllocation)->InitBlockAllocation(
9673  hCurrentPool,
9674  pBestRequestBlock,
9675  bestRequest.offset,
9676  alignment,
9677  size,
9678  suballocType,
9679  mapped,
9680  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
9681  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
9682  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
9683  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
9684  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
9685  {
9686  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
9687  }
9688  if(IsCorruptionDetectionEnabled())
9689  {
9690  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
9691  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
9692  }
9693  return VK_SUCCESS;
9694  }
9695  // else: Some allocations must have been touched while we are here. Next try.
9696  }
9697  else
9698  {
9699  // Could not find place in any of the blocks - break outer loop.
9700  break;
9701  }
9702  }
9703  /* Maximum number of tries exceeded - a very unlike event when many other
9704  threads are simultaneously touching allocations making it impossible to make
9705  lost at the same time as we try to allocate. */
9706  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
9707  {
9708  return VK_ERROR_TOO_MANY_OBJECTS;
9709  }
9710  }
9711 
9712  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9713 }
9714 
9715 void VmaBlockVector::Free(
9716  VmaAllocation hAllocation)
9717 {
9718  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
9719 
9720  // Scope for lock.
9721  {
9722  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9723 
9724  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
9725 
9726  if(IsCorruptionDetectionEnabled())
9727  {
9728  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
9729  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
9730  }
9731 
9732  if(hAllocation->IsPersistentMap())
9733  {
9734  pBlock->Unmap(m_hAllocator, 1);
9735  }
9736 
9737  pBlock->m_pMetadata->Free(hAllocation);
9738  VMA_HEAVY_ASSERT(pBlock->Validate());
9739 
9740  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
9741 
9742  // pBlock became empty after this deallocation.
9743  if(pBlock->m_pMetadata->IsEmpty())
9744  {
9745  // Already has empty Allocation. We don't want to have two, so delete this one.
9746  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
9747  {
9748  pBlockToDelete = pBlock;
9749  Remove(pBlock);
9750  }
9751  // We now have first empty block.
9752  else
9753  {
9754  m_HasEmptyBlock = true;
9755  }
9756  }
9757  // pBlock didn't become empty, but we have another empty block - find and free that one.
9758  // (This is optional, heuristics.)
9759  else if(m_HasEmptyBlock)
9760  {
9761  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
9762  if(pLastBlock->m_pMetadata->IsEmpty() && m_Blocks.size() > m_MinBlockCount)
9763  {
9764  pBlockToDelete = pLastBlock;
9765  m_Blocks.pop_back();
9766  m_HasEmptyBlock = false;
9767  }
9768  }
9769 
9770  IncrementallySortBlocks();
9771  }
9772 
9773  // Destruction of a free Allocation. Deferred until this point, outside of mutex
9774  // lock, for performance reason.
9775  if(pBlockToDelete != VMA_NULL)
9776  {
9777  VMA_DEBUG_LOG(" Deleted empty allocation");
9778  pBlockToDelete->Destroy(m_hAllocator);
9779  vma_delete(m_hAllocator, pBlockToDelete);
9780  }
9781 }
9782 
9783 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
9784 {
9785  VkDeviceSize result = 0;
9786  for(size_t i = m_Blocks.size(); i--; )
9787  {
9788  result = VMA_MAX(result, m_Blocks[i]->m_pMetadata->GetSize());
9789  if(result >= m_PreferredBlockSize)
9790  {
9791  break;
9792  }
9793  }
9794  return result;
9795 }
9796 
9797 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
9798 {
9799  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
9800  {
9801  if(m_Blocks[blockIndex] == pBlock)
9802  {
9803  VmaVectorRemove(m_Blocks, blockIndex);
9804  return;
9805  }
9806  }
9807  VMA_ASSERT(0);
9808 }
9809 
9810 void VmaBlockVector::IncrementallySortBlocks()
9811 {
9812  // Bubble sort only until first swap.
9813  for(size_t i = 1; i < m_Blocks.size(); ++i)
9814  {
9815  if(m_Blocks[i - 1]->m_pMetadata->GetSumFreeSize() > m_Blocks[i]->m_pMetadata->GetSumFreeSize())
9816  {
9817  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
9818  return;
9819  }
9820  }
9821 }
9822 
9823 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
9824 {
9825  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
9826  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
9827  allocInfo.allocationSize = blockSize;
9828  VkDeviceMemory mem = VK_NULL_HANDLE;
9829  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
9830  if(res < 0)
9831  {
9832  return res;
9833  }
9834 
9835  // New VkDeviceMemory successfully created.
9836 
9837  // Create new Allocation for it.
9838  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
9839  pBlock->Init(
9840  m_hAllocator,
9841  m_MemoryTypeIndex,
9842  mem,
9843  allocInfo.allocationSize,
9844  m_NextBlockId++,
9845  m_LinearAlgorithm);
9846 
9847  m_Blocks.push_back(pBlock);
9848  if(pNewBlockIndex != VMA_NULL)
9849  {
9850  *pNewBlockIndex = m_Blocks.size() - 1;
9851  }
9852 
9853  return VK_SUCCESS;
9854 }
9855 
9856 #if VMA_STATS_STRING_ENABLED
9857 
9858 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
9859 {
9860  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9861 
9862  json.BeginObject();
9863 
9864  if(m_IsCustomPool)
9865  {
9866  json.WriteString("MemoryTypeIndex");
9867  json.WriteNumber(m_MemoryTypeIndex);
9868 
9869  json.WriteString("BlockSize");
9870  json.WriteNumber(m_PreferredBlockSize);
9871 
9872  json.WriteString("BlockCount");
9873  json.BeginObject(true);
9874  if(m_MinBlockCount > 0)
9875  {
9876  json.WriteString("Min");
9877  json.WriteNumber((uint64_t)m_MinBlockCount);
9878  }
9879  if(m_MaxBlockCount < SIZE_MAX)
9880  {
9881  json.WriteString("Max");
9882  json.WriteNumber((uint64_t)m_MaxBlockCount);
9883  }
9884  json.WriteString("Cur");
9885  json.WriteNumber((uint64_t)m_Blocks.size());
9886  json.EndObject();
9887 
9888  if(m_FrameInUseCount > 0)
9889  {
9890  json.WriteString("FrameInUseCount");
9891  json.WriteNumber(m_FrameInUseCount);
9892  }
9893 
9894  if(m_LinearAlgorithm)
9895  {
9896  json.WriteString("LinearAlgorithm");
9897  json.WriteBool(true);
9898  }
9899  }
9900  else
9901  {
9902  json.WriteString("PreferredBlockSize");
9903  json.WriteNumber(m_PreferredBlockSize);
9904  }
9905 
9906  json.WriteString("Blocks");
9907  json.BeginObject();
9908  for(size_t i = 0; i < m_Blocks.size(); ++i)
9909  {
9910  json.BeginString();
9911  json.ContinueString(m_Blocks[i]->GetId());
9912  json.EndString();
9913 
9914  m_Blocks[i]->m_pMetadata->PrintDetailedMap(json);
9915  }
9916  json.EndObject();
9917 
9918  json.EndObject();
9919 }
9920 
9921 #endif // #if VMA_STATS_STRING_ENABLED
9922 
9923 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
9924  VmaAllocator hAllocator,
9925  uint32_t currentFrameIndex)
9926 {
9927  if(m_pDefragmentator == VMA_NULL)
9928  {
9929  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
9930  hAllocator,
9931  this,
9932  currentFrameIndex);
9933  }
9934 
9935  return m_pDefragmentator;
9936 }
9937 
9938 VkResult VmaBlockVector::Defragment(
9939  VmaDefragmentationStats* pDefragmentationStats,
9940  VkDeviceSize& maxBytesToMove,
9941  uint32_t& maxAllocationsToMove)
9942 {
9943  if(m_pDefragmentator == VMA_NULL)
9944  {
9945  return VK_SUCCESS;
9946  }
9947 
9948  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9949 
9950  // Defragment.
9951  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
9952 
9953  // Accumulate statistics.
9954  if(pDefragmentationStats != VMA_NULL)
9955  {
9956  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
9957  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
9958  pDefragmentationStats->bytesMoved += bytesMoved;
9959  pDefragmentationStats->allocationsMoved += allocationsMoved;
9960  VMA_ASSERT(bytesMoved <= maxBytesToMove);
9961  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
9962  maxBytesToMove -= bytesMoved;
9963  maxAllocationsToMove -= allocationsMoved;
9964  }
9965 
9966  // Free empty blocks.
9967  m_HasEmptyBlock = false;
9968  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
9969  {
9970  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
9971  if(pBlock->m_pMetadata->IsEmpty())
9972  {
9973  if(m_Blocks.size() > m_MinBlockCount)
9974  {
9975  if(pDefragmentationStats != VMA_NULL)
9976  {
9977  ++pDefragmentationStats->deviceMemoryBlocksFreed;
9978  pDefragmentationStats->bytesFreed += pBlock->m_pMetadata->GetSize();
9979  }
9980 
9981  VmaVectorRemove(m_Blocks, blockIndex);
9982  pBlock->Destroy(m_hAllocator);
9983  vma_delete(m_hAllocator, pBlock);
9984  }
9985  else
9986  {
9987  m_HasEmptyBlock = true;
9988  }
9989  }
9990  }
9991 
9992  return result;
9993 }
9994 
9995 void VmaBlockVector::DestroyDefragmentator()
9996 {
9997  if(m_pDefragmentator != VMA_NULL)
9998  {
9999  vma_delete(m_hAllocator, m_pDefragmentator);
10000  m_pDefragmentator = VMA_NULL;
10001  }
10002 }
10003 
10004 void VmaBlockVector::MakePoolAllocationsLost(
10005  uint32_t currentFrameIndex,
10006  size_t* pLostAllocationCount)
10007 {
10008  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
10009  size_t lostAllocationCount = 0;
10010  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
10011  {
10012  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
10013  VMA_ASSERT(pBlock);
10014  lostAllocationCount += pBlock->m_pMetadata->MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
10015  }
10016  if(pLostAllocationCount != VMA_NULL)
10017  {
10018  *pLostAllocationCount = lostAllocationCount;
10019  }
10020 }
10021 
10022 VkResult VmaBlockVector::CheckCorruption()
10023 {
10024  if(!IsCorruptionDetectionEnabled())
10025  {
10026  return VK_ERROR_FEATURE_NOT_PRESENT;
10027  }
10028 
10029  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
10030  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
10031  {
10032  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
10033  VMA_ASSERT(pBlock);
10034  VkResult res = pBlock->CheckCorruption(m_hAllocator);
10035  if(res != VK_SUCCESS)
10036  {
10037  return res;
10038  }
10039  }
10040  return VK_SUCCESS;
10041 }
10042 
10043 void VmaBlockVector::AddStats(VmaStats* pStats)
10044 {
10045  const uint32_t memTypeIndex = m_MemoryTypeIndex;
10046  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
10047 
10048  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
10049 
10050  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
10051  {
10052  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
10053  VMA_ASSERT(pBlock);
10054  VMA_HEAVY_ASSERT(pBlock->Validate());
10055  VmaStatInfo allocationStatInfo;
10056  pBlock->m_pMetadata->CalcAllocationStatInfo(allocationStatInfo);
10057  VmaAddStatInfo(pStats->total, allocationStatInfo);
10058  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
10059  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
10060  }
10061 }
10062 
10064 // VmaDefragmentator members definition
10065 
10066 VmaDefragmentator::VmaDefragmentator(
10067  VmaAllocator hAllocator,
10068  VmaBlockVector* pBlockVector,
10069  uint32_t currentFrameIndex) :
10070  m_hAllocator(hAllocator),
10071  m_pBlockVector(pBlockVector),
10072  m_CurrentFrameIndex(currentFrameIndex),
10073  m_BytesMoved(0),
10074  m_AllocationsMoved(0),
10075  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
10076  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
10077 {
10078  VMA_ASSERT(!pBlockVector->UsesLinearAlgorithm());
10079 }
10080 
10081 VmaDefragmentator::~VmaDefragmentator()
10082 {
10083  for(size_t i = m_Blocks.size(); i--; )
10084  {
10085  vma_delete(m_hAllocator, m_Blocks[i]);
10086  }
10087 }
10088 
10089 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
10090 {
10091  AllocationInfo allocInfo;
10092  allocInfo.m_hAllocation = hAlloc;
10093  allocInfo.m_pChanged = pChanged;
10094  m_Allocations.push_back(allocInfo);
10095 }
10096 
10097 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
10098 {
10099  // It has already been mapped for defragmentation.
10100  if(m_pMappedDataForDefragmentation)
10101  {
10102  *ppMappedData = m_pMappedDataForDefragmentation;
10103  return VK_SUCCESS;
10104  }
10105 
10106  // It is originally mapped.
10107  if(m_pBlock->GetMappedData())
10108  {
10109  *ppMappedData = m_pBlock->GetMappedData();
10110  return VK_SUCCESS;
10111  }
10112 
10113  // Map on first usage.
10114  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
10115  *ppMappedData = m_pMappedDataForDefragmentation;
10116  return res;
10117 }
10118 
10119 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
10120 {
10121  if(m_pMappedDataForDefragmentation != VMA_NULL)
10122  {
10123  m_pBlock->Unmap(hAllocator, 1);
10124  }
10125 }
10126 
10127 VkResult VmaDefragmentator::DefragmentRound(
10128  VkDeviceSize maxBytesToMove,
10129  uint32_t maxAllocationsToMove)
10130 {
10131  if(m_Blocks.empty())
10132  {
10133  return VK_SUCCESS;
10134  }
10135 
10136  size_t srcBlockIndex = m_Blocks.size() - 1;
10137  size_t srcAllocIndex = SIZE_MAX;
10138  for(;;)
10139  {
10140  // 1. Find next allocation to move.
10141  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
10142  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
10143  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
10144  {
10145  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
10146  {
10147  // Finished: no more allocations to process.
10148  if(srcBlockIndex == 0)
10149  {
10150  return VK_SUCCESS;
10151  }
10152  else
10153  {
10154  --srcBlockIndex;
10155  srcAllocIndex = SIZE_MAX;
10156  }
10157  }
10158  else
10159  {
10160  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
10161  }
10162  }
10163 
10164  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
10165  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
10166 
10167  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
10168  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
10169  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
10170  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
10171 
10172  // 2. Try to find new place for this allocation in preceding or current block.
10173  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
10174  {
10175  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
10176  VmaAllocationRequest dstAllocRequest;
10177  if(pDstBlockInfo->m_pBlock->m_pMetadata->CreateAllocationRequest(
10178  m_CurrentFrameIndex,
10179  m_pBlockVector->GetFrameInUseCount(),
10180  m_pBlockVector->GetBufferImageGranularity(),
10181  size,
10182  alignment,
10183  false, // upperAddress
10184  suballocType,
10185  false, // canMakeOtherLost
10186  &dstAllocRequest) &&
10187  MoveMakesSense(
10188  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
10189  {
10190  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
10191 
10192  // Reached limit on number of allocations or bytes to move.
10193  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
10194  (m_BytesMoved + size > maxBytesToMove))
10195  {
10196  return VK_INCOMPLETE;
10197  }
10198 
10199  void* pDstMappedData = VMA_NULL;
10200  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
10201  if(res != VK_SUCCESS)
10202  {
10203  return res;
10204  }
10205 
10206  void* pSrcMappedData = VMA_NULL;
10207  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
10208  if(res != VK_SUCCESS)
10209  {
10210  return res;
10211  }
10212 
10213  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
10214  memcpy(
10215  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
10216  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
10217  static_cast<size_t>(size));
10218 
10219  if(VMA_DEBUG_MARGIN > 0)
10220  {
10221  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
10222  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
10223  }
10224 
10225  pDstBlockInfo->m_pBlock->m_pMetadata->Alloc(
10226  dstAllocRequest,
10227  suballocType,
10228  size,
10229  false, // upperAddress
10230  allocInfo.m_hAllocation);
10231  pSrcBlockInfo->m_pBlock->m_pMetadata->FreeAtOffset(srcOffset);
10232 
10233  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
10234 
10235  if(allocInfo.m_pChanged != VMA_NULL)
10236  {
10237  *allocInfo.m_pChanged = VK_TRUE;
10238  }
10239 
10240  ++m_AllocationsMoved;
10241  m_BytesMoved += size;
10242 
10243  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
10244 
10245  break;
10246  }
10247  }
10248 
10249  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
10250 
10251  if(srcAllocIndex > 0)
10252  {
10253  --srcAllocIndex;
10254  }
10255  else
10256  {
10257  if(srcBlockIndex > 0)
10258  {
10259  --srcBlockIndex;
10260  srcAllocIndex = SIZE_MAX;
10261  }
10262  else
10263  {
10264  return VK_SUCCESS;
10265  }
10266  }
10267  }
10268 }
10269 
10270 VkResult VmaDefragmentator::Defragment(
10271  VkDeviceSize maxBytesToMove,
10272  uint32_t maxAllocationsToMove)
10273 {
10274  if(m_Allocations.empty())
10275  {
10276  return VK_SUCCESS;
10277  }
10278 
10279  // Create block info for each block.
10280  const size_t blockCount = m_pBlockVector->m_Blocks.size();
10281  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
10282  {
10283  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
10284  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
10285  m_Blocks.push_back(pBlockInfo);
10286  }
10287 
10288  // Sort them by m_pBlock pointer value.
10289  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
10290 
10291  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
10292  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
10293  {
10294  AllocationInfo& allocInfo = m_Allocations[blockIndex];
10295  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
10296  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
10297  {
10298  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
10299  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
10300  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
10301  {
10302  (*it)->m_Allocations.push_back(allocInfo);
10303  }
10304  else
10305  {
10306  VMA_ASSERT(0);
10307  }
10308  }
10309  }
10310  m_Allocations.clear();
10311 
10312  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
10313  {
10314  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
10315  pBlockInfo->CalcHasNonMovableAllocations();
10316  pBlockInfo->SortAllocationsBySizeDescecnding();
10317  }
10318 
10319  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
10320  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
10321 
10322  // Execute defragmentation rounds (the main part).
10323  VkResult result = VK_SUCCESS;
10324  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
10325  {
10326  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
10327  }
10328 
10329  // Unmap blocks that were mapped for defragmentation.
10330  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
10331  {
10332  m_Blocks[blockIndex]->Unmap(m_hAllocator);
10333  }
10334 
10335  return result;
10336 }
10337 
10338 bool VmaDefragmentator::MoveMakesSense(
10339  size_t dstBlockIndex, VkDeviceSize dstOffset,
10340  size_t srcBlockIndex, VkDeviceSize srcOffset)
10341 {
10342  if(dstBlockIndex < srcBlockIndex)
10343  {
10344  return true;
10345  }
10346  if(dstBlockIndex > srcBlockIndex)
10347  {
10348  return false;
10349  }
10350  if(dstOffset < srcOffset)
10351  {
10352  return true;
10353  }
10354  return false;
10355 }
10356 
10358 // VmaRecorder
10359 
10360 #if VMA_RECORDING_ENABLED
10361 
10362 VmaRecorder::VmaRecorder() :
10363  m_UseMutex(true),
10364  m_Flags(0),
10365  m_File(VMA_NULL),
10366  m_Freq(INT64_MAX),
10367  m_StartCounter(INT64_MAX)
10368 {
10369 }
10370 
10371 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
10372 {
10373  m_UseMutex = useMutex;
10374  m_Flags = settings.flags;
10375 
10376  QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq);
10377  QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter);
10378 
10379  // Open file for writing.
10380  errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
10381  if(err != 0)
10382  {
10383  return VK_ERROR_INITIALIZATION_FAILED;
10384  }
10385 
10386  // Write header.
10387  fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
10388  fprintf(m_File, "%s\n", "1,3");
10389 
10390  return VK_SUCCESS;
10391 }
10392 
10393 VmaRecorder::~VmaRecorder()
10394 {
10395  if(m_File != VMA_NULL)
10396  {
10397  fclose(m_File);
10398  }
10399 }
10400 
10401 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
10402 {
10403  CallParams callParams;
10404  GetBasicParams(callParams);
10405 
10406  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10407  fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
10408  Flush();
10409 }
10410 
10411 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
10412 {
10413  CallParams callParams;
10414  GetBasicParams(callParams);
10415 
10416  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10417  fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
10418  Flush();
10419 }
10420 
10421 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
10422 {
10423  CallParams callParams;
10424  GetBasicParams(callParams);
10425 
10426  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10427  fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
10428  createInfo.memoryTypeIndex,
10429  createInfo.flags,
10430  createInfo.blockSize,
10431  createInfo.minBlockCount,
10432  createInfo.maxBlockCount,
10433  createInfo.frameInUseCount,
10434  pool);
10435  Flush();
10436 }
10437 
10438 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
10439 {
10440  CallParams callParams;
10441  GetBasicParams(callParams);
10442 
10443  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10444  fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
10445  pool);
10446  Flush();
10447 }
10448 
10449 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
10450  const VkMemoryRequirements& vkMemReq,
10451  const VmaAllocationCreateInfo& createInfo,
10452  VmaAllocation allocation)
10453 {
10454  CallParams callParams;
10455  GetBasicParams(callParams);
10456 
10457  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10458  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
10459  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10460  vkMemReq.size,
10461  vkMemReq.alignment,
10462  vkMemReq.memoryTypeBits,
10463  createInfo.flags,
10464  createInfo.usage,
10465  createInfo.requiredFlags,
10466  createInfo.preferredFlags,
10467  createInfo.memoryTypeBits,
10468  createInfo.pool,
10469  allocation,
10470  userDataStr.GetString());
10471  Flush();
10472 }
10473 
10474 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
10475  const VkMemoryRequirements& vkMemReq,
10476  bool requiresDedicatedAllocation,
10477  bool prefersDedicatedAllocation,
10478  const VmaAllocationCreateInfo& createInfo,
10479  VmaAllocation allocation)
10480 {
10481  CallParams callParams;
10482  GetBasicParams(callParams);
10483 
10484  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10485  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
10486  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10487  vkMemReq.size,
10488  vkMemReq.alignment,
10489  vkMemReq.memoryTypeBits,
10490  requiresDedicatedAllocation ? 1 : 0,
10491  prefersDedicatedAllocation ? 1 : 0,
10492  createInfo.flags,
10493  createInfo.usage,
10494  createInfo.requiredFlags,
10495  createInfo.preferredFlags,
10496  createInfo.memoryTypeBits,
10497  createInfo.pool,
10498  allocation,
10499  userDataStr.GetString());
10500  Flush();
10501 }
10502 
10503 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
10504  const VkMemoryRequirements& vkMemReq,
10505  bool requiresDedicatedAllocation,
10506  bool prefersDedicatedAllocation,
10507  const VmaAllocationCreateInfo& createInfo,
10508  VmaAllocation allocation)
10509 {
10510  CallParams callParams;
10511  GetBasicParams(callParams);
10512 
10513  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10514  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
10515  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10516  vkMemReq.size,
10517  vkMemReq.alignment,
10518  vkMemReq.memoryTypeBits,
10519  requiresDedicatedAllocation ? 1 : 0,
10520  prefersDedicatedAllocation ? 1 : 0,
10521  createInfo.flags,
10522  createInfo.usage,
10523  createInfo.requiredFlags,
10524  createInfo.preferredFlags,
10525  createInfo.memoryTypeBits,
10526  createInfo.pool,
10527  allocation,
10528  userDataStr.GetString());
10529  Flush();
10530 }
10531 
10532 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
10533  VmaAllocation allocation)
10534 {
10535  CallParams callParams;
10536  GetBasicParams(callParams);
10537 
10538  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10539  fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
10540  allocation);
10541  Flush();
10542 }
10543 
10544 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
10545  VmaAllocation allocation,
10546  const void* pUserData)
10547 {
10548  CallParams callParams;
10549  GetBasicParams(callParams);
10550 
10551  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10552  UserDataString userDataStr(
10553  allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
10554  pUserData);
10555  fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10556  allocation,
10557  userDataStr.GetString());
10558  Flush();
10559 }
10560 
10561 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
10562  VmaAllocation allocation)
10563 {
10564  CallParams callParams;
10565  GetBasicParams(callParams);
10566 
10567  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10568  fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
10569  allocation);
10570  Flush();
10571 }
10572 
10573 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
10574  VmaAllocation allocation)
10575 {
10576  CallParams callParams;
10577  GetBasicParams(callParams);
10578 
10579  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10580  fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
10581  allocation);
10582  Flush();
10583 }
10584 
10585 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
10586  VmaAllocation allocation)
10587 {
10588  CallParams callParams;
10589  GetBasicParams(callParams);
10590 
10591  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10592  fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
10593  allocation);
10594  Flush();
10595 }
10596 
10597 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
10598  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
10599 {
10600  CallParams callParams;
10601  GetBasicParams(callParams);
10602 
10603  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10604  fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
10605  allocation,
10606  offset,
10607  size);
10608  Flush();
10609 }
10610 
10611 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
10612  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
10613 {
10614  CallParams callParams;
10615  GetBasicParams(callParams);
10616 
10617  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10618  fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
10619  allocation,
10620  offset,
10621  size);
10622  Flush();
10623 }
10624 
10625 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
10626  const VkBufferCreateInfo& bufCreateInfo,
10627  const VmaAllocationCreateInfo& allocCreateInfo,
10628  VmaAllocation allocation)
10629 {
10630  CallParams callParams;
10631  GetBasicParams(callParams);
10632 
10633  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10634  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
10635  fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10636  bufCreateInfo.flags,
10637  bufCreateInfo.size,
10638  bufCreateInfo.usage,
10639  bufCreateInfo.sharingMode,
10640  allocCreateInfo.flags,
10641  allocCreateInfo.usage,
10642  allocCreateInfo.requiredFlags,
10643  allocCreateInfo.preferredFlags,
10644  allocCreateInfo.memoryTypeBits,
10645  allocCreateInfo.pool,
10646  allocation,
10647  userDataStr.GetString());
10648  Flush();
10649 }
10650 
10651 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
10652  const VkImageCreateInfo& imageCreateInfo,
10653  const VmaAllocationCreateInfo& allocCreateInfo,
10654  VmaAllocation allocation)
10655 {
10656  CallParams callParams;
10657  GetBasicParams(callParams);
10658 
10659  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10660  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
10661  fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10662  imageCreateInfo.flags,
10663  imageCreateInfo.imageType,
10664  imageCreateInfo.format,
10665  imageCreateInfo.extent.width,
10666  imageCreateInfo.extent.height,
10667  imageCreateInfo.extent.depth,
10668  imageCreateInfo.mipLevels,
10669  imageCreateInfo.arrayLayers,
10670  imageCreateInfo.samples,
10671  imageCreateInfo.tiling,
10672  imageCreateInfo.usage,
10673  imageCreateInfo.sharingMode,
10674  imageCreateInfo.initialLayout,
10675  allocCreateInfo.flags,
10676  allocCreateInfo.usage,
10677  allocCreateInfo.requiredFlags,
10678  allocCreateInfo.preferredFlags,
10679  allocCreateInfo.memoryTypeBits,
10680  allocCreateInfo.pool,
10681  allocation,
10682  userDataStr.GetString());
10683  Flush();
10684 }
10685 
10686 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
10687  VmaAllocation allocation)
10688 {
10689  CallParams callParams;
10690  GetBasicParams(callParams);
10691 
10692  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10693  fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
10694  allocation);
10695  Flush();
10696 }
10697 
10698 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
10699  VmaAllocation allocation)
10700 {
10701  CallParams callParams;
10702  GetBasicParams(callParams);
10703 
10704  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10705  fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
10706  allocation);
10707  Flush();
10708 }
10709 
10710 void VmaRecorder::RecordTouchAllocation(uint32_t frameIndex,
10711  VmaAllocation allocation)
10712 {
10713  CallParams callParams;
10714  GetBasicParams(callParams);
10715 
10716  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10717  fprintf(m_File, "%u,%.3f,%u,vmaTouchAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
10718  allocation);
10719  Flush();
10720 }
10721 
10722 void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex,
10723  VmaAllocation allocation)
10724 {
10725  CallParams callParams;
10726  GetBasicParams(callParams);
10727 
10728  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10729  fprintf(m_File, "%u,%.3f,%u,vmaGetAllocationInfo,%p\n", callParams.threadId, callParams.time, frameIndex,
10730  allocation);
10731  Flush();
10732 }
10733 
10734 void VmaRecorder::RecordMakePoolAllocationsLost(uint32_t frameIndex,
10735  VmaPool pool)
10736 {
10737  CallParams callParams;
10738  GetBasicParams(callParams);
10739 
10740  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10741  fprintf(m_File, "%u,%.3f,%u,vmaMakePoolAllocationsLost,%p\n", callParams.threadId, callParams.time, frameIndex,
10742  pool);
10743  Flush();
10744 }
10745 
10746 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
10747 {
10748  if(pUserData != VMA_NULL)
10749  {
10750  if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
10751  {
10752  m_Str = (const char*)pUserData;
10753  }
10754  else
10755  {
10756  sprintf_s(m_PtrStr, "%p", pUserData);
10757  m_Str = m_PtrStr;
10758  }
10759  }
10760  else
10761  {
10762  m_Str = "";
10763  }
10764 }
10765 
10766 void VmaRecorder::WriteConfiguration(
10767  const VkPhysicalDeviceProperties& devProps,
10768  const VkPhysicalDeviceMemoryProperties& memProps,
10769  bool dedicatedAllocationExtensionEnabled)
10770 {
10771  fprintf(m_File, "Config,Begin\n");
10772 
10773  fprintf(m_File, "PhysicalDevice,apiVersion,%u\n", devProps.apiVersion);
10774  fprintf(m_File, "PhysicalDevice,driverVersion,%u\n", devProps.driverVersion);
10775  fprintf(m_File, "PhysicalDevice,vendorID,%u\n", devProps.vendorID);
10776  fprintf(m_File, "PhysicalDevice,deviceID,%u\n", devProps.deviceID);
10777  fprintf(m_File, "PhysicalDevice,deviceType,%u\n", devProps.deviceType);
10778  fprintf(m_File, "PhysicalDevice,deviceName,%s\n", devProps.deviceName);
10779 
10780  fprintf(m_File, "PhysicalDeviceLimits,maxMemoryAllocationCount,%u\n", devProps.limits.maxMemoryAllocationCount);
10781  fprintf(m_File, "PhysicalDeviceLimits,bufferImageGranularity,%llu\n", devProps.limits.bufferImageGranularity);
10782  fprintf(m_File, "PhysicalDeviceLimits,nonCoherentAtomSize,%llu\n", devProps.limits.nonCoherentAtomSize);
10783 
10784  fprintf(m_File, "PhysicalDeviceMemory,HeapCount,%u\n", memProps.memoryHeapCount);
10785  for(uint32_t i = 0; i < memProps.memoryHeapCount; ++i)
10786  {
10787  fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,size,%llu\n", i, memProps.memoryHeaps[i].size);
10788  fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,flags,%u\n", i, memProps.memoryHeaps[i].flags);
10789  }
10790  fprintf(m_File, "PhysicalDeviceMemory,TypeCount,%u\n", memProps.memoryTypeCount);
10791  for(uint32_t i = 0; i < memProps.memoryTypeCount; ++i)
10792  {
10793  fprintf(m_File, "PhysicalDeviceMemory,Type,%u,heapIndex,%u\n", i, memProps.memoryTypes[i].heapIndex);
10794  fprintf(m_File, "PhysicalDeviceMemory,Type,%u,propertyFlags,%u\n", i, memProps.memoryTypes[i].propertyFlags);
10795  }
10796 
10797  fprintf(m_File, "Extension,VK_KHR_dedicated_allocation,%u\n", dedicatedAllocationExtensionEnabled ? 1 : 0);
10798 
10799  fprintf(m_File, "Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY,%u\n", VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ? 1 : 0);
10800  fprintf(m_File, "Macro,VMA_DEBUG_ALIGNMENT,%llu\n", (VkDeviceSize)VMA_DEBUG_ALIGNMENT);
10801  fprintf(m_File, "Macro,VMA_DEBUG_MARGIN,%llu\n", (VkDeviceSize)VMA_DEBUG_MARGIN);
10802  fprintf(m_File, "Macro,VMA_DEBUG_INITIALIZE_ALLOCATIONS,%u\n", VMA_DEBUG_INITIALIZE_ALLOCATIONS ? 1 : 0);
10803  fprintf(m_File, "Macro,VMA_DEBUG_DETECT_CORRUPTION,%u\n", VMA_DEBUG_DETECT_CORRUPTION ? 1 : 0);
10804  fprintf(m_File, "Macro,VMA_DEBUG_GLOBAL_MUTEX,%u\n", VMA_DEBUG_GLOBAL_MUTEX ? 1 : 0);
10805  fprintf(m_File, "Macro,VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY,%llu\n", (VkDeviceSize)VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY);
10806  fprintf(m_File, "Macro,VMA_SMALL_HEAP_MAX_SIZE,%llu\n", (VkDeviceSize)VMA_SMALL_HEAP_MAX_SIZE);
10807  fprintf(m_File, "Macro,VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE,%llu\n", (VkDeviceSize)VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
10808 
10809  fprintf(m_File, "Config,End\n");
10810 }
10811 
10812 void VmaRecorder::GetBasicParams(CallParams& outParams)
10813 {
10814  outParams.threadId = GetCurrentThreadId();
10815 
10816  LARGE_INTEGER counter;
10817  QueryPerformanceCounter(&counter);
10818  outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq;
10819 }
10820 
10821 void VmaRecorder::Flush()
10822 {
10823  if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
10824  {
10825  fflush(m_File);
10826  }
10827 }
10828 
10829 #endif // #if VMA_RECORDING_ENABLED
10830 
10832 // VmaAllocator_T
10833 
10834 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
10835  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
10836  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
10837  m_hDevice(pCreateInfo->device),
10838  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
10839  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
10840  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
10841  m_PreferredLargeHeapBlockSize(0),
10842  m_PhysicalDevice(pCreateInfo->physicalDevice),
10843  m_CurrentFrameIndex(0),
10844  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
10845  m_NextPoolId(0)
10846 #if VMA_RECORDING_ENABLED
10847  ,m_pRecorder(VMA_NULL)
10848 #endif
10849 {
10850  if(VMA_DEBUG_DETECT_CORRUPTION)
10851  {
10852  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
10853  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
10854  }
10855 
10856  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
10857 
10858 #if !(VMA_DEDICATED_ALLOCATION)
10859  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
10860  {
10861  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
10862  }
10863 #endif
10864 
10865  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
10866  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
10867  memset(&m_MemProps, 0, sizeof(m_MemProps));
10868 
10869  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
10870  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
10871 
10872  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
10873  {
10874  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
10875  }
10876 
10877  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
10878  {
10879  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
10880  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
10881  }
10882 
10883  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
10884 
10885  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
10886  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
10887 
10888  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
10889  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
10890 
10891  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
10892  {
10893  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
10894  {
10895  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
10896  if(limit != VK_WHOLE_SIZE)
10897  {
10898  m_HeapSizeLimit[heapIndex] = limit;
10899  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
10900  {
10901  m_MemProps.memoryHeaps[heapIndex].size = limit;
10902  }
10903  }
10904  }
10905  }
10906 
10907  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
10908  {
10909  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
10910 
10911  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
10912  this,
10913  memTypeIndex,
10914  preferredBlockSize,
10915  0,
10916  SIZE_MAX,
10917  GetBufferImageGranularity(),
10918  pCreateInfo->frameInUseCount,
10919  false, // isCustomPool
10920  false); // linearAlgorithm
10921  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
10922  // becase minBlockCount is 0.
10923  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
10924 
10925  }
10926 }
10927 
10928 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
10929 {
10930  VkResult res = VK_SUCCESS;
10931 
10932  if(pCreateInfo->pRecordSettings != VMA_NULL &&
10933  !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
10934  {
10935 #if VMA_RECORDING_ENABLED
10936  m_pRecorder = vma_new(this, VmaRecorder)();
10937  res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
10938  if(res != VK_SUCCESS)
10939  {
10940  return res;
10941  }
10942  m_pRecorder->WriteConfiguration(
10943  m_PhysicalDeviceProperties,
10944  m_MemProps,
10945  m_UseKhrDedicatedAllocation);
10946  m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
10947 #else
10948  VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
10949  return VK_ERROR_FEATURE_NOT_PRESENT;
10950 #endif
10951  }
10952 
10953  return res;
10954 }
10955 
10956 VmaAllocator_T::~VmaAllocator_T()
10957 {
10958 #if VMA_RECORDING_ENABLED
10959  if(m_pRecorder != VMA_NULL)
10960  {
10961  m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
10962  vma_delete(this, m_pRecorder);
10963  }
10964 #endif
10965 
10966  VMA_ASSERT(m_Pools.empty());
10967 
10968  for(size_t i = GetMemoryTypeCount(); i--; )
10969  {
10970  vma_delete(this, m_pDedicatedAllocations[i]);
10971  vma_delete(this, m_pBlockVectors[i]);
10972  }
10973 }
10974 
10975 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
10976 {
10977 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
10978  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
10979  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
10980  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
10981  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
10982  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
10983  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
10984  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
10985  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
10986  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
10987  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
10988  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
10989  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
10990  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
10991  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
10992  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
10993  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
10994 #if VMA_DEDICATED_ALLOCATION
10995  if(m_UseKhrDedicatedAllocation)
10996  {
10997  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
10998  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
10999  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
11000  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
11001  }
11002 #endif // #if VMA_DEDICATED_ALLOCATION
11003 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
11004 
11005 #define VMA_COPY_IF_NOT_NULL(funcName) \
11006  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
11007 
11008  if(pVulkanFunctions != VMA_NULL)
11009  {
11010  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
11011  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
11012  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
11013  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
11014  VMA_COPY_IF_NOT_NULL(vkMapMemory);
11015  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
11016  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
11017  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
11018  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
11019  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
11020  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
11021  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
11022  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
11023  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
11024  VMA_COPY_IF_NOT_NULL(vkCreateImage);
11025  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
11026 #if VMA_DEDICATED_ALLOCATION
11027  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
11028  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
11029 #endif
11030  }
11031 
11032 #undef VMA_COPY_IF_NOT_NULL
11033 
11034  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
11035  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
11036  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
11037  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
11038  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
11039  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
11040  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
11041  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
11042  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
11043  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
11044  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
11045  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
11046  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
11047  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
11048  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
11049  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
11050  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
11051  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
11052 #if VMA_DEDICATED_ALLOCATION
11053  if(m_UseKhrDedicatedAllocation)
11054  {
11055  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
11056  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
11057  }
11058 #endif
11059 }
11060 
11061 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
11062 {
11063  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
11064  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
11065  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
11066  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
11067 }
11068 
11069 VkResult VmaAllocator_T::AllocateMemoryOfType(
11070  VkDeviceSize size,
11071  VkDeviceSize alignment,
11072  bool dedicatedAllocation,
11073  VkBuffer dedicatedBuffer,
11074  VkImage dedicatedImage,
11075  const VmaAllocationCreateInfo& createInfo,
11076  uint32_t memTypeIndex,
11077  VmaSuballocationType suballocType,
11078  VmaAllocation* pAllocation)
11079 {
11080  VMA_ASSERT(pAllocation != VMA_NULL);
11081  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
11082 
11083  VmaAllocationCreateInfo finalCreateInfo = createInfo;
11084 
11085  // If memory type is not HOST_VISIBLE, disable MAPPED.
11086  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
11087  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
11088  {
11089  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
11090  }
11091 
11092  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
11093  VMA_ASSERT(blockVector);
11094 
11095  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
11096  bool preferDedicatedMemory =
11097  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
11098  dedicatedAllocation ||
11099  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
11100  size > preferredBlockSize / 2;
11101 
11102  if(preferDedicatedMemory &&
11103  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
11104  finalCreateInfo.pool == VK_NULL_HANDLE)
11105  {
11106  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
11107  }
11108 
11109  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
11110  {
11111  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
11112  {
11113  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11114  }
11115  else
11116  {
11117  return AllocateDedicatedMemory(
11118  size,
11119  suballocType,
11120  memTypeIndex,
11121  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
11122  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
11123  finalCreateInfo.pUserData,
11124  dedicatedBuffer,
11125  dedicatedImage,
11126  pAllocation);
11127  }
11128  }
11129  else
11130  {
11131  VkResult res = blockVector->Allocate(
11132  VK_NULL_HANDLE, // hCurrentPool
11133  m_CurrentFrameIndex.load(),
11134  size,
11135  alignment,
11136  finalCreateInfo,
11137  suballocType,
11138  pAllocation);
11139  if(res == VK_SUCCESS)
11140  {
11141  return res;
11142  }
11143 
11144  // 5. Try dedicated memory.
11145  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
11146  {
11147  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11148  }
11149  else
11150  {
11151  res = AllocateDedicatedMemory(
11152  size,
11153  suballocType,
11154  memTypeIndex,
11155  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
11156  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
11157  finalCreateInfo.pUserData,
11158  dedicatedBuffer,
11159  dedicatedImage,
11160  pAllocation);
11161  if(res == VK_SUCCESS)
11162  {
11163  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
11164  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
11165  return VK_SUCCESS;
11166  }
11167  else
11168  {
11169  // Everything failed: Return error code.
11170  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
11171  return res;
11172  }
11173  }
11174  }
11175 }
11176 
11177 VkResult VmaAllocator_T::AllocateDedicatedMemory(
11178  VkDeviceSize size,
11179  VmaSuballocationType suballocType,
11180  uint32_t memTypeIndex,
11181  bool map,
11182  bool isUserDataString,
11183  void* pUserData,
11184  VkBuffer dedicatedBuffer,
11185  VkImage dedicatedImage,
11186  VmaAllocation* pAllocation)
11187 {
11188  VMA_ASSERT(pAllocation);
11189 
11190  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
11191  allocInfo.memoryTypeIndex = memTypeIndex;
11192  allocInfo.allocationSize = size;
11193 
11194 #if VMA_DEDICATED_ALLOCATION
11195  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
11196  if(m_UseKhrDedicatedAllocation)
11197  {
11198  if(dedicatedBuffer != VK_NULL_HANDLE)
11199  {
11200  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
11201  dedicatedAllocInfo.buffer = dedicatedBuffer;
11202  allocInfo.pNext = &dedicatedAllocInfo;
11203  }
11204  else if(dedicatedImage != VK_NULL_HANDLE)
11205  {
11206  dedicatedAllocInfo.image = dedicatedImage;
11207  allocInfo.pNext = &dedicatedAllocInfo;
11208  }
11209  }
11210 #endif // #if VMA_DEDICATED_ALLOCATION
11211 
11212  // Allocate VkDeviceMemory.
11213  VkDeviceMemory hMemory = VK_NULL_HANDLE;
11214  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
11215  if(res < 0)
11216  {
11217  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
11218  return res;
11219  }
11220 
11221  void* pMappedData = VMA_NULL;
11222  if(map)
11223  {
11224  res = (*m_VulkanFunctions.vkMapMemory)(
11225  m_hDevice,
11226  hMemory,
11227  0,
11228  VK_WHOLE_SIZE,
11229  0,
11230  &pMappedData);
11231  if(res < 0)
11232  {
11233  VMA_DEBUG_LOG(" vkMapMemory FAILED");
11234  FreeVulkanMemory(memTypeIndex, size, hMemory);
11235  return res;
11236  }
11237  }
11238 
11239  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
11240  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
11241  (*pAllocation)->SetUserData(this, pUserData);
11242  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
11243  {
11244  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
11245  }
11246 
11247  // Register it in m_pDedicatedAllocations.
11248  {
11249  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
11250  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
11251  VMA_ASSERT(pDedicatedAllocations);
11252  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
11253  }
11254 
11255  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
11256 
11257  return VK_SUCCESS;
11258 }
11259 
11260 void VmaAllocator_T::GetBufferMemoryRequirements(
11261  VkBuffer hBuffer,
11262  VkMemoryRequirements& memReq,
11263  bool& requiresDedicatedAllocation,
11264  bool& prefersDedicatedAllocation) const
11265 {
11266 #if VMA_DEDICATED_ALLOCATION
11267  if(m_UseKhrDedicatedAllocation)
11268  {
11269  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
11270  memReqInfo.buffer = hBuffer;
11271 
11272  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
11273 
11274  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
11275  memReq2.pNext = &memDedicatedReq;
11276 
11277  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
11278 
11279  memReq = memReq2.memoryRequirements;
11280  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
11281  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
11282  }
11283  else
11284 #endif // #if VMA_DEDICATED_ALLOCATION
11285  {
11286  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
11287  requiresDedicatedAllocation = false;
11288  prefersDedicatedAllocation = false;
11289  }
11290 }
11291 
11292 void VmaAllocator_T::GetImageMemoryRequirements(
11293  VkImage hImage,
11294  VkMemoryRequirements& memReq,
11295  bool& requiresDedicatedAllocation,
11296  bool& prefersDedicatedAllocation) const
11297 {
11298 #if VMA_DEDICATED_ALLOCATION
11299  if(m_UseKhrDedicatedAllocation)
11300  {
11301  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
11302  memReqInfo.image = hImage;
11303 
11304  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
11305 
11306  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
11307  memReq2.pNext = &memDedicatedReq;
11308 
11309  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
11310 
11311  memReq = memReq2.memoryRequirements;
11312  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
11313  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
11314  }
11315  else
11316 #endif // #if VMA_DEDICATED_ALLOCATION
11317  {
11318  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
11319  requiresDedicatedAllocation = false;
11320  prefersDedicatedAllocation = false;
11321  }
11322 }
11323 
11324 VkResult VmaAllocator_T::AllocateMemory(
11325  const VkMemoryRequirements& vkMemReq,
11326  bool requiresDedicatedAllocation,
11327  bool prefersDedicatedAllocation,
11328  VkBuffer dedicatedBuffer,
11329  VkImage dedicatedImage,
11330  const VmaAllocationCreateInfo& createInfo,
11331  VmaSuballocationType suballocType,
11332  VmaAllocation* pAllocation)
11333 {
11334  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
11335  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
11336  {
11337  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
11338  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11339  }
11340  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
11341  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
11342  {
11343  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
11344  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11345  }
11346  if(requiresDedicatedAllocation)
11347  {
11348  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
11349  {
11350  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
11351  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11352  }
11353  if(createInfo.pool != VK_NULL_HANDLE)
11354  {
11355  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
11356  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11357  }
11358  }
11359  if((createInfo.pool != VK_NULL_HANDLE) &&
11360  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
11361  {
11362  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
11363  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11364  }
11365 
11366  if(createInfo.pool != VK_NULL_HANDLE)
11367  {
11368  const VkDeviceSize alignmentForPool = VMA_MAX(
11369  vkMemReq.alignment,
11370  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
11371  return createInfo.pool->m_BlockVector.Allocate(
11372  createInfo.pool,
11373  m_CurrentFrameIndex.load(),
11374  vkMemReq.size,
11375  alignmentForPool,
11376  createInfo,
11377  suballocType,
11378  pAllocation);
11379  }
11380  else
11381  {
11382  // Bit mask of memory Vulkan types acceptable for this allocation.
11383  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
11384  uint32_t memTypeIndex = UINT32_MAX;
11385  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
11386  if(res == VK_SUCCESS)
11387  {
11388  VkDeviceSize alignmentForMemType = VMA_MAX(
11389  vkMemReq.alignment,
11390  GetMemoryTypeMinAlignment(memTypeIndex));
11391 
11392  res = AllocateMemoryOfType(
11393  vkMemReq.size,
11394  alignmentForMemType,
11395  requiresDedicatedAllocation || prefersDedicatedAllocation,
11396  dedicatedBuffer,
11397  dedicatedImage,
11398  createInfo,
11399  memTypeIndex,
11400  suballocType,
11401  pAllocation);
11402  // Succeeded on first try.
11403  if(res == VK_SUCCESS)
11404  {
11405  return res;
11406  }
11407  // Allocation from this memory type failed. Try other compatible memory types.
11408  else
11409  {
11410  for(;;)
11411  {
11412  // Remove old memTypeIndex from list of possibilities.
11413  memoryTypeBits &= ~(1u << memTypeIndex);
11414  // Find alternative memTypeIndex.
11415  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
11416  if(res == VK_SUCCESS)
11417  {
11418  alignmentForMemType = VMA_MAX(
11419  vkMemReq.alignment,
11420  GetMemoryTypeMinAlignment(memTypeIndex));
11421 
11422  res = AllocateMemoryOfType(
11423  vkMemReq.size,
11424  alignmentForMemType,
11425  requiresDedicatedAllocation || prefersDedicatedAllocation,
11426  dedicatedBuffer,
11427  dedicatedImage,
11428  createInfo,
11429  memTypeIndex,
11430  suballocType,
11431  pAllocation);
11432  // Allocation from this alternative memory type succeeded.
11433  if(res == VK_SUCCESS)
11434  {
11435  return res;
11436  }
11437  // else: Allocation from this memory type failed. Try next one - next loop iteration.
11438  }
11439  // No other matching memory type index could be found.
11440  else
11441  {
11442  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
11443  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11444  }
11445  }
11446  }
11447  }
11448  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
11449  else
11450  return res;
11451  }
11452 }
11453 
11454 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
11455 {
11456  VMA_ASSERT(allocation);
11457 
11458  if(allocation->CanBecomeLost() == false ||
11459  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
11460  {
11461  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
11462  {
11463  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
11464  }
11465 
11466  switch(allocation->GetType())
11467  {
11468  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
11469  {
11470  VmaBlockVector* pBlockVector = VMA_NULL;
11471  VmaPool hPool = allocation->GetPool();
11472  if(hPool != VK_NULL_HANDLE)
11473  {
11474  pBlockVector = &hPool->m_BlockVector;
11475  }
11476  else
11477  {
11478  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
11479  pBlockVector = m_pBlockVectors[memTypeIndex];
11480  }
11481  pBlockVector->Free(allocation);
11482  }
11483  break;
11484  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
11485  FreeDedicatedMemory(allocation);
11486  break;
11487  default:
11488  VMA_ASSERT(0);
11489  }
11490  }
11491 
11492  allocation->SetUserData(this, VMA_NULL);
11493  vma_delete(this, allocation);
11494 }
11495 
11496 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
11497 {
11498  // Initialize.
11499  InitStatInfo(pStats->total);
11500  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
11501  InitStatInfo(pStats->memoryType[i]);
11502  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
11503  InitStatInfo(pStats->memoryHeap[i]);
11504 
11505  // Process default pools.
11506  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
11507  {
11508  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
11509  VMA_ASSERT(pBlockVector);
11510  pBlockVector->AddStats(pStats);
11511  }
11512 
11513  // Process custom pools.
11514  {
11515  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
11516  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
11517  {
11518  m_Pools[poolIndex]->m_BlockVector.AddStats(pStats);
11519  }
11520  }
11521 
11522  // Process dedicated allocations.
11523  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
11524  {
11525  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
11526  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
11527  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
11528  VMA_ASSERT(pDedicatedAllocVector);
11529  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
11530  {
11531  VmaStatInfo allocationStatInfo;
11532  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
11533  VmaAddStatInfo(pStats->total, allocationStatInfo);
11534  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
11535  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
11536  }
11537  }
11538 
11539  // Postprocess.
11540  VmaPostprocessCalcStatInfo(pStats->total);
11541  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
11542  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
11543  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
11544  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
11545 }
11546 
11547 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
11548 
11549 VkResult VmaAllocator_T::Defragment(
11550  VmaAllocation* pAllocations,
11551  size_t allocationCount,
11552  VkBool32* pAllocationsChanged,
11553  const VmaDefragmentationInfo* pDefragmentationInfo,
11554  VmaDefragmentationStats* pDefragmentationStats)
11555 {
11556  if(pAllocationsChanged != VMA_NULL)
11557  {
11558  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
11559  }
11560  if(pDefragmentationStats != VMA_NULL)
11561  {
11562  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
11563  }
11564 
11565  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
11566 
11567  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
11568 
11569  const size_t poolCount = m_Pools.size();
11570 
11571  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
11572  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
11573  {
11574  VmaAllocation hAlloc = pAllocations[allocIndex];
11575  VMA_ASSERT(hAlloc);
11576  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
11577  // DedicatedAlloc cannot be defragmented.
11578  const VkMemoryPropertyFlags requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
11579  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
11580  // Only HOST_VISIBLE and HOST_COHERENT memory types can be defragmented.
11581  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags) &&
11582  // Lost allocation cannot be defragmented.
11583  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
11584  {
11585  VmaBlockVector* pAllocBlockVector = VMA_NULL;
11586 
11587  const VmaPool hAllocPool = hAlloc->GetPool();
11588  // This allocation belongs to custom pool.
11589  if(hAllocPool != VK_NULL_HANDLE)
11590  {
11591  // Pools with linear algorithm are not defragmented.
11592  if(!hAllocPool->m_BlockVector.UsesLinearAlgorithm())
11593  {
11594  pAllocBlockVector = &hAllocPool->m_BlockVector;
11595  }
11596  }
11597  // This allocation belongs to general pool.
11598  else
11599  {
11600  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
11601  }
11602 
11603  if(pAllocBlockVector != VMA_NULL)
11604  {
11605  VmaDefragmentator* const pDefragmentator =
11606  pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
11607  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
11608  &pAllocationsChanged[allocIndex] : VMA_NULL;
11609  pDefragmentator->AddAllocation(hAlloc, pChanged);
11610  }
11611  }
11612  }
11613 
11614  VkResult result = VK_SUCCESS;
11615 
11616  // ======== Main processing.
11617 
11618  VkDeviceSize maxBytesToMove = SIZE_MAX;
11619  uint32_t maxAllocationsToMove = UINT32_MAX;
11620  if(pDefragmentationInfo != VMA_NULL)
11621  {
11622  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
11623  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
11624  }
11625 
11626  // Process standard memory.
11627  for(uint32_t memTypeIndex = 0;
11628  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
11629  ++memTypeIndex)
11630  {
11631  // Only HOST_VISIBLE memory types can be defragmented.
11632  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
11633  {
11634  result = m_pBlockVectors[memTypeIndex]->Defragment(
11635  pDefragmentationStats,
11636  maxBytesToMove,
11637  maxAllocationsToMove);
11638  }
11639  }
11640 
11641  // Process custom pools.
11642  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
11643  {
11644  result = m_Pools[poolIndex]->m_BlockVector.Defragment(
11645  pDefragmentationStats,
11646  maxBytesToMove,
11647  maxAllocationsToMove);
11648  }
11649 
11650  // ======== Destroy defragmentators.
11651 
11652  // Process custom pools.
11653  for(size_t poolIndex = poolCount; poolIndex--; )
11654  {
11655  m_Pools[poolIndex]->m_BlockVector.DestroyDefragmentator();
11656  }
11657 
11658  // Process standard memory.
11659  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
11660  {
11661  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
11662  {
11663  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
11664  }
11665  }
11666 
11667  return result;
11668 }
11669 
11670 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
11671 {
11672  if(hAllocation->CanBecomeLost())
11673  {
11674  /*
11675  Warning: This is a carefully designed algorithm.
11676  Do not modify unless you really know what you're doing :)
11677  */
11678  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
11679  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
11680  for(;;)
11681  {
11682  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
11683  {
11684  pAllocationInfo->memoryType = UINT32_MAX;
11685  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
11686  pAllocationInfo->offset = 0;
11687  pAllocationInfo->size = hAllocation->GetSize();
11688  pAllocationInfo->pMappedData = VMA_NULL;
11689  pAllocationInfo->pUserData = hAllocation->GetUserData();
11690  return;
11691  }
11692  else if(localLastUseFrameIndex == localCurrFrameIndex)
11693  {
11694  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
11695  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
11696  pAllocationInfo->offset = hAllocation->GetOffset();
11697  pAllocationInfo->size = hAllocation->GetSize();
11698  pAllocationInfo->pMappedData = VMA_NULL;
11699  pAllocationInfo->pUserData = hAllocation->GetUserData();
11700  return;
11701  }
11702  else // Last use time earlier than current time.
11703  {
11704  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
11705  {
11706  localLastUseFrameIndex = localCurrFrameIndex;
11707  }
11708  }
11709  }
11710  }
11711  else
11712  {
11713 #if VMA_STATS_STRING_ENABLED
11714  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
11715  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
11716  for(;;)
11717  {
11718  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
11719  if(localLastUseFrameIndex == localCurrFrameIndex)
11720  {
11721  break;
11722  }
11723  else // Last use time earlier than current time.
11724  {
11725  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
11726  {
11727  localLastUseFrameIndex = localCurrFrameIndex;
11728  }
11729  }
11730  }
11731 #endif
11732 
11733  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
11734  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
11735  pAllocationInfo->offset = hAllocation->GetOffset();
11736  pAllocationInfo->size = hAllocation->GetSize();
11737  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
11738  pAllocationInfo->pUserData = hAllocation->GetUserData();
11739  }
11740 }
11741 
11742 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
11743 {
11744  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
11745  if(hAllocation->CanBecomeLost())
11746  {
11747  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
11748  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
11749  for(;;)
11750  {
11751  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
11752  {
11753  return false;
11754  }
11755  else if(localLastUseFrameIndex == localCurrFrameIndex)
11756  {
11757  return true;
11758  }
11759  else // Last use time earlier than current time.
11760  {
11761  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
11762  {
11763  localLastUseFrameIndex = localCurrFrameIndex;
11764  }
11765  }
11766  }
11767  }
11768  else
11769  {
11770 #if VMA_STATS_STRING_ENABLED
11771  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
11772  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
11773  for(;;)
11774  {
11775  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
11776  if(localLastUseFrameIndex == localCurrFrameIndex)
11777  {
11778  break;
11779  }
11780  else // Last use time earlier than current time.
11781  {
11782  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
11783  {
11784  localLastUseFrameIndex = localCurrFrameIndex;
11785  }
11786  }
11787  }
11788 #endif
11789 
11790  return true;
11791  }
11792 }
11793 
11794 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
11795 {
11796  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u, flags=%u", pCreateInfo->memoryTypeIndex, pCreateInfo->flags);
11797 
11798  const bool isLinearAlgorithm = (pCreateInfo->flags & VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT) != 0;
11799 
11800  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
11801 
11802  if(newCreateInfo.maxBlockCount == 0)
11803  {
11804  newCreateInfo.maxBlockCount = isLinearAlgorithm ? 1 : SIZE_MAX;
11805  }
11806  if(newCreateInfo.minBlockCount > newCreateInfo.maxBlockCount ||
11807  isLinearAlgorithm && newCreateInfo.maxBlockCount > 1)
11808  {
11809  return VK_ERROR_INITIALIZATION_FAILED;
11810  }
11811  if(newCreateInfo.blockSize == 0)
11812  {
11813  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
11814  }
11815 
11816  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
11817 
11818  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
11819  if(res != VK_SUCCESS)
11820  {
11821  vma_delete(this, *pPool);
11822  *pPool = VMA_NULL;
11823  return res;
11824  }
11825 
11826  // Add to m_Pools.
11827  {
11828  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
11829  (*pPool)->SetId(m_NextPoolId++);
11830  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
11831  }
11832 
11833  return VK_SUCCESS;
11834 }
11835 
11836 void VmaAllocator_T::DestroyPool(VmaPool pool)
11837 {
11838  // Remove from m_Pools.
11839  {
11840  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
11841  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
11842  VMA_ASSERT(success && "Pool not found in Allocator.");
11843  }
11844 
11845  vma_delete(this, pool);
11846 }
11847 
11848 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
11849 {
11850  pool->m_BlockVector.GetPoolStats(pPoolStats);
11851 }
11852 
11853 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
11854 {
11855  m_CurrentFrameIndex.store(frameIndex);
11856 }
11857 
11858 void VmaAllocator_T::MakePoolAllocationsLost(
11859  VmaPool hPool,
11860  size_t* pLostAllocationCount)
11861 {
11862  hPool->m_BlockVector.MakePoolAllocationsLost(
11863  m_CurrentFrameIndex.load(),
11864  pLostAllocationCount);
11865 }
11866 
11867 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
11868 {
11869  return hPool->m_BlockVector.CheckCorruption();
11870 }
11871 
11872 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
11873 {
11874  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
11875 
11876  // Process default pools.
11877  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
11878  {
11879  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
11880  {
11881  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
11882  VMA_ASSERT(pBlockVector);
11883  VkResult localRes = pBlockVector->CheckCorruption();
11884  switch(localRes)
11885  {
11886  case VK_ERROR_FEATURE_NOT_PRESENT:
11887  break;
11888  case VK_SUCCESS:
11889  finalRes = VK_SUCCESS;
11890  break;
11891  default:
11892  return localRes;
11893  }
11894  }
11895  }
11896 
11897  // Process custom pools.
11898  {
11899  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
11900  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
11901  {
11902  if(((1u << m_Pools[poolIndex]->m_BlockVector.GetMemoryTypeIndex()) & memoryTypeBits) != 0)
11903  {
11904  VkResult localRes = m_Pools[poolIndex]->m_BlockVector.CheckCorruption();
11905  switch(localRes)
11906  {
11907  case VK_ERROR_FEATURE_NOT_PRESENT:
11908  break;
11909  case VK_SUCCESS:
11910  finalRes = VK_SUCCESS;
11911  break;
11912  default:
11913  return localRes;
11914  }
11915  }
11916  }
11917  }
11918 
11919  return finalRes;
11920 }
11921 
11922 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
11923 {
11924  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
11925  (*pAllocation)->InitLost();
11926 }
11927 
11928 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
11929 {
11930  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
11931 
11932  VkResult res;
11933  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
11934  {
11935  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
11936  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
11937  {
11938  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
11939  if(res == VK_SUCCESS)
11940  {
11941  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
11942  }
11943  }
11944  else
11945  {
11946  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
11947  }
11948  }
11949  else
11950  {
11951  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
11952  }
11953 
11954  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
11955  {
11956  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
11957  }
11958 
11959  return res;
11960 }
11961 
11962 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
11963 {
11964  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
11965  {
11966  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
11967  }
11968 
11969  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
11970 
11971  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
11972  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
11973  {
11974  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
11975  m_HeapSizeLimit[heapIndex] += size;
11976  }
11977 }
11978 
11979 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
11980 {
11981  if(hAllocation->CanBecomeLost())
11982  {
11983  return VK_ERROR_MEMORY_MAP_FAILED;
11984  }
11985 
11986  switch(hAllocation->GetType())
11987  {
11988  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
11989  {
11990  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
11991  char *pBytes = VMA_NULL;
11992  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
11993  if(res == VK_SUCCESS)
11994  {
11995  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
11996  hAllocation->BlockAllocMap();
11997  }
11998  return res;
11999  }
12000  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12001  return hAllocation->DedicatedAllocMap(this, ppData);
12002  default:
12003  VMA_ASSERT(0);
12004  return VK_ERROR_MEMORY_MAP_FAILED;
12005  }
12006 }
12007 
12008 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
12009 {
12010  switch(hAllocation->GetType())
12011  {
12012  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
12013  {
12014  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
12015  hAllocation->BlockAllocUnmap();
12016  pBlock->Unmap(this, 1);
12017  }
12018  break;
12019  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12020  hAllocation->DedicatedAllocUnmap(this);
12021  break;
12022  default:
12023  VMA_ASSERT(0);
12024  }
12025 }
12026 
12027 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
12028 {
12029  VkResult res = VK_SUCCESS;
12030  switch(hAllocation->GetType())
12031  {
12032  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12033  res = GetVulkanFunctions().vkBindBufferMemory(
12034  m_hDevice,
12035  hBuffer,
12036  hAllocation->GetMemory(),
12037  0); //memoryOffset
12038  break;
12039  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
12040  {
12041  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
12042  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
12043  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
12044  break;
12045  }
12046  default:
12047  VMA_ASSERT(0);
12048  }
12049  return res;
12050 }
12051 
12052 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
12053 {
12054  VkResult res = VK_SUCCESS;
12055  switch(hAllocation->GetType())
12056  {
12057  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12058  res = GetVulkanFunctions().vkBindImageMemory(
12059  m_hDevice,
12060  hImage,
12061  hAllocation->GetMemory(),
12062  0); //memoryOffset
12063  break;
12064  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
12065  {
12066  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
12067  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
12068  res = pBlock->BindImageMemory(this, hAllocation, hImage);
12069  break;
12070  }
12071  default:
12072  VMA_ASSERT(0);
12073  }
12074  return res;
12075 }
12076 
12077 void VmaAllocator_T::FlushOrInvalidateAllocation(
12078  VmaAllocation hAllocation,
12079  VkDeviceSize offset, VkDeviceSize size,
12080  VMA_CACHE_OPERATION op)
12081 {
12082  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
12083  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
12084  {
12085  const VkDeviceSize allocationSize = hAllocation->GetSize();
12086  VMA_ASSERT(offset <= allocationSize);
12087 
12088  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
12089 
12090  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
12091  memRange.memory = hAllocation->GetMemory();
12092 
12093  switch(hAllocation->GetType())
12094  {
12095  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12096  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
12097  if(size == VK_WHOLE_SIZE)
12098  {
12099  memRange.size = allocationSize - memRange.offset;
12100  }
12101  else
12102  {
12103  VMA_ASSERT(offset + size <= allocationSize);
12104  memRange.size = VMA_MIN(
12105  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
12106  allocationSize - memRange.offset);
12107  }
12108  break;
12109 
12110  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
12111  {
12112  // 1. Still within this allocation.
12113  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
12114  if(size == VK_WHOLE_SIZE)
12115  {
12116  size = allocationSize - offset;
12117  }
12118  else
12119  {
12120  VMA_ASSERT(offset + size <= allocationSize);
12121  }
12122  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
12123 
12124  // 2. Adjust to whole block.
12125  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
12126  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
12127  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_pMetadata->GetSize();
12128  memRange.offset += allocationOffset;
12129  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
12130 
12131  break;
12132  }
12133 
12134  default:
12135  VMA_ASSERT(0);
12136  }
12137 
12138  switch(op)
12139  {
12140  case VMA_CACHE_FLUSH:
12141  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
12142  break;
12143  case VMA_CACHE_INVALIDATE:
12144  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
12145  break;
12146  default:
12147  VMA_ASSERT(0);
12148  }
12149  }
12150  // else: Just ignore this call.
12151 }
12152 
12153 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
12154 {
12155  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
12156 
12157  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
12158  {
12159  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
12160  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
12161  VMA_ASSERT(pDedicatedAllocations);
12162  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
12163  VMA_ASSERT(success);
12164  }
12165 
12166  VkDeviceMemory hMemory = allocation->GetMemory();
12167 
12168  if(allocation->GetMappedData() != VMA_NULL)
12169  {
12170  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
12171  }
12172 
12173  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
12174 
12175  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
12176 }
12177 
12178 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
12179 {
12180  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
12181  !hAllocation->CanBecomeLost() &&
12182  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
12183  {
12184  void* pData = VMA_NULL;
12185  VkResult res = Map(hAllocation, &pData);
12186  if(res == VK_SUCCESS)
12187  {
12188  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
12189  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
12190  Unmap(hAllocation);
12191  }
12192  else
12193  {
12194  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
12195  }
12196  }
12197 }
12198 
12199 #if VMA_STATS_STRING_ENABLED
12200 
12201 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
12202 {
12203  bool dedicatedAllocationsStarted = false;
12204  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
12205  {
12206  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
12207  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
12208  VMA_ASSERT(pDedicatedAllocVector);
12209  if(pDedicatedAllocVector->empty() == false)
12210  {
12211  if(dedicatedAllocationsStarted == false)
12212  {
12213  dedicatedAllocationsStarted = true;
12214  json.WriteString("DedicatedAllocations");
12215  json.BeginObject();
12216  }
12217 
12218  json.BeginString("Type ");
12219  json.ContinueString(memTypeIndex);
12220  json.EndString();
12221 
12222  json.BeginArray();
12223 
12224  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
12225  {
12226  json.BeginObject(true);
12227  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
12228  hAlloc->PrintParameters(json);
12229  json.EndObject();
12230  }
12231 
12232  json.EndArray();
12233  }
12234  }
12235  if(dedicatedAllocationsStarted)
12236  {
12237  json.EndObject();
12238  }
12239 
12240  {
12241  bool allocationsStarted = false;
12242  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
12243  {
12244  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
12245  {
12246  if(allocationsStarted == false)
12247  {
12248  allocationsStarted = true;
12249  json.WriteString("DefaultPools");
12250  json.BeginObject();
12251  }
12252 
12253  json.BeginString("Type ");
12254  json.ContinueString(memTypeIndex);
12255  json.EndString();
12256 
12257  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
12258  }
12259  }
12260  if(allocationsStarted)
12261  {
12262  json.EndObject();
12263  }
12264  }
12265 
12266  // Custom pools
12267  {
12268  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
12269  const size_t poolCount = m_Pools.size();
12270  if(poolCount > 0)
12271  {
12272  json.WriteString("Pools");
12273  json.BeginObject();
12274  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
12275  {
12276  json.BeginString();
12277  json.ContinueString(m_Pools[poolIndex]->GetId());
12278  json.EndString();
12279 
12280  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
12281  }
12282  json.EndObject();
12283  }
12284  }
12285 }
12286 
12287 #endif // #if VMA_STATS_STRING_ENABLED
12288 
12290 // Public interface
12291 
12292 VkResult vmaCreateAllocator(
12293  const VmaAllocatorCreateInfo* pCreateInfo,
12294  VmaAllocator* pAllocator)
12295 {
12296  VMA_ASSERT(pCreateInfo && pAllocator);
12297  VMA_DEBUG_LOG("vmaCreateAllocator");
12298  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
12299  return (*pAllocator)->Init(pCreateInfo);
12300 }
12301 
12302 void vmaDestroyAllocator(
12303  VmaAllocator allocator)
12304 {
12305  if(allocator != VK_NULL_HANDLE)
12306  {
12307  VMA_DEBUG_LOG("vmaDestroyAllocator");
12308  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
12309  vma_delete(&allocationCallbacks, allocator);
12310  }
12311 }
12312 
12313 void vmaGetPhysicalDeviceProperties(
12314  VmaAllocator allocator,
12315  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
12316 {
12317  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
12318  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
12319 }
12320 
12321 void vmaGetMemoryProperties(
12322  VmaAllocator allocator,
12323  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
12324 {
12325  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
12326  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
12327 }
12328 
12329 void vmaGetMemoryTypeProperties(
12330  VmaAllocator allocator,
12331  uint32_t memoryTypeIndex,
12332  VkMemoryPropertyFlags* pFlags)
12333 {
12334  VMA_ASSERT(allocator && pFlags);
12335  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
12336  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
12337 }
12338 
12339 void vmaSetCurrentFrameIndex(
12340  VmaAllocator allocator,
12341  uint32_t frameIndex)
12342 {
12343  VMA_ASSERT(allocator);
12344  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
12345 
12346  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12347 
12348  allocator->SetCurrentFrameIndex(frameIndex);
12349 }
12350 
12351 void vmaCalculateStats(
12352  VmaAllocator allocator,
12353  VmaStats* pStats)
12354 {
12355  VMA_ASSERT(allocator && pStats);
12356  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12357  allocator->CalculateStats(pStats);
12358 }
12359 
12360 #if VMA_STATS_STRING_ENABLED
12361 
12362 void vmaBuildStatsString(
12363  VmaAllocator allocator,
12364  char** ppStatsString,
12365  VkBool32 detailedMap)
12366 {
12367  VMA_ASSERT(allocator && ppStatsString);
12368  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12369 
12370  VmaStringBuilder sb(allocator);
12371  {
12372  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
12373  json.BeginObject();
12374 
12375  VmaStats stats;
12376  allocator->CalculateStats(&stats);
12377 
12378  json.WriteString("Total");
12379  VmaPrintStatInfo(json, stats.total);
12380 
12381  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
12382  {
12383  json.BeginString("Heap ");
12384  json.ContinueString(heapIndex);
12385  json.EndString();
12386  json.BeginObject();
12387 
12388  json.WriteString("Size");
12389  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
12390 
12391  json.WriteString("Flags");
12392  json.BeginArray(true);
12393  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
12394  {
12395  json.WriteString("DEVICE_LOCAL");
12396  }
12397  json.EndArray();
12398 
12399  if(stats.memoryHeap[heapIndex].blockCount > 0)
12400  {
12401  json.WriteString("Stats");
12402  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
12403  }
12404 
12405  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
12406  {
12407  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
12408  {
12409  json.BeginString("Type ");
12410  json.ContinueString(typeIndex);
12411  json.EndString();
12412 
12413  json.BeginObject();
12414 
12415  json.WriteString("Flags");
12416  json.BeginArray(true);
12417  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
12418  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
12419  {
12420  json.WriteString("DEVICE_LOCAL");
12421  }
12422  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
12423  {
12424  json.WriteString("HOST_VISIBLE");
12425  }
12426  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
12427  {
12428  json.WriteString("HOST_COHERENT");
12429  }
12430  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
12431  {
12432  json.WriteString("HOST_CACHED");
12433  }
12434  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
12435  {
12436  json.WriteString("LAZILY_ALLOCATED");
12437  }
12438  json.EndArray();
12439 
12440  if(stats.memoryType[typeIndex].blockCount > 0)
12441  {
12442  json.WriteString("Stats");
12443  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
12444  }
12445 
12446  json.EndObject();
12447  }
12448  }
12449 
12450  json.EndObject();
12451  }
12452  if(detailedMap == VK_TRUE)
12453  {
12454  allocator->PrintDetailedMap(json);
12455  }
12456 
12457  json.EndObject();
12458  }
12459 
12460  const size_t len = sb.GetLength();
12461  char* const pChars = vma_new_array(allocator, char, len + 1);
12462  if(len > 0)
12463  {
12464  memcpy(pChars, sb.GetData(), len);
12465  }
12466  pChars[len] = '\0';
12467  *ppStatsString = pChars;
12468 }
12469 
12470 void vmaFreeStatsString(
12471  VmaAllocator allocator,
12472  char* pStatsString)
12473 {
12474  if(pStatsString != VMA_NULL)
12475  {
12476  VMA_ASSERT(allocator);
12477  size_t len = strlen(pStatsString);
12478  vma_delete_array(allocator, pStatsString, len + 1);
12479  }
12480 }
12481 
12482 #endif // #if VMA_STATS_STRING_ENABLED
12483 
12484 /*
12485 This function is not protected by any mutex because it just reads immutable data.
12486 */
12487 VkResult vmaFindMemoryTypeIndex(
12488  VmaAllocator allocator,
12489  uint32_t memoryTypeBits,
12490  const VmaAllocationCreateInfo* pAllocationCreateInfo,
12491  uint32_t* pMemoryTypeIndex)
12492 {
12493  VMA_ASSERT(allocator != VK_NULL_HANDLE);
12494  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
12495  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
12496 
12497  if(pAllocationCreateInfo->memoryTypeBits != 0)
12498  {
12499  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
12500  }
12501 
12502  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
12503  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
12504 
12505  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
12506  if(mapped)
12507  {
12508  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
12509  }
12510 
12511  // Convert usage to requiredFlags and preferredFlags.
12512  switch(pAllocationCreateInfo->usage)
12513  {
12514  case VMA_MEMORY_USAGE_UNKNOWN:
12515  break;
12516  case VMA_MEMORY_USAGE_GPU_ONLY:
12517  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
12518  {
12519  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
12520  }
12521  break;
12522  case VMA_MEMORY_USAGE_CPU_ONLY:
12523  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
12524  break;
12525  case VMA_MEMORY_USAGE_CPU_TO_GPU:
12526  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
12527  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
12528  {
12529  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
12530  }
12531  break;
12532  case VMA_MEMORY_USAGE_GPU_TO_CPU:
12533  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
12534  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
12535  break;
12536  default:
12537  break;
12538  }
12539 
12540  *pMemoryTypeIndex = UINT32_MAX;
12541  uint32_t minCost = UINT32_MAX;
12542  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
12543  memTypeIndex < allocator->GetMemoryTypeCount();
12544  ++memTypeIndex, memTypeBit <<= 1)
12545  {
12546  // This memory type is acceptable according to memoryTypeBits bitmask.
12547  if((memTypeBit & memoryTypeBits) != 0)
12548  {
12549  const VkMemoryPropertyFlags currFlags =
12550  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
12551  // This memory type contains requiredFlags.
12552  if((requiredFlags & ~currFlags) == 0)
12553  {
12554  // Calculate cost as number of bits from preferredFlags not present in this memory type.
12555  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
12556  // Remember memory type with lowest cost.
12557  if(currCost < minCost)
12558  {
12559  *pMemoryTypeIndex = memTypeIndex;
12560  if(currCost == 0)
12561  {
12562  return VK_SUCCESS;
12563  }
12564  minCost = currCost;
12565  }
12566  }
12567  }
12568  }
12569  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
12570 }
12571 
12572 VkResult vmaFindMemoryTypeIndexForBufferInfo(
12573  VmaAllocator allocator,
12574  const VkBufferCreateInfo* pBufferCreateInfo,
12575  const VmaAllocationCreateInfo* pAllocationCreateInfo,
12576  uint32_t* pMemoryTypeIndex)
12577 {
12578  VMA_ASSERT(allocator != VK_NULL_HANDLE);
12579  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
12580  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
12581  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
12582 
12583  const VkDevice hDev = allocator->m_hDevice;
12584  VkBuffer hBuffer = VK_NULL_HANDLE;
12585  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
12586  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
12587  if(res == VK_SUCCESS)
12588  {
12589  VkMemoryRequirements memReq = {};
12590  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
12591  hDev, hBuffer, &memReq);
12592 
12593  res = vmaFindMemoryTypeIndex(
12594  allocator,
12595  memReq.memoryTypeBits,
12596  pAllocationCreateInfo,
12597  pMemoryTypeIndex);
12598 
12599  allocator->GetVulkanFunctions().vkDestroyBuffer(
12600  hDev, hBuffer, allocator->GetAllocationCallbacks());
12601  }
12602  return res;
12603 }
12604 
12605 VkResult vmaFindMemoryTypeIndexForImageInfo(
12606  VmaAllocator allocator,
12607  const VkImageCreateInfo* pImageCreateInfo,
12608  const VmaAllocationCreateInfo* pAllocationCreateInfo,
12609  uint32_t* pMemoryTypeIndex)
12610 {
12611  VMA_ASSERT(allocator != VK_NULL_HANDLE);
12612  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
12613  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
12614  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
12615 
12616  const VkDevice hDev = allocator->m_hDevice;
12617  VkImage hImage = VK_NULL_HANDLE;
12618  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
12619  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
12620  if(res == VK_SUCCESS)
12621  {
12622  VkMemoryRequirements memReq = {};
12623  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
12624  hDev, hImage, &memReq);
12625 
12626  res = vmaFindMemoryTypeIndex(
12627  allocator,
12628  memReq.memoryTypeBits,
12629  pAllocationCreateInfo,
12630  pMemoryTypeIndex);
12631 
12632  allocator->GetVulkanFunctions().vkDestroyImage(
12633  hDev, hImage, allocator->GetAllocationCallbacks());
12634  }
12635  return res;
12636 }
12637 
12638 VkResult vmaCreatePool(
12639  VmaAllocator allocator,
12640  const VmaPoolCreateInfo* pCreateInfo,
12641  VmaPool* pPool)
12642 {
12643  VMA_ASSERT(allocator && pCreateInfo && pPool);
12644 
12645  VMA_DEBUG_LOG("vmaCreatePool");
12646 
12647  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12648 
12649  VkResult res = allocator->CreatePool(pCreateInfo, pPool);
12650 
12651 #if VMA_RECORDING_ENABLED
12652  if(allocator->GetRecorder() != VMA_NULL)
12653  {
12654  allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
12655  }
12656 #endif
12657 
12658  return res;
12659 }
12660 
12661 void vmaDestroyPool(
12662  VmaAllocator allocator,
12663  VmaPool pool)
12664 {
12665  VMA_ASSERT(allocator);
12666 
12667  if(pool == VK_NULL_HANDLE)
12668  {
12669  return;
12670  }
12671 
12672  VMA_DEBUG_LOG("vmaDestroyPool");
12673 
12674  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12675 
12676 #if VMA_RECORDING_ENABLED
12677  if(allocator->GetRecorder() != VMA_NULL)
12678  {
12679  allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
12680  }
12681 #endif
12682 
12683  allocator->DestroyPool(pool);
12684 }
12685 
12686 void vmaGetPoolStats(
12687  VmaAllocator allocator,
12688  VmaPool pool,
12689  VmaPoolStats* pPoolStats)
12690 {
12691  VMA_ASSERT(allocator && pool && pPoolStats);
12692 
12693  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12694 
12695  allocator->GetPoolStats(pool, pPoolStats);
12696 }
12697 
12698 void vmaMakePoolAllocationsLost(
12699  VmaAllocator allocator,
12700  VmaPool pool,
12701  size_t* pLostAllocationCount)
12702 {
12703  VMA_ASSERT(allocator && pool);
12704 
12705  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12706 
12707 #if VMA_RECORDING_ENABLED
12708  if(allocator->GetRecorder() != VMA_NULL)
12709  {
12710  allocator->GetRecorder()->RecordMakePoolAllocationsLost(allocator->GetCurrentFrameIndex(), pool);
12711  }
12712 #endif
12713 
12714  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
12715 }
12716 
12717 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
12718 {
12719  VMA_ASSERT(allocator && pool);
12720 
12721  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12722 
12723  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
12724 
12725  return allocator->CheckPoolCorruption(pool);
12726 }
12727 
12728 VkResult vmaAllocateMemory(
12729  VmaAllocator allocator,
12730  const VkMemoryRequirements* pVkMemoryRequirements,
12731  const VmaAllocationCreateInfo* pCreateInfo,
12732  VmaAllocation* pAllocation,
12733  VmaAllocationInfo* pAllocationInfo)
12734 {
12735  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
12736 
12737  VMA_DEBUG_LOG("vmaAllocateMemory");
12738 
12739  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12740 
12741  VkResult result = allocator->AllocateMemory(
12742  *pVkMemoryRequirements,
12743  false, // requiresDedicatedAllocation
12744  false, // prefersDedicatedAllocation
12745  VK_NULL_HANDLE, // dedicatedBuffer
12746  VK_NULL_HANDLE, // dedicatedImage
12747  *pCreateInfo,
12748  VMA_SUBALLOCATION_TYPE_UNKNOWN,
12749  pAllocation);
12750 
12751 #if VMA_RECORDING_ENABLED
12752  if(allocator->GetRecorder() != VMA_NULL)
12753  {
12754  allocator->GetRecorder()->RecordAllocateMemory(
12755  allocator->GetCurrentFrameIndex(),
12756  *pVkMemoryRequirements,
12757  *pCreateInfo,
12758  *pAllocation);
12759  }
12760 #endif
12761 
12762  if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
12763  {
12764  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
12765  }
12766 
12767  return result;
12768 }
12769 
12770 VkResult vmaAllocateMemoryForBuffer(
12771  VmaAllocator allocator,
12772  VkBuffer buffer,
12773  const VmaAllocationCreateInfo* pCreateInfo,
12774  VmaAllocation* pAllocation,
12775  VmaAllocationInfo* pAllocationInfo)
12776 {
12777  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
12778 
12779  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
12780 
12781  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12782 
12783  VkMemoryRequirements vkMemReq = {};
12784  bool requiresDedicatedAllocation = false;
12785  bool prefersDedicatedAllocation = false;
12786  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
12787  requiresDedicatedAllocation,
12788  prefersDedicatedAllocation);
12789 
12790  VkResult result = allocator->AllocateMemory(
12791  vkMemReq,
12792  requiresDedicatedAllocation,
12793  prefersDedicatedAllocation,
12794  buffer, // dedicatedBuffer
12795  VK_NULL_HANDLE, // dedicatedImage
12796  *pCreateInfo,
12797  VMA_SUBALLOCATION_TYPE_BUFFER,
12798  pAllocation);
12799 
12800 #if VMA_RECORDING_ENABLED
12801  if(allocator->GetRecorder() != VMA_NULL)
12802  {
12803  allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
12804  allocator->GetCurrentFrameIndex(),
12805  vkMemReq,
12806  requiresDedicatedAllocation,
12807  prefersDedicatedAllocation,
12808  *pCreateInfo,
12809  *pAllocation);
12810  }
12811 #endif
12812 
12813  if(pAllocationInfo && result == VK_SUCCESS)
12814  {
12815  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
12816  }
12817 
12818  return result;
12819 }
12820 
12821 VkResult vmaAllocateMemoryForImage(
12822  VmaAllocator allocator,
12823  VkImage image,
12824  const VmaAllocationCreateInfo* pCreateInfo,
12825  VmaAllocation* pAllocation,
12826  VmaAllocationInfo* pAllocationInfo)
12827 {
12828  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
12829 
12830  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
12831 
12832  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12833 
12834  VkMemoryRequirements vkMemReq = {};
12835  bool requiresDedicatedAllocation = false;
12836  bool prefersDedicatedAllocation = false;
12837  allocator->GetImageMemoryRequirements(image, vkMemReq,
12838  requiresDedicatedAllocation, prefersDedicatedAllocation);
12839 
12840  VkResult result = allocator->AllocateMemory(
12841  vkMemReq,
12842  requiresDedicatedAllocation,
12843  prefersDedicatedAllocation,
12844  VK_NULL_HANDLE, // dedicatedBuffer
12845  image, // dedicatedImage
12846  *pCreateInfo,
12847  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
12848  pAllocation);
12849 
12850 #if VMA_RECORDING_ENABLED
12851  if(allocator->GetRecorder() != VMA_NULL)
12852  {
12853  allocator->GetRecorder()->RecordAllocateMemoryForImage(
12854  allocator->GetCurrentFrameIndex(),
12855  vkMemReq,
12856  requiresDedicatedAllocation,
12857  prefersDedicatedAllocation,
12858  *pCreateInfo,
12859  *pAllocation);
12860  }
12861 #endif
12862 
12863  if(pAllocationInfo && result == VK_SUCCESS)
12864  {
12865  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
12866  }
12867 
12868  return result;
12869 }
12870 
12871 void vmaFreeMemory(
12872  VmaAllocator allocator,
12873  VmaAllocation allocation)
12874 {
12875  VMA_ASSERT(allocator);
12876 
12877  if(allocation == VK_NULL_HANDLE)
12878  {
12879  return;
12880  }
12881 
12882  VMA_DEBUG_LOG("vmaFreeMemory");
12883 
12884  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12885 
12886 #if VMA_RECORDING_ENABLED
12887  if(allocator->GetRecorder() != VMA_NULL)
12888  {
12889  allocator->GetRecorder()->RecordFreeMemory(
12890  allocator->GetCurrentFrameIndex(),
12891  allocation);
12892  }
12893 #endif
12894 
12895  allocator->FreeMemory(allocation);
12896 }
12897 
12898 void vmaGetAllocationInfo(
12899  VmaAllocator allocator,
12900  VmaAllocation allocation,
12901  VmaAllocationInfo* pAllocationInfo)
12902 {
12903  VMA_ASSERT(allocator && allocation && pAllocationInfo);
12904 
12905  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12906 
12907 #if VMA_RECORDING_ENABLED
12908  if(allocator->GetRecorder() != VMA_NULL)
12909  {
12910  allocator->GetRecorder()->RecordGetAllocationInfo(
12911  allocator->GetCurrentFrameIndex(),
12912  allocation);
12913  }
12914 #endif
12915 
12916  allocator->GetAllocationInfo(allocation, pAllocationInfo);
12917 }
12918 
12919 VkBool32 vmaTouchAllocation(
12920  VmaAllocator allocator,
12921  VmaAllocation allocation)
12922 {
12923  VMA_ASSERT(allocator && allocation);
12924 
12925  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12926 
12927 #if VMA_RECORDING_ENABLED
12928  if(allocator->GetRecorder() != VMA_NULL)
12929  {
12930  allocator->GetRecorder()->RecordTouchAllocation(
12931  allocator->GetCurrentFrameIndex(),
12932  allocation);
12933  }
12934 #endif
12935 
12936  return allocator->TouchAllocation(allocation);
12937 }
12938 
12939 void vmaSetAllocationUserData(
12940  VmaAllocator allocator,
12941  VmaAllocation allocation,
12942  void* pUserData)
12943 {
12944  VMA_ASSERT(allocator && allocation);
12945 
12946  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12947 
12948  allocation->SetUserData(allocator, pUserData);
12949 
12950 #if VMA_RECORDING_ENABLED
12951  if(allocator->GetRecorder() != VMA_NULL)
12952  {
12953  allocator->GetRecorder()->RecordSetAllocationUserData(
12954  allocator->GetCurrentFrameIndex(),
12955  allocation,
12956  pUserData);
12957  }
12958 #endif
12959 }
12960 
12961 void vmaCreateLostAllocation(
12962  VmaAllocator allocator,
12963  VmaAllocation* pAllocation)
12964 {
12965  VMA_ASSERT(allocator && pAllocation);
12966 
12967  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
12968 
12969  allocator->CreateLostAllocation(pAllocation);
12970 
12971 #if VMA_RECORDING_ENABLED
12972  if(allocator->GetRecorder() != VMA_NULL)
12973  {
12974  allocator->GetRecorder()->RecordCreateLostAllocation(
12975  allocator->GetCurrentFrameIndex(),
12976  *pAllocation);
12977  }
12978 #endif
12979 }
12980 
12981 VkResult vmaMapMemory(
12982  VmaAllocator allocator,
12983  VmaAllocation allocation,
12984  void** ppData)
12985 {
12986  VMA_ASSERT(allocator && allocation && ppData);
12987 
12988  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12989 
12990  VkResult res = allocator->Map(allocation, ppData);
12991 
12992 #if VMA_RECORDING_ENABLED
12993  if(allocator->GetRecorder() != VMA_NULL)
12994  {
12995  allocator->GetRecorder()->RecordMapMemory(
12996  allocator->GetCurrentFrameIndex(),
12997  allocation);
12998  }
12999 #endif
13000 
13001  return res;
13002 }
13003 
13004 void vmaUnmapMemory(
13005  VmaAllocator allocator,
13006  VmaAllocation allocation)
13007 {
13008  VMA_ASSERT(allocator && allocation);
13009 
13010  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13011 
13012 #if VMA_RECORDING_ENABLED
13013  if(allocator->GetRecorder() != VMA_NULL)
13014  {
13015  allocator->GetRecorder()->RecordUnmapMemory(
13016  allocator->GetCurrentFrameIndex(),
13017  allocation);
13018  }
13019 #endif
13020 
13021  allocator->Unmap(allocation);
13022 }
13023 
13024 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
13025 {
13026  VMA_ASSERT(allocator && allocation);
13027 
13028  VMA_DEBUG_LOG("vmaFlushAllocation");
13029 
13030  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13031 
13032  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
13033 
13034 #if VMA_RECORDING_ENABLED
13035  if(allocator->GetRecorder() != VMA_NULL)
13036  {
13037  allocator->GetRecorder()->RecordFlushAllocation(
13038  allocator->GetCurrentFrameIndex(),
13039  allocation, offset, size);
13040  }
13041 #endif
13042 }
13043 
13044 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
13045 {
13046  VMA_ASSERT(allocator && allocation);
13047 
13048  VMA_DEBUG_LOG("vmaInvalidateAllocation");
13049 
13050  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13051 
13052  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
13053 
13054 #if VMA_RECORDING_ENABLED
13055  if(allocator->GetRecorder() != VMA_NULL)
13056  {
13057  allocator->GetRecorder()->RecordInvalidateAllocation(
13058  allocator->GetCurrentFrameIndex(),
13059  allocation, offset, size);
13060  }
13061 #endif
13062 }
13063 
13064 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
13065 {
13066  VMA_ASSERT(allocator);
13067 
13068  VMA_DEBUG_LOG("vmaCheckCorruption");
13069 
13070  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13071 
13072  return allocator->CheckCorruption(memoryTypeBits);
13073 }
13074 
13075 VkResult vmaDefragment(
13076  VmaAllocator allocator,
13077  VmaAllocation* pAllocations,
13078  size_t allocationCount,
13079  VkBool32* pAllocationsChanged,
13080  const VmaDefragmentationInfo *pDefragmentationInfo,
13081  VmaDefragmentationStats* pDefragmentationStats)
13082 {
13083  VMA_ASSERT(allocator && pAllocations);
13084 
13085  VMA_DEBUG_LOG("vmaDefragment");
13086 
13087  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13088 
13089  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
13090 }
13091 
13092 VkResult vmaBindBufferMemory(
13093  VmaAllocator allocator,
13094  VmaAllocation allocation,
13095  VkBuffer buffer)
13096 {
13097  VMA_ASSERT(allocator && allocation && buffer);
13098 
13099  VMA_DEBUG_LOG("vmaBindBufferMemory");
13100 
13101  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13102 
13103  return allocator->BindBufferMemory(allocation, buffer);
13104 }
13105 
13106 VkResult vmaBindImageMemory(
13107  VmaAllocator allocator,
13108  VmaAllocation allocation,
13109  VkImage image)
13110 {
13111  VMA_ASSERT(allocator && allocation && image);
13112 
13113  VMA_DEBUG_LOG("vmaBindImageMemory");
13114 
13115  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13116 
13117  return allocator->BindImageMemory(allocation, image);
13118 }
13119 
13120 VkResult vmaCreateBuffer(
13121  VmaAllocator allocator,
13122  const VkBufferCreateInfo* pBufferCreateInfo,
13123  const VmaAllocationCreateInfo* pAllocationCreateInfo,
13124  VkBuffer* pBuffer,
13125  VmaAllocation* pAllocation,
13126  VmaAllocationInfo* pAllocationInfo)
13127 {
13128  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
13129 
13130  VMA_DEBUG_LOG("vmaCreateBuffer");
13131 
13132  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13133 
13134  *pBuffer = VK_NULL_HANDLE;
13135  *pAllocation = VK_NULL_HANDLE;
13136 
13137  // 1. Create VkBuffer.
13138  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
13139  allocator->m_hDevice,
13140  pBufferCreateInfo,
13141  allocator->GetAllocationCallbacks(),
13142  pBuffer);
13143  if(res >= 0)
13144  {
13145  // 2. vkGetBufferMemoryRequirements.
13146  VkMemoryRequirements vkMemReq = {};
13147  bool requiresDedicatedAllocation = false;
13148  bool prefersDedicatedAllocation = false;
13149  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
13150  requiresDedicatedAllocation, prefersDedicatedAllocation);
13151 
13152  // Make sure alignment requirements for specific buffer usages reported
13153  // in Physical Device Properties are included in alignment reported by memory requirements.
13154  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
13155  {
13156  VMA_ASSERT(vkMemReq.alignment %
13157  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
13158  }
13159  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
13160  {
13161  VMA_ASSERT(vkMemReq.alignment %
13162  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
13163  }
13164  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
13165  {
13166  VMA_ASSERT(vkMemReq.alignment %
13167  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
13168  }
13169 
13170  // 3. Allocate memory using allocator.
13171  res = allocator->AllocateMemory(
13172  vkMemReq,
13173  requiresDedicatedAllocation,
13174  prefersDedicatedAllocation,
13175  *pBuffer, // dedicatedBuffer
13176  VK_NULL_HANDLE, // dedicatedImage
13177  *pAllocationCreateInfo,
13178  VMA_SUBALLOCATION_TYPE_BUFFER,
13179  pAllocation);
13180 
13181 #if VMA_RECORDING_ENABLED
13182  if(allocator->GetRecorder() != VMA_NULL)
13183  {
13184  allocator->GetRecorder()->RecordCreateBuffer(
13185  allocator->GetCurrentFrameIndex(),
13186  *pBufferCreateInfo,
13187  *pAllocationCreateInfo,
13188  *pAllocation);
13189  }
13190 #endif
13191 
13192  if(res >= 0)
13193  {
13194  // 3. Bind buffer with memory.
13195  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
13196  if(res >= 0)
13197  {
13198  // All steps succeeded.
13199  #if VMA_STATS_STRING_ENABLED
13200  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
13201  #endif
13202  if(pAllocationInfo != VMA_NULL)
13203  {
13204  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
13205  }
13206 
13207  return VK_SUCCESS;
13208  }
13209  allocator->FreeMemory(*pAllocation);
13210  *pAllocation = VK_NULL_HANDLE;
13211  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
13212  *pBuffer = VK_NULL_HANDLE;
13213  return res;
13214  }
13215  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
13216  *pBuffer = VK_NULL_HANDLE;
13217  return res;
13218  }
13219  return res;
13220 }
13221 
13222 void vmaDestroyBuffer(
13223  VmaAllocator allocator,
13224  VkBuffer buffer,
13225  VmaAllocation allocation)
13226 {
13227  VMA_ASSERT(allocator);
13228 
13229  if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
13230  {
13231  return;
13232  }
13233 
13234  VMA_DEBUG_LOG("vmaDestroyBuffer");
13235 
13236  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13237 
13238 #if VMA_RECORDING_ENABLED
13239  if(allocator->GetRecorder() != VMA_NULL)
13240  {
13241  allocator->GetRecorder()->RecordDestroyBuffer(
13242  allocator->GetCurrentFrameIndex(),
13243  allocation);
13244  }
13245 #endif
13246 
13247  if(buffer != VK_NULL_HANDLE)
13248  {
13249  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
13250  }
13251 
13252  if(allocation != VK_NULL_HANDLE)
13253  {
13254  allocator->FreeMemory(allocation);
13255  }
13256 }
13257 
13258 VkResult vmaCreateImage(
13259  VmaAllocator allocator,
13260  const VkImageCreateInfo* pImageCreateInfo,
13261  const VmaAllocationCreateInfo* pAllocationCreateInfo,
13262  VkImage* pImage,
13263  VmaAllocation* pAllocation,
13264  VmaAllocationInfo* pAllocationInfo)
13265 {
13266  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
13267 
13268  VMA_DEBUG_LOG("vmaCreateImage");
13269 
13270  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13271 
13272  *pImage = VK_NULL_HANDLE;
13273  *pAllocation = VK_NULL_HANDLE;
13274 
13275  // 1. Create VkImage.
13276  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
13277  allocator->m_hDevice,
13278  pImageCreateInfo,
13279  allocator->GetAllocationCallbacks(),
13280  pImage);
13281  if(res >= 0)
13282  {
13283  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
13284  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
13285  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
13286 
13287  // 2. Allocate memory using allocator.
13288  VkMemoryRequirements vkMemReq = {};
13289  bool requiresDedicatedAllocation = false;
13290  bool prefersDedicatedAllocation = false;
13291  allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
13292  requiresDedicatedAllocation, prefersDedicatedAllocation);
13293 
13294  res = allocator->AllocateMemory(
13295  vkMemReq,
13296  requiresDedicatedAllocation,
13297  prefersDedicatedAllocation,
13298  VK_NULL_HANDLE, // dedicatedBuffer
13299  *pImage, // dedicatedImage
13300  *pAllocationCreateInfo,
13301  suballocType,
13302  pAllocation);
13303 
13304 #if VMA_RECORDING_ENABLED
13305  if(allocator->GetRecorder() != VMA_NULL)
13306  {
13307  allocator->GetRecorder()->RecordCreateImage(
13308  allocator->GetCurrentFrameIndex(),
13309  *pImageCreateInfo,
13310  *pAllocationCreateInfo,
13311  *pAllocation);
13312  }
13313 #endif
13314 
13315  if(res >= 0)
13316  {
13317  // 3. Bind image with memory.
13318  res = allocator->BindImageMemory(*pAllocation, *pImage);
13319  if(res >= 0)
13320  {
13321  // All steps succeeded.
13322  #if VMA_STATS_STRING_ENABLED
13323  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
13324  #endif
13325  if(pAllocationInfo != VMA_NULL)
13326  {
13327  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
13328  }
13329 
13330  return VK_SUCCESS;
13331  }
13332  allocator->FreeMemory(*pAllocation);
13333  *pAllocation = VK_NULL_HANDLE;
13334  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
13335  *pImage = VK_NULL_HANDLE;
13336  return res;
13337  }
13338  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
13339  *pImage = VK_NULL_HANDLE;
13340  return res;
13341  }
13342  return res;
13343 }
13344 
13345 void vmaDestroyImage(
13346  VmaAllocator allocator,
13347  VkImage image,
13348  VmaAllocation allocation)
13349 {
13350  VMA_ASSERT(allocator);
13351 
13352  if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
13353  {
13354  return;
13355  }
13356 
13357  VMA_DEBUG_LOG("vmaDestroyImage");
13358 
13359  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13360 
13361 #if VMA_RECORDING_ENABLED
13362  if(allocator->GetRecorder() != VMA_NULL)
13363  {
13364  allocator->GetRecorder()->RecordDestroyImage(
13365  allocator->GetCurrentFrameIndex(),
13366  allocation);
13367  }
13368 #endif
13369 
13370  if(image != VK_NULL_HANDLE)
13371  {
13372  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
13373  }
13374  if(allocation != VK_NULL_HANDLE)
13375  {
13376  allocator->FreeMemory(allocation);
13377  }
13378 }
13379 
13380 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:1429
+Go to the documentation of this file.
1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22 
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25 
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29 
1342 #include <vulkan/vulkan.h>
1343 
1344 #if !defined(VMA_DEDICATED_ALLOCATION)
1345  #if VK_KHR_get_memory_requirements2 && VK_KHR_dedicated_allocation
1346  #define VMA_DEDICATED_ALLOCATION 1
1347  #else
1348  #define VMA_DEDICATED_ALLOCATION 0
1349  #endif
1350 #endif
1351 
1361 VK_DEFINE_HANDLE(VmaAllocator)
1362 
1363 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
1365  VmaAllocator allocator,
1366  uint32_t memoryType,
1367  VkDeviceMemory memory,
1368  VkDeviceSize size);
1370 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
1371  VmaAllocator allocator,
1372  uint32_t memoryType,
1373  VkDeviceMemory memory,
1374  VkDeviceSize size);
1375 
1383 typedef struct VmaDeviceMemoryCallbacks {
1385  PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
1387  PFN_vmaFreeDeviceMemoryFunction pfnFree;
1388 } VmaDeviceMemoryCallbacks;
1389 
1391 typedef enum VmaAllocatorCreateFlagBits {
1396  VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
1418  VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
1419 
1420  VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1421 } VmaAllocatorCreateFlagBits;
1422 typedef VkFlags VmaAllocatorCreateFlags;
1423 
1428 typedef struct VmaVulkanFunctions {
1429  PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
1430  PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
1431  PFN_vkAllocateMemory vkAllocateMemory;
1432  PFN_vkFreeMemory vkFreeMemory;
1433  PFN_vkMapMemory vkMapMemory;
1434  PFN_vkUnmapMemory vkUnmapMemory;
1435  PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges;
1436  PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges;
1437  PFN_vkBindBufferMemory vkBindBufferMemory;
1438  PFN_vkBindImageMemory vkBindImageMemory;
1439  PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
1440  PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
1441  PFN_vkCreateBuffer vkCreateBuffer;
1442  PFN_vkDestroyBuffer vkDestroyBuffer;
1443  PFN_vkCreateImage vkCreateImage;
1444  PFN_vkDestroyImage vkDestroyImage;
1445 #if VMA_DEDICATED_ALLOCATION
1446  PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
1447  PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
1448 #endif
1449 } VmaVulkanFunctions;
1450 
1452 typedef enum VmaRecordFlagBits {
1458  VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001,
1459 
1460  VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1461 } VmaRecordFlagBits;
1462 typedef VkFlags VmaRecordFlags;
1463 
1464 /*
1465 Define this macro to 0/1 to disable/enable support for recording functionality,
1466 available through VmaAllocatorCreateInfo::pRecordSettings.
1467 */
1468 #ifndef VMA_RECORDING_ENABLED
1469  #ifdef _WIN32
1470  #define VMA_RECORDING_ENABLED 1
1471  #else
1472  #define VMA_RECORDING_ENABLED 0
1473  #endif
1474 #endif
1475 
1477 typedef struct VmaRecordSettings
1478 {
1480  VmaRecordFlags flags;
1488  const char* pFilePath;
1489 } VmaRecordSettings;
1490 
1492 typedef struct VmaAllocatorCreateInfo
1493 {
1495  VmaAllocatorCreateFlags flags;
1497 
1498  VkPhysicalDevice physicalDevice;
1500 
1501  VkDevice device;
1503 
1504  VkDeviceSize preferredLargeHeapBlockSize;
1506 
1507  const VkAllocationCallbacks* pAllocationCallbacks;
1509 
1510  const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
1524  uint32_t frameInUseCount;
1548  const VkDeviceSize* pHeapSizeLimit;
1560  const VmaVulkanFunctions* pVulkanFunctions;
1567  const VmaRecordSettings* pRecordSettings;
1568 } VmaAllocatorCreateInfo;
1569 
1571 VkResult vmaCreateAllocator(
1572  const VmaAllocatorCreateInfo* pCreateInfo,
1573  VmaAllocator* pAllocator);
1574 
1576 void vmaDestroyAllocator(
1577  VmaAllocator allocator);
1578 
1583 void vmaGetPhysicalDeviceProperties(
1584  VmaAllocator allocator,
1585  const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
1586 
1591 void vmaGetMemoryProperties(
1592  VmaAllocator allocator,
1593  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1594 
1601 void vmaGetMemoryTypeProperties(
1602  VmaAllocator allocator,
1603  uint32_t memoryTypeIndex,
1604  VkMemoryPropertyFlags* pFlags);
1605 
1614 void vmaSetCurrentFrameIndex(
1615  VmaAllocator allocator,
1616  uint32_t frameIndex);
1617 
1620 typedef struct VmaStatInfo
1621 {
1623  uint32_t blockCount;
1625  uint32_t allocationCount;
1627  uint32_t unusedRangeCount;
1629  VkDeviceSize usedBytes;
1631  VkDeviceSize unusedBytes;
1632  VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1633  VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1634 } VmaStatInfo;
1635 
1637 typedef struct VmaStats
1638 {
1639  VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1640  VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1641  VmaStatInfo total;
1642 } VmaStats;
1643 
1645 void vmaCalculateStats(
1646  VmaAllocator allocator,
1647  VmaStats* pStats);
1648 
1649 #define VMA_STATS_STRING_ENABLED 1
1650 
1651 #if VMA_STATS_STRING_ENABLED
1652 
1654 
1656 void vmaBuildStatsString(
1657  VmaAllocator allocator,
1658  char** ppStatsString,
1659  VkBool32 detailedMap);
1660 
1661 void vmaFreeStatsString(
1662  VmaAllocator allocator,
1663  char* pStatsString);
1664 
1665 #endif // #if VMA_STATS_STRING_ENABLED
1666 
1675 VK_DEFINE_HANDLE(VmaPool)
1676 
1677 typedef enum VmaMemoryUsage
1678 {
1682  VMA_MEMORY_USAGE_UNKNOWN = 0,
1699  VMA_MEMORY_USAGE_GPU_ONLY = 1,
1709  VMA_MEMORY_USAGE_CPU_ONLY = 2,
1716  VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1725  VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1726  VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1727 } VmaMemoryUsage;
1728 
1730 typedef enum VmaAllocationCreateFlagBits {
1742  VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1743 
1753  VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1766  VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1779  VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1786  VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1792  VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1797  VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT = 0x00000040,
1798 
1799  VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1800 } VmaAllocationCreateFlagBits;
1801 typedef VkFlags VmaAllocationCreateFlags;
1802 
1803 typedef struct VmaAllocationCreateInfo
1804 {
1806  VmaAllocationCreateFlags flags;
1812  VmaMemoryUsage usage;
1817  VkMemoryPropertyFlags requiredFlags;
1822  VkMemoryPropertyFlags preferredFlags;
1830  uint32_t memoryTypeBits;
1836  VmaPool pool;
1843  void* pUserData;
1844 } VmaAllocationCreateInfo;
1845 
1862 VkResult vmaFindMemoryTypeIndex(
1863  VmaAllocator allocator,
1864  uint32_t memoryTypeBits,
1865  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1866  uint32_t* pMemoryTypeIndex);
1867 
1880 VkResult vmaFindMemoryTypeIndexForBufferInfo(
1881  VmaAllocator allocator,
1882  const VkBufferCreateInfo* pBufferCreateInfo,
1883  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1884  uint32_t* pMemoryTypeIndex);
1885 
1898 VkResult vmaFindMemoryTypeIndexForImageInfo(
1899  VmaAllocator allocator,
1900  const VkImageCreateInfo* pImageCreateInfo,
1901  const VmaAllocationCreateInfo* pAllocationCreateInfo,
1902  uint32_t* pMemoryTypeIndex);
1903 
1905 typedef enum VmaPoolCreateFlagBits {
1923  VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1924 
1938  VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT = 0x00000004,
1939 
1940  VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1941 } VmaPoolCreateFlagBits;
1942 typedef VkFlags VmaPoolCreateFlags;
1943 
1946 typedef struct VmaPoolCreateInfo {
1949  uint32_t memoryTypeIndex;
1952  VmaPoolCreateFlags flags;
1957  VkDeviceSize blockSize;
1962  size_t minBlockCount;
1971  size_t maxBlockCount;
1985  uint32_t frameInUseCount;
1986 } VmaPoolCreateInfo;
1987 
1990 typedef struct VmaPoolStats {
1993  VkDeviceSize size;
1996  VkDeviceSize unusedSize;
1999  size_t allocationCount;
2002  size_t unusedRangeCount;
2009  VkDeviceSize unusedRangeSizeMax;
2010 } VmaPoolStats;
2011 
2018 VkResult vmaCreatePool(
2019  VmaAllocator allocator,
2020  const VmaPoolCreateInfo* pCreateInfo,
2021  VmaPool* pPool);
2022 
2025 void vmaDestroyPool(
2026  VmaAllocator allocator,
2027  VmaPool pool);
2028 
2035 void vmaGetPoolStats(
2036  VmaAllocator allocator,
2037  VmaPool pool,
2038  VmaPoolStats* pPoolStats);
2039 
2046 void vmaMakePoolAllocationsLost(
2047  VmaAllocator allocator,
2048  VmaPool pool,
2049  size_t* pLostAllocationCount);
2050 
2065 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool);
2066 
2091 VK_DEFINE_HANDLE(VmaAllocation)
2092 
2093 
2095 typedef struct VmaAllocationInfo {
2100  uint32_t memoryType;
2109  VkDeviceMemory deviceMemory;
2114  VkDeviceSize offset;
2119  VkDeviceSize size;
2128  void* pMappedData;
2133  void* pUserData;
2134 } VmaAllocationInfo;
2135 
2146 VkResult vmaAllocateMemory(
2147  VmaAllocator allocator,
2148  const VkMemoryRequirements* pVkMemoryRequirements,
2149  const VmaAllocationCreateInfo* pCreateInfo,
2150  VmaAllocation* pAllocation,
2151  VmaAllocationInfo* pAllocationInfo);
2152 
2159 VkResult vmaAllocateMemoryForBuffer(
2160  VmaAllocator allocator,
2161  VkBuffer buffer,
2162  const VmaAllocationCreateInfo* pCreateInfo,
2163  VmaAllocation* pAllocation,
2164  VmaAllocationInfo* pAllocationInfo);
2165 
2167 VkResult vmaAllocateMemoryForImage(
2168  VmaAllocator allocator,
2169  VkImage image,
2170  const VmaAllocationCreateInfo* pCreateInfo,
2171  VmaAllocation* pAllocation,
2172  VmaAllocationInfo* pAllocationInfo);
2173 
2175 void vmaFreeMemory(
2176  VmaAllocator allocator,
2177  VmaAllocation allocation);
2178 
2195 void vmaGetAllocationInfo(
2196  VmaAllocator allocator,
2197  VmaAllocation allocation,
2198  VmaAllocationInfo* pAllocationInfo);
2199 
2214 VkBool32 vmaTouchAllocation(
2215  VmaAllocator allocator,
2216  VmaAllocation allocation);
2217 
2231 void vmaSetAllocationUserData(
2232  VmaAllocator allocator,
2233  VmaAllocation allocation,
2234  void* pUserData);
2235 
2246 void vmaCreateLostAllocation(
2247  VmaAllocator allocator,
2248  VmaAllocation* pAllocation);
2249 
2284 VkResult vmaMapMemory(
2285  VmaAllocator allocator,
2286  VmaAllocation allocation,
2287  void** ppData);
2288 
2293 void vmaUnmapMemory(
2294  VmaAllocator allocator,
2295  VmaAllocation allocation);
2296 
2309 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
2310 
2323 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
2324 
2341 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits);
2342 
2344 typedef struct VmaDefragmentationInfo {
2349  VkDeviceSize maxBytesToMove;
2354  uint32_t maxAllocationsToMove;
2355 } VmaDefragmentationInfo;
2356 
2358 typedef struct VmaDefragmentationStats {
2360  VkDeviceSize bytesMoved;
2362  VkDeviceSize bytesFreed;
2364  uint32_t allocationsMoved;
2366  uint32_t deviceMemoryBlocksFreed;
2367 } VmaDefragmentationStats;
2368 
2455 VkResult vmaDefragment(
2456  VmaAllocator allocator,
2457  VmaAllocation* pAllocations,
2458  size_t allocationCount,
2459  VkBool32* pAllocationsChanged,
2460  const VmaDefragmentationInfo *pDefragmentationInfo,
2461  VmaDefragmentationStats* pDefragmentationStats);
2462 
2475 VkResult vmaBindBufferMemory(
2476  VmaAllocator allocator,
2477  VmaAllocation allocation,
2478  VkBuffer buffer);
2479 
2492 VkResult vmaBindImageMemory(
2493  VmaAllocator allocator,
2494  VmaAllocation allocation,
2495  VkImage image);
2496 
2523 VkResult vmaCreateBuffer(
2524  VmaAllocator allocator,
2525  const VkBufferCreateInfo* pBufferCreateInfo,
2526  const VmaAllocationCreateInfo* pAllocationCreateInfo,
2527  VkBuffer* pBuffer,
2528  VmaAllocation* pAllocation,
2529  VmaAllocationInfo* pAllocationInfo);
2530 
2542 void vmaDestroyBuffer(
2543  VmaAllocator allocator,
2544  VkBuffer buffer,
2545  VmaAllocation allocation);
2546 
2548 VkResult vmaCreateImage(
2549  VmaAllocator allocator,
2550  const VkImageCreateInfo* pImageCreateInfo,
2551  const VmaAllocationCreateInfo* pAllocationCreateInfo,
2552  VkImage* pImage,
2553  VmaAllocation* pAllocation,
2554  VmaAllocationInfo* pAllocationInfo);
2555 
2567 void vmaDestroyImage(
2568  VmaAllocator allocator,
2569  VkImage image,
2570  VmaAllocation allocation);
2571 
2572 #ifdef __cplusplus
2573 }
2574 #endif
2575 
2576 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
2577 
2578 // For Visual Studio IntelliSense.
2579 #if defined(__cplusplus) && defined(__INTELLISENSE__)
2580 #define VMA_IMPLEMENTATION
2581 #endif
2582 
2583 #ifdef VMA_IMPLEMENTATION
2584 #undef VMA_IMPLEMENTATION
2585 
2586 #include <cstdint>
2587 #include <cstdlib>
2588 #include <cstring>
2589 
2590 /*******************************************************************************
2591 CONFIGURATION SECTION
2592 
2593 Define some of these macros before each #include of this header or change them
2594 here if you need other then default behavior depending on your environment.
2595 */
2596 
2597 /*
2598 Define this macro to 1 to make the library fetch pointers to Vulkan functions
2599 internally, like:
2600 
2601  vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
2602 
2603 Define to 0 if you are going to provide you own pointers to Vulkan functions via
2604 VmaAllocatorCreateInfo::pVulkanFunctions.
2605 */
2606 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
2607 #define VMA_STATIC_VULKAN_FUNCTIONS 1
2608 #endif
2609 
2610 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
2611 //#define VMA_USE_STL_CONTAINERS 1
2612 
2613 /* Set this macro to 1 to make the library including and using STL containers:
2614 std::pair, std::vector, std::list, std::unordered_map.
2615 
2616 Set it to 0 or undefined to make the library using its own implementation of
2617 the containers.
2618 */
2619 #if VMA_USE_STL_CONTAINERS
2620  #define VMA_USE_STL_VECTOR 1
2621  #define VMA_USE_STL_UNORDERED_MAP 1
2622  #define VMA_USE_STL_LIST 1
2623 #endif
2624 
2625 #if VMA_USE_STL_VECTOR
2626  #include <vector>
2627 #endif
2628 
2629 #if VMA_USE_STL_UNORDERED_MAP
2630  #include <unordered_map>
2631 #endif
2632 
2633 #if VMA_USE_STL_LIST
2634  #include <list>
2635 #endif
2636 
2637 /*
2638 Following headers are used in this CONFIGURATION section only, so feel free to
2639 remove them if not needed.
2640 */
2641 #include <cassert> // for assert
2642 #include <algorithm> // for min, max
2643 #include <mutex> // for std::mutex
2644 #include <atomic> // for std::atomic
2645 
2646 #ifndef VMA_NULL
2647  // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
2648  #define VMA_NULL nullptr
2649 #endif
2650 
2651 #if defined(__APPLE__) || defined(__ANDROID__)
2652 #include <cstdlib>
2653 void *aligned_alloc(size_t alignment, size_t size)
2654 {
2655  // alignment must be >= sizeof(void*)
2656  if(alignment < sizeof(void*))
2657  {
2658  alignment = sizeof(void*);
2659  }
2660 
2661  void *pointer;
2662  if(posix_memalign(&pointer, alignment, size) == 0)
2663  return pointer;
2664  return VMA_NULL;
2665 }
2666 #endif
2667 
2668 // If your compiler is not compatible with C++11 and definition of
2669 // aligned_alloc() function is missing, uncommeting following line may help:
2670 
2671 //#include <malloc.h>
2672 
2673 // Normal assert to check for programmer's errors, especially in Debug configuration.
2674 #ifndef VMA_ASSERT
2675  #ifdef _DEBUG
2676  #define VMA_ASSERT(expr) assert(expr)
2677  #else
2678  #define VMA_ASSERT(expr)
2679  #endif
2680 #endif
2681 
2682 // Assert that will be called very often, like inside data structures e.g. operator[].
2683 // Making it non-empty can make program slow.
2684 #ifndef VMA_HEAVY_ASSERT
2685  #ifdef _DEBUG
2686  #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
2687  #else
2688  #define VMA_HEAVY_ASSERT(expr)
2689  #endif
2690 #endif
2691 
2692 #ifndef VMA_ALIGN_OF
2693  #define VMA_ALIGN_OF(type) (__alignof(type))
2694 #endif
2695 
2696 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
2697  #if defined(_WIN32)
2698  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
2699  #else
2700  #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
2701  #endif
2702 #endif
2703 
2704 #ifndef VMA_SYSTEM_FREE
2705  #if defined(_WIN32)
2706  #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
2707  #else
2708  #define VMA_SYSTEM_FREE(ptr) free(ptr)
2709  #endif
2710 #endif
2711 
2712 #ifndef VMA_MIN
2713  #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
2714 #endif
2715 
2716 #ifndef VMA_MAX
2717  #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
2718 #endif
2719 
2720 #ifndef VMA_SWAP
2721  #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
2722 #endif
2723 
2724 #ifndef VMA_SORT
2725  #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
2726 #endif
2727 
2728 #ifndef VMA_DEBUG_LOG
2729  #define VMA_DEBUG_LOG(format, ...)
2730  /*
2731  #define VMA_DEBUG_LOG(format, ...) do { \
2732  printf(format, __VA_ARGS__); \
2733  printf("\n"); \
2734  } while(false)
2735  */
2736 #endif
2737 
2738 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
2739 #if VMA_STATS_STRING_ENABLED
2740  static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
2741  {
2742  snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
2743  }
2744  static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
2745  {
2746  snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
2747  }
2748  static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
2749  {
2750  snprintf(outStr, strLen, "%p", ptr);
2751  }
2752 #endif
2753 
2754 #ifndef VMA_MUTEX
2755  class VmaMutex
2756  {
2757  public:
2758  VmaMutex() { }
2759  ~VmaMutex() { }
2760  void Lock() { m_Mutex.lock(); }
2761  void Unlock() { m_Mutex.unlock(); }
2762  private:
2763  std::mutex m_Mutex;
2764  };
2765  #define VMA_MUTEX VmaMutex
2766 #endif
2767 
2768 /*
2769 If providing your own implementation, you need to implement a subset of std::atomic:
2770 
2771 - Constructor(uint32_t desired)
2772 - uint32_t load() const
2773 - void store(uint32_t desired)
2774 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
2775 */
2776 #ifndef VMA_ATOMIC_UINT32
2777  #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
2778 #endif
2779 
2780 #ifndef VMA_BEST_FIT
2781 
2793  #define VMA_BEST_FIT (1)
2794 #endif
2795 
2796 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
2797 
2801  #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
2802 #endif
2803 
2804 #ifndef VMA_DEBUG_ALIGNMENT
2805 
2809  #define VMA_DEBUG_ALIGNMENT (1)
2810 #endif
2811 
2812 #ifndef VMA_DEBUG_MARGIN
2813 
2817  #define VMA_DEBUG_MARGIN (0)
2818 #endif
2819 
2820 #ifndef VMA_DEBUG_INITIALIZE_ALLOCATIONS
2821 
2825  #define VMA_DEBUG_INITIALIZE_ALLOCATIONS (0)
2826 #endif
2827 
2828 #ifndef VMA_DEBUG_DETECT_CORRUPTION
2829 
2834  #define VMA_DEBUG_DETECT_CORRUPTION (0)
2835 #endif
2836 
2837 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2838 
2842  #define VMA_DEBUG_GLOBAL_MUTEX (0)
2843 #endif
2844 
2845 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2846 
2850  #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2851 #endif
2852 
2853 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2854  #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2856 #endif
2857 
2858 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2859  #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2861 #endif
2862 
2863 #ifndef VMA_CLASS_NO_COPY
2864  #define VMA_CLASS_NO_COPY(className) \
2865  private: \
2866  className(const className&) = delete; \
2867  className& operator=(const className&) = delete;
2868 #endif
2869 
2870 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2871 
2872 // Decimal 2139416166, float NaN, little-endian binary 66 E6 84 7F.
2873 static const uint32_t VMA_CORRUPTION_DETECTION_MAGIC_VALUE = 0x7F84E666;
2874 
2875 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_CREATED = 0xDC;
2876 static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF;
2877 
2878 /*******************************************************************************
2879 END OF CONFIGURATION
2880 */
2881 
2882 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2883  VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2884 
2885 // Returns number of bits set to 1 in (v).
2886 static inline uint32_t VmaCountBitsSet(uint32_t v)
2887 {
2888  uint32_t c = v - ((v >> 1) & 0x55555555);
2889  c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2890  c = ((c >> 4) + c) & 0x0F0F0F0F;
2891  c = ((c >> 8) + c) & 0x00FF00FF;
2892  c = ((c >> 16) + c) & 0x0000FFFF;
2893  return c;
2894 }
2895 
2896 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2897 // Use types like uint32_t, uint64_t as T.
2898 template <typename T>
2899 static inline T VmaAlignUp(T val, T align)
2900 {
2901  return (val + align - 1) / align * align;
2902 }
2903 // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8.
2904 // Use types like uint32_t, uint64_t as T.
2905 template <typename T>
2906 static inline T VmaAlignDown(T val, T align)
2907 {
2908  return val / align * align;
2909 }
2910 
2911 // Division with mathematical rounding to nearest number.
2912 template <typename T>
2913 inline T VmaRoundDiv(T x, T y)
2914 {
2915  return (x + (y / (T)2)) / y;
2916 }
2917 
2918 static inline bool VmaStrIsEmpty(const char* pStr)
2919 {
2920  return pStr == VMA_NULL || *pStr == '\0';
2921 }
2922 
2923 #ifndef VMA_SORT
2924 
2925 template<typename Iterator, typename Compare>
2926 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2927 {
2928  Iterator centerValue = end; --centerValue;
2929  Iterator insertIndex = beg;
2930  for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2931  {
2932  if(cmp(*memTypeIndex, *centerValue))
2933  {
2934  if(insertIndex != memTypeIndex)
2935  {
2936  VMA_SWAP(*memTypeIndex, *insertIndex);
2937  }
2938  ++insertIndex;
2939  }
2940  }
2941  if(insertIndex != centerValue)
2942  {
2943  VMA_SWAP(*insertIndex, *centerValue);
2944  }
2945  return insertIndex;
2946 }
2947 
2948 template<typename Iterator, typename Compare>
2949 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2950 {
2951  if(beg < end)
2952  {
2953  Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2954  VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2955  VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2956  }
2957 }
2958 
2959 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2960 
2961 #endif // #ifndef VMA_SORT
2962 
2963 /*
2964 Returns true if two memory blocks occupy overlapping pages.
2965 ResourceA must be in less memory offset than ResourceB.
2966 
2967 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2968 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2969 */
2970 static inline bool VmaBlocksOnSamePage(
2971  VkDeviceSize resourceAOffset,
2972  VkDeviceSize resourceASize,
2973  VkDeviceSize resourceBOffset,
2974  VkDeviceSize pageSize)
2975 {
2976  VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2977  VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2978  VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2979  VkDeviceSize resourceBStart = resourceBOffset;
2980  VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2981  return resourceAEndPage == resourceBStartPage;
2982 }
2983 
2984 enum VmaSuballocationType
2985 {
2986  VMA_SUBALLOCATION_TYPE_FREE = 0,
2987  VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2988  VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2989  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2990  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2991  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2992  VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2993 };
2994 
2995 /*
2996 Returns true if given suballocation types could conflict and must respect
2997 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2998 or linear image and another one is optimal image. If type is unknown, behave
2999 conservatively.
3000 */
3001 static inline bool VmaIsBufferImageGranularityConflict(
3002  VmaSuballocationType suballocType1,
3003  VmaSuballocationType suballocType2)
3004 {
3005  if(suballocType1 > suballocType2)
3006  {
3007  VMA_SWAP(suballocType1, suballocType2);
3008  }
3009 
3010  switch(suballocType1)
3011  {
3012  case VMA_SUBALLOCATION_TYPE_FREE:
3013  return false;
3014  case VMA_SUBALLOCATION_TYPE_UNKNOWN:
3015  return true;
3016  case VMA_SUBALLOCATION_TYPE_BUFFER:
3017  return
3018  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
3019  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
3020  case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
3021  return
3022  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
3023  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
3024  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
3025  case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
3026  return
3027  suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
3028  case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
3029  return false;
3030  default:
3031  VMA_ASSERT(0);
3032  return true;
3033  }
3034 }
3035 
3036 static void VmaWriteMagicValue(void* pData, VkDeviceSize offset)
3037 {
3038  uint32_t* pDst = (uint32_t*)((char*)pData + offset);
3039  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
3040  for(size_t i = 0; i < numberCount; ++i, ++pDst)
3041  {
3042  *pDst = VMA_CORRUPTION_DETECTION_MAGIC_VALUE;
3043  }
3044 }
3045 
3046 static bool VmaValidateMagicValue(const void* pData, VkDeviceSize offset)
3047 {
3048  const uint32_t* pSrc = (const uint32_t*)((const char*)pData + offset);
3049  const size_t numberCount = VMA_DEBUG_MARGIN / sizeof(uint32_t);
3050  for(size_t i = 0; i < numberCount; ++i, ++pSrc)
3051  {
3052  if(*pSrc != VMA_CORRUPTION_DETECTION_MAGIC_VALUE)
3053  {
3054  return false;
3055  }
3056  }
3057  return true;
3058 }
3059 
3060 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
3061 struct VmaMutexLock
3062 {
3063  VMA_CLASS_NO_COPY(VmaMutexLock)
3064 public:
3065  VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
3066  m_pMutex(useMutex ? &mutex : VMA_NULL)
3067  {
3068  if(m_pMutex)
3069  {
3070  m_pMutex->Lock();
3071  }
3072  }
3073 
3074  ~VmaMutexLock()
3075  {
3076  if(m_pMutex)
3077  {
3078  m_pMutex->Unlock();
3079  }
3080  }
3081 
3082 private:
3083  VMA_MUTEX* m_pMutex;
3084 };
3085 
3086 #if VMA_DEBUG_GLOBAL_MUTEX
3087  static VMA_MUTEX gDebugGlobalMutex;
3088  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
3089 #else
3090  #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
3091 #endif
3092 
3093 // Minimum size of a free suballocation to register it in the free suballocation collection.
3094 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
3095 
3096 /*
3097 Performs binary search and returns iterator to first element that is greater or
3098 equal to (key), according to comparison (cmp).
3099 
3100 Cmp should return true if first argument is less than second argument.
3101 
3102 Returned value is the found element, if present in the collection or place where
3103 new element with value (key) should be inserted.
3104 */
3105 template <typename CmpLess, typename IterT, typename KeyT>
3106 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpLess cmp)
3107 {
3108  size_t down = 0, up = (end - beg);
3109  while(down < up)
3110  {
3111  const size_t mid = (down + up) / 2;
3112  if(cmp(*(beg+mid), key))
3113  {
3114  down = mid + 1;
3115  }
3116  else
3117  {
3118  up = mid;
3119  }
3120  }
3121  return beg + down;
3122 }
3123 
3125 // Memory allocation
3126 
3127 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
3128 {
3129  if((pAllocationCallbacks != VMA_NULL) &&
3130  (pAllocationCallbacks->pfnAllocation != VMA_NULL))
3131  {
3132  return (*pAllocationCallbacks->pfnAllocation)(
3133  pAllocationCallbacks->pUserData,
3134  size,
3135  alignment,
3136  VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
3137  }
3138  else
3139  {
3140  return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
3141  }
3142 }
3143 
3144 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
3145 {
3146  if((pAllocationCallbacks != VMA_NULL) &&
3147  (pAllocationCallbacks->pfnFree != VMA_NULL))
3148  {
3149  (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
3150  }
3151  else
3152  {
3153  VMA_SYSTEM_FREE(ptr);
3154  }
3155 }
3156 
3157 template<typename T>
3158 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
3159 {
3160  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
3161 }
3162 
3163 template<typename T>
3164 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
3165 {
3166  return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
3167 }
3168 
3169 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
3170 
3171 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
3172 
3173 template<typename T>
3174 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
3175 {
3176  ptr->~T();
3177  VmaFree(pAllocationCallbacks, ptr);
3178 }
3179 
3180 template<typename T>
3181 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
3182 {
3183  if(ptr != VMA_NULL)
3184  {
3185  for(size_t i = count; i--; )
3186  {
3187  ptr[i].~T();
3188  }
3189  VmaFree(pAllocationCallbacks, ptr);
3190  }
3191 }
3192 
3193 // STL-compatible allocator.
3194 template<typename T>
3195 class VmaStlAllocator
3196 {
3197 public:
3198  const VkAllocationCallbacks* const m_pCallbacks;
3199  typedef T value_type;
3200 
3201  VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
3202  template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
3203 
3204  T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
3205  void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
3206 
3207  template<typename U>
3208  bool operator==(const VmaStlAllocator<U>& rhs) const
3209  {
3210  return m_pCallbacks == rhs.m_pCallbacks;
3211  }
3212  template<typename U>
3213  bool operator!=(const VmaStlAllocator<U>& rhs) const
3214  {
3215  return m_pCallbacks != rhs.m_pCallbacks;
3216  }
3217 
3218  VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
3219 };
3220 
3221 #if VMA_USE_STL_VECTOR
3222 
3223 #define VmaVector std::vector
3224 
3225 template<typename T, typename allocatorT>
3226 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
3227 {
3228  vec.insert(vec.begin() + index, item);
3229 }
3230 
3231 template<typename T, typename allocatorT>
3232 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
3233 {
3234  vec.erase(vec.begin() + index);
3235 }
3236 
3237 #else // #if VMA_USE_STL_VECTOR
3238 
3239 /* Class with interface compatible with subset of std::vector.
3240 T must be POD because constructors and destructors are not called and memcpy is
3241 used for these objects. */
3242 template<typename T, typename AllocatorT>
3243 class VmaVector
3244 {
3245 public:
3246  typedef T value_type;
3247 
3248  VmaVector(const AllocatorT& allocator) :
3249  m_Allocator(allocator),
3250  m_pArray(VMA_NULL),
3251  m_Count(0),
3252  m_Capacity(0)
3253  {
3254  }
3255 
3256  VmaVector(size_t count, const AllocatorT& allocator) :
3257  m_Allocator(allocator),
3258  m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
3259  m_Count(count),
3260  m_Capacity(count)
3261  {
3262  }
3263 
3264  VmaVector(const VmaVector<T, AllocatorT>& src) :
3265  m_Allocator(src.m_Allocator),
3266  m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
3267  m_Count(src.m_Count),
3268  m_Capacity(src.m_Count)
3269  {
3270  if(m_Count != 0)
3271  {
3272  memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
3273  }
3274  }
3275 
3276  ~VmaVector()
3277  {
3278  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3279  }
3280 
3281  VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
3282  {
3283  if(&rhs != this)
3284  {
3285  resize(rhs.m_Count);
3286  if(m_Count != 0)
3287  {
3288  memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
3289  }
3290  }
3291  return *this;
3292  }
3293 
3294  bool empty() const { return m_Count == 0; }
3295  size_t size() const { return m_Count; }
3296  T* data() { return m_pArray; }
3297  const T* data() const { return m_pArray; }
3298 
3299  T& operator[](size_t index)
3300  {
3301  VMA_HEAVY_ASSERT(index < m_Count);
3302  return m_pArray[index];
3303  }
3304  const T& operator[](size_t index) const
3305  {
3306  VMA_HEAVY_ASSERT(index < m_Count);
3307  return m_pArray[index];
3308  }
3309 
3310  T& front()
3311  {
3312  VMA_HEAVY_ASSERT(m_Count > 0);
3313  return m_pArray[0];
3314  }
3315  const T& front() const
3316  {
3317  VMA_HEAVY_ASSERT(m_Count > 0);
3318  return m_pArray[0];
3319  }
3320  T& back()
3321  {
3322  VMA_HEAVY_ASSERT(m_Count > 0);
3323  return m_pArray[m_Count - 1];
3324  }
3325  const T& back() const
3326  {
3327  VMA_HEAVY_ASSERT(m_Count > 0);
3328  return m_pArray[m_Count - 1];
3329  }
3330 
3331  void reserve(size_t newCapacity, bool freeMemory = false)
3332  {
3333  newCapacity = VMA_MAX(newCapacity, m_Count);
3334 
3335  if((newCapacity < m_Capacity) && !freeMemory)
3336  {
3337  newCapacity = m_Capacity;
3338  }
3339 
3340  if(newCapacity != m_Capacity)
3341  {
3342  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
3343  if(m_Count != 0)
3344  {
3345  memcpy(newArray, m_pArray, m_Count * sizeof(T));
3346  }
3347  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3348  m_Capacity = newCapacity;
3349  m_pArray = newArray;
3350  }
3351  }
3352 
3353  void resize(size_t newCount, bool freeMemory = false)
3354  {
3355  size_t newCapacity = m_Capacity;
3356  if(newCount > m_Capacity)
3357  {
3358  newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
3359  }
3360  else if(freeMemory)
3361  {
3362  newCapacity = newCount;
3363  }
3364 
3365  if(newCapacity != m_Capacity)
3366  {
3367  T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
3368  const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
3369  if(elementsToCopy != 0)
3370  {
3371  memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
3372  }
3373  VmaFree(m_Allocator.m_pCallbacks, m_pArray);
3374  m_Capacity = newCapacity;
3375  m_pArray = newArray;
3376  }
3377 
3378  m_Count = newCount;
3379  }
3380 
3381  void clear(bool freeMemory = false)
3382  {
3383  resize(0, freeMemory);
3384  }
3385 
3386  void insert(size_t index, const T& src)
3387  {
3388  VMA_HEAVY_ASSERT(index <= m_Count);
3389  const size_t oldCount = size();
3390  resize(oldCount + 1);
3391  if(index < oldCount)
3392  {
3393  memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
3394  }
3395  m_pArray[index] = src;
3396  }
3397 
3398  void remove(size_t index)
3399  {
3400  VMA_HEAVY_ASSERT(index < m_Count);
3401  const size_t oldCount = size();
3402  if(index < oldCount - 1)
3403  {
3404  memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
3405  }
3406  resize(oldCount - 1);
3407  }
3408 
3409  void push_back(const T& src)
3410  {
3411  const size_t newIndex = size();
3412  resize(newIndex + 1);
3413  m_pArray[newIndex] = src;
3414  }
3415 
3416  void pop_back()
3417  {
3418  VMA_HEAVY_ASSERT(m_Count > 0);
3419  resize(size() - 1);
3420  }
3421 
3422  void push_front(const T& src)
3423  {
3424  insert(0, src);
3425  }
3426 
3427  void pop_front()
3428  {
3429  VMA_HEAVY_ASSERT(m_Count > 0);
3430  remove(0);
3431  }
3432 
3433  typedef T* iterator;
3434 
3435  iterator begin() { return m_pArray; }
3436  iterator end() { return m_pArray + m_Count; }
3437 
3438 private:
3439  AllocatorT m_Allocator;
3440  T* m_pArray;
3441  size_t m_Count;
3442  size_t m_Capacity;
3443 };
3444 
3445 template<typename T, typename allocatorT>
3446 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
3447 {
3448  vec.insert(index, item);
3449 }
3450 
3451 template<typename T, typename allocatorT>
3452 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
3453 {
3454  vec.remove(index);
3455 }
3456 
3457 #endif // #if VMA_USE_STL_VECTOR
3458 
3459 template<typename CmpLess, typename VectorT>
3460 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
3461 {
3462  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3463  vector.data(),
3464  vector.data() + vector.size(),
3465  value,
3466  CmpLess()) - vector.data();
3467  VmaVectorInsert(vector, indexToInsert, value);
3468  return indexToInsert;
3469 }
3470 
3471 template<typename CmpLess, typename VectorT>
3472 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
3473 {
3474  CmpLess comparator;
3475  typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
3476  vector.begin(),
3477  vector.end(),
3478  value,
3479  comparator);
3480  if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
3481  {
3482  size_t indexToRemove = it - vector.begin();
3483  VmaVectorRemove(vector, indexToRemove);
3484  return true;
3485  }
3486  return false;
3487 }
3488 
3489 template<typename CmpLess, typename IterT, typename KeyT>
3490 IterT VmaVectorFindSorted(const IterT& beg, const IterT& end, const KeyT& value)
3491 {
3492  CmpLess comparator;
3493  typename IterT it = VmaBinaryFindFirstNotLess<CmpLess, IterT, KeyT>(
3494  beg, end, value, comparator);
3495  if(it == end ||
3496  !comparator(*it, value) && !comparator(value, *it))
3497  {
3498  return it;
3499  }
3500  return end;
3501 }
3502 
3504 // class VmaPoolAllocator
3505 
3506 /*
3507 Allocator for objects of type T using a list of arrays (pools) to speed up
3508 allocation. Number of elements that can be allocated is not bounded because
3509 allocator can create multiple blocks.
3510 */
3511 template<typename T>
3512 class VmaPoolAllocator
3513 {
3514  VMA_CLASS_NO_COPY(VmaPoolAllocator)
3515 public:
3516  VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
3517  ~VmaPoolAllocator();
3518  void Clear();
3519  T* Alloc();
3520  void Free(T* ptr);
3521 
3522 private:
3523  union Item
3524  {
3525  uint32_t NextFreeIndex;
3526  T Value;
3527  };
3528 
3529  struct ItemBlock
3530  {
3531  Item* pItems;
3532  uint32_t FirstFreeIndex;
3533  };
3534 
3535  const VkAllocationCallbacks* m_pAllocationCallbacks;
3536  size_t m_ItemsPerBlock;
3537  VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
3538 
3539  ItemBlock& CreateNewBlock();
3540 };
3541 
3542 template<typename T>
3543 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
3544  m_pAllocationCallbacks(pAllocationCallbacks),
3545  m_ItemsPerBlock(itemsPerBlock),
3546  m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
3547 {
3548  VMA_ASSERT(itemsPerBlock > 0);
3549 }
3550 
3551 template<typename T>
3552 VmaPoolAllocator<T>::~VmaPoolAllocator()
3553 {
3554  Clear();
3555 }
3556 
3557 template<typename T>
3558 void VmaPoolAllocator<T>::Clear()
3559 {
3560  for(size_t i = m_ItemBlocks.size(); i--; )
3561  vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
3562  m_ItemBlocks.clear();
3563 }
3564 
3565 template<typename T>
3566 T* VmaPoolAllocator<T>::Alloc()
3567 {
3568  for(size_t i = m_ItemBlocks.size(); i--; )
3569  {
3570  ItemBlock& block = m_ItemBlocks[i];
3571  // This block has some free items: Use first one.
3572  if(block.FirstFreeIndex != UINT32_MAX)
3573  {
3574  Item* const pItem = &block.pItems[block.FirstFreeIndex];
3575  block.FirstFreeIndex = pItem->NextFreeIndex;
3576  return &pItem->Value;
3577  }
3578  }
3579 
3580  // No block has free item: Create new one and use it.
3581  ItemBlock& newBlock = CreateNewBlock();
3582  Item* const pItem = &newBlock.pItems[0];
3583  newBlock.FirstFreeIndex = pItem->NextFreeIndex;
3584  return &pItem->Value;
3585 }
3586 
3587 template<typename T>
3588 void VmaPoolAllocator<T>::Free(T* ptr)
3589 {
3590  // Search all memory blocks to find ptr.
3591  for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
3592  {
3593  ItemBlock& block = m_ItemBlocks[i];
3594 
3595  // Casting to union.
3596  Item* pItemPtr;
3597  memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
3598 
3599  // Check if pItemPtr is in address range of this block.
3600  if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
3601  {
3602  const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
3603  pItemPtr->NextFreeIndex = block.FirstFreeIndex;
3604  block.FirstFreeIndex = index;
3605  return;
3606  }
3607  }
3608  VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
3609 }
3610 
3611 template<typename T>
3612 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
3613 {
3614  ItemBlock newBlock = {
3615  vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
3616 
3617  m_ItemBlocks.push_back(newBlock);
3618 
3619  // Setup singly-linked list of all free items in this block.
3620  for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
3621  newBlock.pItems[i].NextFreeIndex = i + 1;
3622  newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
3623  return m_ItemBlocks.back();
3624 }
3625 
3627 // class VmaRawList, VmaList
3628 
3629 #if VMA_USE_STL_LIST
3630 
3631 #define VmaList std::list
3632 
3633 #else // #if VMA_USE_STL_LIST
3634 
3635 template<typename T>
3636 struct VmaListItem
3637 {
3638  VmaListItem* pPrev;
3639  VmaListItem* pNext;
3640  T Value;
3641 };
3642 
3643 // Doubly linked list.
3644 template<typename T>
3645 class VmaRawList
3646 {
3647  VMA_CLASS_NO_COPY(VmaRawList)
3648 public:
3649  typedef VmaListItem<T> ItemType;
3650 
3651  VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
3652  ~VmaRawList();
3653  void Clear();
3654 
3655  size_t GetCount() const { return m_Count; }
3656  bool IsEmpty() const { return m_Count == 0; }
3657 
3658  ItemType* Front() { return m_pFront; }
3659  const ItemType* Front() const { return m_pFront; }
3660  ItemType* Back() { return m_pBack; }
3661  const ItemType* Back() const { return m_pBack; }
3662 
3663  ItemType* PushBack();
3664  ItemType* PushFront();
3665  ItemType* PushBack(const T& value);
3666  ItemType* PushFront(const T& value);
3667  void PopBack();
3668  void PopFront();
3669 
3670  // Item can be null - it means PushBack.
3671  ItemType* InsertBefore(ItemType* pItem);
3672  // Item can be null - it means PushFront.
3673  ItemType* InsertAfter(ItemType* pItem);
3674 
3675  ItemType* InsertBefore(ItemType* pItem, const T& value);
3676  ItemType* InsertAfter(ItemType* pItem, const T& value);
3677 
3678  void Remove(ItemType* pItem);
3679 
3680 private:
3681  const VkAllocationCallbacks* const m_pAllocationCallbacks;
3682  VmaPoolAllocator<ItemType> m_ItemAllocator;
3683  ItemType* m_pFront;
3684  ItemType* m_pBack;
3685  size_t m_Count;
3686 };
3687 
3688 template<typename T>
3689 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
3690  m_pAllocationCallbacks(pAllocationCallbacks),
3691  m_ItemAllocator(pAllocationCallbacks, 128),
3692  m_pFront(VMA_NULL),
3693  m_pBack(VMA_NULL),
3694  m_Count(0)
3695 {
3696 }
3697 
3698 template<typename T>
3699 VmaRawList<T>::~VmaRawList()
3700 {
3701  // Intentionally not calling Clear, because that would be unnecessary
3702  // computations to return all items to m_ItemAllocator as free.
3703 }
3704 
3705 template<typename T>
3706 void VmaRawList<T>::Clear()
3707 {
3708  if(IsEmpty() == false)
3709  {
3710  ItemType* pItem = m_pBack;
3711  while(pItem != VMA_NULL)
3712  {
3713  ItemType* const pPrevItem = pItem->pPrev;
3714  m_ItemAllocator.Free(pItem);
3715  pItem = pPrevItem;
3716  }
3717  m_pFront = VMA_NULL;
3718  m_pBack = VMA_NULL;
3719  m_Count = 0;
3720  }
3721 }
3722 
3723 template<typename T>
3724 VmaListItem<T>* VmaRawList<T>::PushBack()
3725 {
3726  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3727  pNewItem->pNext = VMA_NULL;
3728  if(IsEmpty())
3729  {
3730  pNewItem->pPrev = VMA_NULL;
3731  m_pFront = pNewItem;
3732  m_pBack = pNewItem;
3733  m_Count = 1;
3734  }
3735  else
3736  {
3737  pNewItem->pPrev = m_pBack;
3738  m_pBack->pNext = pNewItem;
3739  m_pBack = pNewItem;
3740  ++m_Count;
3741  }
3742  return pNewItem;
3743 }
3744 
3745 template<typename T>
3746 VmaListItem<T>* VmaRawList<T>::PushFront()
3747 {
3748  ItemType* const pNewItem = m_ItemAllocator.Alloc();
3749  pNewItem->pPrev = VMA_NULL;
3750  if(IsEmpty())
3751  {
3752  pNewItem->pNext = VMA_NULL;
3753  m_pFront = pNewItem;
3754  m_pBack = pNewItem;
3755  m_Count = 1;
3756  }
3757  else
3758  {
3759  pNewItem->pNext = m_pFront;
3760  m_pFront->pPrev = pNewItem;
3761  m_pFront = pNewItem;
3762  ++m_Count;
3763  }
3764  return pNewItem;
3765 }
3766 
3767 template<typename T>
3768 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
3769 {
3770  ItemType* const pNewItem = PushBack();
3771  pNewItem->Value = value;
3772  return pNewItem;
3773 }
3774 
3775 template<typename T>
3776 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
3777 {
3778  ItemType* const pNewItem = PushFront();
3779  pNewItem->Value = value;
3780  return pNewItem;
3781 }
3782 
3783 template<typename T>
3784 void VmaRawList<T>::PopBack()
3785 {
3786  VMA_HEAVY_ASSERT(m_Count > 0);
3787  ItemType* const pBackItem = m_pBack;
3788  ItemType* const pPrevItem = pBackItem->pPrev;
3789  if(pPrevItem != VMA_NULL)
3790  {
3791  pPrevItem->pNext = VMA_NULL;
3792  }
3793  m_pBack = pPrevItem;
3794  m_ItemAllocator.Free(pBackItem);
3795  --m_Count;
3796 }
3797 
3798 template<typename T>
3799 void VmaRawList<T>::PopFront()
3800 {
3801  VMA_HEAVY_ASSERT(m_Count > 0);
3802  ItemType* const pFrontItem = m_pFront;
3803  ItemType* const pNextItem = pFrontItem->pNext;
3804  if(pNextItem != VMA_NULL)
3805  {
3806  pNextItem->pPrev = VMA_NULL;
3807  }
3808  m_pFront = pNextItem;
3809  m_ItemAllocator.Free(pFrontItem);
3810  --m_Count;
3811 }
3812 
3813 template<typename T>
3814 void VmaRawList<T>::Remove(ItemType* pItem)
3815 {
3816  VMA_HEAVY_ASSERT(pItem != VMA_NULL);
3817  VMA_HEAVY_ASSERT(m_Count > 0);
3818 
3819  if(pItem->pPrev != VMA_NULL)
3820  {
3821  pItem->pPrev->pNext = pItem->pNext;
3822  }
3823  else
3824  {
3825  VMA_HEAVY_ASSERT(m_pFront == pItem);
3826  m_pFront = pItem->pNext;
3827  }
3828 
3829  if(pItem->pNext != VMA_NULL)
3830  {
3831  pItem->pNext->pPrev = pItem->pPrev;
3832  }
3833  else
3834  {
3835  VMA_HEAVY_ASSERT(m_pBack == pItem);
3836  m_pBack = pItem->pPrev;
3837  }
3838 
3839  m_ItemAllocator.Free(pItem);
3840  --m_Count;
3841 }
3842 
3843 template<typename T>
3844 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
3845 {
3846  if(pItem != VMA_NULL)
3847  {
3848  ItemType* const prevItem = pItem->pPrev;
3849  ItemType* const newItem = m_ItemAllocator.Alloc();
3850  newItem->pPrev = prevItem;
3851  newItem->pNext = pItem;
3852  pItem->pPrev = newItem;
3853  if(prevItem != VMA_NULL)
3854  {
3855  prevItem->pNext = newItem;
3856  }
3857  else
3858  {
3859  VMA_HEAVY_ASSERT(m_pFront == pItem);
3860  m_pFront = newItem;
3861  }
3862  ++m_Count;
3863  return newItem;
3864  }
3865  else
3866  return PushBack();
3867 }
3868 
3869 template<typename T>
3870 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
3871 {
3872  if(pItem != VMA_NULL)
3873  {
3874  ItemType* const nextItem = pItem->pNext;
3875  ItemType* const newItem = m_ItemAllocator.Alloc();
3876  newItem->pNext = nextItem;
3877  newItem->pPrev = pItem;
3878  pItem->pNext = newItem;
3879  if(nextItem != VMA_NULL)
3880  {
3881  nextItem->pPrev = newItem;
3882  }
3883  else
3884  {
3885  VMA_HEAVY_ASSERT(m_pBack == pItem);
3886  m_pBack = newItem;
3887  }
3888  ++m_Count;
3889  return newItem;
3890  }
3891  else
3892  return PushFront();
3893 }
3894 
3895 template<typename T>
3896 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3897 {
3898  ItemType* const newItem = InsertBefore(pItem);
3899  newItem->Value = value;
3900  return newItem;
3901 }
3902 
3903 template<typename T>
3904 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3905 {
3906  ItemType* const newItem = InsertAfter(pItem);
3907  newItem->Value = value;
3908  return newItem;
3909 }
3910 
3911 template<typename T, typename AllocatorT>
3912 class VmaList
3913 {
3914  VMA_CLASS_NO_COPY(VmaList)
3915 public:
3916  class iterator
3917  {
3918  public:
3919  iterator() :
3920  m_pList(VMA_NULL),
3921  m_pItem(VMA_NULL)
3922  {
3923  }
3924 
3925  T& operator*() const
3926  {
3927  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3928  return m_pItem->Value;
3929  }
3930  T* operator->() const
3931  {
3932  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3933  return &m_pItem->Value;
3934  }
3935 
3936  iterator& operator++()
3937  {
3938  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3939  m_pItem = m_pItem->pNext;
3940  return *this;
3941  }
3942  iterator& operator--()
3943  {
3944  if(m_pItem != VMA_NULL)
3945  {
3946  m_pItem = m_pItem->pPrev;
3947  }
3948  else
3949  {
3950  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3951  m_pItem = m_pList->Back();
3952  }
3953  return *this;
3954  }
3955 
3956  iterator operator++(int)
3957  {
3958  iterator result = *this;
3959  ++*this;
3960  return result;
3961  }
3962  iterator operator--(int)
3963  {
3964  iterator result = *this;
3965  --*this;
3966  return result;
3967  }
3968 
3969  bool operator==(const iterator& rhs) const
3970  {
3971  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3972  return m_pItem == rhs.m_pItem;
3973  }
3974  bool operator!=(const iterator& rhs) const
3975  {
3976  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3977  return m_pItem != rhs.m_pItem;
3978  }
3979 
3980  private:
3981  VmaRawList<T>* m_pList;
3982  VmaListItem<T>* m_pItem;
3983 
3984  iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3985  m_pList(pList),
3986  m_pItem(pItem)
3987  {
3988  }
3989 
3990  friend class VmaList<T, AllocatorT>;
3991  };
3992 
3993  class const_iterator
3994  {
3995  public:
3996  const_iterator() :
3997  m_pList(VMA_NULL),
3998  m_pItem(VMA_NULL)
3999  {
4000  }
4001 
4002  const_iterator(const iterator& src) :
4003  m_pList(src.m_pList),
4004  m_pItem(src.m_pItem)
4005  {
4006  }
4007 
4008  const T& operator*() const
4009  {
4010  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
4011  return m_pItem->Value;
4012  }
4013  const T* operator->() const
4014  {
4015  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
4016  return &m_pItem->Value;
4017  }
4018 
4019  const_iterator& operator++()
4020  {
4021  VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
4022  m_pItem = m_pItem->pNext;
4023  return *this;
4024  }
4025  const_iterator& operator--()
4026  {
4027  if(m_pItem != VMA_NULL)
4028  {
4029  m_pItem = m_pItem->pPrev;
4030  }
4031  else
4032  {
4033  VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
4034  m_pItem = m_pList->Back();
4035  }
4036  return *this;
4037  }
4038 
4039  const_iterator operator++(int)
4040  {
4041  const_iterator result = *this;
4042  ++*this;
4043  return result;
4044  }
4045  const_iterator operator--(int)
4046  {
4047  const_iterator result = *this;
4048  --*this;
4049  return result;
4050  }
4051 
4052  bool operator==(const const_iterator& rhs) const
4053  {
4054  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
4055  return m_pItem == rhs.m_pItem;
4056  }
4057  bool operator!=(const const_iterator& rhs) const
4058  {
4059  VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
4060  return m_pItem != rhs.m_pItem;
4061  }
4062 
4063  private:
4064  const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
4065  m_pList(pList),
4066  m_pItem(pItem)
4067  {
4068  }
4069 
4070  const VmaRawList<T>* m_pList;
4071  const VmaListItem<T>* m_pItem;
4072 
4073  friend class VmaList<T, AllocatorT>;
4074  };
4075 
4076  VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
4077 
4078  bool empty() const { return m_RawList.IsEmpty(); }
4079  size_t size() const { return m_RawList.GetCount(); }
4080 
4081  iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
4082  iterator end() { return iterator(&m_RawList, VMA_NULL); }
4083 
4084  const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
4085  const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
4086 
4087  void clear() { m_RawList.Clear(); }
4088  void push_back(const T& value) { m_RawList.PushBack(value); }
4089  void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
4090  iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
4091 
4092 private:
4093  VmaRawList<T> m_RawList;
4094 };
4095 
4096 #endif // #if VMA_USE_STL_LIST
4097 
4099 // class VmaMap
4100 
4101 // Unused in this version.
4102 #if 0
4103 
4104 #if VMA_USE_STL_UNORDERED_MAP
4105 
4106 #define VmaPair std::pair
4107 
4108 #define VMA_MAP_TYPE(KeyT, ValueT) \
4109  std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
4110 
4111 #else // #if VMA_USE_STL_UNORDERED_MAP
4112 
4113 template<typename T1, typename T2>
4114 struct VmaPair
4115 {
4116  T1 first;
4117  T2 second;
4118 
4119  VmaPair() : first(), second() { }
4120  VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
4121 };
4122 
4123 /* Class compatible with subset of interface of std::unordered_map.
4124 KeyT, ValueT must be POD because they will be stored in VmaVector.
4125 */
4126 template<typename KeyT, typename ValueT>
4127 class VmaMap
4128 {
4129 public:
4130  typedef VmaPair<KeyT, ValueT> PairType;
4131  typedef PairType* iterator;
4132 
4133  VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
4134 
4135  iterator begin() { return m_Vector.begin(); }
4136  iterator end() { return m_Vector.end(); }
4137 
4138  void insert(const PairType& pair);
4139  iterator find(const KeyT& key);
4140  void erase(iterator it);
4141 
4142 private:
4143  VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
4144 };
4145 
4146 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
4147 
4148 template<typename FirstT, typename SecondT>
4149 struct VmaPairFirstLess
4150 {
4151  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
4152  {
4153  return lhs.first < rhs.first;
4154  }
4155  bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
4156  {
4157  return lhs.first < rhsFirst;
4158  }
4159 };
4160 
4161 template<typename KeyT, typename ValueT>
4162 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
4163 {
4164  const size_t indexToInsert = VmaBinaryFindFirstNotLess(
4165  m_Vector.data(),
4166  m_Vector.data() + m_Vector.size(),
4167  pair,
4168  VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
4169  VmaVectorInsert(m_Vector, indexToInsert, pair);
4170 }
4171 
4172 template<typename KeyT, typename ValueT>
4173 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
4174 {
4175  PairType* it = VmaBinaryFindFirstNotLess(
4176  m_Vector.data(),
4177  m_Vector.data() + m_Vector.size(),
4178  key,
4179  VmaPairFirstLess<KeyT, ValueT>());
4180  if((it != m_Vector.end()) && (it->first == key))
4181  {
4182  return it;
4183  }
4184  else
4185  {
4186  return m_Vector.end();
4187  }
4188 }
4189 
4190 template<typename KeyT, typename ValueT>
4191 void VmaMap<KeyT, ValueT>::erase(iterator it)
4192 {
4193  VmaVectorRemove(m_Vector, it - m_Vector.begin());
4194 }
4195 
4196 #endif // #if VMA_USE_STL_UNORDERED_MAP
4197 
4198 #endif // #if 0
4199 
4201 
4202 class VmaDeviceMemoryBlock;
4203 
4204 enum VMA_CACHE_OPERATION { VMA_CACHE_FLUSH, VMA_CACHE_INVALIDATE };
4205 
4206 struct VmaAllocation_T
4207 {
4208  VMA_CLASS_NO_COPY(VmaAllocation_T)
4209 private:
4210  static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
4211 
4212  enum FLAGS
4213  {
4214  FLAG_USER_DATA_STRING = 0x01,
4215  };
4216 
4217 public:
4218  enum ALLOCATION_TYPE
4219  {
4220  ALLOCATION_TYPE_NONE,
4221  ALLOCATION_TYPE_BLOCK,
4222  ALLOCATION_TYPE_DEDICATED,
4223  };
4224 
4225  VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
4226  m_Alignment(1),
4227  m_Size(0),
4228  m_pUserData(VMA_NULL),
4229  m_LastUseFrameIndex(currentFrameIndex),
4230  m_Type((uint8_t)ALLOCATION_TYPE_NONE),
4231  m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
4232  m_MapCount(0),
4233  m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
4234  {
4235 #if VMA_STATS_STRING_ENABLED
4236  m_CreationFrameIndex = currentFrameIndex;
4237  m_BufferImageUsage = 0;
4238 #endif
4239  }
4240 
4241  ~VmaAllocation_T()
4242  {
4243  VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
4244 
4245  // Check if owned string was freed.
4246  VMA_ASSERT(m_pUserData == VMA_NULL);
4247  }
4248 
4249  void InitBlockAllocation(
4250  VmaPool hPool,
4251  VmaDeviceMemoryBlock* block,
4252  VkDeviceSize offset,
4253  VkDeviceSize alignment,
4254  VkDeviceSize size,
4255  VmaSuballocationType suballocationType,
4256  bool mapped,
4257  bool canBecomeLost)
4258  {
4259  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4260  VMA_ASSERT(block != VMA_NULL);
4261  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
4262  m_Alignment = alignment;
4263  m_Size = size;
4264  m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
4265  m_SuballocationType = (uint8_t)suballocationType;
4266  m_BlockAllocation.m_hPool = hPool;
4267  m_BlockAllocation.m_Block = block;
4268  m_BlockAllocation.m_Offset = offset;
4269  m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
4270  }
4271 
4272  void InitLost()
4273  {
4274  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4275  VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
4276  m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
4277  m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
4278  m_BlockAllocation.m_Block = VMA_NULL;
4279  m_BlockAllocation.m_Offset = 0;
4280  m_BlockAllocation.m_CanBecomeLost = true;
4281  }
4282 
4283  void ChangeBlockAllocation(
4284  VmaAllocator hAllocator,
4285  VmaDeviceMemoryBlock* block,
4286  VkDeviceSize offset);
4287 
4288  // pMappedData not null means allocation is created with MAPPED flag.
4289  void InitDedicatedAllocation(
4290  uint32_t memoryTypeIndex,
4291  VkDeviceMemory hMemory,
4292  VmaSuballocationType suballocationType,
4293  void* pMappedData,
4294  VkDeviceSize size)
4295  {
4296  VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
4297  VMA_ASSERT(hMemory != VK_NULL_HANDLE);
4298  m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
4299  m_Alignment = 0;
4300  m_Size = size;
4301  m_SuballocationType = (uint8_t)suballocationType;
4302  m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
4303  m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
4304  m_DedicatedAllocation.m_hMemory = hMemory;
4305  m_DedicatedAllocation.m_pMappedData = pMappedData;
4306  }
4307 
4308  ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
4309  VkDeviceSize GetAlignment() const { return m_Alignment; }
4310  VkDeviceSize GetSize() const { return m_Size; }
4311  bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
4312  void* GetUserData() const { return m_pUserData; }
4313  void SetUserData(VmaAllocator hAllocator, void* pUserData);
4314  VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
4315 
4316  VmaDeviceMemoryBlock* GetBlock() const
4317  {
4318  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4319  return m_BlockAllocation.m_Block;
4320  }
4321  VkDeviceSize GetOffset() const;
4322  VkDeviceMemory GetMemory() const;
4323  uint32_t GetMemoryTypeIndex() const;
4324  bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
4325  void* GetMappedData() const;
4326  bool CanBecomeLost() const;
4327  VmaPool GetPool() const;
4328 
4329  uint32_t GetLastUseFrameIndex() const
4330  {
4331  return m_LastUseFrameIndex.load();
4332  }
4333  bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
4334  {
4335  return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
4336  }
4337  /*
4338  - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
4339  makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
4340  - Else, returns false.
4341 
4342  If hAllocation is already lost, assert - you should not call it then.
4343  If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
4344  */
4345  bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4346 
4347  void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
4348  {
4349  VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
4350  outInfo.blockCount = 1;
4351  outInfo.allocationCount = 1;
4352  outInfo.unusedRangeCount = 0;
4353  outInfo.usedBytes = m_Size;
4354  outInfo.unusedBytes = 0;
4355  outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
4356  outInfo.unusedRangeSizeMin = UINT64_MAX;
4357  outInfo.unusedRangeSizeMax = 0;
4358  }
4359 
4360  void BlockAllocMap();
4361  void BlockAllocUnmap();
4362  VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
4363  void DedicatedAllocUnmap(VmaAllocator hAllocator);
4364 
4365 #if VMA_STATS_STRING_ENABLED
4366  uint32_t GetCreationFrameIndex() const { return m_CreationFrameIndex; }
4367  uint32_t GetBufferImageUsage() const { return m_BufferImageUsage; }
4368 
4369  void InitBufferImageUsage(uint32_t bufferImageUsage)
4370  {
4371  VMA_ASSERT(m_BufferImageUsage == 0);
4372  m_BufferImageUsage = bufferImageUsage;
4373  }
4374 
4375  void PrintParameters(class VmaJsonWriter& json) const;
4376 #endif
4377 
4378 private:
4379  VkDeviceSize m_Alignment;
4380  VkDeviceSize m_Size;
4381  void* m_pUserData;
4382  VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
4383  uint8_t m_Type; // ALLOCATION_TYPE
4384  uint8_t m_SuballocationType; // VmaSuballocationType
4385  // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
4386  // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
4387  uint8_t m_MapCount;
4388  uint8_t m_Flags; // enum FLAGS
4389 
4390  // Allocation out of VmaDeviceMemoryBlock.
4391  struct BlockAllocation
4392  {
4393  VmaPool m_hPool; // Null if belongs to general memory.
4394  VmaDeviceMemoryBlock* m_Block;
4395  VkDeviceSize m_Offset;
4396  bool m_CanBecomeLost;
4397  };
4398 
4399  // Allocation for an object that has its own private VkDeviceMemory.
4400  struct DedicatedAllocation
4401  {
4402  uint32_t m_MemoryTypeIndex;
4403  VkDeviceMemory m_hMemory;
4404  void* m_pMappedData; // Not null means memory is mapped.
4405  };
4406 
4407  union
4408  {
4409  // Allocation out of VmaDeviceMemoryBlock.
4410  BlockAllocation m_BlockAllocation;
4411  // Allocation for an object that has its own private VkDeviceMemory.
4412  DedicatedAllocation m_DedicatedAllocation;
4413  };
4414 
4415 #if VMA_STATS_STRING_ENABLED
4416  uint32_t m_CreationFrameIndex;
4417  uint32_t m_BufferImageUsage; // 0 if unknown.
4418 #endif
4419 
4420  void FreeUserDataString(VmaAllocator hAllocator);
4421 };
4422 
4423 /*
4424 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
4425 allocated memory block or free.
4426 */
4427 struct VmaSuballocation
4428 {
4429  VkDeviceSize offset;
4430  VkDeviceSize size;
4431  VmaAllocation hAllocation;
4432  VmaSuballocationType type;
4433 };
4434 
4435 // Comparator for offsets.
4436 struct VmaSuballocationOffsetLess
4437 {
4438  bool operator()(const VmaSuballocation& lhs, const VmaSuballocation& rhs) const
4439  {
4440  return lhs.offset < rhs.offset;
4441  }
4442 };
4443 struct VmaSuballocationOffsetGreater
4444 {
4445  bool operator()(const VmaSuballocation& lhs, const VmaSuballocation& rhs) const
4446  {
4447  return lhs.offset > rhs.offset;
4448  }
4449 };
4450 
4451 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
4452 
4453 // Cost of one additional allocation lost, as equivalent in bytes.
4454 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
4455 
4456 /*
4457 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
4458 
4459 If canMakeOtherLost was false:
4460 - item points to a FREE suballocation.
4461 - itemsToMakeLostCount is 0.
4462 
4463 If canMakeOtherLost was true:
4464 - item points to first of sequence of suballocations, which are either FREE,
4465  or point to VmaAllocations that can become lost.
4466 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
4467  the requested allocation to succeed.
4468 */
4469 struct VmaAllocationRequest
4470 {
4471  VkDeviceSize offset;
4472  VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
4473  VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
4474  VmaSuballocationList::iterator item;
4475  size_t itemsToMakeLostCount;
4476 
4477  VkDeviceSize CalcCost() const
4478  {
4479  return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
4480  }
4481 };
4482 
4483 /*
4484 Data structure used for bookkeeping of allocations and unused ranges of memory
4485 in a single VkDeviceMemory block.
4486 */
4487 class VmaBlockMetadata
4488 {
4489 public:
4490  VmaBlockMetadata() : m_Size(0) { }
4491  virtual ~VmaBlockMetadata() { }
4492  virtual void Init(VkDeviceSize size) { m_Size = size; }
4493 
4494  // Validates all data structures inside this object. If not valid, returns false.
4495  virtual bool Validate() const = 0;
4496  VkDeviceSize GetSize() const { return m_Size; }
4497  virtual size_t GetAllocationCount() const = 0;
4498  virtual VkDeviceSize GetSumFreeSize() const = 0;
4499  virtual VkDeviceSize GetUnusedRangeSizeMax() const = 0;
4500  // Returns true if this block is empty - contains only single free suballocation.
4501  virtual bool IsEmpty() const = 0;
4502 
4503  virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const = 0;
4504  virtual void AddPoolStats(VmaPoolStats& inoutStats) const = 0;
4505 
4506 #if VMA_STATS_STRING_ENABLED
4507  virtual void PrintDetailedMap(class VmaJsonWriter& json) const = 0;
4508 #endif
4509 
4510  // Tries to find a place for suballocation with given parameters inside this block.
4511  // If succeeded, fills pAllocationRequest and returns true.
4512  // If failed, returns false.
4513  virtual bool CreateAllocationRequest(
4514  uint32_t currentFrameIndex,
4515  uint32_t frameInUseCount,
4516  VkDeviceSize bufferImageGranularity,
4517  VkDeviceSize allocSize,
4518  VkDeviceSize allocAlignment,
4519  bool upperAddress,
4520  VmaSuballocationType allocType,
4521  bool canMakeOtherLost,
4522  VmaAllocationRequest* pAllocationRequest) = 0;
4523 
4524  virtual bool MakeRequestedAllocationsLost(
4525  uint32_t currentFrameIndex,
4526  uint32_t frameInUseCount,
4527  VmaAllocationRequest* pAllocationRequest) = 0;
4528 
4529  virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) = 0;
4530 
4531  virtual VkResult CheckCorruption(const void* pBlockData) = 0;
4532 
4533  // Makes actual allocation based on request. Request must already be checked and valid.
4534  virtual void Alloc(
4535  const VmaAllocationRequest& request,
4536  VmaSuballocationType type,
4537  VkDeviceSize allocSize,
4538  bool upperAddress,
4539  VmaAllocation hAllocation) = 0;
4540 
4541  // Frees suballocation assigned to given memory region.
4542  virtual void Free(const VmaAllocation allocation) = 0;
4543  virtual void FreeAtOffset(VkDeviceSize offset) = 0;
4544 
4545 protected:
4546 #if VMA_STATS_STRING_ENABLED
4547  void PrintDetailedMap_Begin(class VmaJsonWriter& json,
4548  VkDeviceSize unusedBytes,
4549  size_t allocationCount,
4550  size_t unusedRangeCount) const;
4551  void PrintDetailedMap_Allocation(class VmaJsonWriter& json,
4552  VkDeviceSize offset,
4553  VmaAllocation hAllocation) const;
4554  void PrintDetailedMap_UnusedRange(class VmaJsonWriter& json,
4555  VkDeviceSize offset,
4556  VkDeviceSize size) const;
4557  void PrintDetailedMap_End(class VmaJsonWriter& json) const;
4558 #endif
4559 
4560 private:
4561  VkDeviceSize m_Size;
4562 };
4563 
4564 class VmaBlockMetadata_Generic : public VmaBlockMetadata
4565 {
4566  VMA_CLASS_NO_COPY(VmaBlockMetadata_Generic)
4567 public:
4568  VmaBlockMetadata_Generic(VmaAllocator hAllocator);
4569  virtual ~VmaBlockMetadata_Generic();
4570  virtual void Init(VkDeviceSize size);
4571 
4572  virtual bool Validate() const;
4573  virtual size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
4574  virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
4575  virtual VkDeviceSize GetUnusedRangeSizeMax() const;
4576  virtual bool IsEmpty() const;
4577 
4578  virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
4579  virtual void AddPoolStats(VmaPoolStats& inoutStats) const;
4580 
4581 #if VMA_STATS_STRING_ENABLED
4582  virtual void PrintDetailedMap(class VmaJsonWriter& json) const;
4583 #endif
4584 
4585  virtual bool CreateAllocationRequest(
4586  uint32_t currentFrameIndex,
4587  uint32_t frameInUseCount,
4588  VkDeviceSize bufferImageGranularity,
4589  VkDeviceSize allocSize,
4590  VkDeviceSize allocAlignment,
4591  bool upperAddress,
4592  VmaSuballocationType allocType,
4593  bool canMakeOtherLost,
4594  VmaAllocationRequest* pAllocationRequest);
4595 
4596  virtual bool MakeRequestedAllocationsLost(
4597  uint32_t currentFrameIndex,
4598  uint32_t frameInUseCount,
4599  VmaAllocationRequest* pAllocationRequest);
4600 
4601  virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4602 
4603  virtual VkResult CheckCorruption(const void* pBlockData);
4604 
4605  virtual void Alloc(
4606  const VmaAllocationRequest& request,
4607  VmaSuballocationType type,
4608  VkDeviceSize allocSize,
4609  bool upperAddress,
4610  VmaAllocation hAllocation);
4611 
4612  virtual void Free(const VmaAllocation allocation);
4613  virtual void FreeAtOffset(VkDeviceSize offset);
4614 
4615 private:
4616  uint32_t m_FreeCount;
4617  VkDeviceSize m_SumFreeSize;
4618  VmaSuballocationList m_Suballocations;
4619  // Suballocations that are free and have size greater than certain threshold.
4620  // Sorted by size, ascending.
4621  VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
4622 
4623  bool ValidateFreeSuballocationList() const;
4624 
4625  // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
4626  // If yes, fills pOffset and returns true. If no, returns false.
4627  bool CheckAllocation(
4628  uint32_t currentFrameIndex,
4629  uint32_t frameInUseCount,
4630  VkDeviceSize bufferImageGranularity,
4631  VkDeviceSize allocSize,
4632  VkDeviceSize allocAlignment,
4633  VmaSuballocationType allocType,
4634  VmaSuballocationList::const_iterator suballocItem,
4635  bool canMakeOtherLost,
4636  VkDeviceSize* pOffset,
4637  size_t* itemsToMakeLostCount,
4638  VkDeviceSize* pSumFreeSize,
4639  VkDeviceSize* pSumItemSize) const;
4640  // Given free suballocation, it merges it with following one, which must also be free.
4641  void MergeFreeWithNext(VmaSuballocationList::iterator item);
4642  // Releases given suballocation, making it free.
4643  // Merges it with adjacent free suballocations if applicable.
4644  // Returns iterator to new free suballocation at this place.
4645  VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
4646  // Given free suballocation, it inserts it into sorted list of
4647  // m_FreeSuballocationsBySize if it's suitable.
4648  void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
4649  // Given free suballocation, it removes it from sorted list of
4650  // m_FreeSuballocationsBySize if it's suitable.
4651  void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
4652 };
4653 
4654 /*
4655 Allocations and their references in internal data structure look like this:
4656 
4657 if(m_2ndVectorMode == SECOND_VECTOR_EMPTY):
4658 
4659  0 +-------+
4660  | |
4661  | |
4662  | |
4663  +-------+
4664  | Alloc | 1st[m_1stNullItemsBeginCount]
4665  +-------+
4666  | Alloc | 1st[m_1stNullItemsBeginCount + 1]
4667  +-------+
4668  | ... |
4669  +-------+
4670  | Alloc | 1st[1st.size() - 1]
4671  +-------+
4672  | |
4673  | |
4674  | |
4675 GetSize() +-------+
4676 
4677 if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER):
4678 
4679  0 +-------+
4680  | Alloc | 2nd[0]
4681  +-------+
4682  | Alloc | 2nd[1]
4683  +-------+
4684  | ... |
4685  +-------+
4686  | Alloc | 2nd[2nd.size() - 1]
4687  +-------+
4688  | |
4689  | |
4690  | |
4691  +-------+
4692  | Alloc | 1st[m_1stNullItemsBeginCount]
4693  +-------+
4694  | Alloc | 1st[m_1stNullItemsBeginCount + 1]
4695  +-------+
4696  | ... |
4697  +-------+
4698  | Alloc | 1st[1st.size() - 1]
4699  +-------+
4700  | |
4701 GetSize() +-------+
4702 
4703 if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK):
4704 
4705  0 +-------+
4706  | |
4707  | |
4708  | |
4709  +-------+
4710  | Alloc | 1st[m_1stNullItemsBeginCount]
4711  +-------+
4712  | Alloc | 1st[m_1stNullItemsBeginCount + 1]
4713  +-------+
4714  | ... |
4715  +-------+
4716  | Alloc | 1st[1st.size() - 1]
4717  +-------+
4718  | |
4719  | |
4720  | |
4721  +-------+
4722  | Alloc | 2nd[2nd.size() - 1]
4723  +-------+
4724  | ... |
4725  +-------+
4726  | Alloc | 2nd[1]
4727  +-------+
4728  | Alloc | 2nd[0]
4729 GetSize() +-------+
4730 
4731 */
4732 class VmaBlockMetadata_Linear : public VmaBlockMetadata
4733 {
4734  VMA_CLASS_NO_COPY(VmaBlockMetadata_Linear)
4735 public:
4736  VmaBlockMetadata_Linear(VmaAllocator hAllocator);
4737  virtual ~VmaBlockMetadata_Linear();
4738  virtual void Init(VkDeviceSize size);
4739 
4740  virtual bool Validate() const;
4741  virtual size_t GetAllocationCount() const;
4742  virtual VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
4743  virtual VkDeviceSize GetUnusedRangeSizeMax() const;
4744  virtual bool IsEmpty() const { return GetAllocationCount() == 0; }
4745 
4746  virtual void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
4747  virtual void AddPoolStats(VmaPoolStats& inoutStats) const;
4748 
4749 #if VMA_STATS_STRING_ENABLED
4750  virtual void PrintDetailedMap(class VmaJsonWriter& json) const;
4751 #endif
4752 
4753  virtual bool CreateAllocationRequest(
4754  uint32_t currentFrameIndex,
4755  uint32_t frameInUseCount,
4756  VkDeviceSize bufferImageGranularity,
4757  VkDeviceSize allocSize,
4758  VkDeviceSize allocAlignment,
4759  bool upperAddress,
4760  VmaSuballocationType allocType,
4761  bool canMakeOtherLost,
4762  VmaAllocationRequest* pAllocationRequest);
4763 
4764  virtual bool MakeRequestedAllocationsLost(
4765  uint32_t currentFrameIndex,
4766  uint32_t frameInUseCount,
4767  VmaAllocationRequest* pAllocationRequest);
4768 
4769  virtual uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4770 
4771  virtual VkResult CheckCorruption(const void* pBlockData);
4772 
4773  virtual void Alloc(
4774  const VmaAllocationRequest& request,
4775  VmaSuballocationType type,
4776  VkDeviceSize allocSize,
4777  bool upperAddress,
4778  VmaAllocation hAllocation);
4779 
4780  virtual void Free(const VmaAllocation allocation);
4781  virtual void FreeAtOffset(VkDeviceSize offset);
4782 
4783 private:
4784  /*
4785  There are two suballocation vectors, used in ping-pong way.
4786  The one with index m_1stVectorIndex is called 1st.
4787  The one with index (m_1stVectorIndex ^ 1) is called 2nd.
4788  2nd can be non-empty only when 1st is not empty.
4789  When 2nd is not empty, m_2ndVectorMode indicates its mode of operation.
4790  */
4791  typedef VmaVector< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > SuballocationVectorType;
4792 
4793  enum SECOND_VECTOR_MODE
4794  {
4795  SECOND_VECTOR_EMPTY,
4796  /*
4797  Suballocations in 2nd vector are created later than the ones in 1st, but they
4798  all have smaller offset.
4799  */
4800  SECOND_VECTOR_RING_BUFFER,
4801  /*
4802  Suballocations in 2nd vector are upper side of double stack.
4803  They all have offsets higher than those in 1st vector.
4804  Top of this stack means smaller offsets, but higher indices in this vector.
4805  */
4806  SECOND_VECTOR_DOUBLE_STACK,
4807  };
4808 
4809  VkDeviceSize m_SumFreeSize;
4810  SuballocationVectorType m_Suballocations0, m_Suballocations1;
4811  uint32_t m_1stVectorIndex;
4812  SECOND_VECTOR_MODE m_2ndVectorMode;
4813 
4814  SuballocationVectorType& AccessSuballocations1st() { return m_1stVectorIndex ? m_Suballocations1 : m_Suballocations0; }
4815  SuballocationVectorType& AccessSuballocations2nd() { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; }
4816  const SuballocationVectorType& AccessSuballocations1st() const { return m_1stVectorIndex ? m_Suballocations1 : m_Suballocations0; }
4817  const SuballocationVectorType& AccessSuballocations2nd() const { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; }
4818 
4819  // Number of items in 1st vector with hAllocation = null at the beginning.
4820  size_t m_1stNullItemsBeginCount;
4821  // Number of other items in 1st vector with hAllocation = null somewhere in the middle.
4822  size_t m_1stNullItemsMiddleCount;
4823  // Number of items in 2nd vector with hAllocation = null.
4824  size_t m_2ndNullItemsCount;
4825 
4826  bool ShouldCompact1st() const;
4827  void CleanupAfterFree();
4828 };
4829 
4830 /*
4831 Represents a single block of device memory (`VkDeviceMemory`) with all the
4832 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
4833 
4834 Thread-safety: This class must be externally synchronized.
4835 */
4836 class VmaDeviceMemoryBlock
4837 {
4838  VMA_CLASS_NO_COPY(VmaDeviceMemoryBlock)
4839 public:
4840  VmaBlockMetadata* m_pMetadata;
4841 
4842  VmaDeviceMemoryBlock(VmaAllocator hAllocator);
4843 
4844  ~VmaDeviceMemoryBlock()
4845  {
4846  VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
4847  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
4848  }
4849 
4850  // Always call after construction.
4851  void Init(
4852  VmaAllocator hAllocator,
4853  uint32_t newMemoryTypeIndex,
4854  VkDeviceMemory newMemory,
4855  VkDeviceSize newSize,
4856  uint32_t id,
4857  bool linearAlgorithm);
4858  // Always call before destruction.
4859  void Destroy(VmaAllocator allocator);
4860 
4861  VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
4862  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
4863  uint32_t GetId() const { return m_Id; }
4864  void* GetMappedData() const { return m_pMappedData; }
4865 
4866  // Validates all data structures inside this object. If not valid, returns false.
4867  bool Validate() const;
4868 
4869  VkResult CheckCorruption(VmaAllocator hAllocator);
4870 
4871  // ppData can be null.
4872  VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
4873  void Unmap(VmaAllocator hAllocator, uint32_t count);
4874 
4875  VkResult WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
4876  VkResult ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize);
4877 
4878  VkResult BindBufferMemory(
4879  const VmaAllocator hAllocator,
4880  const VmaAllocation hAllocation,
4881  VkBuffer hBuffer);
4882  VkResult BindImageMemory(
4883  const VmaAllocator hAllocator,
4884  const VmaAllocation hAllocation,
4885  VkImage hImage);
4886 
4887 private:
4888  uint32_t m_MemoryTypeIndex;
4889  uint32_t m_Id;
4890  VkDeviceMemory m_hMemory;
4891 
4892  // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
4893  // Also protects m_MapCount, m_pMappedData.
4894  VMA_MUTEX m_Mutex;
4895  uint32_t m_MapCount;
4896  void* m_pMappedData;
4897 };
4898 
4899 struct VmaPointerLess
4900 {
4901  bool operator()(const void* lhs, const void* rhs) const
4902  {
4903  return lhs < rhs;
4904  }
4905 };
4906 
4907 class VmaDefragmentator;
4908 
4909 /*
4910 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
4911 Vulkan memory type.
4912 
4913 Synchronized internally with a mutex.
4914 */
4915 struct VmaBlockVector
4916 {
4917  VMA_CLASS_NO_COPY(VmaBlockVector)
4918 public:
4919  VmaBlockVector(
4920  VmaAllocator hAllocator,
4921  uint32_t memoryTypeIndex,
4922  VkDeviceSize preferredBlockSize,
4923  size_t minBlockCount,
4924  size_t maxBlockCount,
4925  VkDeviceSize bufferImageGranularity,
4926  uint32_t frameInUseCount,
4927  bool isCustomPool,
4928  bool linearAlgorithm);
4929  ~VmaBlockVector();
4930 
4931  VkResult CreateMinBlocks();
4932 
4933  uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
4934  VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
4935  VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
4936  uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
4937  bool UsesLinearAlgorithm() const { return m_LinearAlgorithm; }
4938 
4939  void GetPoolStats(VmaPoolStats* pStats);
4940 
4941  bool IsEmpty() const { return m_Blocks.empty(); }
4942  bool IsCorruptionDetectionEnabled() const;
4943 
4944  VkResult Allocate(
4945  VmaPool hCurrentPool,
4946  uint32_t currentFrameIndex,
4947  VkDeviceSize size,
4948  VkDeviceSize alignment,
4949  const VmaAllocationCreateInfo& createInfo,
4950  VmaSuballocationType suballocType,
4951  VmaAllocation* pAllocation);
4952 
4953  void Free(
4954  VmaAllocation hAllocation);
4955 
4956  // Adds statistics of this BlockVector to pStats.
4957  void AddStats(VmaStats* pStats);
4958 
4959 #if VMA_STATS_STRING_ENABLED
4960  void PrintDetailedMap(class VmaJsonWriter& json);
4961 #endif
4962 
4963  void MakePoolAllocationsLost(
4964  uint32_t currentFrameIndex,
4965  size_t* pLostAllocationCount);
4966  VkResult CheckCorruption();
4967 
4968  VmaDefragmentator* EnsureDefragmentator(
4969  VmaAllocator hAllocator,
4970  uint32_t currentFrameIndex);
4971 
4972  VkResult Defragment(
4973  VmaDefragmentationStats* pDefragmentationStats,
4974  VkDeviceSize& maxBytesToMove,
4975  uint32_t& maxAllocationsToMove);
4976 
4977  void DestroyDefragmentator();
4978 
4979 private:
4980  friend class VmaDefragmentator;
4981 
4982  const VmaAllocator m_hAllocator;
4983  const uint32_t m_MemoryTypeIndex;
4984  const VkDeviceSize m_PreferredBlockSize;
4985  const size_t m_MinBlockCount;
4986  const size_t m_MaxBlockCount;
4987  const VkDeviceSize m_BufferImageGranularity;
4988  const uint32_t m_FrameInUseCount;
4989  const bool m_IsCustomPool;
4990  const bool m_LinearAlgorithm;
4991  bool m_HasEmptyBlock;
4992  VMA_MUTEX m_Mutex;
4993  // Incrementally sorted by sumFreeSize, ascending.
4994  VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
4995  /* There can be at most one allocation that is completely empty - a
4996  hysteresis to avoid pessimistic case of alternating creation and destruction
4997  of a VkDeviceMemory. */
4998  VmaDefragmentator* m_pDefragmentator;
4999  uint32_t m_NextBlockId;
5000 
5001  VkDeviceSize CalcMaxBlockSize() const;
5002 
5003  // Finds and removes given block from vector.
5004  void Remove(VmaDeviceMemoryBlock* pBlock);
5005 
5006  // Performs single step in sorting m_Blocks. They may not be fully sorted
5007  // after this call.
5008  void IncrementallySortBlocks();
5009 
5010  VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
5011 };
5012 
5013 struct VmaPool_T
5014 {
5015  VMA_CLASS_NO_COPY(VmaPool_T)
5016 public:
5017  VmaBlockVector m_BlockVector;
5018 
5019  VmaPool_T(
5020  VmaAllocator hAllocator,
5021  const VmaPoolCreateInfo& createInfo);
5022  ~VmaPool_T();
5023 
5024  uint32_t GetId() const { return m_Id; }
5025  void SetId(uint32_t id) { VMA_ASSERT(m_Id == 0); m_Id = id; }
5026 
5027 #if VMA_STATS_STRING_ENABLED
5028  //void PrintDetailedMap(class VmaStringBuilder& sb);
5029 #endif
5030 
5031 private:
5032  uint32_t m_Id;
5033 };
5034 
5035 class VmaDefragmentator
5036 {
5037  VMA_CLASS_NO_COPY(VmaDefragmentator)
5038 private:
5039  const VmaAllocator m_hAllocator;
5040  VmaBlockVector* const m_pBlockVector;
5041  uint32_t m_CurrentFrameIndex;
5042  VkDeviceSize m_BytesMoved;
5043  uint32_t m_AllocationsMoved;
5044 
5045  struct AllocationInfo
5046  {
5047  VmaAllocation m_hAllocation;
5048  VkBool32* m_pChanged;
5049 
5050  AllocationInfo() :
5051  m_hAllocation(VK_NULL_HANDLE),
5052  m_pChanged(VMA_NULL)
5053  {
5054  }
5055  };
5056 
5057  struct AllocationInfoSizeGreater
5058  {
5059  bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
5060  {
5061  return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
5062  }
5063  };
5064 
5065  // Used between AddAllocation and Defragment.
5066  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
5067 
5068  struct BlockInfo
5069  {
5070  VmaDeviceMemoryBlock* m_pBlock;
5071  bool m_HasNonMovableAllocations;
5072  VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
5073 
5074  BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
5075  m_pBlock(VMA_NULL),
5076  m_HasNonMovableAllocations(true),
5077  m_Allocations(pAllocationCallbacks),
5078  m_pMappedDataForDefragmentation(VMA_NULL)
5079  {
5080  }
5081 
5082  void CalcHasNonMovableAllocations()
5083  {
5084  const size_t blockAllocCount = m_pBlock->m_pMetadata->GetAllocationCount();
5085  const size_t defragmentAllocCount = m_Allocations.size();
5086  m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
5087  }
5088 
5089  void SortAllocationsBySizeDescecnding()
5090  {
5091  VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
5092  }
5093 
5094  VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
5095  void Unmap(VmaAllocator hAllocator);
5096 
5097  private:
5098  // Not null if mapped for defragmentation only, not originally mapped.
5099  void* m_pMappedDataForDefragmentation;
5100  };
5101 
5102  struct BlockPointerLess
5103  {
5104  bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
5105  {
5106  return pLhsBlockInfo->m_pBlock < pRhsBlock;
5107  }
5108  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
5109  {
5110  return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
5111  }
5112  };
5113 
5114  // 1. Blocks with some non-movable allocations go first.
5115  // 2. Blocks with smaller sumFreeSize go first.
5116  struct BlockInfoCompareMoveDestination
5117  {
5118  bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
5119  {
5120  if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
5121  {
5122  return true;
5123  }
5124  if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
5125  {
5126  return false;
5127  }
5128  if(pLhsBlockInfo->m_pBlock->m_pMetadata->GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_pMetadata->GetSumFreeSize())
5129  {
5130  return true;
5131  }
5132  return false;
5133  }
5134  };
5135 
5136  typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
5137  BlockInfoVector m_Blocks;
5138 
5139  VkResult DefragmentRound(
5140  VkDeviceSize maxBytesToMove,
5141  uint32_t maxAllocationsToMove);
5142 
5143  static bool MoveMakesSense(
5144  size_t dstBlockIndex, VkDeviceSize dstOffset,
5145  size_t srcBlockIndex, VkDeviceSize srcOffset);
5146 
5147 public:
5148  VmaDefragmentator(
5149  VmaAllocator hAllocator,
5150  VmaBlockVector* pBlockVector,
5151  uint32_t currentFrameIndex);
5152 
5153  ~VmaDefragmentator();
5154 
5155  VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
5156  uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
5157 
5158  void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
5159 
5160  VkResult Defragment(
5161  VkDeviceSize maxBytesToMove,
5162  uint32_t maxAllocationsToMove);
5163 };
5164 
5165 #if VMA_RECORDING_ENABLED
5166 
5167 class VmaRecorder
5168 {
5169 public:
5170  VmaRecorder();
5171  VkResult Init(const VmaRecordSettings& settings, bool useMutex);
5172  void WriteConfiguration(
5173  const VkPhysicalDeviceProperties& devProps,
5174  const VkPhysicalDeviceMemoryProperties& memProps,
5175  bool dedicatedAllocationExtensionEnabled);
5176  ~VmaRecorder();
5177 
5178  void RecordCreateAllocator(uint32_t frameIndex);
5179  void RecordDestroyAllocator(uint32_t frameIndex);
5180  void RecordCreatePool(uint32_t frameIndex,
5181  const VmaPoolCreateInfo& createInfo,
5182  VmaPool pool);
5183  void RecordDestroyPool(uint32_t frameIndex, VmaPool pool);
5184  void RecordAllocateMemory(uint32_t frameIndex,
5185  const VkMemoryRequirements& vkMemReq,
5186  const VmaAllocationCreateInfo& createInfo,
5187  VmaAllocation allocation);
5188  void RecordAllocateMemoryForBuffer(uint32_t frameIndex,
5189  const VkMemoryRequirements& vkMemReq,
5190  bool requiresDedicatedAllocation,
5191  bool prefersDedicatedAllocation,
5192  const VmaAllocationCreateInfo& createInfo,
5193  VmaAllocation allocation);
5194  void RecordAllocateMemoryForImage(uint32_t frameIndex,
5195  const VkMemoryRequirements& vkMemReq,
5196  bool requiresDedicatedAllocation,
5197  bool prefersDedicatedAllocation,
5198  const VmaAllocationCreateInfo& createInfo,
5199  VmaAllocation allocation);
5200  void RecordFreeMemory(uint32_t frameIndex,
5201  VmaAllocation allocation);
5202  void RecordSetAllocationUserData(uint32_t frameIndex,
5203  VmaAllocation allocation,
5204  const void* pUserData);
5205  void RecordCreateLostAllocation(uint32_t frameIndex,
5206  VmaAllocation allocation);
5207  void RecordMapMemory(uint32_t frameIndex,
5208  VmaAllocation allocation);
5209  void RecordUnmapMemory(uint32_t frameIndex,
5210  VmaAllocation allocation);
5211  void RecordFlushAllocation(uint32_t frameIndex,
5212  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
5213  void RecordInvalidateAllocation(uint32_t frameIndex,
5214  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size);
5215  void RecordCreateBuffer(uint32_t frameIndex,
5216  const VkBufferCreateInfo& bufCreateInfo,
5217  const VmaAllocationCreateInfo& allocCreateInfo,
5218  VmaAllocation allocation);
5219  void RecordCreateImage(uint32_t frameIndex,
5220  const VkImageCreateInfo& imageCreateInfo,
5221  const VmaAllocationCreateInfo& allocCreateInfo,
5222  VmaAllocation allocation);
5223  void RecordDestroyBuffer(uint32_t frameIndex,
5224  VmaAllocation allocation);
5225  void RecordDestroyImage(uint32_t frameIndex,
5226  VmaAllocation allocation);
5227  void RecordTouchAllocation(uint32_t frameIndex,
5228  VmaAllocation allocation);
5229  void RecordGetAllocationInfo(uint32_t frameIndex,
5230  VmaAllocation allocation);
5231  void RecordMakePoolAllocationsLost(uint32_t frameIndex,
5232  VmaPool pool);
5233 
5234 private:
5235  struct CallParams
5236  {
5237  uint32_t threadId;
5238  double time;
5239  };
5240 
5241  class UserDataString
5242  {
5243  public:
5244  UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData);
5245  const char* GetString() const { return m_Str; }
5246 
5247  private:
5248  char m_PtrStr[17];
5249  const char* m_Str;
5250  };
5251 
5252  bool m_UseMutex;
5253  VmaRecordFlags m_Flags;
5254  FILE* m_File;
5255  VMA_MUTEX m_FileMutex;
5256  int64_t m_Freq;
5257  int64_t m_StartCounter;
5258 
5259  void GetBasicParams(CallParams& outParams);
5260  void Flush();
5261 };
5262 
5263 #endif // #if VMA_RECORDING_ENABLED
5264 
5265 // Main allocator object.
5266 struct VmaAllocator_T
5267 {
5268  VMA_CLASS_NO_COPY(VmaAllocator_T)
5269 public:
5270  bool m_UseMutex;
5271  bool m_UseKhrDedicatedAllocation;
5272  VkDevice m_hDevice;
5273  bool m_AllocationCallbacksSpecified;
5274  VkAllocationCallbacks m_AllocationCallbacks;
5275  VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
5276 
5277  // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
5278  VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
5279  VMA_MUTEX m_HeapSizeLimitMutex;
5280 
5281  VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
5282  VkPhysicalDeviceMemoryProperties m_MemProps;
5283 
5284  // Default pools.
5285  VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
5286 
5287  // Each vector is sorted by memory (handle value).
5288  typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
5289  AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
5290  VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
5291 
5292  VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
5293  VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo);
5294  ~VmaAllocator_T();
5295 
5296  const VkAllocationCallbacks* GetAllocationCallbacks() const
5297  {
5298  return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
5299  }
5300  const VmaVulkanFunctions& GetVulkanFunctions() const
5301  {
5302  return m_VulkanFunctions;
5303  }
5304 
5305  VkDeviceSize GetBufferImageGranularity() const
5306  {
5307  return VMA_MAX(
5308  static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
5309  m_PhysicalDeviceProperties.limits.bufferImageGranularity);
5310  }
5311 
5312  uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
5313  uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
5314 
5315  uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
5316  {
5317  VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
5318  return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
5319  }
5320  // True when specific memory type is HOST_VISIBLE but not HOST_COHERENT.
5321  bool IsMemoryTypeNonCoherent(uint32_t memTypeIndex) const
5322  {
5323  return (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & (VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) ==
5324  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
5325  }
5326  // Minimum alignment for all allocations in specific memory type.
5327  VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const
5328  {
5329  return IsMemoryTypeNonCoherent(memTypeIndex) ?
5330  VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) :
5331  (VkDeviceSize)VMA_DEBUG_ALIGNMENT;
5332  }
5333 
5334  bool IsIntegratedGpu() const
5335  {
5336  return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU;
5337  }
5338 
5339 #if VMA_RECORDING_ENABLED
5340  VmaRecorder* GetRecorder() const { return m_pRecorder; }
5341 #endif
5342 
5343  void GetBufferMemoryRequirements(
5344  VkBuffer hBuffer,
5345  VkMemoryRequirements& memReq,
5346  bool& requiresDedicatedAllocation,
5347  bool& prefersDedicatedAllocation) const;
5348  void GetImageMemoryRequirements(
5349  VkImage hImage,
5350  VkMemoryRequirements& memReq,
5351  bool& requiresDedicatedAllocation,
5352  bool& prefersDedicatedAllocation) const;
5353 
5354  // Main allocation function.
5355  VkResult AllocateMemory(
5356  const VkMemoryRequirements& vkMemReq,
5357  bool requiresDedicatedAllocation,
5358  bool prefersDedicatedAllocation,
5359  VkBuffer dedicatedBuffer,
5360  VkImage dedicatedImage,
5361  const VmaAllocationCreateInfo& createInfo,
5362  VmaSuballocationType suballocType,
5363  VmaAllocation* pAllocation);
5364 
5365  // Main deallocation function.
5366  void FreeMemory(const VmaAllocation allocation);
5367 
5368  void CalculateStats(VmaStats* pStats);
5369 
5370 #if VMA_STATS_STRING_ENABLED
5371  void PrintDetailedMap(class VmaJsonWriter& json);
5372 #endif
5373 
5374  VkResult Defragment(
5375  VmaAllocation* pAllocations,
5376  size_t allocationCount,
5377  VkBool32* pAllocationsChanged,
5378  const VmaDefragmentationInfo* pDefragmentationInfo,
5379  VmaDefragmentationStats* pDefragmentationStats);
5380 
5381  void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
5382  bool TouchAllocation(VmaAllocation hAllocation);
5383 
5384  VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
5385  void DestroyPool(VmaPool pool);
5386  void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
5387 
5388  void SetCurrentFrameIndex(uint32_t frameIndex);
5389  uint32_t GetCurrentFrameIndex() const { return m_CurrentFrameIndex.load(); }
5390 
5391  void MakePoolAllocationsLost(
5392  VmaPool hPool,
5393  size_t* pLostAllocationCount);
5394  VkResult CheckPoolCorruption(VmaPool hPool);
5395  VkResult CheckCorruption(uint32_t memoryTypeBits);
5396 
5397  void CreateLostAllocation(VmaAllocation* pAllocation);
5398 
5399  VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
5400  void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
5401 
5402  VkResult Map(VmaAllocation hAllocation, void** ppData);
5403  void Unmap(VmaAllocation hAllocation);
5404 
5405  VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
5406  VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
5407 
5408  void FlushOrInvalidateAllocation(
5409  VmaAllocation hAllocation,
5410  VkDeviceSize offset, VkDeviceSize size,
5411  VMA_CACHE_OPERATION op);
5412 
5413  void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern);
5414 
5415 private:
5416  VkDeviceSize m_PreferredLargeHeapBlockSize;
5417 
5418  VkPhysicalDevice m_PhysicalDevice;
5419  VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
5420 
5421  VMA_MUTEX m_PoolsMutex;
5422  // Protected by m_PoolsMutex. Sorted by pointer value.
5423  VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
5424  uint32_t m_NextPoolId;
5425 
5426  VmaVulkanFunctions m_VulkanFunctions;
5427 
5428 #if VMA_RECORDING_ENABLED
5429  VmaRecorder* m_pRecorder;
5430 #endif
5431 
5432  void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
5433 
5434  VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
5435 
5436  VkResult AllocateMemoryOfType(
5437  VkDeviceSize size,
5438  VkDeviceSize alignment,
5439  bool dedicatedAllocation,
5440  VkBuffer dedicatedBuffer,
5441  VkImage dedicatedImage,
5442  const VmaAllocationCreateInfo& createInfo,
5443  uint32_t memTypeIndex,
5444  VmaSuballocationType suballocType,
5445  VmaAllocation* pAllocation);
5446 
5447  // Allocates and registers new VkDeviceMemory specifically for single allocation.
5448  VkResult AllocateDedicatedMemory(
5449  VkDeviceSize size,
5450  VmaSuballocationType suballocType,
5451  uint32_t memTypeIndex,
5452  bool map,
5453  bool isUserDataString,
5454  void* pUserData,
5455  VkBuffer dedicatedBuffer,
5456  VkImage dedicatedImage,
5457  VmaAllocation* pAllocation);
5458 
5459  // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
5460  void FreeDedicatedMemory(VmaAllocation allocation);
5461 };
5462 
5464 // Memory allocation #2 after VmaAllocator_T definition
5465 
5466 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
5467 {
5468  return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
5469 }
5470 
5471 static void VmaFree(VmaAllocator hAllocator, void* ptr)
5472 {
5473  VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
5474 }
5475 
5476 template<typename T>
5477 static T* VmaAllocate(VmaAllocator hAllocator)
5478 {
5479  return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
5480 }
5481 
5482 template<typename T>
5483 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
5484 {
5485  return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
5486 }
5487 
5488 template<typename T>
5489 static void vma_delete(VmaAllocator hAllocator, T* ptr)
5490 {
5491  if(ptr != VMA_NULL)
5492  {
5493  ptr->~T();
5494  VmaFree(hAllocator, ptr);
5495  }
5496 }
5497 
5498 template<typename T>
5499 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
5500 {
5501  if(ptr != VMA_NULL)
5502  {
5503  for(size_t i = count; i--; )
5504  ptr[i].~T();
5505  VmaFree(hAllocator, ptr);
5506  }
5507 }
5508 
5510 // VmaStringBuilder
5511 
5512 #if VMA_STATS_STRING_ENABLED
5513 
5514 class VmaStringBuilder
5515 {
5516 public:
5517  VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
5518  size_t GetLength() const { return m_Data.size(); }
5519  const char* GetData() const { return m_Data.data(); }
5520 
5521  void Add(char ch) { m_Data.push_back(ch); }
5522  void Add(const char* pStr);
5523  void AddNewLine() { Add('\n'); }
5524  void AddNumber(uint32_t num);
5525  void AddNumber(uint64_t num);
5526  void AddPointer(const void* ptr);
5527 
5528 private:
5529  VmaVector< char, VmaStlAllocator<char> > m_Data;
5530 };
5531 
5532 void VmaStringBuilder::Add(const char* pStr)
5533 {
5534  const size_t strLen = strlen(pStr);
5535  if(strLen > 0)
5536  {
5537  const size_t oldCount = m_Data.size();
5538  m_Data.resize(oldCount + strLen);
5539  memcpy(m_Data.data() + oldCount, pStr, strLen);
5540  }
5541 }
5542 
5543 void VmaStringBuilder::AddNumber(uint32_t num)
5544 {
5545  char buf[11];
5546  VmaUint32ToStr(buf, sizeof(buf), num);
5547  Add(buf);
5548 }
5549 
5550 void VmaStringBuilder::AddNumber(uint64_t num)
5551 {
5552  char buf[21];
5553  VmaUint64ToStr(buf, sizeof(buf), num);
5554  Add(buf);
5555 }
5556 
5557 void VmaStringBuilder::AddPointer(const void* ptr)
5558 {
5559  char buf[21];
5560  VmaPtrToStr(buf, sizeof(buf), ptr);
5561  Add(buf);
5562 }
5563 
5564 #endif // #if VMA_STATS_STRING_ENABLED
5565 
5567 // VmaJsonWriter
5568 
5569 #if VMA_STATS_STRING_ENABLED
5570 
5571 class VmaJsonWriter
5572 {
5573  VMA_CLASS_NO_COPY(VmaJsonWriter)
5574 public:
5575  VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
5576  ~VmaJsonWriter();
5577 
5578  void BeginObject(bool singleLine = false);
5579  void EndObject();
5580 
5581  void BeginArray(bool singleLine = false);
5582  void EndArray();
5583 
5584  void WriteString(const char* pStr);
5585  void BeginString(const char* pStr = VMA_NULL);
5586  void ContinueString(const char* pStr);
5587  void ContinueString(uint32_t n);
5588  void ContinueString(uint64_t n);
5589  void ContinueString_Pointer(const void* ptr);
5590  void EndString(const char* pStr = VMA_NULL);
5591 
5592  void WriteNumber(uint32_t n);
5593  void WriteNumber(uint64_t n);
5594  void WriteBool(bool b);
5595  void WriteNull();
5596 
5597 private:
5598  static const char* const INDENT;
5599 
5600  enum COLLECTION_TYPE
5601  {
5602  COLLECTION_TYPE_OBJECT,
5603  COLLECTION_TYPE_ARRAY,
5604  };
5605  struct StackItem
5606  {
5607  COLLECTION_TYPE type;
5608  uint32_t valueCount;
5609  bool singleLineMode;
5610  };
5611 
5612  VmaStringBuilder& m_SB;
5613  VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
5614  bool m_InsideString;
5615 
5616  void BeginValue(bool isString);
5617  void WriteIndent(bool oneLess = false);
5618 };
5619 
5620 const char* const VmaJsonWriter::INDENT = " ";
5621 
5622 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
5623  m_SB(sb),
5624  m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
5625  m_InsideString(false)
5626 {
5627 }
5628 
5629 VmaJsonWriter::~VmaJsonWriter()
5630 {
5631  VMA_ASSERT(!m_InsideString);
5632  VMA_ASSERT(m_Stack.empty());
5633 }
5634 
5635 void VmaJsonWriter::BeginObject(bool singleLine)
5636 {
5637  VMA_ASSERT(!m_InsideString);
5638 
5639  BeginValue(false);
5640  m_SB.Add('{');
5641 
5642  StackItem item;
5643  item.type = COLLECTION_TYPE_OBJECT;
5644  item.valueCount = 0;
5645  item.singleLineMode = singleLine;
5646  m_Stack.push_back(item);
5647 }
5648 
5649 void VmaJsonWriter::EndObject()
5650 {
5651  VMA_ASSERT(!m_InsideString);
5652 
5653  WriteIndent(true);
5654  m_SB.Add('}');
5655 
5656  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
5657  m_Stack.pop_back();
5658 }
5659 
5660 void VmaJsonWriter::BeginArray(bool singleLine)
5661 {
5662  VMA_ASSERT(!m_InsideString);
5663 
5664  BeginValue(false);
5665  m_SB.Add('[');
5666 
5667  StackItem item;
5668  item.type = COLLECTION_TYPE_ARRAY;
5669  item.valueCount = 0;
5670  item.singleLineMode = singleLine;
5671  m_Stack.push_back(item);
5672 }
5673 
5674 void VmaJsonWriter::EndArray()
5675 {
5676  VMA_ASSERT(!m_InsideString);
5677 
5678  WriteIndent(true);
5679  m_SB.Add(']');
5680 
5681  VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
5682  m_Stack.pop_back();
5683 }
5684 
5685 void VmaJsonWriter::WriteString(const char* pStr)
5686 {
5687  BeginString(pStr);
5688  EndString();
5689 }
5690 
5691 void VmaJsonWriter::BeginString(const char* pStr)
5692 {
5693  VMA_ASSERT(!m_InsideString);
5694 
5695  BeginValue(true);
5696  m_SB.Add('"');
5697  m_InsideString = true;
5698  if(pStr != VMA_NULL && pStr[0] != '\0')
5699  {
5700  ContinueString(pStr);
5701  }
5702 }
5703 
5704 void VmaJsonWriter::ContinueString(const char* pStr)
5705 {
5706  VMA_ASSERT(m_InsideString);
5707 
5708  const size_t strLen = strlen(pStr);
5709  for(size_t i = 0; i < strLen; ++i)
5710  {
5711  char ch = pStr[i];
5712  if(ch == '\\')
5713  {
5714  m_SB.Add("\\\\");
5715  }
5716  else if(ch == '"')
5717  {
5718  m_SB.Add("\\\"");
5719  }
5720  else if(ch >= 32)
5721  {
5722  m_SB.Add(ch);
5723  }
5724  else switch(ch)
5725  {
5726  case '\b':
5727  m_SB.Add("\\b");
5728  break;
5729  case '\f':
5730  m_SB.Add("\\f");
5731  break;
5732  case '\n':
5733  m_SB.Add("\\n");
5734  break;
5735  case '\r':
5736  m_SB.Add("\\r");
5737  break;
5738  case '\t':
5739  m_SB.Add("\\t");
5740  break;
5741  default:
5742  VMA_ASSERT(0 && "Character not currently supported.");
5743  break;
5744  }
5745  }
5746 }
5747 
5748 void VmaJsonWriter::ContinueString(uint32_t n)
5749 {
5750  VMA_ASSERT(m_InsideString);
5751  m_SB.AddNumber(n);
5752 }
5753 
5754 void VmaJsonWriter::ContinueString(uint64_t n)
5755 {
5756  VMA_ASSERT(m_InsideString);
5757  m_SB.AddNumber(n);
5758 }
5759 
5760 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
5761 {
5762  VMA_ASSERT(m_InsideString);
5763  m_SB.AddPointer(ptr);
5764 }
5765 
5766 void VmaJsonWriter::EndString(const char* pStr)
5767 {
5768  VMA_ASSERT(m_InsideString);
5769  if(pStr != VMA_NULL && pStr[0] != '\0')
5770  {
5771  ContinueString(pStr);
5772  }
5773  m_SB.Add('"');
5774  m_InsideString = false;
5775 }
5776 
5777 void VmaJsonWriter::WriteNumber(uint32_t n)
5778 {
5779  VMA_ASSERT(!m_InsideString);
5780  BeginValue(false);
5781  m_SB.AddNumber(n);
5782 }
5783 
5784 void VmaJsonWriter::WriteNumber(uint64_t n)
5785 {
5786  VMA_ASSERT(!m_InsideString);
5787  BeginValue(false);
5788  m_SB.AddNumber(n);
5789 }
5790 
5791 void VmaJsonWriter::WriteBool(bool b)
5792 {
5793  VMA_ASSERT(!m_InsideString);
5794  BeginValue(false);
5795  m_SB.Add(b ? "true" : "false");
5796 }
5797 
5798 void VmaJsonWriter::WriteNull()
5799 {
5800  VMA_ASSERT(!m_InsideString);
5801  BeginValue(false);
5802  m_SB.Add("null");
5803 }
5804 
5805 void VmaJsonWriter::BeginValue(bool isString)
5806 {
5807  if(!m_Stack.empty())
5808  {
5809  StackItem& currItem = m_Stack.back();
5810  if(currItem.type == COLLECTION_TYPE_OBJECT &&
5811  currItem.valueCount % 2 == 0)
5812  {
5813  VMA_ASSERT(isString);
5814  }
5815 
5816  if(currItem.type == COLLECTION_TYPE_OBJECT &&
5817  currItem.valueCount % 2 != 0)
5818  {
5819  m_SB.Add(": ");
5820  }
5821  else if(currItem.valueCount > 0)
5822  {
5823  m_SB.Add(", ");
5824  WriteIndent();
5825  }
5826  else
5827  {
5828  WriteIndent();
5829  }
5830  ++currItem.valueCount;
5831  }
5832 }
5833 
5834 void VmaJsonWriter::WriteIndent(bool oneLess)
5835 {
5836  if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
5837  {
5838  m_SB.AddNewLine();
5839 
5840  size_t count = m_Stack.size();
5841  if(count > 0 && oneLess)
5842  {
5843  --count;
5844  }
5845  for(size_t i = 0; i < count; ++i)
5846  {
5847  m_SB.Add(INDENT);
5848  }
5849  }
5850 }
5851 
5852 #endif // #if VMA_STATS_STRING_ENABLED
5853 
5855 
5856 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
5857 {
5858  if(IsUserDataString())
5859  {
5860  VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
5861 
5862  FreeUserDataString(hAllocator);
5863 
5864  if(pUserData != VMA_NULL)
5865  {
5866  const char* const newStrSrc = (char*)pUserData;
5867  const size_t newStrLen = strlen(newStrSrc);
5868  char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
5869  memcpy(newStrDst, newStrSrc, newStrLen + 1);
5870  m_pUserData = newStrDst;
5871  }
5872  }
5873  else
5874  {
5875  m_pUserData = pUserData;
5876  }
5877 }
5878 
5879 void VmaAllocation_T::ChangeBlockAllocation(
5880  VmaAllocator hAllocator,
5881  VmaDeviceMemoryBlock* block,
5882  VkDeviceSize offset)
5883 {
5884  VMA_ASSERT(block != VMA_NULL);
5885  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
5886 
5887  // Move mapping reference counter from old block to new block.
5888  if(block != m_BlockAllocation.m_Block)
5889  {
5890  uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
5891  if(IsPersistentMap())
5892  ++mapRefCount;
5893  m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
5894  block->Map(hAllocator, mapRefCount, VMA_NULL);
5895  }
5896 
5897  m_BlockAllocation.m_Block = block;
5898  m_BlockAllocation.m_Offset = offset;
5899 }
5900 
5901 VkDeviceSize VmaAllocation_T::GetOffset() const
5902 {
5903  switch(m_Type)
5904  {
5905  case ALLOCATION_TYPE_BLOCK:
5906  return m_BlockAllocation.m_Offset;
5907  case ALLOCATION_TYPE_DEDICATED:
5908  return 0;
5909  default:
5910  VMA_ASSERT(0);
5911  return 0;
5912  }
5913 }
5914 
5915 VkDeviceMemory VmaAllocation_T::GetMemory() const
5916 {
5917  switch(m_Type)
5918  {
5919  case ALLOCATION_TYPE_BLOCK:
5920  return m_BlockAllocation.m_Block->GetDeviceMemory();
5921  case ALLOCATION_TYPE_DEDICATED:
5922  return m_DedicatedAllocation.m_hMemory;
5923  default:
5924  VMA_ASSERT(0);
5925  return VK_NULL_HANDLE;
5926  }
5927 }
5928 
5929 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
5930 {
5931  switch(m_Type)
5932  {
5933  case ALLOCATION_TYPE_BLOCK:
5934  return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
5935  case ALLOCATION_TYPE_DEDICATED:
5936  return m_DedicatedAllocation.m_MemoryTypeIndex;
5937  default:
5938  VMA_ASSERT(0);
5939  return UINT32_MAX;
5940  }
5941 }
5942 
5943 void* VmaAllocation_T::GetMappedData() const
5944 {
5945  switch(m_Type)
5946  {
5947  case ALLOCATION_TYPE_BLOCK:
5948  if(m_MapCount != 0)
5949  {
5950  void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
5951  VMA_ASSERT(pBlockData != VMA_NULL);
5952  return (char*)pBlockData + m_BlockAllocation.m_Offset;
5953  }
5954  else
5955  {
5956  return VMA_NULL;
5957  }
5958  break;
5959  case ALLOCATION_TYPE_DEDICATED:
5960  VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
5961  return m_DedicatedAllocation.m_pMappedData;
5962  default:
5963  VMA_ASSERT(0);
5964  return VMA_NULL;
5965  }
5966 }
5967 
5968 bool VmaAllocation_T::CanBecomeLost() const
5969 {
5970  switch(m_Type)
5971  {
5972  case ALLOCATION_TYPE_BLOCK:
5973  return m_BlockAllocation.m_CanBecomeLost;
5974  case ALLOCATION_TYPE_DEDICATED:
5975  return false;
5976  default:
5977  VMA_ASSERT(0);
5978  return false;
5979  }
5980 }
5981 
5982 VmaPool VmaAllocation_T::GetPool() const
5983 {
5984  VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
5985  return m_BlockAllocation.m_hPool;
5986 }
5987 
5988 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5989 {
5990  VMA_ASSERT(CanBecomeLost());
5991 
5992  /*
5993  Warning: This is a carefully designed algorithm.
5994  Do not modify unless you really know what you're doing :)
5995  */
5996  uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
5997  for(;;)
5998  {
5999  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
6000  {
6001  VMA_ASSERT(0);
6002  return false;
6003  }
6004  else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
6005  {
6006  return false;
6007  }
6008  else // Last use time earlier than current time.
6009  {
6010  if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
6011  {
6012  // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
6013  // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
6014  return true;
6015  }
6016  }
6017  }
6018 }
6019 
6020 #if VMA_STATS_STRING_ENABLED
6021 
6022 // Correspond to values of enum VmaSuballocationType.
6023 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
6024  "FREE",
6025  "UNKNOWN",
6026  "BUFFER",
6027  "IMAGE_UNKNOWN",
6028  "IMAGE_LINEAR",
6029  "IMAGE_OPTIMAL",
6030 };
6031 
6032 void VmaAllocation_T::PrintParameters(class VmaJsonWriter& json) const
6033 {
6034  json.WriteString("Type");
6035  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[m_SuballocationType]);
6036 
6037  json.WriteString("Size");
6038  json.WriteNumber(m_Size);
6039 
6040  if(m_pUserData != VMA_NULL)
6041  {
6042  json.WriteString("UserData");
6043  if(IsUserDataString())
6044  {
6045  json.WriteString((const char*)m_pUserData);
6046  }
6047  else
6048  {
6049  json.BeginString();
6050  json.ContinueString_Pointer(m_pUserData);
6051  json.EndString();
6052  }
6053  }
6054 
6055  json.WriteString("CreationFrameIndex");
6056  json.WriteNumber(m_CreationFrameIndex);
6057 
6058  json.WriteString("LastUseFrameIndex");
6059  json.WriteNumber(GetLastUseFrameIndex());
6060 
6061  if(m_BufferImageUsage != 0)
6062  {
6063  json.WriteString("Usage");
6064  json.WriteNumber(m_BufferImageUsage);
6065  }
6066 }
6067 
6068 #endif
6069 
6070 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
6071 {
6072  VMA_ASSERT(IsUserDataString());
6073  if(m_pUserData != VMA_NULL)
6074  {
6075  char* const oldStr = (char*)m_pUserData;
6076  const size_t oldStrLen = strlen(oldStr);
6077  vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
6078  m_pUserData = VMA_NULL;
6079  }
6080 }
6081 
6082 void VmaAllocation_T::BlockAllocMap()
6083 {
6084  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
6085 
6086  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
6087  {
6088  ++m_MapCount;
6089  }
6090  else
6091  {
6092  VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
6093  }
6094 }
6095 
6096 void VmaAllocation_T::BlockAllocUnmap()
6097 {
6098  VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
6099 
6100  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
6101  {
6102  --m_MapCount;
6103  }
6104  else
6105  {
6106  VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
6107  }
6108 }
6109 
6110 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
6111 {
6112  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
6113 
6114  if(m_MapCount != 0)
6115  {
6116  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
6117  {
6118  VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
6119  *ppData = m_DedicatedAllocation.m_pMappedData;
6120  ++m_MapCount;
6121  return VK_SUCCESS;
6122  }
6123  else
6124  {
6125  VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
6126  return VK_ERROR_MEMORY_MAP_FAILED;
6127  }
6128  }
6129  else
6130  {
6131  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
6132  hAllocator->m_hDevice,
6133  m_DedicatedAllocation.m_hMemory,
6134  0, // offset
6135  VK_WHOLE_SIZE,
6136  0, // flags
6137  ppData);
6138  if(result == VK_SUCCESS)
6139  {
6140  m_DedicatedAllocation.m_pMappedData = *ppData;
6141  m_MapCount = 1;
6142  }
6143  return result;
6144  }
6145 }
6146 
6147 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
6148 {
6149  VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
6150 
6151  if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
6152  {
6153  --m_MapCount;
6154  if(m_MapCount == 0)
6155  {
6156  m_DedicatedAllocation.m_pMappedData = VMA_NULL;
6157  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
6158  hAllocator->m_hDevice,
6159  m_DedicatedAllocation.m_hMemory);
6160  }
6161  }
6162  else
6163  {
6164  VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
6165  }
6166 }
6167 
6168 #if VMA_STATS_STRING_ENABLED
6169 
6170 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
6171 {
6172  json.BeginObject();
6173 
6174  json.WriteString("Blocks");
6175  json.WriteNumber(stat.blockCount);
6176 
6177  json.WriteString("Allocations");
6178  json.WriteNumber(stat.allocationCount);
6179 
6180  json.WriteString("UnusedRanges");
6181  json.WriteNumber(stat.unusedRangeCount);
6182 
6183  json.WriteString("UsedBytes");
6184  json.WriteNumber(stat.usedBytes);
6185 
6186  json.WriteString("UnusedBytes");
6187  json.WriteNumber(stat.unusedBytes);
6188 
6189  if(stat.allocationCount > 1)
6190  {
6191  json.WriteString("AllocationSize");
6192  json.BeginObject(true);
6193  json.WriteString("Min");
6194  json.WriteNumber(stat.allocationSizeMin);
6195  json.WriteString("Avg");
6196  json.WriteNumber(stat.allocationSizeAvg);
6197  json.WriteString("Max");
6198  json.WriteNumber(stat.allocationSizeMax);
6199  json.EndObject();
6200  }
6201 
6202  if(stat.unusedRangeCount > 1)
6203  {
6204  json.WriteString("UnusedRangeSize");
6205  json.BeginObject(true);
6206  json.WriteString("Min");
6207  json.WriteNumber(stat.unusedRangeSizeMin);
6208  json.WriteString("Avg");
6209  json.WriteNumber(stat.unusedRangeSizeAvg);
6210  json.WriteString("Max");
6211  json.WriteNumber(stat.unusedRangeSizeMax);
6212  json.EndObject();
6213  }
6214 
6215  json.EndObject();
6216 }
6217 
6218 #endif // #if VMA_STATS_STRING_ENABLED
6219 
6220 struct VmaSuballocationItemSizeLess
6221 {
6222  bool operator()(
6223  const VmaSuballocationList::iterator lhs,
6224  const VmaSuballocationList::iterator rhs) const
6225  {
6226  return lhs->size < rhs->size;
6227  }
6228  bool operator()(
6229  const VmaSuballocationList::iterator lhs,
6230  VkDeviceSize rhsSize) const
6231  {
6232  return lhs->size < rhsSize;
6233  }
6234 };
6235 
6236 
6238 // class VmaBlockMetadata
6239 
6240 #if VMA_STATS_STRING_ENABLED
6241 
6242 void VmaBlockMetadata::PrintDetailedMap_Begin(class VmaJsonWriter& json,
6243  VkDeviceSize unusedBytes,
6244  size_t allocationCount,
6245  size_t unusedRangeCount) const
6246 {
6247  json.BeginObject();
6248 
6249  json.WriteString("TotalBytes");
6250  json.WriteNumber(GetSize());
6251 
6252  json.WriteString("UnusedBytes");
6253  json.WriteNumber(unusedBytes);
6254 
6255  json.WriteString("Allocations");
6256  json.WriteNumber((uint64_t)allocationCount);
6257 
6258  json.WriteString("UnusedRanges");
6259  json.WriteNumber((uint64_t)unusedRangeCount);
6260 
6261  json.WriteString("Suballocations");
6262  json.BeginArray();
6263 }
6264 
6265 void VmaBlockMetadata::PrintDetailedMap_Allocation(class VmaJsonWriter& json,
6266  VkDeviceSize offset,
6267  VmaAllocation hAllocation) const
6268 {
6269  json.BeginObject(true);
6270 
6271  json.WriteString("Offset");
6272  json.WriteNumber(offset);
6273 
6274  hAllocation->PrintParameters(json);
6275 
6276  json.EndObject();
6277 }
6278 
6279 void VmaBlockMetadata::PrintDetailedMap_UnusedRange(class VmaJsonWriter& json,
6280  VkDeviceSize offset,
6281  VkDeviceSize size) const
6282 {
6283  json.BeginObject(true);
6284 
6285  json.WriteString("Offset");
6286  json.WriteNumber(offset);
6287 
6288  json.WriteString("Type");
6289  json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[VMA_SUBALLOCATION_TYPE_FREE]);
6290 
6291  json.WriteString("Size");
6292  json.WriteNumber(size);
6293 
6294  json.EndObject();
6295 }
6296 
6297 void VmaBlockMetadata::PrintDetailedMap_End(class VmaJsonWriter& json) const
6298 {
6299  json.EndArray();
6300  json.EndObject();
6301 }
6302 
6303 #endif // #if VMA_STATS_STRING_ENABLED
6304 
6306 // class VmaBlockMetadata_Generic
6307 
6308 VmaBlockMetadata_Generic::VmaBlockMetadata_Generic(VmaAllocator hAllocator) :
6309  m_FreeCount(0),
6310  m_SumFreeSize(0),
6311  m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
6312  m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
6313 {
6314 }
6315 
6316 VmaBlockMetadata_Generic::~VmaBlockMetadata_Generic()
6317 {
6318 }
6319 
6320 void VmaBlockMetadata_Generic::Init(VkDeviceSize size)
6321 {
6322  VmaBlockMetadata::Init(size);
6323  m_FreeCount = 1;
6324  m_SumFreeSize = size;
6325 
6326  VmaSuballocation suballoc = {};
6327  suballoc.offset = 0;
6328  suballoc.size = size;
6329  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6330  suballoc.hAllocation = VK_NULL_HANDLE;
6331 
6332  m_Suballocations.push_back(suballoc);
6333  VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
6334  --suballocItem;
6335  m_FreeSuballocationsBySize.push_back(suballocItem);
6336 }
6337 
6338 bool VmaBlockMetadata_Generic::Validate() const
6339 {
6340  if(m_Suballocations.empty())
6341  {
6342  return false;
6343  }
6344 
6345  // Expected offset of new suballocation as calculated from previous ones.
6346  VkDeviceSize calculatedOffset = 0;
6347  // Expected number of free suballocations as calculated from traversing their list.
6348  uint32_t calculatedFreeCount = 0;
6349  // Expected sum size of free suballocations as calculated from traversing their list.
6350  VkDeviceSize calculatedSumFreeSize = 0;
6351  // Expected number of free suballocations that should be registered in
6352  // m_FreeSuballocationsBySize calculated from traversing their list.
6353  size_t freeSuballocationsToRegister = 0;
6354  // True if previous visited suballocation was free.
6355  bool prevFree = false;
6356 
6357  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6358  suballocItem != m_Suballocations.cend();
6359  ++suballocItem)
6360  {
6361  const VmaSuballocation& subAlloc = *suballocItem;
6362 
6363  // Actual offset of this suballocation doesn't match expected one.
6364  if(subAlloc.offset != calculatedOffset)
6365  {
6366  return false;
6367  }
6368 
6369  const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
6370  // Two adjacent free suballocations are invalid. They should be merged.
6371  if(prevFree && currFree)
6372  {
6373  return false;
6374  }
6375 
6376  if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
6377  {
6378  return false;
6379  }
6380 
6381  if(currFree)
6382  {
6383  calculatedSumFreeSize += subAlloc.size;
6384  ++calculatedFreeCount;
6385  if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6386  {
6387  ++freeSuballocationsToRegister;
6388  }
6389 
6390  // Margin required between allocations - every free space must be at least that large.
6391  if(subAlloc.size < VMA_DEBUG_MARGIN)
6392  {
6393  return false;
6394  }
6395  }
6396  else
6397  {
6398  if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
6399  {
6400  return false;
6401  }
6402  if(subAlloc.hAllocation->GetSize() != subAlloc.size)
6403  {
6404  return false;
6405  }
6406 
6407  // Margin required between allocations - previous allocation must be free.
6408  if(VMA_DEBUG_MARGIN > 0 && !prevFree)
6409  {
6410  return false;
6411  }
6412  }
6413 
6414  calculatedOffset += subAlloc.size;
6415  prevFree = currFree;
6416  }
6417 
6418  // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
6419  // match expected one.
6420  if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
6421  {
6422  return false;
6423  }
6424 
6425  VkDeviceSize lastSize = 0;
6426  for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
6427  {
6428  VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
6429 
6430  // Only free suballocations can be registered in m_FreeSuballocationsBySize.
6431  if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
6432  {
6433  return false;
6434  }
6435  // They must be sorted by size ascending.
6436  if(suballocItem->size < lastSize)
6437  {
6438  return false;
6439  }
6440 
6441  lastSize = suballocItem->size;
6442  }
6443 
6444  // Check if totals match calculacted values.
6445  if(!ValidateFreeSuballocationList() ||
6446  (calculatedOffset != GetSize()) ||
6447  (calculatedSumFreeSize != m_SumFreeSize) ||
6448  (calculatedFreeCount != m_FreeCount))
6449  {
6450  return false;
6451  }
6452 
6453  return true;
6454 }
6455 
6456 VkDeviceSize VmaBlockMetadata_Generic::GetUnusedRangeSizeMax() const
6457 {
6458  if(!m_FreeSuballocationsBySize.empty())
6459  {
6460  return m_FreeSuballocationsBySize.back()->size;
6461  }
6462  else
6463  {
6464  return 0;
6465  }
6466 }
6467 
6468 bool VmaBlockMetadata_Generic::IsEmpty() const
6469 {
6470  return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
6471 }
6472 
6473 void VmaBlockMetadata_Generic::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
6474 {
6475  outInfo.blockCount = 1;
6476 
6477  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
6478  outInfo.allocationCount = rangeCount - m_FreeCount;
6479  outInfo.unusedRangeCount = m_FreeCount;
6480 
6481  outInfo.unusedBytes = m_SumFreeSize;
6482  outInfo.usedBytes = GetSize() - outInfo.unusedBytes;
6483 
6484  outInfo.allocationSizeMin = UINT64_MAX;
6485  outInfo.allocationSizeMax = 0;
6486  outInfo.unusedRangeSizeMin = UINT64_MAX;
6487  outInfo.unusedRangeSizeMax = 0;
6488 
6489  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6490  suballocItem != m_Suballocations.cend();
6491  ++suballocItem)
6492  {
6493  const VmaSuballocation& suballoc = *suballocItem;
6494  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
6495  {
6496  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
6497  outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
6498  }
6499  else
6500  {
6501  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
6502  outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
6503  }
6504  }
6505 }
6506 
6507 void VmaBlockMetadata_Generic::AddPoolStats(VmaPoolStats& inoutStats) const
6508 {
6509  const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
6510 
6511  inoutStats.size += GetSize();
6512  inoutStats.unusedSize += m_SumFreeSize;
6513  inoutStats.allocationCount += rangeCount - m_FreeCount;
6514  inoutStats.unusedRangeCount += m_FreeCount;
6515  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
6516 }
6517 
6518 #if VMA_STATS_STRING_ENABLED
6519 
6520 void VmaBlockMetadata_Generic::PrintDetailedMap(class VmaJsonWriter& json) const
6521 {
6522  PrintDetailedMap_Begin(json,
6523  m_SumFreeSize, // unusedBytes
6524  m_Suballocations.size() - (size_t)m_FreeCount, // allocationCount
6525  m_FreeCount); // unusedRangeCount
6526 
6527  size_t i = 0;
6528  for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
6529  suballocItem != m_Suballocations.cend();
6530  ++suballocItem, ++i)
6531  {
6532  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6533  {
6534  PrintDetailedMap_UnusedRange(json, suballocItem->offset, suballocItem->size);
6535  }
6536  else
6537  {
6538  PrintDetailedMap_Allocation(json, suballocItem->offset, suballocItem->hAllocation);
6539  }
6540  }
6541 
6542  PrintDetailedMap_End(json);
6543 }
6544 
6545 #endif // #if VMA_STATS_STRING_ENABLED
6546 
6547 /*
6548 How many suitable free suballocations to analyze before choosing best one.
6549 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
6550  be chosen.
6551 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
6552  suballocations will be analized and best one will be chosen.
6553 - Any other value is also acceptable.
6554 */
6555 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
6556 
6557 bool VmaBlockMetadata_Generic::CreateAllocationRequest(
6558  uint32_t currentFrameIndex,
6559  uint32_t frameInUseCount,
6560  VkDeviceSize bufferImageGranularity,
6561  VkDeviceSize allocSize,
6562  VkDeviceSize allocAlignment,
6563  bool upperAddress,
6564  VmaSuballocationType allocType,
6565  bool canMakeOtherLost,
6566  VmaAllocationRequest* pAllocationRequest)
6567 {
6568  VMA_ASSERT(allocSize > 0);
6569  VMA_ASSERT(!upperAddress);
6570  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
6571  VMA_ASSERT(pAllocationRequest != VMA_NULL);
6572  VMA_HEAVY_ASSERT(Validate());
6573 
6574  // There is not enough total free space in this block to fullfill the request: Early return.
6575  if(canMakeOtherLost == false && m_SumFreeSize < allocSize + 2 * VMA_DEBUG_MARGIN)
6576  {
6577  return false;
6578  }
6579 
6580  // New algorithm, efficiently searching freeSuballocationsBySize.
6581  const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
6582  if(freeSuballocCount > 0)
6583  {
6584  if(VMA_BEST_FIT)
6585  {
6586  // Find first free suballocation with size not less than allocSize + 2 * VMA_DEBUG_MARGIN.
6587  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
6588  m_FreeSuballocationsBySize.data(),
6589  m_FreeSuballocationsBySize.data() + freeSuballocCount,
6590  allocSize + 2 * VMA_DEBUG_MARGIN,
6591  VmaSuballocationItemSizeLess());
6592  size_t index = it - m_FreeSuballocationsBySize.data();
6593  for(; index < freeSuballocCount; ++index)
6594  {
6595  if(CheckAllocation(
6596  currentFrameIndex,
6597  frameInUseCount,
6598  bufferImageGranularity,
6599  allocSize,
6600  allocAlignment,
6601  allocType,
6602  m_FreeSuballocationsBySize[index],
6603  false, // canMakeOtherLost
6604  &pAllocationRequest->offset,
6605  &pAllocationRequest->itemsToMakeLostCount,
6606  &pAllocationRequest->sumFreeSize,
6607  &pAllocationRequest->sumItemSize))
6608  {
6609  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
6610  return true;
6611  }
6612  }
6613  }
6614  else
6615  {
6616  // Search staring from biggest suballocations.
6617  for(size_t index = freeSuballocCount; index--; )
6618  {
6619  if(CheckAllocation(
6620  currentFrameIndex,
6621  frameInUseCount,
6622  bufferImageGranularity,
6623  allocSize,
6624  allocAlignment,
6625  allocType,
6626  m_FreeSuballocationsBySize[index],
6627  false, // canMakeOtherLost
6628  &pAllocationRequest->offset,
6629  &pAllocationRequest->itemsToMakeLostCount,
6630  &pAllocationRequest->sumFreeSize,
6631  &pAllocationRequest->sumItemSize))
6632  {
6633  pAllocationRequest->item = m_FreeSuballocationsBySize[index];
6634  return true;
6635  }
6636  }
6637  }
6638  }
6639 
6640  if(canMakeOtherLost)
6641  {
6642  // Brute-force algorithm. TODO: Come up with something better.
6643 
6644  pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
6645  pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
6646 
6647  VmaAllocationRequest tmpAllocRequest = {};
6648  for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
6649  suballocIt != m_Suballocations.end();
6650  ++suballocIt)
6651  {
6652  if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
6653  suballocIt->hAllocation->CanBecomeLost())
6654  {
6655  if(CheckAllocation(
6656  currentFrameIndex,
6657  frameInUseCount,
6658  bufferImageGranularity,
6659  allocSize,
6660  allocAlignment,
6661  allocType,
6662  suballocIt,
6663  canMakeOtherLost,
6664  &tmpAllocRequest.offset,
6665  &tmpAllocRequest.itemsToMakeLostCount,
6666  &tmpAllocRequest.sumFreeSize,
6667  &tmpAllocRequest.sumItemSize))
6668  {
6669  tmpAllocRequest.item = suballocIt;
6670 
6671  if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
6672  {
6673  *pAllocationRequest = tmpAllocRequest;
6674  }
6675  }
6676  }
6677  }
6678 
6679  if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
6680  {
6681  return true;
6682  }
6683  }
6684 
6685  return false;
6686 }
6687 
6688 bool VmaBlockMetadata_Generic::MakeRequestedAllocationsLost(
6689  uint32_t currentFrameIndex,
6690  uint32_t frameInUseCount,
6691  VmaAllocationRequest* pAllocationRequest)
6692 {
6693  while(pAllocationRequest->itemsToMakeLostCount > 0)
6694  {
6695  if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
6696  {
6697  ++pAllocationRequest->item;
6698  }
6699  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
6700  VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
6701  VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
6702  if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
6703  {
6704  pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
6705  --pAllocationRequest->itemsToMakeLostCount;
6706  }
6707  else
6708  {
6709  return false;
6710  }
6711  }
6712 
6713  VMA_HEAVY_ASSERT(Validate());
6714  VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
6715  VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
6716 
6717  return true;
6718 }
6719 
6720 uint32_t VmaBlockMetadata_Generic::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
6721 {
6722  uint32_t lostAllocationCount = 0;
6723  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
6724  it != m_Suballocations.end();
6725  ++it)
6726  {
6727  if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
6728  it->hAllocation->CanBecomeLost() &&
6729  it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
6730  {
6731  it = FreeSuballocation(it);
6732  ++lostAllocationCount;
6733  }
6734  }
6735  return lostAllocationCount;
6736 }
6737 
6738 VkResult VmaBlockMetadata_Generic::CheckCorruption(const void* pBlockData)
6739 {
6740  for(VmaSuballocationList::iterator it = m_Suballocations.begin();
6741  it != m_Suballocations.end();
6742  ++it)
6743  {
6744  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
6745  {
6746  if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN))
6747  {
6748  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
6749  return VK_ERROR_VALIDATION_FAILED_EXT;
6750  }
6751  if(!VmaValidateMagicValue(pBlockData, it->offset + it->size))
6752  {
6753  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
6754  return VK_ERROR_VALIDATION_FAILED_EXT;
6755  }
6756  }
6757  }
6758 
6759  return VK_SUCCESS;
6760 }
6761 
6762 void VmaBlockMetadata_Generic::Alloc(
6763  const VmaAllocationRequest& request,
6764  VmaSuballocationType type,
6765  VkDeviceSize allocSize,
6766  bool upperAddress,
6767  VmaAllocation hAllocation)
6768 {
6769  VMA_ASSERT(!upperAddress);
6770  VMA_ASSERT(request.item != m_Suballocations.end());
6771  VmaSuballocation& suballoc = *request.item;
6772  // Given suballocation is a free block.
6773  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
6774  // Given offset is inside this suballocation.
6775  VMA_ASSERT(request.offset >= suballoc.offset);
6776  const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
6777  VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
6778  const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
6779 
6780  // Unregister this free suballocation from m_FreeSuballocationsBySize and update
6781  // it to become used.
6782  UnregisterFreeSuballocation(request.item);
6783 
6784  suballoc.offset = request.offset;
6785  suballoc.size = allocSize;
6786  suballoc.type = type;
6787  suballoc.hAllocation = hAllocation;
6788 
6789  // If there are any free bytes remaining at the end, insert new free suballocation after current one.
6790  if(paddingEnd)
6791  {
6792  VmaSuballocation paddingSuballoc = {};
6793  paddingSuballoc.offset = request.offset + allocSize;
6794  paddingSuballoc.size = paddingEnd;
6795  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6796  VmaSuballocationList::iterator next = request.item;
6797  ++next;
6798  const VmaSuballocationList::iterator paddingEndItem =
6799  m_Suballocations.insert(next, paddingSuballoc);
6800  RegisterFreeSuballocation(paddingEndItem);
6801  }
6802 
6803  // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
6804  if(paddingBegin)
6805  {
6806  VmaSuballocation paddingSuballoc = {};
6807  paddingSuballoc.offset = request.offset - paddingBegin;
6808  paddingSuballoc.size = paddingBegin;
6809  paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6810  const VmaSuballocationList::iterator paddingBeginItem =
6811  m_Suballocations.insert(request.item, paddingSuballoc);
6812  RegisterFreeSuballocation(paddingBeginItem);
6813  }
6814 
6815  // Update totals.
6816  m_FreeCount = m_FreeCount - 1;
6817  if(paddingBegin > 0)
6818  {
6819  ++m_FreeCount;
6820  }
6821  if(paddingEnd > 0)
6822  {
6823  ++m_FreeCount;
6824  }
6825  m_SumFreeSize -= allocSize;
6826 }
6827 
6828 void VmaBlockMetadata_Generic::Free(const VmaAllocation allocation)
6829 {
6830  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
6831  suballocItem != m_Suballocations.end();
6832  ++suballocItem)
6833  {
6834  VmaSuballocation& suballoc = *suballocItem;
6835  if(suballoc.hAllocation == allocation)
6836  {
6837  FreeSuballocation(suballocItem);
6838  VMA_HEAVY_ASSERT(Validate());
6839  return;
6840  }
6841  }
6842  VMA_ASSERT(0 && "Not found!");
6843 }
6844 
6845 void VmaBlockMetadata_Generic::FreeAtOffset(VkDeviceSize offset)
6846 {
6847  for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
6848  suballocItem != m_Suballocations.end();
6849  ++suballocItem)
6850  {
6851  VmaSuballocation& suballoc = *suballocItem;
6852  if(suballoc.offset == offset)
6853  {
6854  FreeSuballocation(suballocItem);
6855  return;
6856  }
6857  }
6858  VMA_ASSERT(0 && "Not found!");
6859 }
6860 
6861 bool VmaBlockMetadata_Generic::ValidateFreeSuballocationList() const
6862 {
6863  VkDeviceSize lastSize = 0;
6864  for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
6865  {
6866  const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
6867 
6868  if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
6869  {
6870  VMA_ASSERT(0);
6871  return false;
6872  }
6873  if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6874  {
6875  VMA_ASSERT(0);
6876  return false;
6877  }
6878  if(it->size < lastSize)
6879  {
6880  VMA_ASSERT(0);
6881  return false;
6882  }
6883 
6884  lastSize = it->size;
6885  }
6886  return true;
6887 }
6888 
6889 bool VmaBlockMetadata_Generic::CheckAllocation(
6890  uint32_t currentFrameIndex,
6891  uint32_t frameInUseCount,
6892  VkDeviceSize bufferImageGranularity,
6893  VkDeviceSize allocSize,
6894  VkDeviceSize allocAlignment,
6895  VmaSuballocationType allocType,
6896  VmaSuballocationList::const_iterator suballocItem,
6897  bool canMakeOtherLost,
6898  VkDeviceSize* pOffset,
6899  size_t* itemsToMakeLostCount,
6900  VkDeviceSize* pSumFreeSize,
6901  VkDeviceSize* pSumItemSize) const
6902 {
6903  VMA_ASSERT(allocSize > 0);
6904  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
6905  VMA_ASSERT(suballocItem != m_Suballocations.cend());
6906  VMA_ASSERT(pOffset != VMA_NULL);
6907 
6908  *itemsToMakeLostCount = 0;
6909  *pSumFreeSize = 0;
6910  *pSumItemSize = 0;
6911 
6912  if(canMakeOtherLost)
6913  {
6914  if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6915  {
6916  *pSumFreeSize = suballocItem->size;
6917  }
6918  else
6919  {
6920  if(suballocItem->hAllocation->CanBecomeLost() &&
6921  suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
6922  {
6923  ++*itemsToMakeLostCount;
6924  *pSumItemSize = suballocItem->size;
6925  }
6926  else
6927  {
6928  return false;
6929  }
6930  }
6931 
6932  // Remaining size is too small for this request: Early return.
6933  if(GetSize() - suballocItem->offset < allocSize)
6934  {
6935  return false;
6936  }
6937 
6938  // Start from offset equal to beginning of this suballocation.
6939  *pOffset = suballocItem->offset;
6940 
6941  // Apply VMA_DEBUG_MARGIN at the beginning.
6942  if(VMA_DEBUG_MARGIN > 0)
6943  {
6944  *pOffset += VMA_DEBUG_MARGIN;
6945  }
6946 
6947  // Apply alignment.
6948  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
6949 
6950  // Check previous suballocations for BufferImageGranularity conflicts.
6951  // Make bigger alignment if necessary.
6952  if(bufferImageGranularity > 1)
6953  {
6954  bool bufferImageGranularityConflict = false;
6955  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
6956  while(prevSuballocItem != m_Suballocations.cbegin())
6957  {
6958  --prevSuballocItem;
6959  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
6960  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
6961  {
6962  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
6963  {
6964  bufferImageGranularityConflict = true;
6965  break;
6966  }
6967  }
6968  else
6969  // Already on previous page.
6970  break;
6971  }
6972  if(bufferImageGranularityConflict)
6973  {
6974  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
6975  }
6976  }
6977 
6978  // Now that we have final *pOffset, check if we are past suballocItem.
6979  // If yes, return false - this function should be called for another suballocItem as starting point.
6980  if(*pOffset >= suballocItem->offset + suballocItem->size)
6981  {
6982  return false;
6983  }
6984 
6985  // Calculate padding at the beginning based on current offset.
6986  const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
6987 
6988  // Calculate required margin at the end.
6989  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
6990 
6991  const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
6992  // Another early return check.
6993  if(suballocItem->offset + totalSize > GetSize())
6994  {
6995  return false;
6996  }
6997 
6998  // Advance lastSuballocItem until desired size is reached.
6999  // Update itemsToMakeLostCount.
7000  VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
7001  if(totalSize > suballocItem->size)
7002  {
7003  VkDeviceSize remainingSize = totalSize - suballocItem->size;
7004  while(remainingSize > 0)
7005  {
7006  ++lastSuballocItem;
7007  if(lastSuballocItem == m_Suballocations.cend())
7008  {
7009  return false;
7010  }
7011  if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
7012  {
7013  *pSumFreeSize += lastSuballocItem->size;
7014  }
7015  else
7016  {
7017  VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
7018  if(lastSuballocItem->hAllocation->CanBecomeLost() &&
7019  lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
7020  {
7021  ++*itemsToMakeLostCount;
7022  *pSumItemSize += lastSuballocItem->size;
7023  }
7024  else
7025  {
7026  return false;
7027  }
7028  }
7029  remainingSize = (lastSuballocItem->size < remainingSize) ?
7030  remainingSize - lastSuballocItem->size : 0;
7031  }
7032  }
7033 
7034  // Check next suballocations for BufferImageGranularity conflicts.
7035  // If conflict exists, we must mark more allocations lost or fail.
7036  if(bufferImageGranularity > 1)
7037  {
7038  VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
7039  ++nextSuballocItem;
7040  while(nextSuballocItem != m_Suballocations.cend())
7041  {
7042  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
7043  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
7044  {
7045  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
7046  {
7047  VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
7048  if(nextSuballoc.hAllocation->CanBecomeLost() &&
7049  nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
7050  {
7051  ++*itemsToMakeLostCount;
7052  }
7053  else
7054  {
7055  return false;
7056  }
7057  }
7058  }
7059  else
7060  {
7061  // Already on next page.
7062  break;
7063  }
7064  ++nextSuballocItem;
7065  }
7066  }
7067  }
7068  else
7069  {
7070  const VmaSuballocation& suballoc = *suballocItem;
7071  VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
7072 
7073  *pSumFreeSize = suballoc.size;
7074 
7075  // Size of this suballocation is too small for this request: Early return.
7076  if(suballoc.size < allocSize)
7077  {
7078  return false;
7079  }
7080 
7081  // Start from offset equal to beginning of this suballocation.
7082  *pOffset = suballoc.offset;
7083 
7084  // Apply VMA_DEBUG_MARGIN at the beginning.
7085  if(VMA_DEBUG_MARGIN > 0)
7086  {
7087  *pOffset += VMA_DEBUG_MARGIN;
7088  }
7089 
7090  // Apply alignment.
7091  *pOffset = VmaAlignUp(*pOffset, allocAlignment);
7092 
7093  // Check previous suballocations for BufferImageGranularity conflicts.
7094  // Make bigger alignment if necessary.
7095  if(bufferImageGranularity > 1)
7096  {
7097  bool bufferImageGranularityConflict = false;
7098  VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
7099  while(prevSuballocItem != m_Suballocations.cbegin())
7100  {
7101  --prevSuballocItem;
7102  const VmaSuballocation& prevSuballoc = *prevSuballocItem;
7103  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
7104  {
7105  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
7106  {
7107  bufferImageGranularityConflict = true;
7108  break;
7109  }
7110  }
7111  else
7112  // Already on previous page.
7113  break;
7114  }
7115  if(bufferImageGranularityConflict)
7116  {
7117  *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
7118  }
7119  }
7120 
7121  // Calculate padding at the beginning based on current offset.
7122  const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
7123 
7124  // Calculate required margin at the end.
7125  const VkDeviceSize requiredEndMargin = VMA_DEBUG_MARGIN;
7126 
7127  // Fail if requested size plus margin before and after is bigger than size of this suballocation.
7128  if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
7129  {
7130  return false;
7131  }
7132 
7133  // Check next suballocations for BufferImageGranularity conflicts.
7134  // If conflict exists, allocation cannot be made here.
7135  if(bufferImageGranularity > 1)
7136  {
7137  VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
7138  ++nextSuballocItem;
7139  while(nextSuballocItem != m_Suballocations.cend())
7140  {
7141  const VmaSuballocation& nextSuballoc = *nextSuballocItem;
7142  if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
7143  {
7144  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
7145  {
7146  return false;
7147  }
7148  }
7149  else
7150  {
7151  // Already on next page.
7152  break;
7153  }
7154  ++nextSuballocItem;
7155  }
7156  }
7157  }
7158 
7159  // All tests passed: Success. pOffset is already filled.
7160  return true;
7161 }
7162 
7163 void VmaBlockMetadata_Generic::MergeFreeWithNext(VmaSuballocationList::iterator item)
7164 {
7165  VMA_ASSERT(item != m_Suballocations.end());
7166  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
7167 
7168  VmaSuballocationList::iterator nextItem = item;
7169  ++nextItem;
7170  VMA_ASSERT(nextItem != m_Suballocations.end());
7171  VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
7172 
7173  item->size += nextItem->size;
7174  --m_FreeCount;
7175  m_Suballocations.erase(nextItem);
7176 }
7177 
7178 VmaSuballocationList::iterator VmaBlockMetadata_Generic::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
7179 {
7180  // Change this suballocation to be marked as free.
7181  VmaSuballocation& suballoc = *suballocItem;
7182  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
7183  suballoc.hAllocation = VK_NULL_HANDLE;
7184 
7185  // Update totals.
7186  ++m_FreeCount;
7187  m_SumFreeSize += suballoc.size;
7188 
7189  // Merge with previous and/or next suballocation if it's also free.
7190  bool mergeWithNext = false;
7191  bool mergeWithPrev = false;
7192 
7193  VmaSuballocationList::iterator nextItem = suballocItem;
7194  ++nextItem;
7195  if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
7196  {
7197  mergeWithNext = true;
7198  }
7199 
7200  VmaSuballocationList::iterator prevItem = suballocItem;
7201  if(suballocItem != m_Suballocations.begin())
7202  {
7203  --prevItem;
7204  if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
7205  {
7206  mergeWithPrev = true;
7207  }
7208  }
7209 
7210  if(mergeWithNext)
7211  {
7212  UnregisterFreeSuballocation(nextItem);
7213  MergeFreeWithNext(suballocItem);
7214  }
7215 
7216  if(mergeWithPrev)
7217  {
7218  UnregisterFreeSuballocation(prevItem);
7219  MergeFreeWithNext(prevItem);
7220  RegisterFreeSuballocation(prevItem);
7221  return prevItem;
7222  }
7223  else
7224  {
7225  RegisterFreeSuballocation(suballocItem);
7226  return suballocItem;
7227  }
7228 }
7229 
7230 void VmaBlockMetadata_Generic::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
7231 {
7232  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
7233  VMA_ASSERT(item->size > 0);
7234 
7235  // You may want to enable this validation at the beginning or at the end of
7236  // this function, depending on what do you want to check.
7237  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
7238 
7239  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
7240  {
7241  if(m_FreeSuballocationsBySize.empty())
7242  {
7243  m_FreeSuballocationsBySize.push_back(item);
7244  }
7245  else
7246  {
7247  VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
7248  }
7249  }
7250 
7251  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
7252 }
7253 
7254 
7255 void VmaBlockMetadata_Generic::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
7256 {
7257  VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
7258  VMA_ASSERT(item->size > 0);
7259 
7260  // You may want to enable this validation at the beginning or at the end of
7261  // this function, depending on what do you want to check.
7262  VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
7263 
7264  if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
7265  {
7266  VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
7267  m_FreeSuballocationsBySize.data(),
7268  m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
7269  item,
7270  VmaSuballocationItemSizeLess());
7271  for(size_t index = it - m_FreeSuballocationsBySize.data();
7272  index < m_FreeSuballocationsBySize.size();
7273  ++index)
7274  {
7275  if(m_FreeSuballocationsBySize[index] == item)
7276  {
7277  VmaVectorRemove(m_FreeSuballocationsBySize, index);
7278  return;
7279  }
7280  VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
7281  }
7282  VMA_ASSERT(0 && "Not found.");
7283  }
7284 
7285  //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
7286 }
7287 
7289 // class VmaBlockMetadata_Linear
7290 
7291 VmaBlockMetadata_Linear::VmaBlockMetadata_Linear(VmaAllocator hAllocator) :
7292  m_SumFreeSize(0),
7293  m_Suballocations0(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
7294  m_Suballocations1(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
7295  m_1stVectorIndex(0),
7296  m_2ndVectorMode(SECOND_VECTOR_EMPTY),
7297  m_1stNullItemsBeginCount(0),
7298  m_1stNullItemsMiddleCount(0),
7299  m_2ndNullItemsCount(0)
7300 {
7301 }
7302 
7303 VmaBlockMetadata_Linear::~VmaBlockMetadata_Linear()
7304 {
7305 }
7306 
7307 void VmaBlockMetadata_Linear::Init(VkDeviceSize size)
7308 {
7309  VmaBlockMetadata::Init(size);
7310  m_SumFreeSize = size;
7311 }
7312 
7313 bool VmaBlockMetadata_Linear::Validate() const
7314 {
7315  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7316  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7317 
7318  if(suballocations2nd.empty() != (m_2ndVectorMode == SECOND_VECTOR_EMPTY))
7319  {
7320  return false;
7321  }
7322  if(suballocations1st.empty() && !suballocations2nd.empty() &&
7323  m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7324  {
7325  return false;
7326  }
7327  if(!suballocations1st.empty())
7328  {
7329  // Null item at the beginning should be accounted into m_1stNullItemsBeginCount.
7330  if(suballocations1st[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE)
7331  {
7332  return false;
7333  }
7334  // Null item at the end should be just pop_back().
7335  if(suballocations1st.back().hAllocation == VK_NULL_HANDLE)
7336  {
7337  return false;
7338  }
7339  }
7340  if(!suballocations2nd.empty())
7341  {
7342  // Null item at the end should be just pop_back().
7343  if(suballocations2nd.back().hAllocation == VK_NULL_HANDLE)
7344  {
7345  return false;
7346  }
7347  }
7348 
7349  if(m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount > suballocations1st.size())
7350  {
7351  return false;
7352  }
7353  if(m_2ndNullItemsCount > suballocations2nd.size())
7354  {
7355  return false;
7356  }
7357 
7358  VkDeviceSize sumUsedSize = 0;
7359  const size_t suballoc1stCount = suballocations1st.size();
7360  VkDeviceSize offset = VMA_DEBUG_MARGIN;
7361 
7362  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7363  {
7364  const size_t suballoc2ndCount = suballocations2nd.size();
7365  size_t nullItem2ndCount = 0;
7366  for(size_t i = 0; i < suballoc2ndCount; ++i)
7367  {
7368  const VmaSuballocation& suballoc = suballocations2nd[i];
7369  const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
7370 
7371  if(currFree != (suballoc.hAllocation == VK_NULL_HANDLE))
7372  {
7373  return false;
7374  }
7375  if(suballoc.offset < offset)
7376  {
7377  return false;
7378  }
7379 
7380  if(!currFree)
7381  {
7382  if(suballoc.hAllocation->GetOffset() != suballoc.offset)
7383  {
7384  return false;
7385  }
7386  if(suballoc.hAllocation->GetSize() != suballoc.size)
7387  {
7388  return false;
7389  }
7390  sumUsedSize += suballoc.size;
7391  }
7392  else
7393  {
7394  ++nullItem2ndCount;
7395  }
7396 
7397  offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN;
7398  }
7399 
7400  if(nullItem2ndCount != m_2ndNullItemsCount)
7401  {
7402  return false;
7403  }
7404  }
7405 
7406  for(size_t i = 0; i < m_1stNullItemsBeginCount; ++i)
7407  {
7408  const VmaSuballocation& suballoc = suballocations1st[i];
7409  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE ||
7410  suballoc.hAllocation != VK_NULL_HANDLE)
7411  {
7412  return false;
7413  }
7414  }
7415 
7416  size_t nullItem1stCount = m_1stNullItemsBeginCount;
7417 
7418  for(size_t i = m_1stNullItemsBeginCount; i < suballoc1stCount; ++i)
7419  {
7420  const VmaSuballocation& suballoc = suballocations1st[i];
7421  const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
7422 
7423  if(currFree != (suballoc.hAllocation == VK_NULL_HANDLE))
7424  {
7425  return false;
7426  }
7427  if(suballoc.offset < offset)
7428  {
7429  return false;
7430  }
7431  if(i < m_1stNullItemsBeginCount && !currFree)
7432  {
7433  return false;
7434  }
7435 
7436  if(!currFree)
7437  {
7438  if(suballoc.hAllocation->GetOffset() != suballoc.offset)
7439  {
7440  return false;
7441  }
7442  if(suballoc.hAllocation->GetSize() != suballoc.size)
7443  {
7444  return false;
7445  }
7446  sumUsedSize += suballoc.size;
7447  }
7448  else
7449  {
7450  ++nullItem1stCount;
7451  }
7452 
7453  offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN;
7454  }
7455  if(nullItem1stCount != m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount)
7456  {
7457  return false;
7458  }
7459 
7460  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
7461  {
7462  const size_t suballoc2ndCount = suballocations2nd.size();
7463  size_t nullItem2ndCount = 0;
7464  for(size_t i = suballoc2ndCount; i--; )
7465  {
7466  const VmaSuballocation& suballoc = suballocations2nd[i];
7467  const bool currFree = (suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
7468 
7469  if(currFree != (suballoc.hAllocation == VK_NULL_HANDLE))
7470  {
7471  return false;
7472  }
7473  if(suballoc.offset < offset)
7474  {
7475  return false;
7476  }
7477 
7478  if(!currFree)
7479  {
7480  if(suballoc.hAllocation->GetOffset() != suballoc.offset)
7481  {
7482  return false;
7483  }
7484  if(suballoc.hAllocation->GetSize() != suballoc.size)
7485  {
7486  return false;
7487  }
7488  sumUsedSize += suballoc.size;
7489  }
7490  else
7491  {
7492  ++nullItem2ndCount;
7493  }
7494 
7495  offset = suballoc.offset + suballoc.size + VMA_DEBUG_MARGIN;
7496  }
7497 
7498  if(nullItem2ndCount != m_2ndNullItemsCount)
7499  {
7500  return false;
7501  }
7502  }
7503 
7504  if(offset > GetSize())
7505  {
7506  return false;
7507  }
7508  if(m_SumFreeSize != GetSize() - sumUsedSize)
7509  {
7510  return false;
7511  }
7512 
7513  return true;
7514 }
7515 
7516 size_t VmaBlockMetadata_Linear::GetAllocationCount() const
7517 {
7518  return AccessSuballocations1st().size() - (m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount) +
7519  AccessSuballocations2nd().size() - m_2ndNullItemsCount;
7520 }
7521 
7522 VkDeviceSize VmaBlockMetadata_Linear::GetUnusedRangeSizeMax() const
7523 {
7524  const VkDeviceSize size = GetSize();
7525 
7526  /*
7527  We don't consider gaps inside allocation vectors with freed allocations because
7528  they are not suitable for reuse in linear allocator. We consider only space that
7529  is available for new allocations.
7530  */
7531  if(IsEmpty())
7532  {
7533  return size;
7534  }
7535 
7536  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7537 
7538  switch(m_2ndVectorMode)
7539  {
7540  case SECOND_VECTOR_EMPTY:
7541  /*
7542  Available space is after end of 1st, as well as before beginning of 1st (which
7543  whould make it a ring buffer).
7544  */
7545  {
7546  const size_t suballocations1stCount = suballocations1st.size();
7547  VMA_ASSERT(suballocations1stCount > m_1stNullItemsBeginCount);
7548  const VmaSuballocation& firstSuballoc = suballocations1st[m_1stNullItemsBeginCount];
7549  const VmaSuballocation& lastSuballoc = suballocations1st[suballocations1stCount - 1];
7550  return VMA_MAX(
7551  firstSuballoc.offset,
7552  size - (lastSuballoc.offset + lastSuballoc.size));
7553  }
7554  break;
7555 
7556  case SECOND_VECTOR_RING_BUFFER:
7557  /*
7558  Available space is only between end of 2nd and beginning of 1st.
7559  */
7560  {
7561  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7562  const VmaSuballocation& lastSuballoc2nd = suballocations2nd.back();
7563  const VmaSuballocation& firstSuballoc1st = suballocations1st[m_1stNullItemsBeginCount];
7564  return firstSuballoc1st.offset - (lastSuballoc2nd.offset + lastSuballoc2nd.size);
7565  }
7566  break;
7567 
7568  case SECOND_VECTOR_DOUBLE_STACK:
7569  /*
7570  Available space is only between end of 1st and top of 2nd.
7571  */
7572  {
7573  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7574  const VmaSuballocation& topSuballoc2nd = suballocations2nd.back();
7575  const VmaSuballocation& lastSuballoc1st = suballocations1st.back();
7576  return topSuballoc2nd.offset - (lastSuballoc1st.offset + lastSuballoc1st.size);
7577  }
7578  break;
7579 
7580  default:
7581  VMA_ASSERT(0);
7582  return 0;
7583  }
7584 }
7585 
7586 void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
7587 {
7588  const VkDeviceSize size = GetSize();
7589  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7590  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7591  const size_t suballoc1stCount = suballocations1st.size();
7592  const size_t suballoc2ndCount = suballocations2nd.size();
7593 
7594  outInfo.blockCount = 1;
7595  outInfo.allocationCount = (uint32_t)GetAllocationCount();
7596  outInfo.unusedRangeCount = 0;
7597  outInfo.usedBytes = 0;
7598  outInfo.allocationSizeMin = UINT64_MAX;
7599  outInfo.allocationSizeMax = 0;
7600  outInfo.unusedRangeSizeMin = UINT64_MAX;
7601  outInfo.unusedRangeSizeMax = 0;
7602 
7603  VkDeviceSize lastOffset = 0;
7604 
7605  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7606  {
7607  const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
7608  size_t nextAlloc2ndIndex = 0;
7609  while(lastOffset < freeSpace2ndTo1stEnd)
7610  {
7611  // Find next non-null allocation or move nextAllocIndex to the end.
7612  while(nextAlloc2ndIndex < suballoc2ndCount &&
7613  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7614  {
7615  ++nextAlloc2ndIndex;
7616  }
7617 
7618  // Found non-null allocation.
7619  if(nextAlloc2ndIndex < suballoc2ndCount)
7620  {
7621  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7622 
7623  // 1. Process free space before this allocation.
7624  if(lastOffset < suballoc.offset)
7625  {
7626  // There is free space from lastOffset to suballoc.offset.
7627  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7628  ++outInfo.unusedRangeCount;
7629  outInfo.unusedBytes += unusedRangeSize;
7630  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7631  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7632  }
7633 
7634  // 2. Process this allocation.
7635  // There is allocation with suballoc.offset, suballoc.size.
7636  outInfo.usedBytes += suballoc.size;
7637  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
7638  outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size);
7639 
7640  // 3. Prepare for next iteration.
7641  lastOffset = suballoc.offset + suballoc.size;
7642  ++nextAlloc2ndIndex;
7643  }
7644  // We are at the end.
7645  else
7646  {
7647  // There is free space from lastOffset to freeSpace2ndTo1stEnd.
7648  if(lastOffset < freeSpace2ndTo1stEnd)
7649  {
7650  const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset;
7651  ++outInfo.unusedRangeCount;
7652  outInfo.unusedBytes += unusedRangeSize;
7653  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7654  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7655  }
7656 
7657  // End of loop.
7658  lastOffset = freeSpace2ndTo1stEnd;
7659  }
7660  }
7661  }
7662 
7663  size_t nextAlloc1stIndex = m_1stNullItemsBeginCount;
7664  const VkDeviceSize freeSpace1stTo2ndEnd =
7665  m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size;
7666  while(lastOffset < freeSpace1stTo2ndEnd)
7667  {
7668  // Find next non-null allocation or move nextAllocIndex to the end.
7669  while(nextAlloc1stIndex < suballoc1stCount &&
7670  suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
7671  {
7672  ++nextAlloc1stIndex;
7673  }
7674 
7675  // Found non-null allocation.
7676  if(nextAlloc1stIndex < suballoc1stCount)
7677  {
7678  const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
7679 
7680  // 1. Process free space before this allocation.
7681  if(lastOffset < suballoc.offset)
7682  {
7683  // There is free space from lastOffset to suballoc.offset.
7684  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7685  ++outInfo.unusedRangeCount;
7686  outInfo.unusedBytes += unusedRangeSize;
7687  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7688  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7689  }
7690 
7691  // 2. Process this allocation.
7692  // There is allocation with suballoc.offset, suballoc.size.
7693  outInfo.usedBytes += suballoc.size;
7694  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
7695  outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size);
7696 
7697  // 3. Prepare for next iteration.
7698  lastOffset = suballoc.offset + suballoc.size;
7699  ++nextAlloc1stIndex;
7700  }
7701  // We are at the end.
7702  else
7703  {
7704  // There is free space from lastOffset to freeSpace1stTo2ndEnd.
7705  if(lastOffset < freeSpace1stTo2ndEnd)
7706  {
7707  const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset;
7708  ++outInfo.unusedRangeCount;
7709  outInfo.unusedBytes += unusedRangeSize;
7710  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7711  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7712  }
7713 
7714  // End of loop.
7715  lastOffset = freeSpace1stTo2ndEnd;
7716  }
7717  }
7718 
7719  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
7720  {
7721  size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
7722  while(lastOffset < size)
7723  {
7724  // Find next non-null allocation or move nextAllocIndex to the end.
7725  while(nextAlloc2ndIndex != SIZE_MAX &&
7726  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7727  {
7728  --nextAlloc2ndIndex;
7729  }
7730 
7731  // Found non-null allocation.
7732  if(nextAlloc2ndIndex != SIZE_MAX)
7733  {
7734  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7735 
7736  // 1. Process free space before this allocation.
7737  if(lastOffset < suballoc.offset)
7738  {
7739  // There is free space from lastOffset to suballoc.offset.
7740  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7741  ++outInfo.unusedRangeCount;
7742  outInfo.unusedBytes += unusedRangeSize;
7743  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7744  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7745  }
7746 
7747  // 2. Process this allocation.
7748  // There is allocation with suballoc.offset, suballoc.size.
7749  outInfo.usedBytes += suballoc.size;
7750  outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
7751  outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size);
7752 
7753  // 3. Prepare for next iteration.
7754  lastOffset = suballoc.offset + suballoc.size;
7755  --nextAlloc2ndIndex;
7756  }
7757  // We are at the end.
7758  else
7759  {
7760  // There is free space from lastOffset to size.
7761  if(lastOffset < size)
7762  {
7763  const VkDeviceSize unusedRangeSize = size - lastOffset;
7764  ++outInfo.unusedRangeCount;
7765  outInfo.unusedBytes += unusedRangeSize;
7766  outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize);
7767  outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize);
7768  }
7769 
7770  // End of loop.
7771  lastOffset = size;
7772  }
7773  }
7774  }
7775 
7776  outInfo.unusedBytes = size - outInfo.usedBytes;
7777 }
7778 
7779 void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const
7780 {
7781  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7782  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7783  const VkDeviceSize size = GetSize();
7784  const size_t suballoc1stCount = suballocations1st.size();
7785  const size_t suballoc2ndCount = suballocations2nd.size();
7786 
7787  inoutStats.size += size;
7788 
7789  VkDeviceSize lastOffset = 0;
7790 
7791  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7792  {
7793  const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
7794  size_t nextAlloc2ndIndex = m_1stNullItemsBeginCount;
7795  while(lastOffset < freeSpace2ndTo1stEnd)
7796  {
7797  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
7798  while(nextAlloc2ndIndex < suballoc2ndCount &&
7799  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7800  {
7801  ++nextAlloc2ndIndex;
7802  }
7803 
7804  // Found non-null allocation.
7805  if(nextAlloc2ndIndex < suballoc2ndCount)
7806  {
7807  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7808 
7809  // 1. Process free space before this allocation.
7810  if(lastOffset < suballoc.offset)
7811  {
7812  // There is free space from lastOffset to suballoc.offset.
7813  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7814  inoutStats.unusedSize += unusedRangeSize;
7815  ++inoutStats.unusedRangeCount;
7816  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7817  }
7818 
7819  // 2. Process this allocation.
7820  // There is allocation with suballoc.offset, suballoc.size.
7821  ++inoutStats.allocationCount;
7822 
7823  // 3. Prepare for next iteration.
7824  lastOffset = suballoc.offset + suballoc.size;
7825  ++nextAlloc2ndIndex;
7826  }
7827  // We are at the end.
7828  else
7829  {
7830  if(lastOffset < freeSpace2ndTo1stEnd)
7831  {
7832  // There is free space from lastOffset to freeSpace2ndTo1stEnd.
7833  const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset;
7834  inoutStats.unusedSize += unusedRangeSize;
7835  ++inoutStats.unusedRangeCount;
7836  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7837  }
7838 
7839  // End of loop.
7840  lastOffset = freeSpace2ndTo1stEnd;
7841  }
7842  }
7843  }
7844 
7845  size_t nextAlloc1stIndex = m_1stNullItemsBeginCount;
7846  const VkDeviceSize freeSpace1stTo2ndEnd =
7847  m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size;
7848  while(lastOffset < freeSpace1stTo2ndEnd)
7849  {
7850  // Find next non-null allocation or move nextAllocIndex to the end.
7851  while(nextAlloc1stIndex < suballoc1stCount &&
7852  suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
7853  {
7854  ++nextAlloc1stIndex;
7855  }
7856 
7857  // Found non-null allocation.
7858  if(nextAlloc1stIndex < suballoc1stCount)
7859  {
7860  const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
7861 
7862  // 1. Process free space before this allocation.
7863  if(lastOffset < suballoc.offset)
7864  {
7865  // There is free space from lastOffset to suballoc.offset.
7866  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7867  inoutStats.unusedSize += unusedRangeSize;
7868  ++inoutStats.unusedRangeCount;
7869  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7870  }
7871 
7872  // 2. Process this allocation.
7873  // There is allocation with suballoc.offset, suballoc.size.
7874  ++inoutStats.allocationCount;
7875 
7876  // 3. Prepare for next iteration.
7877  lastOffset = suballoc.offset + suballoc.size;
7878  ++nextAlloc1stIndex;
7879  }
7880  // We are at the end.
7881  else
7882  {
7883  if(lastOffset < freeSpace1stTo2ndEnd)
7884  {
7885  // There is free space from lastOffset to freeSpace1stTo2ndEnd.
7886  const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset;
7887  inoutStats.unusedSize += unusedRangeSize;
7888  ++inoutStats.unusedRangeCount;
7889  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7890  }
7891 
7892  // End of loop.
7893  lastOffset = freeSpace1stTo2ndEnd;
7894  }
7895  }
7896 
7897  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
7898  {
7899  size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
7900  while(lastOffset < size)
7901  {
7902  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
7903  while(nextAlloc2ndIndex != SIZE_MAX &&
7904  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7905  {
7906  --nextAlloc2ndIndex;
7907  }
7908 
7909  // Found non-null allocation.
7910  if(nextAlloc2ndIndex != SIZE_MAX)
7911  {
7912  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7913 
7914  // 1. Process free space before this allocation.
7915  if(lastOffset < suballoc.offset)
7916  {
7917  // There is free space from lastOffset to suballoc.offset.
7918  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
7919  inoutStats.unusedSize += unusedRangeSize;
7920  ++inoutStats.unusedRangeCount;
7921  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7922  }
7923 
7924  // 2. Process this allocation.
7925  // There is allocation with suballoc.offset, suballoc.size.
7926  ++inoutStats.allocationCount;
7927 
7928  // 3. Prepare for next iteration.
7929  lastOffset = suballoc.offset + suballoc.size;
7930  --nextAlloc2ndIndex;
7931  }
7932  // We are at the end.
7933  else
7934  {
7935  if(lastOffset < size)
7936  {
7937  // There is free space from lastOffset to size.
7938  const VkDeviceSize unusedRangeSize = size - lastOffset;
7939  inoutStats.unusedSize += unusedRangeSize;
7940  ++inoutStats.unusedRangeCount;
7941  inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize);
7942  }
7943 
7944  // End of loop.
7945  lastOffset = size;
7946  }
7947  }
7948  }
7949 }
7950 
7951 #if VMA_STATS_STRING_ENABLED
7952 void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const
7953 {
7954  const VkDeviceSize size = GetSize();
7955  const SuballocationVectorType& suballocations1st = AccessSuballocations1st();
7956  const SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
7957  const size_t suballoc1stCount = suballocations1st.size();
7958  const size_t suballoc2ndCount = suballocations2nd.size();
7959 
7960  // FIRST PASS
7961 
7962  size_t unusedRangeCount = 0;
7963  VkDeviceSize usedBytes = 0;
7964 
7965  VkDeviceSize lastOffset = 0;
7966 
7967  size_t alloc2ndCount = 0;
7968  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
7969  {
7970  const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
7971  size_t nextAlloc2ndIndex = 0;
7972  while(lastOffset < freeSpace2ndTo1stEnd)
7973  {
7974  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
7975  while(nextAlloc2ndIndex < suballoc2ndCount &&
7976  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
7977  {
7978  ++nextAlloc2ndIndex;
7979  }
7980 
7981  // Found non-null allocation.
7982  if(nextAlloc2ndIndex < suballoc2ndCount)
7983  {
7984  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
7985 
7986  // 1. Process free space before this allocation.
7987  if(lastOffset < suballoc.offset)
7988  {
7989  // There is free space from lastOffset to suballoc.offset.
7990  ++unusedRangeCount;
7991  }
7992 
7993  // 2. Process this allocation.
7994  // There is allocation with suballoc.offset, suballoc.size.
7995  ++alloc2ndCount;
7996  usedBytes += suballoc.size;
7997 
7998  // 3. Prepare for next iteration.
7999  lastOffset = suballoc.offset + suballoc.size;
8000  ++nextAlloc2ndIndex;
8001  }
8002  // We are at the end.
8003  else
8004  {
8005  if(lastOffset < freeSpace2ndTo1stEnd)
8006  {
8007  // There is free space from lastOffset to freeSpace2ndTo1stEnd.
8008  ++unusedRangeCount;
8009  }
8010 
8011  // End of loop.
8012  lastOffset = freeSpace2ndTo1stEnd;
8013  }
8014  }
8015  }
8016 
8017  size_t nextAlloc1stIndex = m_1stNullItemsBeginCount;
8018  size_t alloc1stCount = 0;
8019  const VkDeviceSize freeSpace1stTo2ndEnd =
8020  m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ? suballocations2nd.back().offset : size;
8021  while(lastOffset < freeSpace1stTo2ndEnd)
8022  {
8023  // Find next non-null allocation or move nextAllocIndex to the end.
8024  while(nextAlloc1stIndex < suballoc1stCount &&
8025  suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
8026  {
8027  ++nextAlloc1stIndex;
8028  }
8029 
8030  // Found non-null allocation.
8031  if(nextAlloc1stIndex < suballoc1stCount)
8032  {
8033  const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
8034 
8035  // 1. Process free space before this allocation.
8036  if(lastOffset < suballoc.offset)
8037  {
8038  // There is free space from lastOffset to suballoc.offset.
8039  ++unusedRangeCount;
8040  }
8041 
8042  // 2. Process this allocation.
8043  // There is allocation with suballoc.offset, suballoc.size.
8044  ++alloc1stCount;
8045  usedBytes += suballoc.size;
8046 
8047  // 3. Prepare for next iteration.
8048  lastOffset = suballoc.offset + suballoc.size;
8049  ++nextAlloc1stIndex;
8050  }
8051  // We are at the end.
8052  else
8053  {
8054  if(lastOffset < size)
8055  {
8056  // There is free space from lastOffset to freeSpace1stTo2ndEnd.
8057  ++unusedRangeCount;
8058  }
8059 
8060  // End of loop.
8061  lastOffset = freeSpace1stTo2ndEnd;
8062  }
8063  }
8064 
8065  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8066  {
8067  size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
8068  while(lastOffset < size)
8069  {
8070  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
8071  while(nextAlloc2ndIndex != SIZE_MAX &&
8072  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
8073  {
8074  --nextAlloc2ndIndex;
8075  }
8076 
8077  // Found non-null allocation.
8078  if(nextAlloc2ndIndex != SIZE_MAX)
8079  {
8080  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
8081 
8082  // 1. Process free space before this allocation.
8083  if(lastOffset < suballoc.offset)
8084  {
8085  // There is free space from lastOffset to suballoc.offset.
8086  ++unusedRangeCount;
8087  }
8088 
8089  // 2. Process this allocation.
8090  // There is allocation with suballoc.offset, suballoc.size.
8091  ++alloc2ndCount;
8092  usedBytes += suballoc.size;
8093 
8094  // 3. Prepare for next iteration.
8095  lastOffset = suballoc.offset + suballoc.size;
8096  --nextAlloc2ndIndex;
8097  }
8098  // We are at the end.
8099  else
8100  {
8101  if(lastOffset < size)
8102  {
8103  // There is free space from lastOffset to size.
8104  ++unusedRangeCount;
8105  }
8106 
8107  // End of loop.
8108  lastOffset = size;
8109  }
8110  }
8111  }
8112 
8113  const VkDeviceSize unusedBytes = size - usedBytes;
8114  PrintDetailedMap_Begin(json, unusedBytes, alloc1stCount + alloc2ndCount, unusedRangeCount);
8115 
8116  // SECOND PASS
8117  lastOffset = 0;
8118 
8119  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
8120  {
8121  const VkDeviceSize freeSpace2ndTo1stEnd = suballocations1st[m_1stNullItemsBeginCount].offset;
8122  size_t nextAlloc2ndIndex = 0;
8123  while(lastOffset < freeSpace2ndTo1stEnd)
8124  {
8125  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
8126  while(nextAlloc2ndIndex < suballoc2ndCount &&
8127  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
8128  {
8129  ++nextAlloc2ndIndex;
8130  }
8131 
8132  // Found non-null allocation.
8133  if(nextAlloc2ndIndex < suballoc2ndCount)
8134  {
8135  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
8136 
8137  // 1. Process free space before this allocation.
8138  if(lastOffset < suballoc.offset)
8139  {
8140  // There is free space from lastOffset to suballoc.offset.
8141  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
8142  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8143  }
8144 
8145  // 2. Process this allocation.
8146  // There is allocation with suballoc.offset, suballoc.size.
8147  PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation);
8148 
8149  // 3. Prepare for next iteration.
8150  lastOffset = suballoc.offset + suballoc.size;
8151  ++nextAlloc2ndIndex;
8152  }
8153  // We are at the end.
8154  else
8155  {
8156  if(lastOffset < freeSpace2ndTo1stEnd)
8157  {
8158  // There is free space from lastOffset to freeSpace2ndTo1stEnd.
8159  const VkDeviceSize unusedRangeSize = freeSpace2ndTo1stEnd - lastOffset;
8160  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8161  }
8162 
8163  // End of loop.
8164  lastOffset = freeSpace2ndTo1stEnd;
8165  }
8166  }
8167  }
8168 
8169  nextAlloc1stIndex = m_1stNullItemsBeginCount;
8170  while(lastOffset < freeSpace1stTo2ndEnd)
8171  {
8172  // Find next non-null allocation or move nextAllocIndex to the end.
8173  while(nextAlloc1stIndex < suballoc1stCount &&
8174  suballocations1st[nextAlloc1stIndex].hAllocation == VK_NULL_HANDLE)
8175  {
8176  ++nextAlloc1stIndex;
8177  }
8178 
8179  // Found non-null allocation.
8180  if(nextAlloc1stIndex < suballoc1stCount)
8181  {
8182  const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex];
8183 
8184  // 1. Process free space before this allocation.
8185  if(lastOffset < suballoc.offset)
8186  {
8187  // There is free space from lastOffset to suballoc.offset.
8188  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
8189  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8190  }
8191 
8192  // 2. Process this allocation.
8193  // There is allocation with suballoc.offset, suballoc.size.
8194  PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation);
8195 
8196  // 3. Prepare for next iteration.
8197  lastOffset = suballoc.offset + suballoc.size;
8198  ++nextAlloc1stIndex;
8199  }
8200  // We are at the end.
8201  else
8202  {
8203  if(lastOffset < freeSpace1stTo2ndEnd)
8204  {
8205  // There is free space from lastOffset to freeSpace1stTo2ndEnd.
8206  const VkDeviceSize unusedRangeSize = freeSpace1stTo2ndEnd - lastOffset;
8207  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8208  }
8209 
8210  // End of loop.
8211  lastOffset = freeSpace1stTo2ndEnd;
8212  }
8213  }
8214 
8215  if(m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8216  {
8217  size_t nextAlloc2ndIndex = suballocations2nd.size() - 1;
8218  while(lastOffset < size)
8219  {
8220  // Find next non-null allocation or move nextAlloc2ndIndex to the end.
8221  while(nextAlloc2ndIndex != SIZE_MAX &&
8222  suballocations2nd[nextAlloc2ndIndex].hAllocation == VK_NULL_HANDLE)
8223  {
8224  --nextAlloc2ndIndex;
8225  }
8226 
8227  // Found non-null allocation.
8228  if(nextAlloc2ndIndex != SIZE_MAX)
8229  {
8230  const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex];
8231 
8232  // 1. Process free space before this allocation.
8233  if(lastOffset < suballoc.offset)
8234  {
8235  // There is free space from lastOffset to suballoc.offset.
8236  const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset;
8237  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8238  }
8239 
8240  // 2. Process this allocation.
8241  // There is allocation with suballoc.offset, suballoc.size.
8242  PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation);
8243 
8244  // 3. Prepare for next iteration.
8245  lastOffset = suballoc.offset + suballoc.size;
8246  --nextAlloc2ndIndex;
8247  }
8248  // We are at the end.
8249  else
8250  {
8251  if(lastOffset < size)
8252  {
8253  // There is free space from lastOffset to size.
8254  const VkDeviceSize unusedRangeSize = size - lastOffset;
8255  PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize);
8256  }
8257 
8258  // End of loop.
8259  lastOffset = size;
8260  }
8261  }
8262  }
8263 
8264  PrintDetailedMap_End(json);
8265 }
8266 #endif // #if VMA_STATS_STRING_ENABLED
8267 
8268 bool VmaBlockMetadata_Linear::CreateAllocationRequest(
8269  uint32_t currentFrameIndex,
8270  uint32_t frameInUseCount,
8271  VkDeviceSize bufferImageGranularity,
8272  VkDeviceSize allocSize,
8273  VkDeviceSize allocAlignment,
8274  bool upperAddress,
8275  VmaSuballocationType allocType,
8276  bool canMakeOtherLost,
8277  VmaAllocationRequest* pAllocationRequest)
8278 {
8279  VMA_ASSERT(allocSize > 0);
8280  VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
8281  VMA_ASSERT(pAllocationRequest != VMA_NULL);
8282  VMA_HEAVY_ASSERT(Validate());
8283 
8284  const VkDeviceSize size = GetSize();
8285  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8286  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8287 
8288  if(upperAddress)
8289  {
8290  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
8291  {
8292  VMA_ASSERT(0 && "Trying to use pool with linear algorithm as double stack, while it is already being used as ring buffer.");
8293  return false;
8294  }
8295 
8296  // Try to allocate before 2nd.back(), or end of block if 2nd.empty().
8297  if(allocSize > size)
8298  {
8299  return false;
8300  }
8301  VkDeviceSize resultBaseOffset = size - allocSize;
8302  if(!suballocations2nd.empty())
8303  {
8304  const VmaSuballocation& lastSuballoc = suballocations2nd.back();
8305  resultBaseOffset = lastSuballoc.offset - allocSize;
8306  if(allocSize > lastSuballoc.offset)
8307  {
8308  return false;
8309  }
8310  }
8311 
8312  // Start from offset equal to end of free space.
8313  VkDeviceSize resultOffset = resultBaseOffset;
8314 
8315  // Apply VMA_DEBUG_MARGIN at the end.
8316  if(VMA_DEBUG_MARGIN > 0)
8317  {
8318  if(resultOffset < VMA_DEBUG_MARGIN)
8319  {
8320  return false;
8321  }
8322  resultOffset -= VMA_DEBUG_MARGIN;
8323  }
8324 
8325  // Apply alignment.
8326  resultOffset = VmaAlignDown(resultOffset, allocAlignment);
8327 
8328  // Check next suballocations from 2nd for BufferImageGranularity conflicts.
8329  // Make bigger alignment if necessary.
8330  if(bufferImageGranularity > 1 && !suballocations2nd.empty())
8331  {
8332  bool bufferImageGranularityConflict = false;
8333  for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; )
8334  {
8335  const VmaSuballocation& nextSuballoc = suballocations2nd[nextSuballocIndex];
8336  if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
8337  {
8338  if(VmaIsBufferImageGranularityConflict(nextSuballoc.type, allocType))
8339  {
8340  bufferImageGranularityConflict = true;
8341  break;
8342  }
8343  }
8344  else
8345  // Already on previous page.
8346  break;
8347  }
8348  if(bufferImageGranularityConflict)
8349  {
8350  resultOffset = VmaAlignDown(resultOffset, bufferImageGranularity);
8351  }
8352  }
8353 
8354  // There is enough free space.
8355  const VkDeviceSize endOf1st = !suballocations1st.empty() ?
8356  suballocations1st.back().offset + suballocations1st.back().size :
8357  0;
8358  if(endOf1st + VMA_DEBUG_MARGIN <= resultOffset)
8359  {
8360  // Check previous suballocations for BufferImageGranularity conflicts.
8361  // If conflict exists, allocation cannot be made here.
8362  if(bufferImageGranularity > 1)
8363  {
8364  for(size_t prevSuballocIndex = suballocations1st.size(); prevSuballocIndex--; )
8365  {
8366  const VmaSuballocation& prevSuballoc = suballocations1st[prevSuballocIndex];
8367  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity))
8368  {
8369  if(VmaIsBufferImageGranularityConflict(allocType, prevSuballoc.type))
8370  {
8371  return false;
8372  }
8373  }
8374  else
8375  {
8376  // Already on next page.
8377  break;
8378  }
8379  }
8380  }
8381 
8382  // All tests passed: Success.
8383  pAllocationRequest->offset = resultOffset;
8384  pAllocationRequest->sumFreeSize = resultBaseOffset + allocSize - endOf1st;
8385  pAllocationRequest->sumItemSize = 0;
8386  // pAllocationRequest->item unused.
8387  pAllocationRequest->itemsToMakeLostCount = 0;
8388  return true;
8389  }
8390  }
8391  else // !upperAddress
8392  {
8393  if(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8394  {
8395  // Try to allocate at the end of 1st vector.
8396 
8397  VkDeviceSize resultBaseOffset = 0;
8398  if(!suballocations1st.empty())
8399  {
8400  const VmaSuballocation& lastSuballoc = suballocations1st.back();
8401  resultBaseOffset = lastSuballoc.offset + lastSuballoc.size;
8402  }
8403 
8404  // Start from offset equal to beginning of free space.
8405  VkDeviceSize resultOffset = resultBaseOffset;
8406 
8407  // Apply VMA_DEBUG_MARGIN at the beginning.
8408  if(VMA_DEBUG_MARGIN > 0)
8409  {
8410  resultOffset += VMA_DEBUG_MARGIN;
8411  }
8412 
8413  // Apply alignment.
8414  resultOffset = VmaAlignUp(resultOffset, allocAlignment);
8415 
8416  // Check previous suballocations for BufferImageGranularity conflicts.
8417  // Make bigger alignment if necessary.
8418  if(bufferImageGranularity > 1 && !suballocations1st.empty())
8419  {
8420  bool bufferImageGranularityConflict = false;
8421  for(size_t prevSuballocIndex = suballocations1st.size(); prevSuballocIndex--; )
8422  {
8423  const VmaSuballocation& prevSuballoc = suballocations1st[prevSuballocIndex];
8424  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity))
8425  {
8426  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
8427  {
8428  bufferImageGranularityConflict = true;
8429  break;
8430  }
8431  }
8432  else
8433  // Already on previous page.
8434  break;
8435  }
8436  if(bufferImageGranularityConflict)
8437  {
8438  resultOffset = VmaAlignUp(resultOffset, bufferImageGranularity);
8439  }
8440  }
8441 
8442  const VkDeviceSize freeSpaceEnd = m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK ?
8443  suballocations2nd.back().offset : size;
8444 
8445  // There is enough free space at the end after alignment.
8446  if(resultOffset + allocSize + VMA_DEBUG_MARGIN <= freeSpaceEnd)
8447  {
8448  // Check next suballocations for BufferImageGranularity conflicts.
8449  // If conflict exists, allocation cannot be made here.
8450  if(bufferImageGranularity > 1 && m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8451  {
8452  for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; )
8453  {
8454  const VmaSuballocation& nextSuballoc = suballocations2nd[nextSuballocIndex];
8455  if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
8456  {
8457  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
8458  {
8459  return false;
8460  }
8461  }
8462  else
8463  {
8464  // Already on previous page.
8465  break;
8466  }
8467  }
8468  }
8469 
8470  // All tests passed: Success.
8471  pAllocationRequest->offset = resultOffset;
8472  pAllocationRequest->sumFreeSize = freeSpaceEnd - resultBaseOffset;
8473  pAllocationRequest->sumItemSize = 0;
8474  // pAllocationRequest->item unused.
8475  pAllocationRequest->itemsToMakeLostCount = 0;
8476  return true;
8477  }
8478  }
8479 
8480  // Wrap-around to end of 2nd vector. Try to allocate there, watching for the
8481  // beginning of 1st vector as the end of free space.
8482  if(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
8483  {
8484  VMA_ASSERT(!suballocations1st.empty());
8485 
8486  VkDeviceSize resultBaseOffset = 0;
8487  if(!suballocations2nd.empty())
8488  {
8489  const VmaSuballocation& lastSuballoc = suballocations2nd.back();
8490  resultBaseOffset = lastSuballoc.offset + lastSuballoc.size;
8491  }
8492 
8493  // Start from offset equal to beginning of free space.
8494  VkDeviceSize resultOffset = resultBaseOffset;
8495 
8496  // Apply VMA_DEBUG_MARGIN at the beginning.
8497  if(VMA_DEBUG_MARGIN > 0)
8498  {
8499  resultOffset += VMA_DEBUG_MARGIN;
8500  }
8501 
8502  // Apply alignment.
8503  resultOffset = VmaAlignUp(resultOffset, allocAlignment);
8504 
8505  // Check previous suballocations for BufferImageGranularity conflicts.
8506  // Make bigger alignment if necessary.
8507  if(bufferImageGranularity > 1 && !suballocations2nd.empty())
8508  {
8509  bool bufferImageGranularityConflict = false;
8510  for(size_t prevSuballocIndex = suballocations2nd.size(); prevSuballocIndex--; )
8511  {
8512  const VmaSuballocation& prevSuballoc = suballocations2nd[prevSuballocIndex];
8513  if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, resultOffset, bufferImageGranularity))
8514  {
8515  if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
8516  {
8517  bufferImageGranularityConflict = true;
8518  break;
8519  }
8520  }
8521  else
8522  // Already on previous page.
8523  break;
8524  }
8525  if(bufferImageGranularityConflict)
8526  {
8527  resultOffset = VmaAlignUp(resultOffset, bufferImageGranularity);
8528  }
8529  }
8530 
8531  pAllocationRequest->itemsToMakeLostCount = 0;
8532  pAllocationRequest->sumItemSize = 0;
8533  size_t index1st = m_1stNullItemsBeginCount;
8534 
8535  if(canMakeOtherLost)
8536  {
8537  while(index1st < suballocations1st.size() &&
8538  resultOffset + allocSize + VMA_DEBUG_MARGIN > suballocations1st[index1st].offset)
8539  {
8540  // Next colliding allocation at the beginning of 1st vector found. Try to make it lost.
8541  const VmaSuballocation& suballoc = suballocations1st[index1st];
8542  if(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE)
8543  {
8544  // No problem.
8545  }
8546  else
8547  {
8548  VMA_ASSERT(suballoc.hAllocation != VK_NULL_HANDLE);
8549  if(suballoc.hAllocation->CanBecomeLost() &&
8550  suballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
8551  {
8552  ++pAllocationRequest->itemsToMakeLostCount;
8553  pAllocationRequest->sumItemSize += suballoc.size;
8554  }
8555  else
8556  {
8557  return false;
8558  }
8559  }
8560  ++index1st;
8561  }
8562 
8563  // Check next suballocations for BufferImageGranularity conflicts.
8564  // If conflict exists, we must mark more allocations lost or fail.
8565  if(bufferImageGranularity > 1)
8566  {
8567  while(index1st < suballocations1st.size())
8568  {
8569  const VmaSuballocation& suballoc = suballocations1st[index1st];
8570  if(VmaBlocksOnSamePage(resultOffset, allocSize, suballoc.offset, bufferImageGranularity))
8571  {
8572  if(suballoc.hAllocation != VK_NULL_HANDLE)
8573  {
8574  // Not checking actual VmaIsBufferImageGranularityConflict(allocType, suballoc.type).
8575  if(suballoc.hAllocation->CanBecomeLost() &&
8576  suballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
8577  {
8578  ++pAllocationRequest->itemsToMakeLostCount;
8579  pAllocationRequest->sumItemSize += suballoc.size;
8580  }
8581  else
8582  {
8583  return false;
8584  }
8585  }
8586  }
8587  else
8588  {
8589  // Already on next page.
8590  break;
8591  }
8592  ++index1st;
8593  }
8594  }
8595  }
8596 
8597  // There is enough free space at the end after alignment.
8598  if(index1st == suballocations1st.size() && resultOffset + allocSize + VMA_DEBUG_MARGIN < size ||
8599  index1st < suballocations1st.size() && resultOffset + allocSize + VMA_DEBUG_MARGIN <= suballocations1st[index1st].offset)
8600  {
8601  // Check next suballocations for BufferImageGranularity conflicts.
8602  // If conflict exists, allocation cannot be made here.
8603  if(bufferImageGranularity > 1)
8604  {
8605  for(size_t nextSuballocIndex = index1st;
8606  nextSuballocIndex < suballocations1st.size();
8607  nextSuballocIndex++)
8608  {
8609  const VmaSuballocation& nextSuballoc = suballocations1st[nextSuballocIndex];
8610  if(VmaBlocksOnSamePage(resultOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
8611  {
8612  if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
8613  {
8614  return false;
8615  }
8616  }
8617  else
8618  {
8619  // Already on next page.
8620  break;
8621  }
8622  }
8623  }
8624 
8625  // All tests passed: Success.
8626  pAllocationRequest->offset = resultOffset;
8627  pAllocationRequest->sumFreeSize =
8628  (index1st < suballocations1st.size() ? suballocations1st[index1st].offset : size)
8629  - resultBaseOffset
8630  - pAllocationRequest->sumItemSize;
8631  // pAllocationRequest->item unused.
8632  return true;
8633  }
8634  }
8635  }
8636 
8637  return false;
8638 }
8639 
8640 bool VmaBlockMetadata_Linear::MakeRequestedAllocationsLost(
8641  uint32_t currentFrameIndex,
8642  uint32_t frameInUseCount,
8643  VmaAllocationRequest* pAllocationRequest)
8644 {
8645  if(pAllocationRequest->itemsToMakeLostCount == 0)
8646  {
8647  return true;
8648  }
8649 
8650  VMA_ASSERT(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER);
8651 
8652  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8653  size_t index1st = m_1stNullItemsBeginCount;
8654  size_t madeLostCount = 0;
8655  while(madeLostCount < pAllocationRequest->itemsToMakeLostCount)
8656  {
8657  VMA_ASSERT(index1st < suballocations1st.size());
8658  VmaSuballocation& suballoc = suballocations1st[index1st];
8659  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
8660  {
8661  VMA_ASSERT(suballoc.hAllocation != VK_NULL_HANDLE);
8662  VMA_ASSERT(suballoc.hAllocation->CanBecomeLost());
8663  if(suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
8664  {
8665  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
8666  suballoc.hAllocation = VK_NULL_HANDLE;
8667  m_SumFreeSize += suballoc.size;
8668  ++m_1stNullItemsMiddleCount;
8669  ++madeLostCount;
8670  }
8671  else
8672  {
8673  return false;
8674  }
8675  }
8676  ++index1st;
8677  }
8678 
8679  CleanupAfterFree();
8680  //VMA_HEAVY_ASSERT(Validate()); // Already called by ClanupAfterFree().
8681 
8682  return true;
8683 }
8684 
8685 uint32_t VmaBlockMetadata_Linear::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
8686 {
8687  uint32_t lostAllocationCount = 0;
8688 
8689  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8690  for(size_t i = m_1stNullItemsBeginCount, count = suballocations1st.size(); i < count; ++i)
8691  {
8692  VmaSuballocation& suballoc = suballocations1st[i];
8693  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE &&
8694  suballoc.hAllocation->CanBecomeLost() &&
8695  suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
8696  {
8697  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
8698  suballoc.hAllocation = VK_NULL_HANDLE;
8699  ++m_1stNullItemsMiddleCount;
8700  m_SumFreeSize += suballoc.size;
8701  ++lostAllocationCount;
8702  }
8703  }
8704 
8705  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8706  for(size_t i = 0, count = suballocations2nd.size(); i < count; ++i)
8707  {
8708  VmaSuballocation& suballoc = suballocations2nd[i];
8709  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE &&
8710  suballoc.hAllocation->CanBecomeLost() &&
8711  suballoc.hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
8712  {
8713  suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
8714  suballoc.hAllocation = VK_NULL_HANDLE;
8715  ++m_2ndNullItemsCount;
8716  ++lostAllocationCount;
8717  }
8718  }
8719 
8720  if(lostAllocationCount)
8721  {
8722  CleanupAfterFree();
8723  }
8724 
8725  return lostAllocationCount;
8726 }
8727 
8728 VkResult VmaBlockMetadata_Linear::CheckCorruption(const void* pBlockData)
8729 {
8730  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8731  for(size_t i = m_1stNullItemsBeginCount, count = suballocations1st.size(); i < count; ++i)
8732  {
8733  const VmaSuballocation& suballoc = suballocations1st[i];
8734  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
8735  {
8736  if(!VmaValidateMagicValue(pBlockData, suballoc.offset - VMA_DEBUG_MARGIN))
8737  {
8738  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
8739  return VK_ERROR_VALIDATION_FAILED_EXT;
8740  }
8741  if(!VmaValidateMagicValue(pBlockData, suballoc.offset + suballoc.size))
8742  {
8743  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
8744  return VK_ERROR_VALIDATION_FAILED_EXT;
8745  }
8746  }
8747  }
8748 
8749  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8750  for(size_t i = 0, count = suballocations2nd.size(); i < count; ++i)
8751  {
8752  const VmaSuballocation& suballoc = suballocations2nd[i];
8753  if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
8754  {
8755  if(!VmaValidateMagicValue(pBlockData, suballoc.offset - VMA_DEBUG_MARGIN))
8756  {
8757  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!");
8758  return VK_ERROR_VALIDATION_FAILED_EXT;
8759  }
8760  if(!VmaValidateMagicValue(pBlockData, suballoc.offset + suballoc.size))
8761  {
8762  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!");
8763  return VK_ERROR_VALIDATION_FAILED_EXT;
8764  }
8765  }
8766  }
8767 
8768  return VK_SUCCESS;
8769 }
8770 
8771 void VmaBlockMetadata_Linear::Alloc(
8772  const VmaAllocationRequest& request,
8773  VmaSuballocationType type,
8774  VkDeviceSize allocSize,
8775  bool upperAddress,
8776  VmaAllocation hAllocation)
8777 {
8778  const VmaSuballocation newSuballoc = { request.offset, allocSize, hAllocation, type };
8779 
8780  if(upperAddress)
8781  {
8782  VMA_ASSERT(m_2ndVectorMode != SECOND_VECTOR_RING_BUFFER &&
8783  "CRITICAL ERROR: Trying to use linear allocator as double stack while it was already used as ring buffer.");
8784  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8785  suballocations2nd.push_back(newSuballoc);
8786  m_2ndVectorMode = SECOND_VECTOR_DOUBLE_STACK;
8787  }
8788  else
8789  {
8790  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8791 
8792  // First allocation.
8793  if(suballocations1st.empty())
8794  {
8795  suballocations1st.push_back(newSuballoc);
8796  }
8797  else
8798  {
8799  // New allocation at the end of 1st vector.
8800  if(request.offset >= suballocations1st.back().offset + suballocations1st.back().size)
8801  {
8802  // Check if it fits before the end of the block.
8803  VMA_ASSERT(request.offset + allocSize <= GetSize());
8804  suballocations1st.push_back(newSuballoc);
8805  }
8806  // New allocation at the end of 2-part ring buffer, so before first allocation from 1st vector.
8807  else if(request.offset + allocSize <= suballocations1st[m_1stNullItemsBeginCount].offset)
8808  {
8809  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8810 
8811  switch(m_2ndVectorMode)
8812  {
8813  case SECOND_VECTOR_EMPTY:
8814  // First allocation from second part ring buffer.
8815  VMA_ASSERT(suballocations2nd.empty());
8816  m_2ndVectorMode = SECOND_VECTOR_RING_BUFFER;
8817  break;
8818  case SECOND_VECTOR_RING_BUFFER:
8819  // 2-part ring buffer is already started.
8820  VMA_ASSERT(!suballocations2nd.empty());
8821  break;
8822  case SECOND_VECTOR_DOUBLE_STACK:
8823  VMA_ASSERT(0 && "CRITICAL ERROR: Trying to use linear allocator as ring buffer while it was already used as double stack.");
8824  break;
8825  default:
8826  VMA_ASSERT(0);
8827  }
8828 
8829  suballocations2nd.push_back(newSuballoc);
8830  }
8831  else
8832  {
8833  VMA_ASSERT(0 && "CRITICAL INTERNAL ERROR.");
8834  }
8835  }
8836  }
8837 
8838  m_SumFreeSize -= newSuballoc.size;
8839 }
8840 
8841 void VmaBlockMetadata_Linear::Free(const VmaAllocation allocation)
8842 {
8843  FreeAtOffset(allocation->GetOffset());
8844 }
8845 
8846 void VmaBlockMetadata_Linear::FreeAtOffset(VkDeviceSize offset)
8847 {
8848  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8849  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8850 
8851  if(!suballocations1st.empty())
8852  {
8853  // First allocation: Mark it as next empty at the beginning.
8854  VmaSuballocation& firstSuballoc = suballocations1st[m_1stNullItemsBeginCount];
8855  if(firstSuballoc.offset == offset)
8856  {
8857  firstSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
8858  firstSuballoc.hAllocation = VK_NULL_HANDLE;
8859  m_SumFreeSize += firstSuballoc.size;
8860  ++m_1stNullItemsBeginCount;
8861  CleanupAfterFree();
8862  return;
8863  }
8864  }
8865 
8866  // Last allocation in 2-part ring buffer or top of upper stack (same logic).
8867  if(m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER ||
8868  m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK)
8869  {
8870  VmaSuballocation& lastSuballoc = suballocations2nd.back();
8871  if(lastSuballoc.offset == offset)
8872  {
8873  m_SumFreeSize += lastSuballoc.size;
8874  suballocations2nd.pop_back();
8875  CleanupAfterFree();
8876  return;
8877  }
8878  }
8879  // Last allocation in 1st vector.
8880  else if(m_2ndVectorMode == SECOND_VECTOR_EMPTY)
8881  {
8882  VmaSuballocation& lastSuballoc = suballocations1st.back();
8883  if(lastSuballoc.offset == offset)
8884  {
8885  m_SumFreeSize += lastSuballoc.size;
8886  suballocations1st.pop_back();
8887  CleanupAfterFree();
8888  return;
8889  }
8890  }
8891 
8892  // Item from the middle of 1st vector.
8893  {
8894  VmaSuballocation refSuballoc;
8895  refSuballoc.offset = offset;
8896  // Rest of members stays uninitialized intentionally for better performance.
8897  SuballocationVectorType::iterator it = VmaVectorFindSorted<VmaSuballocationOffsetLess>(
8898  suballocations1st.begin() + m_1stNullItemsBeginCount,
8899  suballocations1st.end(),
8900  refSuballoc);
8901  if(it != suballocations1st.end())
8902  {
8903  it->type = VMA_SUBALLOCATION_TYPE_FREE;
8904  it->hAllocation = VK_NULL_HANDLE;
8905  ++m_1stNullItemsMiddleCount;
8906  m_SumFreeSize += it->size;
8907  CleanupAfterFree();
8908  return;
8909  }
8910  }
8911 
8912  if(m_2ndVectorMode != SECOND_VECTOR_EMPTY)
8913  {
8914  // Item from the middle of 2nd vector.
8915  VmaSuballocation refSuballoc;
8916  refSuballoc.offset = offset;
8917  // Rest of members stays uninitialized intentionally for better performance.
8918  SuballocationVectorType::iterator it = m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER ?
8919  VmaVectorFindSorted<VmaSuballocationOffsetLess>(suballocations2nd.begin(), suballocations2nd.end(), refSuballoc) :
8920  VmaVectorFindSorted<VmaSuballocationOffsetGreater>(suballocations2nd.begin(), suballocations2nd.end(), refSuballoc);
8921  if(it != suballocations2nd.end())
8922  {
8923  it->type = VMA_SUBALLOCATION_TYPE_FREE;
8924  it->hAllocation = VK_NULL_HANDLE;
8925  ++m_2ndNullItemsCount;
8926  m_SumFreeSize += it->size;
8927  CleanupAfterFree();
8928  return;
8929  }
8930  }
8931 
8932  VMA_ASSERT(0 && "Allocation to free not found in linear allocator!");
8933 }
8934 
8935 bool VmaBlockMetadata_Linear::ShouldCompact1st() const
8936 {
8937  const size_t nullItemCount = m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount;
8938  const size_t suballocCount = AccessSuballocations1st().size();
8939  return suballocCount > 32 && nullItemCount * 2 >= (suballocCount - nullItemCount) * 3;
8940 }
8941 
8942 void VmaBlockMetadata_Linear::CleanupAfterFree()
8943 {
8944  SuballocationVectorType& suballocations1st = AccessSuballocations1st();
8945  SuballocationVectorType& suballocations2nd = AccessSuballocations2nd();
8946 
8947  if(IsEmpty())
8948  {
8949  suballocations1st.clear();
8950  suballocations2nd.clear();
8951  m_1stNullItemsBeginCount = 0;
8952  m_1stNullItemsMiddleCount = 0;
8953  m_2ndNullItemsCount = 0;
8954  m_2ndVectorMode = SECOND_VECTOR_EMPTY;
8955  }
8956  else
8957  {
8958  const size_t suballoc1stCount = suballocations1st.size();
8959  const size_t nullItem1stCount = m_1stNullItemsBeginCount + m_1stNullItemsMiddleCount;
8960  VMA_ASSERT(nullItem1stCount <= suballoc1stCount);
8961 
8962  // Find more null items at the beginning of 1st vector.
8963  while(m_1stNullItemsBeginCount < suballoc1stCount &&
8964  suballocations1st[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE)
8965  {
8966  ++m_1stNullItemsBeginCount;
8967  --m_1stNullItemsMiddleCount;
8968  }
8969 
8970  // Find more null items at the end of 1st vector.
8971  while(m_1stNullItemsMiddleCount > 0 &&
8972  suballocations1st.back().hAllocation == VK_NULL_HANDLE)
8973  {
8974  --m_1stNullItemsMiddleCount;
8975  suballocations1st.pop_back();
8976  }
8977 
8978  // Find more null items at the end of 2nd vector.
8979  while(m_2ndNullItemsCount > 0 &&
8980  suballocations2nd.back().hAllocation == VK_NULL_HANDLE)
8981  {
8982  --m_2ndNullItemsCount;
8983  suballocations2nd.pop_back();
8984  }
8985 
8986  if(ShouldCompact1st())
8987  {
8988  const size_t nonNullItemCount = suballoc1stCount - nullItem1stCount;
8989  size_t srcIndex = m_1stNullItemsBeginCount;
8990  for(size_t dstIndex = 0; dstIndex < nonNullItemCount; ++dstIndex)
8991  {
8992  while(suballocations1st[srcIndex].hAllocation == VK_NULL_HANDLE)
8993  {
8994  ++srcIndex;
8995  }
8996  if(dstIndex != srcIndex)
8997  {
8998  suballocations1st[dstIndex] = suballocations1st[srcIndex];
8999  }
9000  ++srcIndex;
9001  }
9002  suballocations1st.resize(nonNullItemCount);
9003  m_1stNullItemsBeginCount = 0;
9004  m_1stNullItemsMiddleCount = 0;
9005  }
9006 
9007  // 2nd vector became empty.
9008  if(suballocations2nd.empty())
9009  {
9010  m_2ndVectorMode = SECOND_VECTOR_EMPTY;
9011  }
9012 
9013  // 1st vector became empty.
9014  if(suballocations1st.size() - m_1stNullItemsBeginCount == 0)
9015  {
9016  suballocations1st.clear();
9017  m_1stNullItemsBeginCount = 0;
9018 
9019  if(!suballocations2nd.empty() && m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER)
9020  {
9021  // Swap 1st with 2nd. Now 2nd is empty.
9022  m_2ndVectorMode = SECOND_VECTOR_EMPTY;
9023  m_1stNullItemsMiddleCount = m_2ndNullItemsCount;
9024  while(m_1stNullItemsBeginCount < suballocations2nd.size() &&
9025  suballocations2nd[m_1stNullItemsBeginCount].hAllocation == VK_NULL_HANDLE)
9026  {
9027  ++m_1stNullItemsBeginCount;
9028  --m_1stNullItemsMiddleCount;
9029  }
9030  m_2ndNullItemsCount = 0;
9031  m_1stVectorIndex ^= 1;
9032  }
9033  }
9034  }
9035 
9036  VMA_HEAVY_ASSERT(Validate());
9037 }
9038 
9039 
9041 // class VmaDeviceMemoryBlock
9042 
9043 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
9044  m_pMetadata(VMA_NULL),
9045  m_MemoryTypeIndex(UINT32_MAX),
9046  m_Id(0),
9047  m_hMemory(VK_NULL_HANDLE),
9048  m_MapCount(0),
9049  m_pMappedData(VMA_NULL)
9050 {
9051 }
9052 
9053 void VmaDeviceMemoryBlock::Init(
9054  VmaAllocator hAllocator,
9055  uint32_t newMemoryTypeIndex,
9056  VkDeviceMemory newMemory,
9057  VkDeviceSize newSize,
9058  uint32_t id,
9059  bool linearAlgorithm)
9060 {
9061  VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
9062 
9063  m_MemoryTypeIndex = newMemoryTypeIndex;
9064  m_Id = id;
9065  m_hMemory = newMemory;
9066 
9067  if(linearAlgorithm)
9068  {
9069  m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Linear)(hAllocator);
9070  }
9071  else
9072  {
9073  m_pMetadata = vma_new(hAllocator, VmaBlockMetadata_Generic)(hAllocator);
9074  }
9075  m_pMetadata->Init(newSize);
9076 }
9077 
9078 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
9079 {
9080  // This is the most important assert in the entire library.
9081  // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
9082  VMA_ASSERT(m_pMetadata->IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
9083 
9084  VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
9085  allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_pMetadata->GetSize(), m_hMemory);
9086  m_hMemory = VK_NULL_HANDLE;
9087 
9088  vma_delete(allocator, m_pMetadata);
9089  m_pMetadata = VMA_NULL;
9090 }
9091 
9092 bool VmaDeviceMemoryBlock::Validate() const
9093 {
9094  if((m_hMemory == VK_NULL_HANDLE) ||
9095  (m_pMetadata->GetSize() == 0))
9096  {
9097  return false;
9098  }
9099 
9100  return m_pMetadata->Validate();
9101 }
9102 
9103 VkResult VmaDeviceMemoryBlock::CheckCorruption(VmaAllocator hAllocator)
9104 {
9105  void* pData = nullptr;
9106  VkResult res = Map(hAllocator, 1, &pData);
9107  if(res != VK_SUCCESS)
9108  {
9109  return res;
9110  }
9111 
9112  res = m_pMetadata->CheckCorruption(pData);
9113 
9114  Unmap(hAllocator, 1);
9115 
9116  return res;
9117 }
9118 
9119 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
9120 {
9121  if(count == 0)
9122  {
9123  return VK_SUCCESS;
9124  }
9125 
9126  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
9127  if(m_MapCount != 0)
9128  {
9129  m_MapCount += count;
9130  VMA_ASSERT(m_pMappedData != VMA_NULL);
9131  if(ppData != VMA_NULL)
9132  {
9133  *ppData = m_pMappedData;
9134  }
9135  return VK_SUCCESS;
9136  }
9137  else
9138  {
9139  VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
9140  hAllocator->m_hDevice,
9141  m_hMemory,
9142  0, // offset
9143  VK_WHOLE_SIZE,
9144  0, // flags
9145  &m_pMappedData);
9146  if(result == VK_SUCCESS)
9147  {
9148  if(ppData != VMA_NULL)
9149  {
9150  *ppData = m_pMappedData;
9151  }
9152  m_MapCount = count;
9153  }
9154  return result;
9155  }
9156 }
9157 
9158 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
9159 {
9160  if(count == 0)
9161  {
9162  return;
9163  }
9164 
9165  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
9166  if(m_MapCount >= count)
9167  {
9168  m_MapCount -= count;
9169  if(m_MapCount == 0)
9170  {
9171  m_pMappedData = VMA_NULL;
9172  (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
9173  }
9174  }
9175  else
9176  {
9177  VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
9178  }
9179 }
9180 
9181 VkResult VmaDeviceMemoryBlock::WriteMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
9182 {
9183  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
9184  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
9185 
9186  void* pData;
9187  VkResult res = Map(hAllocator, 1, &pData);
9188  if(res != VK_SUCCESS)
9189  {
9190  return res;
9191  }
9192 
9193  VmaWriteMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN);
9194  VmaWriteMagicValue(pData, allocOffset + allocSize);
9195 
9196  Unmap(hAllocator, 1);
9197 
9198  return VK_SUCCESS;
9199 }
9200 
9201 VkResult VmaDeviceMemoryBlock::ValidateMagicValueAroundAllocation(VmaAllocator hAllocator, VkDeviceSize allocOffset, VkDeviceSize allocSize)
9202 {
9203  VMA_ASSERT(VMA_DEBUG_MARGIN > 0 && VMA_DEBUG_MARGIN % 4 == 0 && VMA_DEBUG_DETECT_CORRUPTION);
9204  VMA_ASSERT(allocOffset >= VMA_DEBUG_MARGIN);
9205 
9206  void* pData;
9207  VkResult res = Map(hAllocator, 1, &pData);
9208  if(res != VK_SUCCESS)
9209  {
9210  return res;
9211  }
9212 
9213  if(!VmaValidateMagicValue(pData, allocOffset - VMA_DEBUG_MARGIN))
9214  {
9215  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE FREED ALLOCATION!");
9216  }
9217  else if(!VmaValidateMagicValue(pData, allocOffset + allocSize))
9218  {
9219  VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER FREED ALLOCATION!");
9220  }
9221 
9222  Unmap(hAllocator, 1);
9223 
9224  return VK_SUCCESS;
9225 }
9226 
9227 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
9228  const VmaAllocator hAllocator,
9229  const VmaAllocation hAllocation,
9230  VkBuffer hBuffer)
9231 {
9232  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
9233  hAllocation->GetBlock() == this);
9234  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
9235  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
9236  return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
9237  hAllocator->m_hDevice,
9238  hBuffer,
9239  m_hMemory,
9240  hAllocation->GetOffset());
9241 }
9242 
9243 VkResult VmaDeviceMemoryBlock::BindImageMemory(
9244  const VmaAllocator hAllocator,
9245  const VmaAllocation hAllocation,
9246  VkImage hImage)
9247 {
9248  VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
9249  hAllocation->GetBlock() == this);
9250  // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
9251  VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
9252  return hAllocator->GetVulkanFunctions().vkBindImageMemory(
9253  hAllocator->m_hDevice,
9254  hImage,
9255  m_hMemory,
9256  hAllocation->GetOffset());
9257 }
9258 
9259 static void InitStatInfo(VmaStatInfo& outInfo)
9260 {
9261  memset(&outInfo, 0, sizeof(outInfo));
9262  outInfo.allocationSizeMin = UINT64_MAX;
9263  outInfo.unusedRangeSizeMin = UINT64_MAX;
9264 }
9265 
9266 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
9267 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
9268 {
9269  inoutInfo.blockCount += srcInfo.blockCount;
9270  inoutInfo.allocationCount += srcInfo.allocationCount;
9271  inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
9272  inoutInfo.usedBytes += srcInfo.usedBytes;
9273  inoutInfo.unusedBytes += srcInfo.unusedBytes;
9274  inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
9275  inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
9276  inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
9277  inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
9278 }
9279 
9280 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
9281 {
9282  inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
9283  VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
9284  inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
9285  VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
9286 }
9287 
9288 VmaPool_T::VmaPool_T(
9289  VmaAllocator hAllocator,
9290  const VmaPoolCreateInfo& createInfo) :
9291  m_BlockVector(
9292  hAllocator,
9293  createInfo.memoryTypeIndex,
9294  createInfo.blockSize,
9295  createInfo.minBlockCount,
9296  createInfo.maxBlockCount,
9297  (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
9298  createInfo.frameInUseCount,
9299  true, // isCustomPool
9300  (createInfo.flags & VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT) != 0), // linearAlgorithm
9301  m_Id(0)
9302 {
9303 }
9304 
9305 VmaPool_T::~VmaPool_T()
9306 {
9307 }
9308 
9309 #if VMA_STATS_STRING_ENABLED
9310 
9311 #endif // #if VMA_STATS_STRING_ENABLED
9312 
9313 VmaBlockVector::VmaBlockVector(
9314  VmaAllocator hAllocator,
9315  uint32_t memoryTypeIndex,
9316  VkDeviceSize preferredBlockSize,
9317  size_t minBlockCount,
9318  size_t maxBlockCount,
9319  VkDeviceSize bufferImageGranularity,
9320  uint32_t frameInUseCount,
9321  bool isCustomPool,
9322  bool linearAlgorithm) :
9323  m_hAllocator(hAllocator),
9324  m_MemoryTypeIndex(memoryTypeIndex),
9325  m_PreferredBlockSize(preferredBlockSize),
9326  m_MinBlockCount(minBlockCount),
9327  m_MaxBlockCount(maxBlockCount),
9328  m_BufferImageGranularity(bufferImageGranularity),
9329  m_FrameInUseCount(frameInUseCount),
9330  m_IsCustomPool(isCustomPool),
9331  m_LinearAlgorithm(linearAlgorithm),
9332  m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
9333  m_HasEmptyBlock(false),
9334  m_pDefragmentator(VMA_NULL),
9335  m_NextBlockId(0)
9336 {
9337 }
9338 
9339 VmaBlockVector::~VmaBlockVector()
9340 {
9341  VMA_ASSERT(m_pDefragmentator == VMA_NULL);
9342 
9343  for(size_t i = m_Blocks.size(); i--; )
9344  {
9345  m_Blocks[i]->Destroy(m_hAllocator);
9346  vma_delete(m_hAllocator, m_Blocks[i]);
9347  }
9348 }
9349 
9350 VkResult VmaBlockVector::CreateMinBlocks()
9351 {
9352  for(size_t i = 0; i < m_MinBlockCount; ++i)
9353  {
9354  VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
9355  if(res != VK_SUCCESS)
9356  {
9357  return res;
9358  }
9359  }
9360  return VK_SUCCESS;
9361 }
9362 
9363 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
9364 {
9365  pStats->size = 0;
9366  pStats->unusedSize = 0;
9367  pStats->allocationCount = 0;
9368  pStats->unusedRangeCount = 0;
9369  pStats->unusedRangeSizeMax = 0;
9370 
9371  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9372 
9373  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
9374  {
9375  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
9376  VMA_ASSERT(pBlock);
9377  VMA_HEAVY_ASSERT(pBlock->Validate());
9378  pBlock->m_pMetadata->AddPoolStats(*pStats);
9379  }
9380 }
9381 
9382 bool VmaBlockVector::IsCorruptionDetectionEnabled() const
9383 {
9384  const uint32_t requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
9385  return (VMA_DEBUG_DETECT_CORRUPTION != 0) &&
9386  (VMA_DEBUG_MARGIN > 0) &&
9387  (m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags;
9388 }
9389 
9390 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
9391 
9392 VkResult VmaBlockVector::Allocate(
9393  VmaPool hCurrentPool,
9394  uint32_t currentFrameIndex,
9395  VkDeviceSize size,
9396  VkDeviceSize alignment,
9397  const VmaAllocationCreateInfo& createInfo,
9398  VmaSuballocationType suballocType,
9399  VmaAllocation* pAllocation)
9400 {
9401  const bool isUpperAddress = (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0;
9402  const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
9403  const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
9404  const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
9405  const bool canCreateNewBlock =
9406  ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
9407  (m_Blocks.size() < m_MaxBlockCount);
9408 
9409  // Upper address can only be used with linear allocator.
9410  if(isUpperAddress && !m_LinearAlgorithm)
9411  {
9412  return VK_ERROR_FEATURE_NOT_PRESENT;
9413  }
9414 
9415  // Early reject: requested allocation size is larger that maximum block size for this block vector.
9416  if(size + 2 * VMA_DEBUG_MARGIN > m_PreferredBlockSize)
9417  {
9418  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9419  }
9420 
9421  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9422 
9423  /*
9424  Under certain condition, this whole section can be skipped for optimization, so
9425  we move on directly to trying to allocate with canMakeOtherLost. That's the case
9426  e.g. for custom pools with linear algorithm.
9427  */
9428  if(!canMakeOtherLost || canCreateNewBlock)
9429  {
9430  // 1. Search existing allocations. Try to allocate without making other allocations lost.
9431  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
9432  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
9433  {
9434  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
9435  VMA_ASSERT(pCurrBlock);
9436  VmaAllocationRequest currRequest = {};
9437  if(pCurrBlock->m_pMetadata->CreateAllocationRequest(
9438  currentFrameIndex,
9439  m_FrameInUseCount,
9440  m_BufferImageGranularity,
9441  size,
9442  alignment,
9443  isUpperAddress,
9444  suballocType,
9445  false, // canMakeOtherLost
9446  &currRequest))
9447  {
9448  // Allocate from pCurrBlock.
9449  VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
9450 
9451  if(mapped)
9452  {
9453  VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
9454  if(res != VK_SUCCESS)
9455  {
9456  return res;
9457  }
9458  }
9459 
9460  // We no longer have an empty Allocation.
9461  if(pCurrBlock->m_pMetadata->IsEmpty())
9462  {
9463  m_HasEmptyBlock = false;
9464  }
9465 
9466  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
9467  pCurrBlock->m_pMetadata->Alloc(currRequest, suballocType, size, isUpperAddress, *pAllocation);
9468  (*pAllocation)->InitBlockAllocation(
9469  hCurrentPool,
9470  pCurrBlock,
9471  currRequest.offset,
9472  alignment,
9473  size,
9474  suballocType,
9475  mapped,
9476  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
9477  VMA_HEAVY_ASSERT(pCurrBlock->Validate());
9478  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
9479  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
9480  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
9481  {
9482  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
9483  }
9484  if(IsCorruptionDetectionEnabled())
9485  {
9486  VkResult res = pCurrBlock->WriteMagicValueAroundAllocation(m_hAllocator, currRequest.offset, size);
9487  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
9488  }
9489  return VK_SUCCESS;
9490  }
9491  }
9492 
9493  // 2. Try to create new block.
9494  if(canCreateNewBlock)
9495  {
9496  // Calculate optimal size for new block.
9497  VkDeviceSize newBlockSize = m_PreferredBlockSize;
9498  uint32_t newBlockSizeShift = 0;
9499  const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
9500 
9501  // Allocating blocks of other sizes is allowed only in default pools.
9502  // In custom pools block size is fixed.
9503  if(m_IsCustomPool == false)
9504  {
9505  // Allocate 1/8, 1/4, 1/2 as first blocks.
9506  const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
9507  for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
9508  {
9509  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
9510  if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= size * 2)
9511  {
9512  newBlockSize = smallerNewBlockSize;
9513  ++newBlockSizeShift;
9514  }
9515  else
9516  {
9517  break;
9518  }
9519  }
9520  }
9521 
9522  size_t newBlockIndex = 0;
9523  VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
9524  // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
9525  if(m_IsCustomPool == false)
9526  {
9527  while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
9528  {
9529  const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
9530  if(smallerNewBlockSize >= size)
9531  {
9532  newBlockSize = smallerNewBlockSize;
9533  ++newBlockSizeShift;
9534  res = CreateBlock(newBlockSize, &newBlockIndex);
9535  }
9536  else
9537  {
9538  break;
9539  }
9540  }
9541  }
9542 
9543  if(res == VK_SUCCESS)
9544  {
9545  VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
9546  VMA_ASSERT(pBlock->m_pMetadata->GetSize() >= size);
9547 
9548  if(mapped)
9549  {
9550  res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
9551  if(res != VK_SUCCESS)
9552  {
9553  return res;
9554  }
9555  }
9556 
9557  // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
9558  VmaAllocationRequest allocRequest;
9559  if(pBlock->m_pMetadata->CreateAllocationRequest(
9560  currentFrameIndex,
9561  m_FrameInUseCount,
9562  m_BufferImageGranularity,
9563  size,
9564  alignment,
9565  isUpperAddress,
9566  suballocType,
9567  false, // canMakeOtherLost
9568  &allocRequest))
9569  {
9570  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
9571  pBlock->m_pMetadata->Alloc(allocRequest, suballocType, size, isUpperAddress, *pAllocation);
9572  (*pAllocation)->InitBlockAllocation(
9573  hCurrentPool,
9574  pBlock,
9575  allocRequest.offset,
9576  alignment,
9577  size,
9578  suballocType,
9579  mapped,
9580  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
9581  VMA_HEAVY_ASSERT(pBlock->Validate());
9582  VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
9583  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
9584  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
9585  {
9586  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
9587  }
9588  if(IsCorruptionDetectionEnabled())
9589  {
9590  res = pBlock->WriteMagicValueAroundAllocation(m_hAllocator, allocRequest.offset, size);
9591  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
9592  }
9593  return VK_SUCCESS;
9594  }
9595  else
9596  {
9597  // Allocation from empty block failed, possibly due to VMA_DEBUG_MARGIN or alignment.
9598  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9599  }
9600  }
9601  }
9602  }
9603 
9604  // 3. Try to allocate from existing blocks with making other allocations lost.
9605  if(canMakeOtherLost)
9606  {
9607  uint32_t tryIndex = 0;
9608  for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
9609  {
9610  VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
9611  VmaAllocationRequest bestRequest = {};
9612  VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
9613 
9614  // 1. Search existing allocations.
9615  // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
9616  for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
9617  {
9618  VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
9619  VMA_ASSERT(pCurrBlock);
9620  VmaAllocationRequest currRequest = {};
9621  if(pCurrBlock->m_pMetadata->CreateAllocationRequest(
9622  currentFrameIndex,
9623  m_FrameInUseCount,
9624  m_BufferImageGranularity,
9625  size,
9626  alignment,
9627  (createInfo.flags & VMA_ALLOCATION_CREATE_UPPER_ADDRESS_BIT) != 0,
9628  suballocType,
9629  canMakeOtherLost,
9630  &currRequest))
9631  {
9632  const VkDeviceSize currRequestCost = currRequest.CalcCost();
9633  if(pBestRequestBlock == VMA_NULL ||
9634  currRequestCost < bestRequestCost)
9635  {
9636  pBestRequestBlock = pCurrBlock;
9637  bestRequest = currRequest;
9638  bestRequestCost = currRequestCost;
9639 
9640  if(bestRequestCost == 0)
9641  {
9642  break;
9643  }
9644  }
9645  }
9646  }
9647 
9648  if(pBestRequestBlock != VMA_NULL)
9649  {
9650  if(mapped)
9651  {
9652  VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
9653  if(res != VK_SUCCESS)
9654  {
9655  return res;
9656  }
9657  }
9658 
9659  if(pBestRequestBlock->m_pMetadata->MakeRequestedAllocationsLost(
9660  currentFrameIndex,
9661  m_FrameInUseCount,
9662  &bestRequest))
9663  {
9664  // We no longer have an empty Allocation.
9665  if(pBestRequestBlock->m_pMetadata->IsEmpty())
9666  {
9667  m_HasEmptyBlock = false;
9668  }
9669  // Allocate from this pBlock.
9670  *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
9671  pBestRequestBlock->m_pMetadata->Alloc(bestRequest, suballocType, size, isUpperAddress, *pAllocation);
9672  (*pAllocation)->InitBlockAllocation(
9673  hCurrentPool,
9674  pBestRequestBlock,
9675  bestRequest.offset,
9676  alignment,
9677  size,
9678  suballocType,
9679  mapped,
9680  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
9681  VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
9682  VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
9683  (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
9684  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
9685  {
9686  m_hAllocator->FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
9687  }
9688  if(IsCorruptionDetectionEnabled())
9689  {
9690  VkResult res = pBestRequestBlock->WriteMagicValueAroundAllocation(m_hAllocator, bestRequest.offset, size);
9691  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to write magic value.");
9692  }
9693  return VK_SUCCESS;
9694  }
9695  // else: Some allocations must have been touched while we are here. Next try.
9696  }
9697  else
9698  {
9699  // Could not find place in any of the blocks - break outer loop.
9700  break;
9701  }
9702  }
9703  /* Maximum number of tries exceeded - a very unlike event when many other
9704  threads are simultaneously touching allocations making it impossible to make
9705  lost at the same time as we try to allocate. */
9706  if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
9707  {
9708  return VK_ERROR_TOO_MANY_OBJECTS;
9709  }
9710  }
9711 
9712  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
9713 }
9714 
9715 void VmaBlockVector::Free(
9716  VmaAllocation hAllocation)
9717 {
9718  VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
9719 
9720  // Scope for lock.
9721  {
9722  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9723 
9724  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
9725 
9726  if(IsCorruptionDetectionEnabled())
9727  {
9728  VkResult res = pBlock->ValidateMagicValueAroundAllocation(m_hAllocator, hAllocation->GetOffset(), hAllocation->GetSize());
9729  VMA_ASSERT(res == VK_SUCCESS && "Couldn't map block memory to validate magic value.");
9730  }
9731 
9732  if(hAllocation->IsPersistentMap())
9733  {
9734  pBlock->Unmap(m_hAllocator, 1);
9735  }
9736 
9737  pBlock->m_pMetadata->Free(hAllocation);
9738  VMA_HEAVY_ASSERT(pBlock->Validate());
9739 
9740  VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
9741 
9742  // pBlock became empty after this deallocation.
9743  if(pBlock->m_pMetadata->IsEmpty())
9744  {
9745  // Already has empty Allocation. We don't want to have two, so delete this one.
9746  if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
9747  {
9748  pBlockToDelete = pBlock;
9749  Remove(pBlock);
9750  }
9751  // We now have first empty block.
9752  else
9753  {
9754  m_HasEmptyBlock = true;
9755  }
9756  }
9757  // pBlock didn't become empty, but we have another empty block - find and free that one.
9758  // (This is optional, heuristics.)
9759  else if(m_HasEmptyBlock)
9760  {
9761  VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
9762  if(pLastBlock->m_pMetadata->IsEmpty() && m_Blocks.size() > m_MinBlockCount)
9763  {
9764  pBlockToDelete = pLastBlock;
9765  m_Blocks.pop_back();
9766  m_HasEmptyBlock = false;
9767  }
9768  }
9769 
9770  IncrementallySortBlocks();
9771  }
9772 
9773  // Destruction of a free Allocation. Deferred until this point, outside of mutex
9774  // lock, for performance reason.
9775  if(pBlockToDelete != VMA_NULL)
9776  {
9777  VMA_DEBUG_LOG(" Deleted empty allocation");
9778  pBlockToDelete->Destroy(m_hAllocator);
9779  vma_delete(m_hAllocator, pBlockToDelete);
9780  }
9781 }
9782 
9783 VkDeviceSize VmaBlockVector::CalcMaxBlockSize() const
9784 {
9785  VkDeviceSize result = 0;
9786  for(size_t i = m_Blocks.size(); i--; )
9787  {
9788  result = VMA_MAX(result, m_Blocks[i]->m_pMetadata->GetSize());
9789  if(result >= m_PreferredBlockSize)
9790  {
9791  break;
9792  }
9793  }
9794  return result;
9795 }
9796 
9797 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
9798 {
9799  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
9800  {
9801  if(m_Blocks[blockIndex] == pBlock)
9802  {
9803  VmaVectorRemove(m_Blocks, blockIndex);
9804  return;
9805  }
9806  }
9807  VMA_ASSERT(0);
9808 }
9809 
9810 void VmaBlockVector::IncrementallySortBlocks()
9811 {
9812  // Bubble sort only until first swap.
9813  for(size_t i = 1; i < m_Blocks.size(); ++i)
9814  {
9815  if(m_Blocks[i - 1]->m_pMetadata->GetSumFreeSize() > m_Blocks[i]->m_pMetadata->GetSumFreeSize())
9816  {
9817  VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
9818  return;
9819  }
9820  }
9821 }
9822 
9823 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
9824 {
9825  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
9826  allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
9827  allocInfo.allocationSize = blockSize;
9828  VkDeviceMemory mem = VK_NULL_HANDLE;
9829  VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
9830  if(res < 0)
9831  {
9832  return res;
9833  }
9834 
9835  // New VkDeviceMemory successfully created.
9836 
9837  // Create new Allocation for it.
9838  VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
9839  pBlock->Init(
9840  m_hAllocator,
9841  m_MemoryTypeIndex,
9842  mem,
9843  allocInfo.allocationSize,
9844  m_NextBlockId++,
9845  m_LinearAlgorithm);
9846 
9847  m_Blocks.push_back(pBlock);
9848  if(pNewBlockIndex != VMA_NULL)
9849  {
9850  *pNewBlockIndex = m_Blocks.size() - 1;
9851  }
9852 
9853  return VK_SUCCESS;
9854 }
9855 
9856 #if VMA_STATS_STRING_ENABLED
9857 
9858 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
9859 {
9860  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9861 
9862  json.BeginObject();
9863 
9864  if(m_IsCustomPool)
9865  {
9866  json.WriteString("MemoryTypeIndex");
9867  json.WriteNumber(m_MemoryTypeIndex);
9868 
9869  json.WriteString("BlockSize");
9870  json.WriteNumber(m_PreferredBlockSize);
9871 
9872  json.WriteString("BlockCount");
9873  json.BeginObject(true);
9874  if(m_MinBlockCount > 0)
9875  {
9876  json.WriteString("Min");
9877  json.WriteNumber((uint64_t)m_MinBlockCount);
9878  }
9879  if(m_MaxBlockCount < SIZE_MAX)
9880  {
9881  json.WriteString("Max");
9882  json.WriteNumber((uint64_t)m_MaxBlockCount);
9883  }
9884  json.WriteString("Cur");
9885  json.WriteNumber((uint64_t)m_Blocks.size());
9886  json.EndObject();
9887 
9888  if(m_FrameInUseCount > 0)
9889  {
9890  json.WriteString("FrameInUseCount");
9891  json.WriteNumber(m_FrameInUseCount);
9892  }
9893 
9894  if(m_LinearAlgorithm)
9895  {
9896  json.WriteString("LinearAlgorithm");
9897  json.WriteBool(true);
9898  }
9899  }
9900  else
9901  {
9902  json.WriteString("PreferredBlockSize");
9903  json.WriteNumber(m_PreferredBlockSize);
9904  }
9905 
9906  json.WriteString("Blocks");
9907  json.BeginObject();
9908  for(size_t i = 0; i < m_Blocks.size(); ++i)
9909  {
9910  json.BeginString();
9911  json.ContinueString(m_Blocks[i]->GetId());
9912  json.EndString();
9913 
9914  m_Blocks[i]->m_pMetadata->PrintDetailedMap(json);
9915  }
9916  json.EndObject();
9917 
9918  json.EndObject();
9919 }
9920 
9921 #endif // #if VMA_STATS_STRING_ENABLED
9922 
9923 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
9924  VmaAllocator hAllocator,
9925  uint32_t currentFrameIndex)
9926 {
9927  if(m_pDefragmentator == VMA_NULL)
9928  {
9929  m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
9930  hAllocator,
9931  this,
9932  currentFrameIndex);
9933  }
9934 
9935  return m_pDefragmentator;
9936 }
9937 
9938 VkResult VmaBlockVector::Defragment(
9939  VmaDefragmentationStats* pDefragmentationStats,
9940  VkDeviceSize& maxBytesToMove,
9941  uint32_t& maxAllocationsToMove)
9942 {
9943  if(m_pDefragmentator == VMA_NULL)
9944  {
9945  return VK_SUCCESS;
9946  }
9947 
9948  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
9949 
9950  // Defragment.
9951  VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
9952 
9953  // Accumulate statistics.
9954  if(pDefragmentationStats != VMA_NULL)
9955  {
9956  const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
9957  const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
9958  pDefragmentationStats->bytesMoved += bytesMoved;
9959  pDefragmentationStats->allocationsMoved += allocationsMoved;
9960  VMA_ASSERT(bytesMoved <= maxBytesToMove);
9961  VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
9962  maxBytesToMove -= bytesMoved;
9963  maxAllocationsToMove -= allocationsMoved;
9964  }
9965 
9966  // Free empty blocks.
9967  m_HasEmptyBlock = false;
9968  for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
9969  {
9970  VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
9971  if(pBlock->m_pMetadata->IsEmpty())
9972  {
9973  if(m_Blocks.size() > m_MinBlockCount)
9974  {
9975  if(pDefragmentationStats != VMA_NULL)
9976  {
9977  ++pDefragmentationStats->deviceMemoryBlocksFreed;
9978  pDefragmentationStats->bytesFreed += pBlock->m_pMetadata->GetSize();
9979  }
9980 
9981  VmaVectorRemove(m_Blocks, blockIndex);
9982  pBlock->Destroy(m_hAllocator);
9983  vma_delete(m_hAllocator, pBlock);
9984  }
9985  else
9986  {
9987  m_HasEmptyBlock = true;
9988  }
9989  }
9990  }
9991 
9992  return result;
9993 }
9994 
9995 void VmaBlockVector::DestroyDefragmentator()
9996 {
9997  if(m_pDefragmentator != VMA_NULL)
9998  {
9999  vma_delete(m_hAllocator, m_pDefragmentator);
10000  m_pDefragmentator = VMA_NULL;
10001  }
10002 }
10003 
10004 void VmaBlockVector::MakePoolAllocationsLost(
10005  uint32_t currentFrameIndex,
10006  size_t* pLostAllocationCount)
10007 {
10008  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
10009  size_t lostAllocationCount = 0;
10010  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
10011  {
10012  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
10013  VMA_ASSERT(pBlock);
10014  lostAllocationCount += pBlock->m_pMetadata->MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
10015  }
10016  if(pLostAllocationCount != VMA_NULL)
10017  {
10018  *pLostAllocationCount = lostAllocationCount;
10019  }
10020 }
10021 
10022 VkResult VmaBlockVector::CheckCorruption()
10023 {
10024  if(!IsCorruptionDetectionEnabled())
10025  {
10026  return VK_ERROR_FEATURE_NOT_PRESENT;
10027  }
10028 
10029  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
10030  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
10031  {
10032  VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
10033  VMA_ASSERT(pBlock);
10034  VkResult res = pBlock->CheckCorruption(m_hAllocator);
10035  if(res != VK_SUCCESS)
10036  {
10037  return res;
10038  }
10039  }
10040  return VK_SUCCESS;
10041 }
10042 
10043 void VmaBlockVector::AddStats(VmaStats* pStats)
10044 {
10045  const uint32_t memTypeIndex = m_MemoryTypeIndex;
10046  const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
10047 
10048  VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
10049 
10050  for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
10051  {
10052  const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
10053  VMA_ASSERT(pBlock);
10054  VMA_HEAVY_ASSERT(pBlock->Validate());
10055  VmaStatInfo allocationStatInfo;
10056  pBlock->m_pMetadata->CalcAllocationStatInfo(allocationStatInfo);
10057  VmaAddStatInfo(pStats->total, allocationStatInfo);
10058  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
10059  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
10060  }
10061 }
10062 
10064 // VmaDefragmentator members definition
10065 
10066 VmaDefragmentator::VmaDefragmentator(
10067  VmaAllocator hAllocator,
10068  VmaBlockVector* pBlockVector,
10069  uint32_t currentFrameIndex) :
10070  m_hAllocator(hAllocator),
10071  m_pBlockVector(pBlockVector),
10072  m_CurrentFrameIndex(currentFrameIndex),
10073  m_BytesMoved(0),
10074  m_AllocationsMoved(0),
10075  m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
10076  m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
10077 {
10078  VMA_ASSERT(!pBlockVector->UsesLinearAlgorithm());
10079 }
10080 
10081 VmaDefragmentator::~VmaDefragmentator()
10082 {
10083  for(size_t i = m_Blocks.size(); i--; )
10084  {
10085  vma_delete(m_hAllocator, m_Blocks[i]);
10086  }
10087 }
10088 
10089 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
10090 {
10091  AllocationInfo allocInfo;
10092  allocInfo.m_hAllocation = hAlloc;
10093  allocInfo.m_pChanged = pChanged;
10094  m_Allocations.push_back(allocInfo);
10095 }
10096 
10097 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
10098 {
10099  // It has already been mapped for defragmentation.
10100  if(m_pMappedDataForDefragmentation)
10101  {
10102  *ppMappedData = m_pMappedDataForDefragmentation;
10103  return VK_SUCCESS;
10104  }
10105 
10106  // It is originally mapped.
10107  if(m_pBlock->GetMappedData())
10108  {
10109  *ppMappedData = m_pBlock->GetMappedData();
10110  return VK_SUCCESS;
10111  }
10112 
10113  // Map on first usage.
10114  VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
10115  *ppMappedData = m_pMappedDataForDefragmentation;
10116  return res;
10117 }
10118 
10119 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
10120 {
10121  if(m_pMappedDataForDefragmentation != VMA_NULL)
10122  {
10123  m_pBlock->Unmap(hAllocator, 1);
10124  }
10125 }
10126 
10127 VkResult VmaDefragmentator::DefragmentRound(
10128  VkDeviceSize maxBytesToMove,
10129  uint32_t maxAllocationsToMove)
10130 {
10131  if(m_Blocks.empty())
10132  {
10133  return VK_SUCCESS;
10134  }
10135 
10136  size_t srcBlockIndex = m_Blocks.size() - 1;
10137  size_t srcAllocIndex = SIZE_MAX;
10138  for(;;)
10139  {
10140  // 1. Find next allocation to move.
10141  // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
10142  // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
10143  while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
10144  {
10145  if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
10146  {
10147  // Finished: no more allocations to process.
10148  if(srcBlockIndex == 0)
10149  {
10150  return VK_SUCCESS;
10151  }
10152  else
10153  {
10154  --srcBlockIndex;
10155  srcAllocIndex = SIZE_MAX;
10156  }
10157  }
10158  else
10159  {
10160  srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
10161  }
10162  }
10163 
10164  BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
10165  AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
10166 
10167  const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
10168  const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
10169  const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
10170  const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
10171 
10172  // 2. Try to find new place for this allocation in preceding or current block.
10173  for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
10174  {
10175  BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
10176  VmaAllocationRequest dstAllocRequest;
10177  if(pDstBlockInfo->m_pBlock->m_pMetadata->CreateAllocationRequest(
10178  m_CurrentFrameIndex,
10179  m_pBlockVector->GetFrameInUseCount(),
10180  m_pBlockVector->GetBufferImageGranularity(),
10181  size,
10182  alignment,
10183  false, // upperAddress
10184  suballocType,
10185  false, // canMakeOtherLost
10186  &dstAllocRequest) &&
10187  MoveMakesSense(
10188  dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
10189  {
10190  VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
10191 
10192  // Reached limit on number of allocations or bytes to move.
10193  if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
10194  (m_BytesMoved + size > maxBytesToMove))
10195  {
10196  return VK_INCOMPLETE;
10197  }
10198 
10199  void* pDstMappedData = VMA_NULL;
10200  VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
10201  if(res != VK_SUCCESS)
10202  {
10203  return res;
10204  }
10205 
10206  void* pSrcMappedData = VMA_NULL;
10207  res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
10208  if(res != VK_SUCCESS)
10209  {
10210  return res;
10211  }
10212 
10213  // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
10214  memcpy(
10215  reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
10216  reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
10217  static_cast<size_t>(size));
10218 
10219  if(VMA_DEBUG_MARGIN > 0)
10220  {
10221  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset - VMA_DEBUG_MARGIN);
10222  VmaWriteMagicValue(pDstMappedData, dstAllocRequest.offset + size);
10223  }
10224 
10225  pDstBlockInfo->m_pBlock->m_pMetadata->Alloc(
10226  dstAllocRequest,
10227  suballocType,
10228  size,
10229  false, // upperAddress
10230  allocInfo.m_hAllocation);
10231  pSrcBlockInfo->m_pBlock->m_pMetadata->FreeAtOffset(srcOffset);
10232 
10233  allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
10234 
10235  if(allocInfo.m_pChanged != VMA_NULL)
10236  {
10237  *allocInfo.m_pChanged = VK_TRUE;
10238  }
10239 
10240  ++m_AllocationsMoved;
10241  m_BytesMoved += size;
10242 
10243  VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
10244 
10245  break;
10246  }
10247  }
10248 
10249  // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
10250 
10251  if(srcAllocIndex > 0)
10252  {
10253  --srcAllocIndex;
10254  }
10255  else
10256  {
10257  if(srcBlockIndex > 0)
10258  {
10259  --srcBlockIndex;
10260  srcAllocIndex = SIZE_MAX;
10261  }
10262  else
10263  {
10264  return VK_SUCCESS;
10265  }
10266  }
10267  }
10268 }
10269 
10270 VkResult VmaDefragmentator::Defragment(
10271  VkDeviceSize maxBytesToMove,
10272  uint32_t maxAllocationsToMove)
10273 {
10274  if(m_Allocations.empty())
10275  {
10276  return VK_SUCCESS;
10277  }
10278 
10279  // Create block info for each block.
10280  const size_t blockCount = m_pBlockVector->m_Blocks.size();
10281  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
10282  {
10283  BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
10284  pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
10285  m_Blocks.push_back(pBlockInfo);
10286  }
10287 
10288  // Sort them by m_pBlock pointer value.
10289  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
10290 
10291  // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
10292  for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
10293  {
10294  AllocationInfo& allocInfo = m_Allocations[blockIndex];
10295  // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
10296  if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
10297  {
10298  VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
10299  BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
10300  if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
10301  {
10302  (*it)->m_Allocations.push_back(allocInfo);
10303  }
10304  else
10305  {
10306  VMA_ASSERT(0);
10307  }
10308  }
10309  }
10310  m_Allocations.clear();
10311 
10312  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
10313  {
10314  BlockInfo* pBlockInfo = m_Blocks[blockIndex];
10315  pBlockInfo->CalcHasNonMovableAllocations();
10316  pBlockInfo->SortAllocationsBySizeDescecnding();
10317  }
10318 
10319  // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
10320  VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
10321 
10322  // Execute defragmentation rounds (the main part).
10323  VkResult result = VK_SUCCESS;
10324  for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
10325  {
10326  result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
10327  }
10328 
10329  // Unmap blocks that were mapped for defragmentation.
10330  for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
10331  {
10332  m_Blocks[blockIndex]->Unmap(m_hAllocator);
10333  }
10334 
10335  return result;
10336 }
10337 
10338 bool VmaDefragmentator::MoveMakesSense(
10339  size_t dstBlockIndex, VkDeviceSize dstOffset,
10340  size_t srcBlockIndex, VkDeviceSize srcOffset)
10341 {
10342  if(dstBlockIndex < srcBlockIndex)
10343  {
10344  return true;
10345  }
10346  if(dstBlockIndex > srcBlockIndex)
10347  {
10348  return false;
10349  }
10350  if(dstOffset < srcOffset)
10351  {
10352  return true;
10353  }
10354  return false;
10355 }
10356 
10358 // VmaRecorder
10359 
10360 #if VMA_RECORDING_ENABLED
10361 
10362 VmaRecorder::VmaRecorder() :
10363  m_UseMutex(true),
10364  m_Flags(0),
10365  m_File(VMA_NULL),
10366  m_Freq(INT64_MAX),
10367  m_StartCounter(INT64_MAX)
10368 {
10369 }
10370 
10371 VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex)
10372 {
10373  m_UseMutex = useMutex;
10374  m_Flags = settings.flags;
10375 
10376  QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq);
10377  QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter);
10378 
10379  // Open file for writing.
10380  errno_t err = fopen_s(&m_File, settings.pFilePath, "wb");
10381  if(err != 0)
10382  {
10383  return VK_ERROR_INITIALIZATION_FAILED;
10384  }
10385 
10386  // Write header.
10387  fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording");
10388  fprintf(m_File, "%s\n", "1,3");
10389 
10390  return VK_SUCCESS;
10391 }
10392 
10393 VmaRecorder::~VmaRecorder()
10394 {
10395  if(m_File != VMA_NULL)
10396  {
10397  fclose(m_File);
10398  }
10399 }
10400 
10401 void VmaRecorder::RecordCreateAllocator(uint32_t frameIndex)
10402 {
10403  CallParams callParams;
10404  GetBasicParams(callParams);
10405 
10406  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10407  fprintf(m_File, "%u,%.3f,%u,vmaCreateAllocator\n", callParams.threadId, callParams.time, frameIndex);
10408  Flush();
10409 }
10410 
10411 void VmaRecorder::RecordDestroyAllocator(uint32_t frameIndex)
10412 {
10413  CallParams callParams;
10414  GetBasicParams(callParams);
10415 
10416  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10417  fprintf(m_File, "%u,%.3f,%u,vmaDestroyAllocator\n", callParams.threadId, callParams.time, frameIndex);
10418  Flush();
10419 }
10420 
10421 void VmaRecorder::RecordCreatePool(uint32_t frameIndex, const VmaPoolCreateInfo& createInfo, VmaPool pool)
10422 {
10423  CallParams callParams;
10424  GetBasicParams(callParams);
10425 
10426  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10427  fprintf(m_File, "%u,%.3f,%u,vmaCreatePool,%u,%u,%llu,%llu,%llu,%u,%p\n", callParams.threadId, callParams.time, frameIndex,
10428  createInfo.memoryTypeIndex,
10429  createInfo.flags,
10430  createInfo.blockSize,
10431  createInfo.minBlockCount,
10432  createInfo.maxBlockCount,
10433  createInfo.frameInUseCount,
10434  pool);
10435  Flush();
10436 }
10437 
10438 void VmaRecorder::RecordDestroyPool(uint32_t frameIndex, VmaPool pool)
10439 {
10440  CallParams callParams;
10441  GetBasicParams(callParams);
10442 
10443  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10444  fprintf(m_File, "%u,%.3f,%u,vmaDestroyPool,%p\n", callParams.threadId, callParams.time, frameIndex,
10445  pool);
10446  Flush();
10447 }
10448 
10449 void VmaRecorder::RecordAllocateMemory(uint32_t frameIndex,
10450  const VkMemoryRequirements& vkMemReq,
10451  const VmaAllocationCreateInfo& createInfo,
10452  VmaAllocation allocation)
10453 {
10454  CallParams callParams;
10455  GetBasicParams(callParams);
10456 
10457  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10458  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
10459  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemory,%llu,%llu,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10460  vkMemReq.size,
10461  vkMemReq.alignment,
10462  vkMemReq.memoryTypeBits,
10463  createInfo.flags,
10464  createInfo.usage,
10465  createInfo.requiredFlags,
10466  createInfo.preferredFlags,
10467  createInfo.memoryTypeBits,
10468  createInfo.pool,
10469  allocation,
10470  userDataStr.GetString());
10471  Flush();
10472 }
10473 
10474 void VmaRecorder::RecordAllocateMemoryForBuffer(uint32_t frameIndex,
10475  const VkMemoryRequirements& vkMemReq,
10476  bool requiresDedicatedAllocation,
10477  bool prefersDedicatedAllocation,
10478  const VmaAllocationCreateInfo& createInfo,
10479  VmaAllocation allocation)
10480 {
10481  CallParams callParams;
10482  GetBasicParams(callParams);
10483 
10484  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10485  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
10486  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForBuffer,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10487  vkMemReq.size,
10488  vkMemReq.alignment,
10489  vkMemReq.memoryTypeBits,
10490  requiresDedicatedAllocation ? 1 : 0,
10491  prefersDedicatedAllocation ? 1 : 0,
10492  createInfo.flags,
10493  createInfo.usage,
10494  createInfo.requiredFlags,
10495  createInfo.preferredFlags,
10496  createInfo.memoryTypeBits,
10497  createInfo.pool,
10498  allocation,
10499  userDataStr.GetString());
10500  Flush();
10501 }
10502 
10503 void VmaRecorder::RecordAllocateMemoryForImage(uint32_t frameIndex,
10504  const VkMemoryRequirements& vkMemReq,
10505  bool requiresDedicatedAllocation,
10506  bool prefersDedicatedAllocation,
10507  const VmaAllocationCreateInfo& createInfo,
10508  VmaAllocation allocation)
10509 {
10510  CallParams callParams;
10511  GetBasicParams(callParams);
10512 
10513  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10514  UserDataString userDataStr(createInfo.flags, createInfo.pUserData);
10515  fprintf(m_File, "%u,%.3f,%u,vmaAllocateMemoryForImage,%llu,%llu,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10516  vkMemReq.size,
10517  vkMemReq.alignment,
10518  vkMemReq.memoryTypeBits,
10519  requiresDedicatedAllocation ? 1 : 0,
10520  prefersDedicatedAllocation ? 1 : 0,
10521  createInfo.flags,
10522  createInfo.usage,
10523  createInfo.requiredFlags,
10524  createInfo.preferredFlags,
10525  createInfo.memoryTypeBits,
10526  createInfo.pool,
10527  allocation,
10528  userDataStr.GetString());
10529  Flush();
10530 }
10531 
10532 void VmaRecorder::RecordFreeMemory(uint32_t frameIndex,
10533  VmaAllocation allocation)
10534 {
10535  CallParams callParams;
10536  GetBasicParams(callParams);
10537 
10538  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10539  fprintf(m_File, "%u,%.3f,%u,vmaFreeMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
10540  allocation);
10541  Flush();
10542 }
10543 
10544 void VmaRecorder::RecordSetAllocationUserData(uint32_t frameIndex,
10545  VmaAllocation allocation,
10546  const void* pUserData)
10547 {
10548  CallParams callParams;
10549  GetBasicParams(callParams);
10550 
10551  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10552  UserDataString userDataStr(
10553  allocation->IsUserDataString() ? VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT : 0,
10554  pUserData);
10555  fprintf(m_File, "%u,%.3f,%u,vmaSetAllocationUserData,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10556  allocation,
10557  userDataStr.GetString());
10558  Flush();
10559 }
10560 
10561 void VmaRecorder::RecordCreateLostAllocation(uint32_t frameIndex,
10562  VmaAllocation allocation)
10563 {
10564  CallParams callParams;
10565  GetBasicParams(callParams);
10566 
10567  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10568  fprintf(m_File, "%u,%.3f,%u,vmaCreateLostAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
10569  allocation);
10570  Flush();
10571 }
10572 
10573 void VmaRecorder::RecordMapMemory(uint32_t frameIndex,
10574  VmaAllocation allocation)
10575 {
10576  CallParams callParams;
10577  GetBasicParams(callParams);
10578 
10579  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10580  fprintf(m_File, "%u,%.3f,%u,vmaMapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
10581  allocation);
10582  Flush();
10583 }
10584 
10585 void VmaRecorder::RecordUnmapMemory(uint32_t frameIndex,
10586  VmaAllocation allocation)
10587 {
10588  CallParams callParams;
10589  GetBasicParams(callParams);
10590 
10591  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10592  fprintf(m_File, "%u,%.3f,%u,vmaUnmapMemory,%p\n", callParams.threadId, callParams.time, frameIndex,
10593  allocation);
10594  Flush();
10595 }
10596 
10597 void VmaRecorder::RecordFlushAllocation(uint32_t frameIndex,
10598  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
10599 {
10600  CallParams callParams;
10601  GetBasicParams(callParams);
10602 
10603  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10604  fprintf(m_File, "%u,%.3f,%u,vmaFlushAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
10605  allocation,
10606  offset,
10607  size);
10608  Flush();
10609 }
10610 
10611 void VmaRecorder::RecordInvalidateAllocation(uint32_t frameIndex,
10612  VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
10613 {
10614  CallParams callParams;
10615  GetBasicParams(callParams);
10616 
10617  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10618  fprintf(m_File, "%u,%.3f,%u,vmaInvalidateAllocation,%p,%llu,%llu\n", callParams.threadId, callParams.time, frameIndex,
10619  allocation,
10620  offset,
10621  size);
10622  Flush();
10623 }
10624 
10625 void VmaRecorder::RecordCreateBuffer(uint32_t frameIndex,
10626  const VkBufferCreateInfo& bufCreateInfo,
10627  const VmaAllocationCreateInfo& allocCreateInfo,
10628  VmaAllocation allocation)
10629 {
10630  CallParams callParams;
10631  GetBasicParams(callParams);
10632 
10633  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10634  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
10635  fprintf(m_File, "%u,%.3f,%u,vmaCreateBuffer,%u,%llu,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10636  bufCreateInfo.flags,
10637  bufCreateInfo.size,
10638  bufCreateInfo.usage,
10639  bufCreateInfo.sharingMode,
10640  allocCreateInfo.flags,
10641  allocCreateInfo.usage,
10642  allocCreateInfo.requiredFlags,
10643  allocCreateInfo.preferredFlags,
10644  allocCreateInfo.memoryTypeBits,
10645  allocCreateInfo.pool,
10646  allocation,
10647  userDataStr.GetString());
10648  Flush();
10649 }
10650 
10651 void VmaRecorder::RecordCreateImage(uint32_t frameIndex,
10652  const VkImageCreateInfo& imageCreateInfo,
10653  const VmaAllocationCreateInfo& allocCreateInfo,
10654  VmaAllocation allocation)
10655 {
10656  CallParams callParams;
10657  GetBasicParams(callParams);
10658 
10659  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10660  UserDataString userDataStr(allocCreateInfo.flags, allocCreateInfo.pUserData);
10661  fprintf(m_File, "%u,%.3f,%u,vmaCreateImage,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%u,%p,%p,%s\n", callParams.threadId, callParams.time, frameIndex,
10662  imageCreateInfo.flags,
10663  imageCreateInfo.imageType,
10664  imageCreateInfo.format,
10665  imageCreateInfo.extent.width,
10666  imageCreateInfo.extent.height,
10667  imageCreateInfo.extent.depth,
10668  imageCreateInfo.mipLevels,
10669  imageCreateInfo.arrayLayers,
10670  imageCreateInfo.samples,
10671  imageCreateInfo.tiling,
10672  imageCreateInfo.usage,
10673  imageCreateInfo.sharingMode,
10674  imageCreateInfo.initialLayout,
10675  allocCreateInfo.flags,
10676  allocCreateInfo.usage,
10677  allocCreateInfo.requiredFlags,
10678  allocCreateInfo.preferredFlags,
10679  allocCreateInfo.memoryTypeBits,
10680  allocCreateInfo.pool,
10681  allocation,
10682  userDataStr.GetString());
10683  Flush();
10684 }
10685 
10686 void VmaRecorder::RecordDestroyBuffer(uint32_t frameIndex,
10687  VmaAllocation allocation)
10688 {
10689  CallParams callParams;
10690  GetBasicParams(callParams);
10691 
10692  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10693  fprintf(m_File, "%u,%.3f,%u,vmaDestroyBuffer,%p\n", callParams.threadId, callParams.time, frameIndex,
10694  allocation);
10695  Flush();
10696 }
10697 
10698 void VmaRecorder::RecordDestroyImage(uint32_t frameIndex,
10699  VmaAllocation allocation)
10700 {
10701  CallParams callParams;
10702  GetBasicParams(callParams);
10703 
10704  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10705  fprintf(m_File, "%u,%.3f,%u,vmaDestroyImage,%p\n", callParams.threadId, callParams.time, frameIndex,
10706  allocation);
10707  Flush();
10708 }
10709 
10710 void VmaRecorder::RecordTouchAllocation(uint32_t frameIndex,
10711  VmaAllocation allocation)
10712 {
10713  CallParams callParams;
10714  GetBasicParams(callParams);
10715 
10716  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10717  fprintf(m_File, "%u,%.3f,%u,vmaTouchAllocation,%p\n", callParams.threadId, callParams.time, frameIndex,
10718  allocation);
10719  Flush();
10720 }
10721 
10722 void VmaRecorder::RecordGetAllocationInfo(uint32_t frameIndex,
10723  VmaAllocation allocation)
10724 {
10725  CallParams callParams;
10726  GetBasicParams(callParams);
10727 
10728  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10729  fprintf(m_File, "%u,%.3f,%u,vmaGetAllocationInfo,%p\n", callParams.threadId, callParams.time, frameIndex,
10730  allocation);
10731  Flush();
10732 }
10733 
10734 void VmaRecorder::RecordMakePoolAllocationsLost(uint32_t frameIndex,
10735  VmaPool pool)
10736 {
10737  CallParams callParams;
10738  GetBasicParams(callParams);
10739 
10740  VmaMutexLock lock(m_FileMutex, m_UseMutex);
10741  fprintf(m_File, "%u,%.3f,%u,vmaMakePoolAllocationsLost,%p\n", callParams.threadId, callParams.time, frameIndex,
10742  pool);
10743  Flush();
10744 }
10745 
10746 VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, const void* pUserData)
10747 {
10748  if(pUserData != VMA_NULL)
10749  {
10750  if((allocFlags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0)
10751  {
10752  m_Str = (const char*)pUserData;
10753  }
10754  else
10755  {
10756  sprintf_s(m_PtrStr, "%p", pUserData);
10757  m_Str = m_PtrStr;
10758  }
10759  }
10760  else
10761  {
10762  m_Str = "";
10763  }
10764 }
10765 
10766 void VmaRecorder::WriteConfiguration(
10767  const VkPhysicalDeviceProperties& devProps,
10768  const VkPhysicalDeviceMemoryProperties& memProps,
10769  bool dedicatedAllocationExtensionEnabled)
10770 {
10771  fprintf(m_File, "Config,Begin\n");
10772 
10773  fprintf(m_File, "PhysicalDevice,apiVersion,%u\n", devProps.apiVersion);
10774  fprintf(m_File, "PhysicalDevice,driverVersion,%u\n", devProps.driverVersion);
10775  fprintf(m_File, "PhysicalDevice,vendorID,%u\n", devProps.vendorID);
10776  fprintf(m_File, "PhysicalDevice,deviceID,%u\n", devProps.deviceID);
10777  fprintf(m_File, "PhysicalDevice,deviceType,%u\n", devProps.deviceType);
10778  fprintf(m_File, "PhysicalDevice,deviceName,%s\n", devProps.deviceName);
10779 
10780  fprintf(m_File, "PhysicalDeviceLimits,maxMemoryAllocationCount,%u\n", devProps.limits.maxMemoryAllocationCount);
10781  fprintf(m_File, "PhysicalDeviceLimits,bufferImageGranularity,%llu\n", devProps.limits.bufferImageGranularity);
10782  fprintf(m_File, "PhysicalDeviceLimits,nonCoherentAtomSize,%llu\n", devProps.limits.nonCoherentAtomSize);
10783 
10784  fprintf(m_File, "PhysicalDeviceMemory,HeapCount,%u\n", memProps.memoryHeapCount);
10785  for(uint32_t i = 0; i < memProps.memoryHeapCount; ++i)
10786  {
10787  fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,size,%llu\n", i, memProps.memoryHeaps[i].size);
10788  fprintf(m_File, "PhysicalDeviceMemory,Heap,%u,flags,%u\n", i, memProps.memoryHeaps[i].flags);
10789  }
10790  fprintf(m_File, "PhysicalDeviceMemory,TypeCount,%u\n", memProps.memoryTypeCount);
10791  for(uint32_t i = 0; i < memProps.memoryTypeCount; ++i)
10792  {
10793  fprintf(m_File, "PhysicalDeviceMemory,Type,%u,heapIndex,%u\n", i, memProps.memoryTypes[i].heapIndex);
10794  fprintf(m_File, "PhysicalDeviceMemory,Type,%u,propertyFlags,%u\n", i, memProps.memoryTypes[i].propertyFlags);
10795  }
10796 
10797  fprintf(m_File, "Extension,VK_KHR_dedicated_allocation,%u\n", dedicatedAllocationExtensionEnabled ? 1 : 0);
10798 
10799  fprintf(m_File, "Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY,%u\n", VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ? 1 : 0);
10800  fprintf(m_File, "Macro,VMA_DEBUG_ALIGNMENT,%llu\n", (VkDeviceSize)VMA_DEBUG_ALIGNMENT);
10801  fprintf(m_File, "Macro,VMA_DEBUG_MARGIN,%llu\n", (VkDeviceSize)VMA_DEBUG_MARGIN);
10802  fprintf(m_File, "Macro,VMA_DEBUG_INITIALIZE_ALLOCATIONS,%u\n", VMA_DEBUG_INITIALIZE_ALLOCATIONS ? 1 : 0);
10803  fprintf(m_File, "Macro,VMA_DEBUG_DETECT_CORRUPTION,%u\n", VMA_DEBUG_DETECT_CORRUPTION ? 1 : 0);
10804  fprintf(m_File, "Macro,VMA_DEBUG_GLOBAL_MUTEX,%u\n", VMA_DEBUG_GLOBAL_MUTEX ? 1 : 0);
10805  fprintf(m_File, "Macro,VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY,%llu\n", (VkDeviceSize)VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY);
10806  fprintf(m_File, "Macro,VMA_SMALL_HEAP_MAX_SIZE,%llu\n", (VkDeviceSize)VMA_SMALL_HEAP_MAX_SIZE);
10807  fprintf(m_File, "Macro,VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE,%llu\n", (VkDeviceSize)VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
10808 
10809  fprintf(m_File, "Config,End\n");
10810 }
10811 
10812 void VmaRecorder::GetBasicParams(CallParams& outParams)
10813 {
10814  outParams.threadId = GetCurrentThreadId();
10815 
10816  LARGE_INTEGER counter;
10817  QueryPerformanceCounter(&counter);
10818  outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq;
10819 }
10820 
10821 void VmaRecorder::Flush()
10822 {
10823  if((m_Flags & VMA_RECORD_FLUSH_AFTER_CALL_BIT) != 0)
10824  {
10825  fflush(m_File);
10826  }
10827 }
10828 
10829 #endif // #if VMA_RECORDING_ENABLED
10830 
10832 // VmaAllocator_T
10833 
10834 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
10835  m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
10836  m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
10837  m_hDevice(pCreateInfo->device),
10838  m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
10839  m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
10840  *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
10841  m_PreferredLargeHeapBlockSize(0),
10842  m_PhysicalDevice(pCreateInfo->physicalDevice),
10843  m_CurrentFrameIndex(0),
10844  m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())),
10845  m_NextPoolId(0)
10846 #if VMA_RECORDING_ENABLED
10847  ,m_pRecorder(VMA_NULL)
10848 #endif
10849 {
10850  if(VMA_DEBUG_DETECT_CORRUPTION)
10851  {
10852  // Needs to be multiply of uint32_t size because we are going to write VMA_CORRUPTION_DETECTION_MAGIC_VALUE to it.
10853  VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0);
10854  }
10855 
10856  VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
10857 
10858 #if !(VMA_DEDICATED_ALLOCATION)
10859  if((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0)
10860  {
10861  VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT set but required extensions are disabled by preprocessor macros.");
10862  }
10863 #endif
10864 
10865  memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
10866  memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
10867  memset(&m_MemProps, 0, sizeof(m_MemProps));
10868 
10869  memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
10870  memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
10871 
10872  for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
10873  {
10874  m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
10875  }
10876 
10877  if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
10878  {
10879  m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
10880  m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
10881  }
10882 
10883  ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
10884 
10885  (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
10886  (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
10887 
10888  m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
10889  pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
10890 
10891  if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
10892  {
10893  for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
10894  {
10895  const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
10896  if(limit != VK_WHOLE_SIZE)
10897  {
10898  m_HeapSizeLimit[heapIndex] = limit;
10899  if(limit < m_MemProps.memoryHeaps[heapIndex].size)
10900  {
10901  m_MemProps.memoryHeaps[heapIndex].size = limit;
10902  }
10903  }
10904  }
10905  }
10906 
10907  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
10908  {
10909  const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
10910 
10911  m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
10912  this,
10913  memTypeIndex,
10914  preferredBlockSize,
10915  0,
10916  SIZE_MAX,
10917  GetBufferImageGranularity(),
10918  pCreateInfo->frameInUseCount,
10919  false, // isCustomPool
10920  false); // linearAlgorithm
10921  // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
10922  // becase minBlockCount is 0.
10923  m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
10924 
10925  }
10926 }
10927 
10928 VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo)
10929 {
10930  VkResult res = VK_SUCCESS;
10931 
10932  if(pCreateInfo->pRecordSettings != VMA_NULL &&
10933  !VmaStrIsEmpty(pCreateInfo->pRecordSettings->pFilePath))
10934  {
10935 #if VMA_RECORDING_ENABLED
10936  m_pRecorder = vma_new(this, VmaRecorder)();
10937  res = m_pRecorder->Init(*pCreateInfo->pRecordSettings, m_UseMutex);
10938  if(res != VK_SUCCESS)
10939  {
10940  return res;
10941  }
10942  m_pRecorder->WriteConfiguration(
10943  m_PhysicalDeviceProperties,
10944  m_MemProps,
10945  m_UseKhrDedicatedAllocation);
10946  m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex());
10947 #else
10948  VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1.");
10949  return VK_ERROR_FEATURE_NOT_PRESENT;
10950 #endif
10951  }
10952 
10953  return res;
10954 }
10955 
10956 VmaAllocator_T::~VmaAllocator_T()
10957 {
10958 #if VMA_RECORDING_ENABLED
10959  if(m_pRecorder != VMA_NULL)
10960  {
10961  m_pRecorder->RecordDestroyAllocator(GetCurrentFrameIndex());
10962  vma_delete(this, m_pRecorder);
10963  }
10964 #endif
10965 
10966  VMA_ASSERT(m_Pools.empty());
10967 
10968  for(size_t i = GetMemoryTypeCount(); i--; )
10969  {
10970  vma_delete(this, m_pDedicatedAllocations[i]);
10971  vma_delete(this, m_pBlockVectors[i]);
10972  }
10973 }
10974 
10975 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
10976 {
10977 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
10978  m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
10979  m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
10980  m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
10981  m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
10982  m_VulkanFunctions.vkMapMemory = &vkMapMemory;
10983  m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
10984  m_VulkanFunctions.vkFlushMappedMemoryRanges = &vkFlushMappedMemoryRanges;
10985  m_VulkanFunctions.vkInvalidateMappedMemoryRanges = &vkInvalidateMappedMemoryRanges;
10986  m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
10987  m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
10988  m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
10989  m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
10990  m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
10991  m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
10992  m_VulkanFunctions.vkCreateImage = &vkCreateImage;
10993  m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
10994 #if VMA_DEDICATED_ALLOCATION
10995  if(m_UseKhrDedicatedAllocation)
10996  {
10997  m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
10998  (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
10999  m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
11000  (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
11001  }
11002 #endif // #if VMA_DEDICATED_ALLOCATION
11003 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
11004 
11005 #define VMA_COPY_IF_NOT_NULL(funcName) \
11006  if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
11007 
11008  if(pVulkanFunctions != VMA_NULL)
11009  {
11010  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
11011  VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
11012  VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
11013  VMA_COPY_IF_NOT_NULL(vkFreeMemory);
11014  VMA_COPY_IF_NOT_NULL(vkMapMemory);
11015  VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
11016  VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges);
11017  VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges);
11018  VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
11019  VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
11020  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
11021  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
11022  VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
11023  VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
11024  VMA_COPY_IF_NOT_NULL(vkCreateImage);
11025  VMA_COPY_IF_NOT_NULL(vkDestroyImage);
11026 #if VMA_DEDICATED_ALLOCATION
11027  VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
11028  VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
11029 #endif
11030  }
11031 
11032 #undef VMA_COPY_IF_NOT_NULL
11033 
11034  // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
11035  // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
11036  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
11037  VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
11038  VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
11039  VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
11040  VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
11041  VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
11042  VMA_ASSERT(m_VulkanFunctions.vkFlushMappedMemoryRanges != VMA_NULL);
11043  VMA_ASSERT(m_VulkanFunctions.vkInvalidateMappedMemoryRanges != VMA_NULL);
11044  VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
11045  VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
11046  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
11047  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
11048  VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
11049  VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
11050  VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
11051  VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
11052 #if VMA_DEDICATED_ALLOCATION
11053  if(m_UseKhrDedicatedAllocation)
11054  {
11055  VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
11056  VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
11057  }
11058 #endif
11059 }
11060 
11061 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
11062 {
11063  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
11064  const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
11065  const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
11066  return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
11067 }
11068 
11069 VkResult VmaAllocator_T::AllocateMemoryOfType(
11070  VkDeviceSize size,
11071  VkDeviceSize alignment,
11072  bool dedicatedAllocation,
11073  VkBuffer dedicatedBuffer,
11074  VkImage dedicatedImage,
11075  const VmaAllocationCreateInfo& createInfo,
11076  uint32_t memTypeIndex,
11077  VmaSuballocationType suballocType,
11078  VmaAllocation* pAllocation)
11079 {
11080  VMA_ASSERT(pAllocation != VMA_NULL);
11081  VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
11082 
11083  VmaAllocationCreateInfo finalCreateInfo = createInfo;
11084 
11085  // If memory type is not HOST_VISIBLE, disable MAPPED.
11086  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
11087  (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
11088  {
11089  finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
11090  }
11091 
11092  VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
11093  VMA_ASSERT(blockVector);
11094 
11095  const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
11096  bool preferDedicatedMemory =
11097  VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
11098  dedicatedAllocation ||
11099  // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
11100  size > preferredBlockSize / 2;
11101 
11102  if(preferDedicatedMemory &&
11103  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
11104  finalCreateInfo.pool == VK_NULL_HANDLE)
11105  {
11106  finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
11107  }
11108 
11109  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
11110  {
11111  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
11112  {
11113  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11114  }
11115  else
11116  {
11117  return AllocateDedicatedMemory(
11118  size,
11119  suballocType,
11120  memTypeIndex,
11121  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
11122  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
11123  finalCreateInfo.pUserData,
11124  dedicatedBuffer,
11125  dedicatedImage,
11126  pAllocation);
11127  }
11128  }
11129  else
11130  {
11131  VkResult res = blockVector->Allocate(
11132  VK_NULL_HANDLE, // hCurrentPool
11133  m_CurrentFrameIndex.load(),
11134  size,
11135  alignment,
11136  finalCreateInfo,
11137  suballocType,
11138  pAllocation);
11139  if(res == VK_SUCCESS)
11140  {
11141  return res;
11142  }
11143 
11144  // 5. Try dedicated memory.
11145  if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
11146  {
11147  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11148  }
11149  else
11150  {
11151  res = AllocateDedicatedMemory(
11152  size,
11153  suballocType,
11154  memTypeIndex,
11155  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
11156  (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
11157  finalCreateInfo.pUserData,
11158  dedicatedBuffer,
11159  dedicatedImage,
11160  pAllocation);
11161  if(res == VK_SUCCESS)
11162  {
11163  // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
11164  VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
11165  return VK_SUCCESS;
11166  }
11167  else
11168  {
11169  // Everything failed: Return error code.
11170  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
11171  return res;
11172  }
11173  }
11174  }
11175 }
11176 
11177 VkResult VmaAllocator_T::AllocateDedicatedMemory(
11178  VkDeviceSize size,
11179  VmaSuballocationType suballocType,
11180  uint32_t memTypeIndex,
11181  bool map,
11182  bool isUserDataString,
11183  void* pUserData,
11184  VkBuffer dedicatedBuffer,
11185  VkImage dedicatedImage,
11186  VmaAllocation* pAllocation)
11187 {
11188  VMA_ASSERT(pAllocation);
11189 
11190  VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
11191  allocInfo.memoryTypeIndex = memTypeIndex;
11192  allocInfo.allocationSize = size;
11193 
11194 #if VMA_DEDICATED_ALLOCATION
11195  VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
11196  if(m_UseKhrDedicatedAllocation)
11197  {
11198  if(dedicatedBuffer != VK_NULL_HANDLE)
11199  {
11200  VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
11201  dedicatedAllocInfo.buffer = dedicatedBuffer;
11202  allocInfo.pNext = &dedicatedAllocInfo;
11203  }
11204  else if(dedicatedImage != VK_NULL_HANDLE)
11205  {
11206  dedicatedAllocInfo.image = dedicatedImage;
11207  allocInfo.pNext = &dedicatedAllocInfo;
11208  }
11209  }
11210 #endif // #if VMA_DEDICATED_ALLOCATION
11211 
11212  // Allocate VkDeviceMemory.
11213  VkDeviceMemory hMemory = VK_NULL_HANDLE;
11214  VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
11215  if(res < 0)
11216  {
11217  VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
11218  return res;
11219  }
11220 
11221  void* pMappedData = VMA_NULL;
11222  if(map)
11223  {
11224  res = (*m_VulkanFunctions.vkMapMemory)(
11225  m_hDevice,
11226  hMemory,
11227  0,
11228  VK_WHOLE_SIZE,
11229  0,
11230  &pMappedData);
11231  if(res < 0)
11232  {
11233  VMA_DEBUG_LOG(" vkMapMemory FAILED");
11234  FreeVulkanMemory(memTypeIndex, size, hMemory);
11235  return res;
11236  }
11237  }
11238 
11239  *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
11240  (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
11241  (*pAllocation)->SetUserData(this, pUserData);
11242  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
11243  {
11244  FillAllocation(*pAllocation, VMA_ALLOCATION_FILL_PATTERN_CREATED);
11245  }
11246 
11247  // Register it in m_pDedicatedAllocations.
11248  {
11249  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
11250  AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
11251  VMA_ASSERT(pDedicatedAllocations);
11252  VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
11253  }
11254 
11255  VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
11256 
11257  return VK_SUCCESS;
11258 }
11259 
11260 void VmaAllocator_T::GetBufferMemoryRequirements(
11261  VkBuffer hBuffer,
11262  VkMemoryRequirements& memReq,
11263  bool& requiresDedicatedAllocation,
11264  bool& prefersDedicatedAllocation) const
11265 {
11266 #if VMA_DEDICATED_ALLOCATION
11267  if(m_UseKhrDedicatedAllocation)
11268  {
11269  VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
11270  memReqInfo.buffer = hBuffer;
11271 
11272  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
11273 
11274  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
11275  memReq2.pNext = &memDedicatedReq;
11276 
11277  (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
11278 
11279  memReq = memReq2.memoryRequirements;
11280  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
11281  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
11282  }
11283  else
11284 #endif // #if VMA_DEDICATED_ALLOCATION
11285  {
11286  (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
11287  requiresDedicatedAllocation = false;
11288  prefersDedicatedAllocation = false;
11289  }
11290 }
11291 
11292 void VmaAllocator_T::GetImageMemoryRequirements(
11293  VkImage hImage,
11294  VkMemoryRequirements& memReq,
11295  bool& requiresDedicatedAllocation,
11296  bool& prefersDedicatedAllocation) const
11297 {
11298 #if VMA_DEDICATED_ALLOCATION
11299  if(m_UseKhrDedicatedAllocation)
11300  {
11301  VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
11302  memReqInfo.image = hImage;
11303 
11304  VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
11305 
11306  VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
11307  memReq2.pNext = &memDedicatedReq;
11308 
11309  (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
11310 
11311  memReq = memReq2.memoryRequirements;
11312  requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
11313  prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
11314  }
11315  else
11316 #endif // #if VMA_DEDICATED_ALLOCATION
11317  {
11318  (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
11319  requiresDedicatedAllocation = false;
11320  prefersDedicatedAllocation = false;
11321  }
11322 }
11323 
11324 VkResult VmaAllocator_T::AllocateMemory(
11325  const VkMemoryRequirements& vkMemReq,
11326  bool requiresDedicatedAllocation,
11327  bool prefersDedicatedAllocation,
11328  VkBuffer dedicatedBuffer,
11329  VkImage dedicatedImage,
11330  const VmaAllocationCreateInfo& createInfo,
11331  VmaSuballocationType suballocType,
11332  VmaAllocation* pAllocation)
11333 {
11334  if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
11335  (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
11336  {
11337  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
11338  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11339  }
11340  if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
11341  (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
11342  {
11343  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
11344  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11345  }
11346  if(requiresDedicatedAllocation)
11347  {
11348  if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
11349  {
11350  VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
11351  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11352  }
11353  if(createInfo.pool != VK_NULL_HANDLE)
11354  {
11355  VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
11356  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11357  }
11358  }
11359  if((createInfo.pool != VK_NULL_HANDLE) &&
11360  ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
11361  {
11362  VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
11363  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11364  }
11365 
11366  if(createInfo.pool != VK_NULL_HANDLE)
11367  {
11368  const VkDeviceSize alignmentForPool = VMA_MAX(
11369  vkMemReq.alignment,
11370  GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex()));
11371  return createInfo.pool->m_BlockVector.Allocate(
11372  createInfo.pool,
11373  m_CurrentFrameIndex.load(),
11374  vkMemReq.size,
11375  alignmentForPool,
11376  createInfo,
11377  suballocType,
11378  pAllocation);
11379  }
11380  else
11381  {
11382  // Bit mask of memory Vulkan types acceptable for this allocation.
11383  uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
11384  uint32_t memTypeIndex = UINT32_MAX;
11385  VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
11386  if(res == VK_SUCCESS)
11387  {
11388  VkDeviceSize alignmentForMemType = VMA_MAX(
11389  vkMemReq.alignment,
11390  GetMemoryTypeMinAlignment(memTypeIndex));
11391 
11392  res = AllocateMemoryOfType(
11393  vkMemReq.size,
11394  alignmentForMemType,
11395  requiresDedicatedAllocation || prefersDedicatedAllocation,
11396  dedicatedBuffer,
11397  dedicatedImage,
11398  createInfo,
11399  memTypeIndex,
11400  suballocType,
11401  pAllocation);
11402  // Succeeded on first try.
11403  if(res == VK_SUCCESS)
11404  {
11405  return res;
11406  }
11407  // Allocation from this memory type failed. Try other compatible memory types.
11408  else
11409  {
11410  for(;;)
11411  {
11412  // Remove old memTypeIndex from list of possibilities.
11413  memoryTypeBits &= ~(1u << memTypeIndex);
11414  // Find alternative memTypeIndex.
11415  res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
11416  if(res == VK_SUCCESS)
11417  {
11418  alignmentForMemType = VMA_MAX(
11419  vkMemReq.alignment,
11420  GetMemoryTypeMinAlignment(memTypeIndex));
11421 
11422  res = AllocateMemoryOfType(
11423  vkMemReq.size,
11424  alignmentForMemType,
11425  requiresDedicatedAllocation || prefersDedicatedAllocation,
11426  dedicatedBuffer,
11427  dedicatedImage,
11428  createInfo,
11429  memTypeIndex,
11430  suballocType,
11431  pAllocation);
11432  // Allocation from this alternative memory type succeeded.
11433  if(res == VK_SUCCESS)
11434  {
11435  return res;
11436  }
11437  // else: Allocation from this memory type failed. Try next one - next loop iteration.
11438  }
11439  // No other matching memory type index could be found.
11440  else
11441  {
11442  // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
11443  return VK_ERROR_OUT_OF_DEVICE_MEMORY;
11444  }
11445  }
11446  }
11447  }
11448  // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
11449  else
11450  return res;
11451  }
11452 }
11453 
11454 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
11455 {
11456  VMA_ASSERT(allocation);
11457 
11458  if(allocation->CanBecomeLost() == false ||
11459  allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
11460  {
11461  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS)
11462  {
11463  FillAllocation(allocation, VMA_ALLOCATION_FILL_PATTERN_DESTROYED);
11464  }
11465 
11466  switch(allocation->GetType())
11467  {
11468  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
11469  {
11470  VmaBlockVector* pBlockVector = VMA_NULL;
11471  VmaPool hPool = allocation->GetPool();
11472  if(hPool != VK_NULL_HANDLE)
11473  {
11474  pBlockVector = &hPool->m_BlockVector;
11475  }
11476  else
11477  {
11478  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
11479  pBlockVector = m_pBlockVectors[memTypeIndex];
11480  }
11481  pBlockVector->Free(allocation);
11482  }
11483  break;
11484  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
11485  FreeDedicatedMemory(allocation);
11486  break;
11487  default:
11488  VMA_ASSERT(0);
11489  }
11490  }
11491 
11492  allocation->SetUserData(this, VMA_NULL);
11493  vma_delete(this, allocation);
11494 }
11495 
11496 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
11497 {
11498  // Initialize.
11499  InitStatInfo(pStats->total);
11500  for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
11501  InitStatInfo(pStats->memoryType[i]);
11502  for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
11503  InitStatInfo(pStats->memoryHeap[i]);
11504 
11505  // Process default pools.
11506  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
11507  {
11508  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
11509  VMA_ASSERT(pBlockVector);
11510  pBlockVector->AddStats(pStats);
11511  }
11512 
11513  // Process custom pools.
11514  {
11515  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
11516  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
11517  {
11518  m_Pools[poolIndex]->m_BlockVector.AddStats(pStats);
11519  }
11520  }
11521 
11522  // Process dedicated allocations.
11523  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
11524  {
11525  const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
11526  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
11527  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
11528  VMA_ASSERT(pDedicatedAllocVector);
11529  for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
11530  {
11531  VmaStatInfo allocationStatInfo;
11532  (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
11533  VmaAddStatInfo(pStats->total, allocationStatInfo);
11534  VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
11535  VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
11536  }
11537  }
11538 
11539  // Postprocess.
11540  VmaPostprocessCalcStatInfo(pStats->total);
11541  for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
11542  VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
11543  for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
11544  VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
11545 }
11546 
11547 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
11548 
11549 VkResult VmaAllocator_T::Defragment(
11550  VmaAllocation* pAllocations,
11551  size_t allocationCount,
11552  VkBool32* pAllocationsChanged,
11553  const VmaDefragmentationInfo* pDefragmentationInfo,
11554  VmaDefragmentationStats* pDefragmentationStats)
11555 {
11556  if(pAllocationsChanged != VMA_NULL)
11557  {
11558  memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
11559  }
11560  if(pDefragmentationStats != VMA_NULL)
11561  {
11562  memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
11563  }
11564 
11565  const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
11566 
11567  VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
11568 
11569  const size_t poolCount = m_Pools.size();
11570 
11571  // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
11572  for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
11573  {
11574  VmaAllocation hAlloc = pAllocations[allocIndex];
11575  VMA_ASSERT(hAlloc);
11576  const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
11577  // DedicatedAlloc cannot be defragmented.
11578  const VkMemoryPropertyFlags requiredMemFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
11579  if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
11580  // Only HOST_VISIBLE and HOST_COHERENT memory types can be defragmented.
11581  ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & requiredMemFlags) == requiredMemFlags) &&
11582  // Lost allocation cannot be defragmented.
11583  (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
11584  {
11585  VmaBlockVector* pAllocBlockVector = VMA_NULL;
11586 
11587  const VmaPool hAllocPool = hAlloc->GetPool();
11588  // This allocation belongs to custom pool.
11589  if(hAllocPool != VK_NULL_HANDLE)
11590  {
11591  // Pools with linear algorithm are not defragmented.
11592  if(!hAllocPool->m_BlockVector.UsesLinearAlgorithm())
11593  {
11594  pAllocBlockVector = &hAllocPool->m_BlockVector;
11595  }
11596  }
11597  // This allocation belongs to general pool.
11598  else
11599  {
11600  pAllocBlockVector = m_pBlockVectors[memTypeIndex];
11601  }
11602 
11603  if(pAllocBlockVector != VMA_NULL)
11604  {
11605  VmaDefragmentator* const pDefragmentator =
11606  pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
11607  VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
11608  &pAllocationsChanged[allocIndex] : VMA_NULL;
11609  pDefragmentator->AddAllocation(hAlloc, pChanged);
11610  }
11611  }
11612  }
11613 
11614  VkResult result = VK_SUCCESS;
11615 
11616  // ======== Main processing.
11617 
11618  VkDeviceSize maxBytesToMove = SIZE_MAX;
11619  uint32_t maxAllocationsToMove = UINT32_MAX;
11620  if(pDefragmentationInfo != VMA_NULL)
11621  {
11622  maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
11623  maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
11624  }
11625 
11626  // Process standard memory.
11627  for(uint32_t memTypeIndex = 0;
11628  (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
11629  ++memTypeIndex)
11630  {
11631  // Only HOST_VISIBLE memory types can be defragmented.
11632  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
11633  {
11634  result = m_pBlockVectors[memTypeIndex]->Defragment(
11635  pDefragmentationStats,
11636  maxBytesToMove,
11637  maxAllocationsToMove);
11638  }
11639  }
11640 
11641  // Process custom pools.
11642  for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
11643  {
11644  result = m_Pools[poolIndex]->m_BlockVector.Defragment(
11645  pDefragmentationStats,
11646  maxBytesToMove,
11647  maxAllocationsToMove);
11648  }
11649 
11650  // ======== Destroy defragmentators.
11651 
11652  // Process custom pools.
11653  for(size_t poolIndex = poolCount; poolIndex--; )
11654  {
11655  m_Pools[poolIndex]->m_BlockVector.DestroyDefragmentator();
11656  }
11657 
11658  // Process standard memory.
11659  for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
11660  {
11661  if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
11662  {
11663  m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
11664  }
11665  }
11666 
11667  return result;
11668 }
11669 
11670 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
11671 {
11672  if(hAllocation->CanBecomeLost())
11673  {
11674  /*
11675  Warning: This is a carefully designed algorithm.
11676  Do not modify unless you really know what you're doing :)
11677  */
11678  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
11679  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
11680  for(;;)
11681  {
11682  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
11683  {
11684  pAllocationInfo->memoryType = UINT32_MAX;
11685  pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
11686  pAllocationInfo->offset = 0;
11687  pAllocationInfo->size = hAllocation->GetSize();
11688  pAllocationInfo->pMappedData = VMA_NULL;
11689  pAllocationInfo->pUserData = hAllocation->GetUserData();
11690  return;
11691  }
11692  else if(localLastUseFrameIndex == localCurrFrameIndex)
11693  {
11694  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
11695  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
11696  pAllocationInfo->offset = hAllocation->GetOffset();
11697  pAllocationInfo->size = hAllocation->GetSize();
11698  pAllocationInfo->pMappedData = VMA_NULL;
11699  pAllocationInfo->pUserData = hAllocation->GetUserData();
11700  return;
11701  }
11702  else // Last use time earlier than current time.
11703  {
11704  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
11705  {
11706  localLastUseFrameIndex = localCurrFrameIndex;
11707  }
11708  }
11709  }
11710  }
11711  else
11712  {
11713 #if VMA_STATS_STRING_ENABLED
11714  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
11715  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
11716  for(;;)
11717  {
11718  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
11719  if(localLastUseFrameIndex == localCurrFrameIndex)
11720  {
11721  break;
11722  }
11723  else // Last use time earlier than current time.
11724  {
11725  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
11726  {
11727  localLastUseFrameIndex = localCurrFrameIndex;
11728  }
11729  }
11730  }
11731 #endif
11732 
11733  pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
11734  pAllocationInfo->deviceMemory = hAllocation->GetMemory();
11735  pAllocationInfo->offset = hAllocation->GetOffset();
11736  pAllocationInfo->size = hAllocation->GetSize();
11737  pAllocationInfo->pMappedData = hAllocation->GetMappedData();
11738  pAllocationInfo->pUserData = hAllocation->GetUserData();
11739  }
11740 }
11741 
11742 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
11743 {
11744  // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
11745  if(hAllocation->CanBecomeLost())
11746  {
11747  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
11748  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
11749  for(;;)
11750  {
11751  if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
11752  {
11753  return false;
11754  }
11755  else if(localLastUseFrameIndex == localCurrFrameIndex)
11756  {
11757  return true;
11758  }
11759  else // Last use time earlier than current time.
11760  {
11761  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
11762  {
11763  localLastUseFrameIndex = localCurrFrameIndex;
11764  }
11765  }
11766  }
11767  }
11768  else
11769  {
11770 #if VMA_STATS_STRING_ENABLED
11771  uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
11772  uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
11773  for(;;)
11774  {
11775  VMA_ASSERT(localLastUseFrameIndex != VMA_FRAME_INDEX_LOST);
11776  if(localLastUseFrameIndex == localCurrFrameIndex)
11777  {
11778  break;
11779  }
11780  else // Last use time earlier than current time.
11781  {
11782  if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
11783  {
11784  localLastUseFrameIndex = localCurrFrameIndex;
11785  }
11786  }
11787  }
11788 #endif
11789 
11790  return true;
11791  }
11792 }
11793 
11794 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
11795 {
11796  VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u, flags=%u", pCreateInfo->memoryTypeIndex, pCreateInfo->flags);
11797 
11798  const bool isLinearAlgorithm = (pCreateInfo->flags & VMA_POOL_CREATE_LINEAR_ALGORITHM_BIT) != 0;
11799 
11800  VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
11801 
11802  if(newCreateInfo.maxBlockCount == 0)
11803  {
11804  newCreateInfo.maxBlockCount = isLinearAlgorithm ? 1 : SIZE_MAX;
11805  }
11806  if(newCreateInfo.minBlockCount > newCreateInfo.maxBlockCount ||
11807  isLinearAlgorithm && newCreateInfo.maxBlockCount > 1)
11808  {
11809  return VK_ERROR_INITIALIZATION_FAILED;
11810  }
11811  if(newCreateInfo.blockSize == 0)
11812  {
11813  newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
11814  }
11815 
11816  *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
11817 
11818  VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
11819  if(res != VK_SUCCESS)
11820  {
11821  vma_delete(this, *pPool);
11822  *pPool = VMA_NULL;
11823  return res;
11824  }
11825 
11826  // Add to m_Pools.
11827  {
11828  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
11829  (*pPool)->SetId(m_NextPoolId++);
11830  VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
11831  }
11832 
11833  return VK_SUCCESS;
11834 }
11835 
11836 void VmaAllocator_T::DestroyPool(VmaPool pool)
11837 {
11838  // Remove from m_Pools.
11839  {
11840  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
11841  bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
11842  VMA_ASSERT(success && "Pool not found in Allocator.");
11843  }
11844 
11845  vma_delete(this, pool);
11846 }
11847 
11848 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
11849 {
11850  pool->m_BlockVector.GetPoolStats(pPoolStats);
11851 }
11852 
11853 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
11854 {
11855  m_CurrentFrameIndex.store(frameIndex);
11856 }
11857 
11858 void VmaAllocator_T::MakePoolAllocationsLost(
11859  VmaPool hPool,
11860  size_t* pLostAllocationCount)
11861 {
11862  hPool->m_BlockVector.MakePoolAllocationsLost(
11863  m_CurrentFrameIndex.load(),
11864  pLostAllocationCount);
11865 }
11866 
11867 VkResult VmaAllocator_T::CheckPoolCorruption(VmaPool hPool)
11868 {
11869  return hPool->m_BlockVector.CheckCorruption();
11870 }
11871 
11872 VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits)
11873 {
11874  VkResult finalRes = VK_ERROR_FEATURE_NOT_PRESENT;
11875 
11876  // Process default pools.
11877  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
11878  {
11879  if(((1u << memTypeIndex) & memoryTypeBits) != 0)
11880  {
11881  VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
11882  VMA_ASSERT(pBlockVector);
11883  VkResult localRes = pBlockVector->CheckCorruption();
11884  switch(localRes)
11885  {
11886  case VK_ERROR_FEATURE_NOT_PRESENT:
11887  break;
11888  case VK_SUCCESS:
11889  finalRes = VK_SUCCESS;
11890  break;
11891  default:
11892  return localRes;
11893  }
11894  }
11895  }
11896 
11897  // Process custom pools.
11898  {
11899  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
11900  for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
11901  {
11902  if(((1u << m_Pools[poolIndex]->m_BlockVector.GetMemoryTypeIndex()) & memoryTypeBits) != 0)
11903  {
11904  VkResult localRes = m_Pools[poolIndex]->m_BlockVector.CheckCorruption();
11905  switch(localRes)
11906  {
11907  case VK_ERROR_FEATURE_NOT_PRESENT:
11908  break;
11909  case VK_SUCCESS:
11910  finalRes = VK_SUCCESS;
11911  break;
11912  default:
11913  return localRes;
11914  }
11915  }
11916  }
11917  }
11918 
11919  return finalRes;
11920 }
11921 
11922 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
11923 {
11924  *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
11925  (*pAllocation)->InitLost();
11926 }
11927 
11928 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
11929 {
11930  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
11931 
11932  VkResult res;
11933  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
11934  {
11935  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
11936  if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
11937  {
11938  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
11939  if(res == VK_SUCCESS)
11940  {
11941  m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
11942  }
11943  }
11944  else
11945  {
11946  res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
11947  }
11948  }
11949  else
11950  {
11951  res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
11952  }
11953 
11954  if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
11955  {
11956  (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
11957  }
11958 
11959  return res;
11960 }
11961 
11962 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
11963 {
11964  if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
11965  {
11966  (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
11967  }
11968 
11969  (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
11970 
11971  const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
11972  if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
11973  {
11974  VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
11975  m_HeapSizeLimit[heapIndex] += size;
11976  }
11977 }
11978 
11979 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
11980 {
11981  if(hAllocation->CanBecomeLost())
11982  {
11983  return VK_ERROR_MEMORY_MAP_FAILED;
11984  }
11985 
11986  switch(hAllocation->GetType())
11987  {
11988  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
11989  {
11990  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
11991  char *pBytes = VMA_NULL;
11992  VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
11993  if(res == VK_SUCCESS)
11994  {
11995  *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
11996  hAllocation->BlockAllocMap();
11997  }
11998  return res;
11999  }
12000  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12001  return hAllocation->DedicatedAllocMap(this, ppData);
12002  default:
12003  VMA_ASSERT(0);
12004  return VK_ERROR_MEMORY_MAP_FAILED;
12005  }
12006 }
12007 
12008 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
12009 {
12010  switch(hAllocation->GetType())
12011  {
12012  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
12013  {
12014  VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
12015  hAllocation->BlockAllocUnmap();
12016  pBlock->Unmap(this, 1);
12017  }
12018  break;
12019  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12020  hAllocation->DedicatedAllocUnmap(this);
12021  break;
12022  default:
12023  VMA_ASSERT(0);
12024  }
12025 }
12026 
12027 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
12028 {
12029  VkResult res = VK_SUCCESS;
12030  switch(hAllocation->GetType())
12031  {
12032  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12033  res = GetVulkanFunctions().vkBindBufferMemory(
12034  m_hDevice,
12035  hBuffer,
12036  hAllocation->GetMemory(),
12037  0); //memoryOffset
12038  break;
12039  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
12040  {
12041  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
12042  VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
12043  res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
12044  break;
12045  }
12046  default:
12047  VMA_ASSERT(0);
12048  }
12049  return res;
12050 }
12051 
12052 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
12053 {
12054  VkResult res = VK_SUCCESS;
12055  switch(hAllocation->GetType())
12056  {
12057  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12058  res = GetVulkanFunctions().vkBindImageMemory(
12059  m_hDevice,
12060  hImage,
12061  hAllocation->GetMemory(),
12062  0); //memoryOffset
12063  break;
12064  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
12065  {
12066  VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
12067  VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
12068  res = pBlock->BindImageMemory(this, hAllocation, hImage);
12069  break;
12070  }
12071  default:
12072  VMA_ASSERT(0);
12073  }
12074  return res;
12075 }
12076 
12077 void VmaAllocator_T::FlushOrInvalidateAllocation(
12078  VmaAllocation hAllocation,
12079  VkDeviceSize offset, VkDeviceSize size,
12080  VMA_CACHE_OPERATION op)
12081 {
12082  const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex();
12083  if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex))
12084  {
12085  const VkDeviceSize allocationSize = hAllocation->GetSize();
12086  VMA_ASSERT(offset <= allocationSize);
12087 
12088  const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize;
12089 
12090  VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
12091  memRange.memory = hAllocation->GetMemory();
12092 
12093  switch(hAllocation->GetType())
12094  {
12095  case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
12096  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
12097  if(size == VK_WHOLE_SIZE)
12098  {
12099  memRange.size = allocationSize - memRange.offset;
12100  }
12101  else
12102  {
12103  VMA_ASSERT(offset + size <= allocationSize);
12104  memRange.size = VMA_MIN(
12105  VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize),
12106  allocationSize - memRange.offset);
12107  }
12108  break;
12109 
12110  case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
12111  {
12112  // 1. Still within this allocation.
12113  memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize);
12114  if(size == VK_WHOLE_SIZE)
12115  {
12116  size = allocationSize - offset;
12117  }
12118  else
12119  {
12120  VMA_ASSERT(offset + size <= allocationSize);
12121  }
12122  memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize);
12123 
12124  // 2. Adjust to whole block.
12125  const VkDeviceSize allocationOffset = hAllocation->GetOffset();
12126  VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0);
12127  const VkDeviceSize blockSize = hAllocation->GetBlock()->m_pMetadata->GetSize();
12128  memRange.offset += allocationOffset;
12129  memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset);
12130 
12131  break;
12132  }
12133 
12134  default:
12135  VMA_ASSERT(0);
12136  }
12137 
12138  switch(op)
12139  {
12140  case VMA_CACHE_FLUSH:
12141  (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange);
12142  break;
12143  case VMA_CACHE_INVALIDATE:
12144  (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange);
12145  break;
12146  default:
12147  VMA_ASSERT(0);
12148  }
12149  }
12150  // else: Just ignore this call.
12151 }
12152 
12153 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
12154 {
12155  VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
12156 
12157  const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
12158  {
12159  VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
12160  AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
12161  VMA_ASSERT(pDedicatedAllocations);
12162  bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
12163  VMA_ASSERT(success);
12164  }
12165 
12166  VkDeviceMemory hMemory = allocation->GetMemory();
12167 
12168  if(allocation->GetMappedData() != VMA_NULL)
12169  {
12170  (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
12171  }
12172 
12173  FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
12174 
12175  VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
12176 }
12177 
12178 void VmaAllocator_T::FillAllocation(const VmaAllocation hAllocation, uint8_t pattern)
12179 {
12180  if(VMA_DEBUG_INITIALIZE_ALLOCATIONS &&
12181  !hAllocation->CanBecomeLost() &&
12182  (m_MemProps.memoryTypes[hAllocation->GetMemoryTypeIndex()].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
12183  {
12184  void* pData = VMA_NULL;
12185  VkResult res = Map(hAllocation, &pData);
12186  if(res == VK_SUCCESS)
12187  {
12188  memset(pData, (int)pattern, (size_t)hAllocation->GetSize());
12189  FlushOrInvalidateAllocation(hAllocation, 0, VK_WHOLE_SIZE, VMA_CACHE_FLUSH);
12190  Unmap(hAllocation);
12191  }
12192  else
12193  {
12194  VMA_ASSERT(0 && "VMA_DEBUG_INITIALIZE_ALLOCATIONS is enabled, but couldn't map memory to fill allocation.");
12195  }
12196  }
12197 }
12198 
12199 #if VMA_STATS_STRING_ENABLED
12200 
12201 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
12202 {
12203  bool dedicatedAllocationsStarted = false;
12204  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
12205  {
12206  VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
12207  AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
12208  VMA_ASSERT(pDedicatedAllocVector);
12209  if(pDedicatedAllocVector->empty() == false)
12210  {
12211  if(dedicatedAllocationsStarted == false)
12212  {
12213  dedicatedAllocationsStarted = true;
12214  json.WriteString("DedicatedAllocations");
12215  json.BeginObject();
12216  }
12217 
12218  json.BeginString("Type ");
12219  json.ContinueString(memTypeIndex);
12220  json.EndString();
12221 
12222  json.BeginArray();
12223 
12224  for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
12225  {
12226  json.BeginObject(true);
12227  const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
12228  hAlloc->PrintParameters(json);
12229  json.EndObject();
12230  }
12231 
12232  json.EndArray();
12233  }
12234  }
12235  if(dedicatedAllocationsStarted)
12236  {
12237  json.EndObject();
12238  }
12239 
12240  {
12241  bool allocationsStarted = false;
12242  for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
12243  {
12244  if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
12245  {
12246  if(allocationsStarted == false)
12247  {
12248  allocationsStarted = true;
12249  json.WriteString("DefaultPools");
12250  json.BeginObject();
12251  }
12252 
12253  json.BeginString("Type ");
12254  json.ContinueString(memTypeIndex);
12255  json.EndString();
12256 
12257  m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
12258  }
12259  }
12260  if(allocationsStarted)
12261  {
12262  json.EndObject();
12263  }
12264  }
12265 
12266  // Custom pools
12267  {
12268  VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
12269  const size_t poolCount = m_Pools.size();
12270  if(poolCount > 0)
12271  {
12272  json.WriteString("Pools");
12273  json.BeginObject();
12274  for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
12275  {
12276  json.BeginString();
12277  json.ContinueString(m_Pools[poolIndex]->GetId());
12278  json.EndString();
12279 
12280  m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
12281  }
12282  json.EndObject();
12283  }
12284  }
12285 }
12286 
12287 #endif // #if VMA_STATS_STRING_ENABLED
12288 
12290 // Public interface
12291 
12292 VkResult vmaCreateAllocator(
12293  const VmaAllocatorCreateInfo* pCreateInfo,
12294  VmaAllocator* pAllocator)
12295 {
12296  VMA_ASSERT(pCreateInfo && pAllocator);
12297  VMA_DEBUG_LOG("vmaCreateAllocator");
12298  *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
12299  return (*pAllocator)->Init(pCreateInfo);
12300 }
12301 
12302 void vmaDestroyAllocator(
12303  VmaAllocator allocator)
12304 {
12305  if(allocator != VK_NULL_HANDLE)
12306  {
12307  VMA_DEBUG_LOG("vmaDestroyAllocator");
12308  VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
12309  vma_delete(&allocationCallbacks, allocator);
12310  }
12311 }
12312 
12313 void vmaGetPhysicalDeviceProperties(
12314  VmaAllocator allocator,
12315  const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
12316 {
12317  VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
12318  *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
12319 }
12320 
12321 void vmaGetMemoryProperties(
12322  VmaAllocator allocator,
12323  const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
12324 {
12325  VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
12326  *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
12327 }
12328 
12329 void vmaGetMemoryTypeProperties(
12330  VmaAllocator allocator,
12331  uint32_t memoryTypeIndex,
12332  VkMemoryPropertyFlags* pFlags)
12333 {
12334  VMA_ASSERT(allocator && pFlags);
12335  VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
12336  *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
12337 }
12338 
12339 void vmaSetCurrentFrameIndex(
12340  VmaAllocator allocator,
12341  uint32_t frameIndex)
12342 {
12343  VMA_ASSERT(allocator);
12344  VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
12345 
12346  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12347 
12348  allocator->SetCurrentFrameIndex(frameIndex);
12349 }
12350 
12351 void vmaCalculateStats(
12352  VmaAllocator allocator,
12353  VmaStats* pStats)
12354 {
12355  VMA_ASSERT(allocator && pStats);
12356  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12357  allocator->CalculateStats(pStats);
12358 }
12359 
12360 #if VMA_STATS_STRING_ENABLED
12361 
12362 void vmaBuildStatsString(
12363  VmaAllocator allocator,
12364  char** ppStatsString,
12365  VkBool32 detailedMap)
12366 {
12367  VMA_ASSERT(allocator && ppStatsString);
12368  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12369 
12370  VmaStringBuilder sb(allocator);
12371  {
12372  VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
12373  json.BeginObject();
12374 
12375  VmaStats stats;
12376  allocator->CalculateStats(&stats);
12377 
12378  json.WriteString("Total");
12379  VmaPrintStatInfo(json, stats.total);
12380 
12381  for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
12382  {
12383  json.BeginString("Heap ");
12384  json.ContinueString(heapIndex);
12385  json.EndString();
12386  json.BeginObject();
12387 
12388  json.WriteString("Size");
12389  json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
12390 
12391  json.WriteString("Flags");
12392  json.BeginArray(true);
12393  if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
12394  {
12395  json.WriteString("DEVICE_LOCAL");
12396  }
12397  json.EndArray();
12398 
12399  if(stats.memoryHeap[heapIndex].blockCount > 0)
12400  {
12401  json.WriteString("Stats");
12402  VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
12403  }
12404 
12405  for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
12406  {
12407  if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
12408  {
12409  json.BeginString("Type ");
12410  json.ContinueString(typeIndex);
12411  json.EndString();
12412 
12413  json.BeginObject();
12414 
12415  json.WriteString("Flags");
12416  json.BeginArray(true);
12417  VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
12418  if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
12419  {
12420  json.WriteString("DEVICE_LOCAL");
12421  }
12422  if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
12423  {
12424  json.WriteString("HOST_VISIBLE");
12425  }
12426  if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
12427  {
12428  json.WriteString("HOST_COHERENT");
12429  }
12430  if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
12431  {
12432  json.WriteString("HOST_CACHED");
12433  }
12434  if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
12435  {
12436  json.WriteString("LAZILY_ALLOCATED");
12437  }
12438  json.EndArray();
12439 
12440  if(stats.memoryType[typeIndex].blockCount > 0)
12441  {
12442  json.WriteString("Stats");
12443  VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
12444  }
12445 
12446  json.EndObject();
12447  }
12448  }
12449 
12450  json.EndObject();
12451  }
12452  if(detailedMap == VK_TRUE)
12453  {
12454  allocator->PrintDetailedMap(json);
12455  }
12456 
12457  json.EndObject();
12458  }
12459 
12460  const size_t len = sb.GetLength();
12461  char* const pChars = vma_new_array(allocator, char, len + 1);
12462  if(len > 0)
12463  {
12464  memcpy(pChars, sb.GetData(), len);
12465  }
12466  pChars[len] = '\0';
12467  *ppStatsString = pChars;
12468 }
12469 
12470 void vmaFreeStatsString(
12471  VmaAllocator allocator,
12472  char* pStatsString)
12473 {
12474  if(pStatsString != VMA_NULL)
12475  {
12476  VMA_ASSERT(allocator);
12477  size_t len = strlen(pStatsString);
12478  vma_delete_array(allocator, pStatsString, len + 1);
12479  }
12480 }
12481 
12482 #endif // #if VMA_STATS_STRING_ENABLED
12483 
12484 /*
12485 This function is not protected by any mutex because it just reads immutable data.
12486 */
12487 VkResult vmaFindMemoryTypeIndex(
12488  VmaAllocator allocator,
12489  uint32_t memoryTypeBits,
12490  const VmaAllocationCreateInfo* pAllocationCreateInfo,
12491  uint32_t* pMemoryTypeIndex)
12492 {
12493  VMA_ASSERT(allocator != VK_NULL_HANDLE);
12494  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
12495  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
12496 
12497  if(pAllocationCreateInfo->memoryTypeBits != 0)
12498  {
12499  memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
12500  }
12501 
12502  uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
12503  uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
12504 
12505  const bool mapped = (pAllocationCreateInfo->flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
12506  if(mapped)
12507  {
12508  preferredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
12509  }
12510 
12511  // Convert usage to requiredFlags and preferredFlags.
12512  switch(pAllocationCreateInfo->usage)
12513  {
12514  case VMA_MEMORY_USAGE_UNKNOWN:
12515  break;
12516  case VMA_MEMORY_USAGE_GPU_ONLY:
12517  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
12518  {
12519  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
12520  }
12521  break;
12522  case VMA_MEMORY_USAGE_CPU_ONLY:
12523  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
12524  break;
12525  case VMA_MEMORY_USAGE_CPU_TO_GPU:
12526  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
12527  if(!allocator->IsIntegratedGpu() || (preferredFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
12528  {
12529  preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
12530  }
12531  break;
12532  case VMA_MEMORY_USAGE_GPU_TO_CPU:
12533  requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
12534  preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
12535  break;
12536  default:
12537  break;
12538  }
12539 
12540  *pMemoryTypeIndex = UINT32_MAX;
12541  uint32_t minCost = UINT32_MAX;
12542  for(uint32_t memTypeIndex = 0, memTypeBit = 1;
12543  memTypeIndex < allocator->GetMemoryTypeCount();
12544  ++memTypeIndex, memTypeBit <<= 1)
12545  {
12546  // This memory type is acceptable according to memoryTypeBits bitmask.
12547  if((memTypeBit & memoryTypeBits) != 0)
12548  {
12549  const VkMemoryPropertyFlags currFlags =
12550  allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
12551  // This memory type contains requiredFlags.
12552  if((requiredFlags & ~currFlags) == 0)
12553  {
12554  // Calculate cost as number of bits from preferredFlags not present in this memory type.
12555  uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
12556  // Remember memory type with lowest cost.
12557  if(currCost < minCost)
12558  {
12559  *pMemoryTypeIndex = memTypeIndex;
12560  if(currCost == 0)
12561  {
12562  return VK_SUCCESS;
12563  }
12564  minCost = currCost;
12565  }
12566  }
12567  }
12568  }
12569  return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
12570 }
12571 
12572 VkResult vmaFindMemoryTypeIndexForBufferInfo(
12573  VmaAllocator allocator,
12574  const VkBufferCreateInfo* pBufferCreateInfo,
12575  const VmaAllocationCreateInfo* pAllocationCreateInfo,
12576  uint32_t* pMemoryTypeIndex)
12577 {
12578  VMA_ASSERT(allocator != VK_NULL_HANDLE);
12579  VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
12580  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
12581  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
12582 
12583  const VkDevice hDev = allocator->m_hDevice;
12584  VkBuffer hBuffer = VK_NULL_HANDLE;
12585  VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
12586  hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
12587  if(res == VK_SUCCESS)
12588  {
12589  VkMemoryRequirements memReq = {};
12590  allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
12591  hDev, hBuffer, &memReq);
12592 
12593  res = vmaFindMemoryTypeIndex(
12594  allocator,
12595  memReq.memoryTypeBits,
12596  pAllocationCreateInfo,
12597  pMemoryTypeIndex);
12598 
12599  allocator->GetVulkanFunctions().vkDestroyBuffer(
12600  hDev, hBuffer, allocator->GetAllocationCallbacks());
12601  }
12602  return res;
12603 }
12604 
12605 VkResult vmaFindMemoryTypeIndexForImageInfo(
12606  VmaAllocator allocator,
12607  const VkImageCreateInfo* pImageCreateInfo,
12608  const VmaAllocationCreateInfo* pAllocationCreateInfo,
12609  uint32_t* pMemoryTypeIndex)
12610 {
12611  VMA_ASSERT(allocator != VK_NULL_HANDLE);
12612  VMA_ASSERT(pImageCreateInfo != VMA_NULL);
12613  VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
12614  VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
12615 
12616  const VkDevice hDev = allocator->m_hDevice;
12617  VkImage hImage = VK_NULL_HANDLE;
12618  VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
12619  hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
12620  if(res == VK_SUCCESS)
12621  {
12622  VkMemoryRequirements memReq = {};
12623  allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
12624  hDev, hImage, &memReq);
12625 
12626  res = vmaFindMemoryTypeIndex(
12627  allocator,
12628  memReq.memoryTypeBits,
12629  pAllocationCreateInfo,
12630  pMemoryTypeIndex);
12631 
12632  allocator->GetVulkanFunctions().vkDestroyImage(
12633  hDev, hImage, allocator->GetAllocationCallbacks());
12634  }
12635  return res;
12636 }
12637 
12638 VkResult vmaCreatePool(
12639  VmaAllocator allocator,
12640  const VmaPoolCreateInfo* pCreateInfo,
12641  VmaPool* pPool)
12642 {
12643  VMA_ASSERT(allocator && pCreateInfo && pPool);
12644 
12645  VMA_DEBUG_LOG("vmaCreatePool");
12646 
12647  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12648 
12649  VkResult res = allocator->CreatePool(pCreateInfo, pPool);
12650 
12651 #if VMA_RECORDING_ENABLED
12652  if(allocator->GetRecorder() != VMA_NULL)
12653  {
12654  allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool);
12655  }
12656 #endif
12657 
12658  return res;
12659 }
12660 
12661 void vmaDestroyPool(
12662  VmaAllocator allocator,
12663  VmaPool pool)
12664 {
12665  VMA_ASSERT(allocator);
12666 
12667  if(pool == VK_NULL_HANDLE)
12668  {
12669  return;
12670  }
12671 
12672  VMA_DEBUG_LOG("vmaDestroyPool");
12673 
12674  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12675 
12676 #if VMA_RECORDING_ENABLED
12677  if(allocator->GetRecorder() != VMA_NULL)
12678  {
12679  allocator->GetRecorder()->RecordDestroyPool(allocator->GetCurrentFrameIndex(), pool);
12680  }
12681 #endif
12682 
12683  allocator->DestroyPool(pool);
12684 }
12685 
12686 void vmaGetPoolStats(
12687  VmaAllocator allocator,
12688  VmaPool pool,
12689  VmaPoolStats* pPoolStats)
12690 {
12691  VMA_ASSERT(allocator && pool && pPoolStats);
12692 
12693  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12694 
12695  allocator->GetPoolStats(pool, pPoolStats);
12696 }
12697 
12698 void vmaMakePoolAllocationsLost(
12699  VmaAllocator allocator,
12700  VmaPool pool,
12701  size_t* pLostAllocationCount)
12702 {
12703  VMA_ASSERT(allocator && pool);
12704 
12705  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12706 
12707 #if VMA_RECORDING_ENABLED
12708  if(allocator->GetRecorder() != VMA_NULL)
12709  {
12710  allocator->GetRecorder()->RecordMakePoolAllocationsLost(allocator->GetCurrentFrameIndex(), pool);
12711  }
12712 #endif
12713 
12714  allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
12715 }
12716 
12717 VkResult vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool)
12718 {
12719  VMA_ASSERT(allocator && pool);
12720 
12721  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12722 
12723  VMA_DEBUG_LOG("vmaCheckPoolCorruption");
12724 
12725  return allocator->CheckPoolCorruption(pool);
12726 }
12727 
12728 VkResult vmaAllocateMemory(
12729  VmaAllocator allocator,
12730  const VkMemoryRequirements* pVkMemoryRequirements,
12731  const VmaAllocationCreateInfo* pCreateInfo,
12732  VmaAllocation* pAllocation,
12733  VmaAllocationInfo* pAllocationInfo)
12734 {
12735  VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
12736 
12737  VMA_DEBUG_LOG("vmaAllocateMemory");
12738 
12739  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12740 
12741  VkResult result = allocator->AllocateMemory(
12742  *pVkMemoryRequirements,
12743  false, // requiresDedicatedAllocation
12744  false, // prefersDedicatedAllocation
12745  VK_NULL_HANDLE, // dedicatedBuffer
12746  VK_NULL_HANDLE, // dedicatedImage
12747  *pCreateInfo,
12748  VMA_SUBALLOCATION_TYPE_UNKNOWN,
12749  pAllocation);
12750 
12751 #if VMA_RECORDING_ENABLED
12752  if(allocator->GetRecorder() != VMA_NULL)
12753  {
12754  allocator->GetRecorder()->RecordAllocateMemory(
12755  allocator->GetCurrentFrameIndex(),
12756  *pVkMemoryRequirements,
12757  *pCreateInfo,
12758  *pAllocation);
12759  }
12760 #endif
12761 
12762  if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS)
12763  {
12764  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
12765  }
12766 
12767  return result;
12768 }
12769 
12770 VkResult vmaAllocateMemoryForBuffer(
12771  VmaAllocator allocator,
12772  VkBuffer buffer,
12773  const VmaAllocationCreateInfo* pCreateInfo,
12774  VmaAllocation* pAllocation,
12775  VmaAllocationInfo* pAllocationInfo)
12776 {
12777  VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
12778 
12779  VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
12780 
12781  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12782 
12783  VkMemoryRequirements vkMemReq = {};
12784  bool requiresDedicatedAllocation = false;
12785  bool prefersDedicatedAllocation = false;
12786  allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
12787  requiresDedicatedAllocation,
12788  prefersDedicatedAllocation);
12789 
12790  VkResult result = allocator->AllocateMemory(
12791  vkMemReq,
12792  requiresDedicatedAllocation,
12793  prefersDedicatedAllocation,
12794  buffer, // dedicatedBuffer
12795  VK_NULL_HANDLE, // dedicatedImage
12796  *pCreateInfo,
12797  VMA_SUBALLOCATION_TYPE_BUFFER,
12798  pAllocation);
12799 
12800 #if VMA_RECORDING_ENABLED
12801  if(allocator->GetRecorder() != VMA_NULL)
12802  {
12803  allocator->GetRecorder()->RecordAllocateMemoryForBuffer(
12804  allocator->GetCurrentFrameIndex(),
12805  vkMemReq,
12806  requiresDedicatedAllocation,
12807  prefersDedicatedAllocation,
12808  *pCreateInfo,
12809  *pAllocation);
12810  }
12811 #endif
12812 
12813  if(pAllocationInfo && result == VK_SUCCESS)
12814  {
12815  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
12816  }
12817 
12818  return result;
12819 }
12820 
12821 VkResult vmaAllocateMemoryForImage(
12822  VmaAllocator allocator,
12823  VkImage image,
12824  const VmaAllocationCreateInfo* pCreateInfo,
12825  VmaAllocation* pAllocation,
12826  VmaAllocationInfo* pAllocationInfo)
12827 {
12828  VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
12829 
12830  VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
12831 
12832  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12833 
12834  VkMemoryRequirements vkMemReq = {};
12835  bool requiresDedicatedAllocation = false;
12836  bool prefersDedicatedAllocation = false;
12837  allocator->GetImageMemoryRequirements(image, vkMemReq,
12838  requiresDedicatedAllocation, prefersDedicatedAllocation);
12839 
12840  VkResult result = allocator->AllocateMemory(
12841  vkMemReq,
12842  requiresDedicatedAllocation,
12843  prefersDedicatedAllocation,
12844  VK_NULL_HANDLE, // dedicatedBuffer
12845  image, // dedicatedImage
12846  *pCreateInfo,
12847  VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
12848  pAllocation);
12849 
12850 #if VMA_RECORDING_ENABLED
12851  if(allocator->GetRecorder() != VMA_NULL)
12852  {
12853  allocator->GetRecorder()->RecordAllocateMemoryForImage(
12854  allocator->GetCurrentFrameIndex(),
12855  vkMemReq,
12856  requiresDedicatedAllocation,
12857  prefersDedicatedAllocation,
12858  *pCreateInfo,
12859  *pAllocation);
12860  }
12861 #endif
12862 
12863  if(pAllocationInfo && result == VK_SUCCESS)
12864  {
12865  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
12866  }
12867 
12868  return result;
12869 }
12870 
12871 void vmaFreeMemory(
12872  VmaAllocator allocator,
12873  VmaAllocation allocation)
12874 {
12875  VMA_ASSERT(allocator);
12876 
12877  if(allocation == VK_NULL_HANDLE)
12878  {
12879  return;
12880  }
12881 
12882  VMA_DEBUG_LOG("vmaFreeMemory");
12883 
12884  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12885 
12886 #if VMA_RECORDING_ENABLED
12887  if(allocator->GetRecorder() != VMA_NULL)
12888  {
12889  allocator->GetRecorder()->RecordFreeMemory(
12890  allocator->GetCurrentFrameIndex(),
12891  allocation);
12892  }
12893 #endif
12894 
12895  allocator->FreeMemory(allocation);
12896 }
12897 
12898 void vmaGetAllocationInfo(
12899  VmaAllocator allocator,
12900  VmaAllocation allocation,
12901  VmaAllocationInfo* pAllocationInfo)
12902 {
12903  VMA_ASSERT(allocator && allocation && pAllocationInfo);
12904 
12905  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12906 
12907 #if VMA_RECORDING_ENABLED
12908  if(allocator->GetRecorder() != VMA_NULL)
12909  {
12910  allocator->GetRecorder()->RecordGetAllocationInfo(
12911  allocator->GetCurrentFrameIndex(),
12912  allocation);
12913  }
12914 #endif
12915 
12916  allocator->GetAllocationInfo(allocation, pAllocationInfo);
12917 }
12918 
12919 VkBool32 vmaTouchAllocation(
12920  VmaAllocator allocator,
12921  VmaAllocation allocation)
12922 {
12923  VMA_ASSERT(allocator && allocation);
12924 
12925  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12926 
12927 #if VMA_RECORDING_ENABLED
12928  if(allocator->GetRecorder() != VMA_NULL)
12929  {
12930  allocator->GetRecorder()->RecordTouchAllocation(
12931  allocator->GetCurrentFrameIndex(),
12932  allocation);
12933  }
12934 #endif
12935 
12936  return allocator->TouchAllocation(allocation);
12937 }
12938 
12939 void vmaSetAllocationUserData(
12940  VmaAllocator allocator,
12941  VmaAllocation allocation,
12942  void* pUserData)
12943 {
12944  VMA_ASSERT(allocator && allocation);
12945 
12946  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12947 
12948  allocation->SetUserData(allocator, pUserData);
12949 
12950 #if VMA_RECORDING_ENABLED
12951  if(allocator->GetRecorder() != VMA_NULL)
12952  {
12953  allocator->GetRecorder()->RecordSetAllocationUserData(
12954  allocator->GetCurrentFrameIndex(),
12955  allocation,
12956  pUserData);
12957  }
12958 #endif
12959 }
12960 
12961 void vmaCreateLostAllocation(
12962  VmaAllocator allocator,
12963  VmaAllocation* pAllocation)
12964 {
12965  VMA_ASSERT(allocator && pAllocation);
12966 
12967  VMA_DEBUG_GLOBAL_MUTEX_LOCK;
12968 
12969  allocator->CreateLostAllocation(pAllocation);
12970 
12971 #if VMA_RECORDING_ENABLED
12972  if(allocator->GetRecorder() != VMA_NULL)
12973  {
12974  allocator->GetRecorder()->RecordCreateLostAllocation(
12975  allocator->GetCurrentFrameIndex(),
12976  *pAllocation);
12977  }
12978 #endif
12979 }
12980 
12981 VkResult vmaMapMemory(
12982  VmaAllocator allocator,
12983  VmaAllocation allocation,
12984  void** ppData)
12985 {
12986  VMA_ASSERT(allocator && allocation && ppData);
12987 
12988  VMA_DEBUG_GLOBAL_MUTEX_LOCK
12989 
12990  VkResult res = allocator->Map(allocation, ppData);
12991 
12992 #if VMA_RECORDING_ENABLED
12993  if(allocator->GetRecorder() != VMA_NULL)
12994  {
12995  allocator->GetRecorder()->RecordMapMemory(
12996  allocator->GetCurrentFrameIndex(),
12997  allocation);
12998  }
12999 #endif
13000 
13001  return res;
13002 }
13003 
13004 void vmaUnmapMemory(
13005  VmaAllocator allocator,
13006  VmaAllocation allocation)
13007 {
13008  VMA_ASSERT(allocator && allocation);
13009 
13010  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13011 
13012 #if VMA_RECORDING_ENABLED
13013  if(allocator->GetRecorder() != VMA_NULL)
13014  {
13015  allocator->GetRecorder()->RecordUnmapMemory(
13016  allocator->GetCurrentFrameIndex(),
13017  allocation);
13018  }
13019 #endif
13020 
13021  allocator->Unmap(allocation);
13022 }
13023 
13024 void vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
13025 {
13026  VMA_ASSERT(allocator && allocation);
13027 
13028  VMA_DEBUG_LOG("vmaFlushAllocation");
13029 
13030  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13031 
13032  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH);
13033 
13034 #if VMA_RECORDING_ENABLED
13035  if(allocator->GetRecorder() != VMA_NULL)
13036  {
13037  allocator->GetRecorder()->RecordFlushAllocation(
13038  allocator->GetCurrentFrameIndex(),
13039  allocation, offset, size);
13040  }
13041 #endif
13042 }
13043 
13044 void vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size)
13045 {
13046  VMA_ASSERT(allocator && allocation);
13047 
13048  VMA_DEBUG_LOG("vmaInvalidateAllocation");
13049 
13050  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13051 
13052  allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE);
13053 
13054 #if VMA_RECORDING_ENABLED
13055  if(allocator->GetRecorder() != VMA_NULL)
13056  {
13057  allocator->GetRecorder()->RecordInvalidateAllocation(
13058  allocator->GetCurrentFrameIndex(),
13059  allocation, offset, size);
13060  }
13061 #endif
13062 }
13063 
13064 VkResult vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits)
13065 {
13066  VMA_ASSERT(allocator);
13067 
13068  VMA_DEBUG_LOG("vmaCheckCorruption");
13069 
13070  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13071 
13072  return allocator->CheckCorruption(memoryTypeBits);
13073 }
13074 
13075 VkResult vmaDefragment(
13076  VmaAllocator allocator,
13077  VmaAllocation* pAllocations,
13078  size_t allocationCount,
13079  VkBool32* pAllocationsChanged,
13080  const VmaDefragmentationInfo *pDefragmentationInfo,
13081  VmaDefragmentationStats* pDefragmentationStats)
13082 {
13083  VMA_ASSERT(allocator && pAllocations);
13084 
13085  VMA_DEBUG_LOG("vmaDefragment");
13086 
13087  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13088 
13089  return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
13090 }
13091 
13092 VkResult vmaBindBufferMemory(
13093  VmaAllocator allocator,
13094  VmaAllocation allocation,
13095  VkBuffer buffer)
13096 {
13097  VMA_ASSERT(allocator && allocation && buffer);
13098 
13099  VMA_DEBUG_LOG("vmaBindBufferMemory");
13100 
13101  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13102 
13103  return allocator->BindBufferMemory(allocation, buffer);
13104 }
13105 
13106 VkResult vmaBindImageMemory(
13107  VmaAllocator allocator,
13108  VmaAllocation allocation,
13109  VkImage image)
13110 {
13111  VMA_ASSERT(allocator && allocation && image);
13112 
13113  VMA_DEBUG_LOG("vmaBindImageMemory");
13114 
13115  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13116 
13117  return allocator->BindImageMemory(allocation, image);
13118 }
13119 
13120 VkResult vmaCreateBuffer(
13121  VmaAllocator allocator,
13122  const VkBufferCreateInfo* pBufferCreateInfo,
13123  const VmaAllocationCreateInfo* pAllocationCreateInfo,
13124  VkBuffer* pBuffer,
13125  VmaAllocation* pAllocation,
13126  VmaAllocationInfo* pAllocationInfo)
13127 {
13128  VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
13129 
13130  VMA_DEBUG_LOG("vmaCreateBuffer");
13131 
13132  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13133 
13134  *pBuffer = VK_NULL_HANDLE;
13135  *pAllocation = VK_NULL_HANDLE;
13136 
13137  // 1. Create VkBuffer.
13138  VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
13139  allocator->m_hDevice,
13140  pBufferCreateInfo,
13141  allocator->GetAllocationCallbacks(),
13142  pBuffer);
13143  if(res >= 0)
13144  {
13145  // 2. vkGetBufferMemoryRequirements.
13146  VkMemoryRequirements vkMemReq = {};
13147  bool requiresDedicatedAllocation = false;
13148  bool prefersDedicatedAllocation = false;
13149  allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
13150  requiresDedicatedAllocation, prefersDedicatedAllocation);
13151 
13152  // Make sure alignment requirements for specific buffer usages reported
13153  // in Physical Device Properties are included in alignment reported by memory requirements.
13154  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
13155  {
13156  VMA_ASSERT(vkMemReq.alignment %
13157  allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
13158  }
13159  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
13160  {
13161  VMA_ASSERT(vkMemReq.alignment %
13162  allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
13163  }
13164  if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
13165  {
13166  VMA_ASSERT(vkMemReq.alignment %
13167  allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
13168  }
13169 
13170  // 3. Allocate memory using allocator.
13171  res = allocator->AllocateMemory(
13172  vkMemReq,
13173  requiresDedicatedAllocation,
13174  prefersDedicatedAllocation,
13175  *pBuffer, // dedicatedBuffer
13176  VK_NULL_HANDLE, // dedicatedImage
13177  *pAllocationCreateInfo,
13178  VMA_SUBALLOCATION_TYPE_BUFFER,
13179  pAllocation);
13180 
13181 #if VMA_RECORDING_ENABLED
13182  if(allocator->GetRecorder() != VMA_NULL)
13183  {
13184  allocator->GetRecorder()->RecordCreateBuffer(
13185  allocator->GetCurrentFrameIndex(),
13186  *pBufferCreateInfo,
13187  *pAllocationCreateInfo,
13188  *pAllocation);
13189  }
13190 #endif
13191 
13192  if(res >= 0)
13193  {
13194  // 3. Bind buffer with memory.
13195  res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
13196  if(res >= 0)
13197  {
13198  // All steps succeeded.
13199  #if VMA_STATS_STRING_ENABLED
13200  (*pAllocation)->InitBufferImageUsage(pBufferCreateInfo->usage);
13201  #endif
13202  if(pAllocationInfo != VMA_NULL)
13203  {
13204  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
13205  }
13206 
13207  return VK_SUCCESS;
13208  }
13209  allocator->FreeMemory(*pAllocation);
13210  *pAllocation = VK_NULL_HANDLE;
13211  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
13212  *pBuffer = VK_NULL_HANDLE;
13213  return res;
13214  }
13215  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
13216  *pBuffer = VK_NULL_HANDLE;
13217  return res;
13218  }
13219  return res;
13220 }
13221 
13222 void vmaDestroyBuffer(
13223  VmaAllocator allocator,
13224  VkBuffer buffer,
13225  VmaAllocation allocation)
13226 {
13227  VMA_ASSERT(allocator);
13228 
13229  if(buffer == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
13230  {
13231  return;
13232  }
13233 
13234  VMA_DEBUG_LOG("vmaDestroyBuffer");
13235 
13236  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13237 
13238 #if VMA_RECORDING_ENABLED
13239  if(allocator->GetRecorder() != VMA_NULL)
13240  {
13241  allocator->GetRecorder()->RecordDestroyBuffer(
13242  allocator->GetCurrentFrameIndex(),
13243  allocation);
13244  }
13245 #endif
13246 
13247  if(buffer != VK_NULL_HANDLE)
13248  {
13249  (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
13250  }
13251 
13252  if(allocation != VK_NULL_HANDLE)
13253  {
13254  allocator->FreeMemory(allocation);
13255  }
13256 }
13257 
13258 VkResult vmaCreateImage(
13259  VmaAllocator allocator,
13260  const VkImageCreateInfo* pImageCreateInfo,
13261  const VmaAllocationCreateInfo* pAllocationCreateInfo,
13262  VkImage* pImage,
13263  VmaAllocation* pAllocation,
13264  VmaAllocationInfo* pAllocationInfo)
13265 {
13266  VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
13267 
13268  VMA_DEBUG_LOG("vmaCreateImage");
13269 
13270  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13271 
13272  *pImage = VK_NULL_HANDLE;
13273  *pAllocation = VK_NULL_HANDLE;
13274 
13275  // 1. Create VkImage.
13276  VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
13277  allocator->m_hDevice,
13278  pImageCreateInfo,
13279  allocator->GetAllocationCallbacks(),
13280  pImage);
13281  if(res >= 0)
13282  {
13283  VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
13284  VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
13285  VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
13286 
13287  // 2. Allocate memory using allocator.
13288  VkMemoryRequirements vkMemReq = {};
13289  bool requiresDedicatedAllocation = false;
13290  bool prefersDedicatedAllocation = false;
13291  allocator->GetImageMemoryRequirements(*pImage, vkMemReq,
13292  requiresDedicatedAllocation, prefersDedicatedAllocation);
13293 
13294  res = allocator->AllocateMemory(
13295  vkMemReq,
13296  requiresDedicatedAllocation,
13297  prefersDedicatedAllocation,
13298  VK_NULL_HANDLE, // dedicatedBuffer
13299  *pImage, // dedicatedImage
13300  *pAllocationCreateInfo,
13301  suballocType,
13302  pAllocation);
13303 
13304 #if VMA_RECORDING_ENABLED
13305  if(allocator->GetRecorder() != VMA_NULL)
13306  {
13307  allocator->GetRecorder()->RecordCreateImage(
13308  allocator->GetCurrentFrameIndex(),
13309  *pImageCreateInfo,
13310  *pAllocationCreateInfo,
13311  *pAllocation);
13312  }
13313 #endif
13314 
13315  if(res >= 0)
13316  {
13317  // 3. Bind image with memory.
13318  res = allocator->BindImageMemory(*pAllocation, *pImage);
13319  if(res >= 0)
13320  {
13321  // All steps succeeded.
13322  #if VMA_STATS_STRING_ENABLED
13323  (*pAllocation)->InitBufferImageUsage(pImageCreateInfo->usage);
13324  #endif
13325  if(pAllocationInfo != VMA_NULL)
13326  {
13327  allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
13328  }
13329 
13330  return VK_SUCCESS;
13331  }
13332  allocator->FreeMemory(*pAllocation);
13333  *pAllocation = VK_NULL_HANDLE;
13334  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
13335  *pImage = VK_NULL_HANDLE;
13336  return res;
13337  }
13338  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
13339  *pImage = VK_NULL_HANDLE;
13340  return res;
13341  }
13342  return res;
13343 }
13344 
13345 void vmaDestroyImage(
13346  VmaAllocator allocator,
13347  VkImage image,
13348  VmaAllocation allocation)
13349 {
13350  VMA_ASSERT(allocator);
13351 
13352  if(image == VK_NULL_HANDLE && allocation == VK_NULL_HANDLE)
13353  {
13354  return;
13355  }
13356 
13357  VMA_DEBUG_LOG("vmaDestroyImage");
13358 
13359  VMA_DEBUG_GLOBAL_MUTEX_LOCK
13360 
13361 #if VMA_RECORDING_ENABLED
13362  if(allocator->GetRecorder() != VMA_NULL)
13363  {
13364  allocator->GetRecorder()->RecordDestroyImage(
13365  allocator->GetCurrentFrameIndex(),
13366  allocation);
13367  }
13368 #endif
13369 
13370  if(image != VK_NULL_HANDLE)
13371  {
13372  (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
13373  }
13374  if(allocation != VK_NULL_HANDLE)
13375  {
13376  allocator->FreeMemory(allocation);
13377  }
13378 }
13379 
13380 #endif // #ifdef VMA_IMPLEMENTATION
VmaVulkanFunctions::vkGetPhysicalDeviceProperties
PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties
Definition: vk_mem_alloc.h:1429
VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
Set this flag if the allocation should have its own memory block.
Definition: vk_mem_alloc.h:1742
vmaUnmapMemory
void vmaUnmapMemory(VmaAllocator allocator, VmaAllocation allocation)
Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
VmaAllocatorCreateInfo::physicalDevice
VkPhysicalDevice physicalDevice
Vulkan physical device.
Definition: vk_mem_alloc.h:1498
diff --git a/src/vk_mem_alloc.h b/src/vk_mem_alloc.h index e92873b..697b733 100644 --- a/src/vk_mem_alloc.h +++ b/src/vk_mem_alloc.h @@ -29,7 +29,7 @@ extern "C" { /** \mainpage Vulkan Memory Allocator -Version 2.1.0-alpha.4 (2018-08-22) +Version 2.1.0-beta.1 (2018-08-24) Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved. \n License: MIT

Pools with linear algorithm support lost allocations when used as ring buffer. If there is not enough free space for a new allocation, but existing allocations from the front of the queue can become lost, they become lost and the allocation succeeds.