From 5f46b51f12fa53ffaeddd81a8a948a6a7878a794 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 3 Aug 2025 15:59:38 -0500 Subject: [PATCH 01/69] + xo-alloc + xo-object + xo-alloc docs + GC utests --- CMakeLists.txt | 8 +- docs/CMakeLists.txt | 9 + docs/README | 41 ++ docs/_static/README | 1 + docs/_static/img/favicon.ico | Bin 0 -> 309936 bytes docs/conf.py | 39 ++ docs/implementation.rst | 202 +++++++ docs/index.rst | 14 + docs/install.rst | 120 +++++ docs/introduction.rst | 268 ++++++++++ include/xo/alloc/AllocPolicy.hpp | 58 +++ .../alloc/{LinearAlloc.hpp => ArenaAlloc.hpp} | 46 +- include/xo/alloc/Forwarding.hpp | 28 + include/xo/alloc/Forwarding1.hpp | 40 ++ include/xo/alloc/GC.hpp | 310 +++++++++++ include/xo/alloc/GCAlloc.hpp | 20 - include/xo/alloc/IAlloc.hpp | 42 +- include/xo/alloc/ListAlloc.hpp | 53 +- include/xo/alloc/Object.hpp | 232 +++++++++ include/xo/alloc/Stack.hpp | 49 ++ src/alloc/AllocPolicy.cpp | 13 + src/alloc/{LinearAlloc.cpp => ArenaAlloc.cpp} | 79 +-- src/alloc/CMakeLists.txt | 7 +- src/alloc/Forwarding1.cpp | 45 ++ src/alloc/GC.cpp | 492 ++++++++++++++++++ src/alloc/IAlloc.cpp | 54 ++ src/alloc/ListAlloc.cpp | 318 +++++++++++ src/alloc/Object.cpp | 196 +++++++ utest/ArenaAlloc.test.cpp | 87 ++++ utest/CMakeLists.txt | 3 +- utest/GC.test.cpp | 69 +++ utest/LinearAlloc.test.cpp | 42 +- 32 files changed, 2903 insertions(+), 82 deletions(-) create mode 100644 docs/CMakeLists.txt create mode 100644 docs/README create mode 100644 docs/_static/README create mode 100644 docs/_static/img/favicon.ico create mode 100644 docs/conf.py create mode 100644 docs/implementation.rst create mode 100644 docs/index.rst create mode 100644 docs/install.rst create mode 100644 docs/introduction.rst create mode 100644 include/xo/alloc/AllocPolicy.hpp rename include/xo/alloc/{LinearAlloc.hpp => ArenaAlloc.hpp} (61%) create mode 100644 include/xo/alloc/Forwarding.hpp create mode 100644 include/xo/alloc/Forwarding1.hpp create mode 100644 include/xo/alloc/GC.hpp delete mode 100644 include/xo/alloc/GCAlloc.hpp create mode 100644 include/xo/alloc/Object.hpp create mode 100644 include/xo/alloc/Stack.hpp create mode 100644 src/alloc/AllocPolicy.cpp rename src/alloc/{LinearAlloc.cpp => ArenaAlloc.cpp} (52%) create mode 100644 src/alloc/Forwarding1.cpp create mode 100644 src/alloc/GC.cpp create mode 100644 src/alloc/IAlloc.cpp create mode 100644 src/alloc/ListAlloc.cpp create mode 100644 src/alloc/Object.cpp create mode 100644 utest/ArenaAlloc.test.cpp create mode 100644 utest/GC.test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index f93e8b0c..eebf3aff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,9 +20,13 @@ add_definitions(${PROJECT_CXX_FLAGS}) # must complete definition of expression lib before configuring examples add_subdirectory(src/alloc) - +add_subdirectory(utest) xo_export_cmake_config(${PROJECT_NAME} ${PROJECT_VERSION} ${PROJECT_NAME}Targets) # ---------------------------------------------------------------- +# docs targets depend on other library/utest/exec targets above, +# --> must come after them. +# +add_subdirectory(docs) -add_subdirectory(utest) +# end CmakeLists.txt diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt new file mode 100644 index 00000000..e13b26a0 --- /dev/null +++ b/docs/CMakeLists.txt @@ -0,0 +1,9 @@ +# xo-alloc/docs/CMakeLists.txt + +xo_doxygen_collect_deps() +xo_docdir_doxygen_config() +xo_docdir_sphinx_config( + index.rst install.rst introduction.rst implementation.rst) + +# see xo-reader/doc or xo-unit/doc for working examples +# example.rst install.rst implementation.rst diff --git a/docs/README b/docs/README new file mode 100644 index 00000000..6aff5d41 --- /dev/null +++ b/docs/README @@ -0,0 +1,41 @@ +standalone build + + +-----------------------------------------------+ + | cmake | + | CMakeLists.txt | + | $PREFIX/share/cmake/xo_macros/xo_cxx.cmake | + +-----------------------------------------------+ + | + | +----------------------+ + +------------------------------------------------->| .build/docs/Doxyfile | + | +----------------------+ + | ^ + | (cmake) | + | /------------/ + | | + | +---------------------------------------+ +-----------------+ + +---->| doxygen |--------->| .build/docs/dox | + | | $PREFIX/share/xo-macros/Doxyfile.in | (doxygen)| +- html/ | + | +---------------------------------------+ | +- xml/ | + | +-----------------+ + | | + | |(sphinx) + | | + | v + | +---------------------------------------+ +--------------------+ + \---->| sphinx |------->| .build/docs/sphinx | + | +- conf.py | | +- html/ | + | +- _static/ | +--------------------+ + | +- *.rst | + +---------------------------------------+ + +umbrella build relies on top-level cmake macros + +files + + README this file + CMakeLists.txt build entry point + conf.py sphinx config + _static static files for sphinx + + index.rst toplevel sphinx document; entry point diff --git a/docs/_static/README b/docs/_static/README new file mode 100644 index 00000000..7297d046 --- /dev/null +++ b/docs/_static/README @@ -0,0 +1 @@ +add any static {.html, .js, ..} files for sphinx to pickup here diff --git a/docs/_static/img/favicon.ico b/docs/_static/img/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..4163dd69c734f8186cf1ce5e726213cebd231c31 GIT binary patch literal 309936 zcmZQzU}WH800Bk@1%|zv3=GQ{7#I#50EsIwXaq4aBx^A+G&Df@9E=RzHB1Z%2@w8@ zDGUsoTbLOf93XrRCkBRSNfrhJ0|#lWE5$ikqY0O79?U|^U$ zn}tC_0>bZ5U|_Ib!@?jS0O4n_Ffbh6%EHhY;OEXI1#%~^r-w@rND_oO*cccXVv1Iz zF)(Phc)B=-RNQ)dx4b6i>dX%x@9!*6Gnu4wa+A(!HtFQYZ7*je9SRb%TBzP4cZ$tP zVYbt&Pn&i>;q_KHJ!zBSLY3J9UJ4VK77ABwWnMNbtss9=YN_w;@ALjX?m2M7c-yHn zb2QI?|2$Jb#qjf;nbq?w|27CXaVWNsOmI2hTLyV&ero8F#WKIIuVn&NhzE8B9^U#LMNYX4rLd$CPc&nsSS&rGvkI^&_$nF1L# z_d{D)7<)DQJFe|mC~$Iu_x|S_4_;rx#UxfO_dfqc+Q(0(Ypg%a_slVu3-bv2A>zrf zXvLrJsUOz94$zigSokL68AJOnzTeh-m!~H22n0p%%r5;^zh#*{N5pIPJqsUpDm+*z z`nsb2nVibGJsf%U7rtG+rLLghAksRuzu}sNm*LiZ(rVxN`0MS2Ch9vir>{Bw79 zl(7G@ik0@~dUIuM7MR)^Z(o1LMF9jHA_`AEEcku(!@Zza+ojJ-Uex+`v~=dw@6th^ zKgP+jG@c{8zvI-Y3y=8?(UF<-aX6vKr6HmCfFzyMOK4XUi8l ziPZhL5^k@0v z8PmGW{&{5fPXF{H`!$Z~G4Fjg`)c62LfLJaZHM~V zmKMmg-&~wEBRcwQ^uxUoH!raF?@6%U@kgs|dwfX4xpc*ui?(f?C%=Plx~A)uGM}3j zcJf(sY1wz)C(IVYD+*;{?Pa%ZYo z_Pq#>_j+4pBkR_McI>J6<7V-M=|%O#;MJ>tOnrPI!Ft)kF7=A4TFNz{JBqxYF?Q^k zVZ%6W&7y;aKgzEsN(B_P$~$Osv|Ci)k?_l%#jOA5T$#&GRRslyCwKPRH%@b2uydmE zAw|1<>vgZB0!ELhsdKP?}rQHfY z{ytq8Zl7j7OD1mmTLzcP$e$c0G7VCpXD!#A*17%hNmr0Sk!9{Jm2VQU>{a{Tb`&`a zHJn&kn7rV3<+9y+%cmwxuJ4|bR8uPY{LkauJF089Ek4QX$)J>8_g|UgJR7SzV{q>A z$=NS6Pai)&vt-lxHOQL8WZpT^hY_1DTwl-0qB_{ov@f6{ks7h0WK z=oOf3J+G!nwk$6v;o6Oh+mt0OJsFhL)7~@4-*I<7@znTf)L;1pBE9R9^1hnEWTTEb z1T#;TZ8w^&wb^p%fzq4bf6aevSGoCZhTF0xGkc+*A@TgRwLaw{z1xyCS)Oiqypi27 zD*IOMaU-Eg3@&*t3X@XQM2q}eV~_dIb0|Kc`<6w)m%%WGYx8^;4vv+w*>@_MPCmW& z@4iR#d*gd$0?MI+H=}rFa$jNi>L6^O_cw#>7{lVmKnIaxe#@_>D^oT5Wpmi$G@^p9 z-aWqK*`j;5GhVAOc4+x5oX2s7VcBx|$%d`_1cPrV=PW!XQMBiJ!+~!{MY@7^EPgE! z`^r?#lVPfB;=ZlwOxBFMnX41H{#BeY|F>aQtJg%G=~L^QcPDVPuF(AI`&ET;ioJMn z>3!B8>Iw2b59RrHYHxVBL7KaLkB~lJ^Ynj@%0J|~8yQVvI5oNZ=d}C$7A!NmBj-%s zE?K1+U+VlZ|Hn*iDGn2b@Lvor6AKJ-ztuDKvGB1hn=t=;MDnfQb-(8486DmAD&wgN zgUK#?xBA%5{>7e58nvb~w*4tQaPfeT#K(QV?JjVaFuZQOdLSwx_Sb#uM!5y{f45sb zVOuSGRCL?@9;Q!!8ou5t6H!)CP*7N4qPc+8LdJsi2Txi?o<)sMV;oxzgAUU@=4nY> zwdNPtml)jtGyCNZX*c;7i^DfYeg3%XXVLX=+0yxfBb% zAHfn&)NXTKSYh!(^}y)^!3VTwHM=ZGxZFrRl$^Uq`QJq!_yHjIB*XS{#8KvaL`cgd=5yQXz<*WT0E|}5Tm2s4_gZ^{M_a&De1{@bh)5~Q62Vl_%2PFc zvZKC|==*1xYkxXfmb3h_(>5#l>{UMht%sYRki$m~W_HH=jCQ9Vq`lml$n_`pK=%wk zo9v+41*ZN%w_Rl$EGe13l@BF95X}9}Qo}9AqpFTzN!1YG`!wbx=M6llJ*8H)a z+4HtI!-seA3eyxeedhaP_M~Xm+lo()*^fT5ED!B}q`!)1o845YhS^V{vc<#ADqElFt+jO2Je3|`P`hd4^Uq#3M~zSV-DZ5+rovEN z#;~vb<=y+Y429nwxO>3&z-*DogYpHBwkKR&{NT5Z>AKG6s}IR!SudLtD3I~zpTz$B zI}>@A&&jl2a7W`~CXcFS&&+Mr`~`N=dVe;4xj(ldQ)!ReM6rL2=l5|jkz2bfPViR*O#hYg(`0SuQky!%+VZIT@mbch?o9MMaOkFj!VmukpEn%MRf%ep zZj4X7^+$E(f7chi8}pNGzqiF~lg)6RuC!M1W#RKlQ|H8sF3f*rxJ~(#>`KNb^O5S>#uF3_L1%@fIe3xe^C@9#3Gx!IyvHlMh-!8DL%29LK zwXovZuQ$wpJA+N!BJG6!f&W+U>ioS{+Vj;my?3>R{;E6T)lHLL^ht_%GW;|Y+`+cF zNI!b2ecMZ(JzNo))(>hT{~4I{G)*~v@9Wk-dNN7ww+(YIIy}7oV)nlkzxGw{<@Gnw%$L*4r$(;;uI#bN$^kh1TysuKYZ9ptphPVc7?Lg>dP@DIMX^Fo|=Pf$0TCdpGjj z?re?u`+@t*mm@!#+v<|IbC%$us|xzI^Y!76=Hn?-Eu@+$Udjb8W)b#$Gj5)>#n(1^MCn;pVbO7S-S3tFAXXZ_g9M_J@7u zd7F&GXRg#-coc1^G>O42&Hq9D$$gX7OaD;(VDE6KY{SvCpUwuE@e!YxCLiy8*z`&x zUGwM?r#n*`kA1d@^IP(x@;2K)v5a5KeVMi%Ppgk&j^T~*diLoPTiTZ8D%oX|7~1mH z&#=tT$^FOYBl=0qYtP&n`K#qx`||Fr6ux<})Z%!d#l_Maa;=k3ONY$<-}#e~yXyDO zt!tZqFOzq?oxi@d_c2F1=LHH&P4_w; z_~o!!<-(qMKA)1}=Ul%RcV-py^9P}Cjo)nGJFN2cY@5O1Q;|=F=PY_qxxeJpew7AA zDb|?wmyG9@2!B5NUwX-h=RUuKKU|t^VZ43$I;}G|p1DN{dCpp;WYQM1sIPgk>B@C+ zRfiR={xn?c%Zr-6Hu13If1jq-jIGZ*4l@MD?c9qg0 zb01!8eyK4jcG{`i8fW5qX1-(O5B@Ej`r`DAxW7K~HB3M1d;SFY1xlqQeoHx1qOYU; zyG9^z5#N!T4KwWu2JwgEa$we`jMH&V%|XQJ0cHMjp{E*?Pxxj`&9Ro zx5(w|lV8qz!MZQO+QG1>bjH0uIj#Ra+sfkRYv@}uzMuT_UWHi3HrWLdcC9nnX05&a zJ~pB6twL+*^rUN5DX#?n%zFFa$cI|K&3X305A)T3uQ3RWoYk;@@=v=;*_DdZmgE{n z7pDIT2tL3+Vd<4e!p9Ta8qN2HGdAT|Z zv8mnW3*fouIU~5txOT?+#Ahws8&@(L`X2V#+K_*f&yVL^$^-cny+5)CK5ku1{SFnb_yo3U$BAX)$;h#Ean zXIsy=S;yk?-V)^^ebKpd8~acEY23?@lWgq}Saj~-mfz-k=TBp}oXf6mCwOemqsh6o z|2?-ci>PKQ&7987KBHLCWuk@1=GO3KEFZGZbm=R$J&rQEAI`7feZ2dS#smG7e-oxn zs$HG0$oAObj9B69d(mqT$Q`bany9Uwe6H+ti0|$@Wd)0#T%A~(^3v$`!O&fXGq(Pj zX*Y*;#!&~So-5x^)b!i7m@u!gntkT=pUEeBbSGL@Grqq0$1a-rO_KEvud6=amhS6U zzFpmU=j1gWj)u>hNmnB2s*2Gp|O z93H(a|Ns6(qiq5^zgV6zdEoP#M%goxXjGxaRVY%@7%=UjT zQ{<{^`7Zy~ciE?NB**yrhLh*Z51sMM+$tJzQLUk5;rdrx8|OaXTl-q3_v~W2H;37O%S^=sJ7>HSZ&?1ojP-8Ol=UYTs~+Go z^Jfq?{JwcBOF+n_y~=%UZ)+^BH&@q2$T@h;R({<6`Ge`2YX$dAMu8 ze4!%!Wo-eGK@L2N80QEc$d+ni68+OJ&zQk)=W)QbJNDJIkV`W`ua^^Cp) zrxqBt?iW@2`C;{eFL%VZJV^Io=@j{Muvt>7SZ0HOr5R&I`&r=wT3SyxNcO&5^gS`3 zf1X}KY5j)IgoW(wHhfvlZ_T}@%+5Mf@EMdd6&Ca>%@CYX-dL%@>(2HeZlmbG-bn|# z82qk2IB{XY2}RaBPS2(`rr+XPrLZ<`?!GGC;~6^w5AvkG?%M8j(1rE?;)iSa55C+c zJ3&KSDq`Bh=Uqj1PBsr7o0LZ|cz1uDZSU;6;HApRDgj(*t3 zj|TVWG3soJUC};~Wg7d3NcqVpS|k=KH(JbAZ?F$Mvbm`J`SN8KR1%yR`fuxpl{yEt{dD*t<{Jcj*CaT25YhC@9!!KXApUZ9S*1yZTg3M(jPCipm zc))wcYU(G03-7)^Ji@RmNBn`z6(*s=9SV1+D;_E6Wf$96$rRDFwC}*49;PpC3r!EP zJg{D6)92FX@?iF^Ahn#1noi4^-{-D)Hd%BP>(tLI9~1(XFH}4^^OVkB-O5uoX^i3u zpP7$*Sjx0n^v1GP8q--e*k5>SsLJwOE4b{`$(1K2CMNx`F;Z{fSSA)wVCl=im==B4 z%Hm|BY;H_|on~>;g_oz!WQZiKusM_S!?#tVokhU7f8&9ZPrqH=Kk>0?j=q8ek8^{j z+Uz2Yc9sBUmqhynb7}El4z6X1->SruEFMYLJxln-$fcLAps+xEM*gio?u>!b;=u}i zO^ZcbmWU~|DNcQu&gDJ3>4^@bOQYq5H3|QX*8O8%keDyOZSqA1L!P*4^P?@*bkY?R z6b_j7&*WNmW5)!h*$)n^dphrhOI*q-3oV_w7bjj8zq-L%v|!&nqpMpjU#gyS>bY{D z=RnhGPo^6hpXTPow5V=2ZBb?HP&^fO^?NSMHB8#H{%yyc(r#nZPh z^xT%M$YQX(*{U+vbK{b>ZaV$cAmPk4*N90MW|o%=D$SNM;+H*cpw1#1b8ef` zwF}G7^evydNPXRlvdiN7M-;A1+Nq#$p?9hN`8lc#sYW-XK>oh8@NKVQ2|HKDX4zg1 z!O|2zhS-!iwK?UMo(!G}OrE^*4nN;APxDlHaCl|xm$1{591JUxPwWZVC+z*#)AA*= zgv6izgR?{YT7UFT6ZyBx>(w_A<*QE5ST;<1bGqfTR{GoqI{`y>nJIq*t8SIt+Ouk6 z?V+O+denCM?cZiiw4b{|=W*jQ9{c>GJj_uOU)wPVJ^R?vCY~D`vM-}z@#Tm2 z;&e|l9G#GQ+cC>hZ26SuHR_l27D$;feBQED)bHnlgZ>9}g>AyNgq`wXpXhBpq4Lq) z_J~=`43!=Xj}7NGa>+5g)1SU+(ZiQ`P+k8YC!c|JM1G$-zd zX@T(5eHCG!KW=+?nYrRt*~5Fc7Bhdj?$GXFy}|9Ef`UTof5pz4paW-9{1~R+;#;TC z>9w2msyDYYuf1BJXL-lXH>BmVp>VC)-sN%%pPwl_)&I-qVmfDe;>NR0@-vQW z@!r29`|Yq#;58G`5a6`wlHTuy=h z+p>5%SLU+lnlHW{J7eb3#RmCt&Leah3P^R^DXwLju#f2<}R4d=J+Y}e*gX_ zzwaIT_wQ=<|6j|Q{|cN;PG0#$zP9A5n*Lv_0*ht&2X;;JWN=*|wL@Yuht%djz3hJk zgOC2cP<9}x`2Nph?N?f+?~qPn_hIh)#{4}Z!F=f)w}AOt{4Zxc4O=Sg?3BG%WRE)Y z{_pdaJjgcq%Y1WBlcz|3aHRV( z@wMIiK5?+TRaA0+^KRF~V$-e1877Z#A zgy5s+Yag-)Xvt(|-LjPF?NgljMs4B?m)8tF@fWW~++isRp2X0WZn;c2ohfbga-Z66 z(*uqN{AOI9bnPbJu1hgRoyR`^TD8~ZkIt19@lIC_y)3tSt+c6mFCEqVc*m9N{%Mz% z9G+qI)n(S~=K@@$iUWVm38> z{Kxv;4AV{GCmB5%l*QTnj+}WrZ(r+v?}Ro`>R5lwgu82z@QOPzMV)F-oRntXxLxXG z&z!!0Pwid)nQqy86*(q_v)4V>|EnPM`|JAIZvVC@EBH1{ZK|^NkTCQsS z_nW`-3w;*<5lzhx5gK#{4=kKf`AFOF`61TkO2&zIf5u1p<{ox& zTmH!1`H5RWnZ~+9{}mLLhH%WOHF?P9t~&32_Ihn0Zm~bhr$(qSJ_+O5{OfgT0^3b@ z1qFpWU*7j0RB1UX5_rB~rRCEp*ClS-^zOU&Bi(f3>?uWR;+Z%9&$xCf(Ch8WQjJUU zudDLb@74PrSNnBV({V*m=qP+QS>|Bh^fGxSZ%FFT+ffPiKD`Hymx!I3@K8Xu@<(aK z%6YL1eyJ6ti&Z~tZ{%}&nlZ<6VZf5rYMWC2+5P?#u;1ptI%}`9ki)+cX8)vL?8_3{ zc#QAP;*w~eX>K4MEF96dMpI%+e#|bm^RMNlUl{&x=8|XlIpfx3`pv-17UsgAc~V3pv#NoMFkppUCw^XVZ7y9W$Jj`i_24+>-c6>C zLbZB%Q7iw~HNJ08>Msr9U2(H+)#HzC$7-#c)@esxj+=3P8cTr0-2=e~YR-J@oP6@% zJ$Hk%ZI2DLPu?<+?U}@3R`j!cy0HJ&2RnG%9ezlyopUQl=a0dLXAZsLU8_2xe!Td0 zx$eP!lmE|ID|0;=T7?4A7lt>xUVnEeH`iMJ zuc~*q_e!xkgItGRC(ZI@;wJmVEOg(??>s--#`yo-J^Qz*G#u(?{?8zHOypVSjB^Ka zvKsChvQK%_G}Cj3*yjI^uD|{*wO2g$FhkZZc;$(uUMu&=6h(V@HMVKF?@Sgwct5E| zb6>(i{?Gq*O=5U+li`o7apMZH8}oGEo5l1-Hk2@J$p4l8+Ti!HkSVs`c8a&}D?GLG zuY!Vt#qFt|w*NmiC1}AFmy!=<3!L(0ZQMT|vY&VAb@sHHXF7#)Y0{rQH(WN@+O(FJ zxsR!j>7RIozz?|%%A!Sajd6UQ>-p?gSbAlr`|T9@*T3F>wbqo+sp0F_c3V$>^7rc! zwg4HnWo-BRBh@doLhyNm|7QX|YZIP-`Mf3gSh0eFg4G_DMDx(aoiPf2o;^SL zTZ0yu>K@oUYnLjw*q{G<_RYJ((B*AjV4ik);%5WjmbId3tvBo6$(+e^;ZHJCzS;f8 zxli>pze&{cvZFGOf|iBMy>{{Ov$Nc*z4dCW_peX+9duO8G$Bo4UiZ!D1daBITaw#k zE@kz9R%D-FpKfjqS zv)%6J{{6ShpZL#cyu#oW+z@F{B3xi?z;2xUf#b}Yg9pX>q|R)4eDlC`L*KsshSnLq z{`dX&Tivd`{J%A)5D zuN#;Re)Gt5-0V--;JlYhqW7ip4dEZ@XVx4xJy1O(Zz z%Z8q+2UJ?a&g8kBZs0fA+H!S<<|$c+^+k7{Jud3iYn(JNi5NZ-bf7^JMW0&7MjMXAEC!o?%hrzkFFV;hi_@j)fDX%ck(D20Nyl znK5@(zxBI~v$L|*-(>yv)n!?`m32R~*f$ankzW@vT0~UwFPhSB=nQ1~We#Bgh$;#HeV>aS}4tl7I)TX%11>r%tRjfvNblRZ5dl)jrRWB&N|%VR&E z+4a_E9xaMB>`YX97F{diYEY;nr{CpV`Z;gGA^D7^nzBj{CIM!XWwmw_1l}Fg@t5oN zP%ifGkiPk|85O8rOuak<9YW? z`4`C&ES+(zN5|)wO2^5U_4#ba7)ocns%w!^>$ta%@9A$(r-fZRN+&PaZFiUF)Y`5J zaaTW8L)%}{2E|Nl7Bk+S&o)8%KaI$RbRp3$*BVY2|UIkKLV%iFE*=c`LX1|?wq*b z4~+L2^_xEZ+A{rC`F;7=)l;fxzU|g!>AmlGVWZvr;~rj20|3{Q)kOP5u}KUrCj+XO`Sx}zMx%22=Z{qc7@ejKvP z+xz&;p8c~_7y^rwt`@J9t?hfi>z{Ab>(r;494B0p^7#Ah+3e$cCcoIZUtztbf`Wp> zlsWPLf-c_tDQ5rQ^=)O{c+n;JZb3y z`{U=j-Ll2DY}0#m&bnNyf@czg%T0^-d%~FP?*+u({o#Ck?Ou+quYVkT1z6T+zm0uj z@lc%c+1bsL*DnDz&rTi7uQTr3^~*Ch(Y$Quq1{p0ceAaR`DU)Y7-j$X_s8OGH3 z_jujzZyTg0e%Eu3agA*G_HfD%)twiQSIA#?pCD|3AAPwRi#l)4T6~ zcg|;-tG-8U)n0`e-(Lsk>^GI4%%X7W-p=Ri7A@cIn`>Qn>dO-GUtK{HgQpg$KU=J~ zz;uC0#;=3*ZtN$evlhqp|Gd>3`+u|8?fj!@DvTXnh370+cc|~WyX+#baes6k9TNt9Nlv5_116U8cE%<>R*f#CoH149ENXsxFLNQ!~Mfr)`3 zg#m&gBBSJJ2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin zXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD zjE2By2#kinXb6mkz-S1JhQMeDjE2C73;{+41}1(6A#QmFE*5qM4rb;N849DG9^N7F zKRcV@za0a^^NcL6Pg&V&U$e5ker09l{LRYh_>-M8{aaSn+#lIFv;Jjg_x;JrD*Kj| z75X(h$M8i?zTowua>lDg6~jA#MtwIdL*Qdd3d7ypLbi{YSw_FJva0@OXYc--o&Drr zcJ`0|+1dYb;eXlLfBt1>zx|t?efn2+&ZKYIIi6p#vIPJC|IctFICxlw)u^Y2PY8U> z&SiL+SIF@xGt=W&cJ_+D+1amgrFCM&|7B-?`@Fgc#;&*n=?EeJX_PEkCIpY7avycDE%(VR9(ZO&pHg-sc->9R9 zZU{U}Ok{YMk;(8TEnWV1cJ`+K+1dZdNyn7v`InV-`&VYB`~UiShL5SKLpLx-T|T5j zz*t0t;cI3V!{@9l>EBsdTPaDyl?p@NTj2o=t!TyXommUSq$H^vfTbMYuaEHLh zyaI*~c?BY%IoN?tqh9mz6cc6~CZb*bcn}r1fs7>)RJ16{K zcJ@zv>2EN|{m#yweXppT=~I5ukO<#VM-J`~c#)dMa4#u^E(2)3Vlap8sM`j22wabfV)$QJ$nY~WvjLX2h64RJJNxj< ztZbeaSvi9{fJa?NuMl8jU|?WpU|`^5U|*VU|`T?U|=v|U|_IhU|_Ie zU|_I?VoL@FMq@?>HeCUB1qHRI2?-m9Lc)i+@o#qagAZBR3Li!%gz1&Q27^7U3=9lH z3=9ma(8L_hz`y_+1?gd6U|7Pyz_6Wxf#Em<1H%;t28MeK3=Gd07#Q9#FfhDlU|{&f zz`*dCfq~%*lm_wNGcqu}W?*1^T4`+j4VJcs0{t&L`~9!%Y?I&FqZ46+Az{+M4II!y z98_R~W_x-W7#KD%Ffg2BU|@I(O~Zc}7#KjK;1uIZQ`3L{b8@IQSxbjJ^!;CUwhL(B zmIh%zY9xI_fDzho(P3a0l5K0|SFGw9USS zfq~%>0|Ub^TxBvI@xRb^{Bs5dhAWJW%m>+6*;g?#F--y8ch11T0Gb(!XJB9mVqjqK zU|?WyU|?XdVqjn}XJBA3Wnf@11M!)eSwP|*^1QqW_v7O354HBs->fX>(ezJ;R7;r^ zT+lRE&A`C0pMinlEdv7ssB9*-&HkN%f#C_XU%MEZzGD~|80;7r7}P<7Jxq+uT&iMH z;1kwnmXwg>+PlfA4EIw~x&CBj4a!BIxcU+Qva{cP%gi+TIeJ(MN%2i5#h|nZ8hiu| zPJr6+pfpcRdIy#3?-&>u&N47CfYN#~0|SFAbi4;Nun8LEq^cJ#M@BNN@$g{!ote1= zS6Um4;{UR=QRTZ!(Ijk22h*Go`HcujDdlHF6kXxuzg7&;gj81573 z%Yy2HV+;%oRSXOaMhpxL3ePS-@uKm5(kzWF&bOYU3d z=s}^>NdZKe0&3@g%KLU`+a6aN2UONShPLNHeE|WYOdnuHf3mX~erM-^7J+^r?0MjK zcJ`FSumr}7siSRwIwc!W`ySL*zk{c|2aV&OXJBBcWME*>U|?VXjRDcgd6Zl8F(ZTF zLq>++-|Xx|gE{?!4p0A)lWX&5blivXNGEIpXgmkh!2+dgTw^()`PG9A3=AN3;)IPH zoGLC=)HD3gNN4z&l@&LbHv;_2&R*~@JBJO_XCIs)IV@ZSD!W1BGFurK7(nS8TU!St zzMFx80kj5lNVIVX#mu{`9ER7K+5CU9v$qV!IskOR#@DPYQ&3xNZC*Jc?~KZm7Xq>j z3=CZi3=E(#N^I#K)Xo8w_Yu%FYvkE72z9yay~F@oPw*o<+xA~}_VYoO2mXA|%Bp1W zPi8!yKHBf4LGoc^U|;~|#EbAf?@0W?PinxCS969(SMS6Mj>GqTGV ze`ea*5B*JFapcw`J}|6F8XV2A|GSt%$C%4sX8H)uv3Y3gP8osq%tH78f% zZ&ns)c^uu(gZ`JDeep+5uJ!+%Y=&34pqX=;d3U543>uUj?->~kKQl4}e`ja+{L9Y%Li6Guv<)*S=kK4atX}2931}{85u!G zSV#`HFdRVt2tv1&f5Xz=dBMQIP{6>z0NS27fKD75R^QIZW;m6c&iWxM+wxa-&h&rT z*$@6_Xa6O;PRPl5+iz=om5GrNv|Sli9{pfsVDK9nAwn%zg1yej!0?}$f#E+Z1H*q7 z28RDkh!Z}@wg)t9xeUh=WzcaVpeHL1_}sbNTyrA{SyLuPR?x97;6wl_qZk<&`iF10 zf=*<5#>l|%ho6Drzaay||0o8A|Md(E|7SBW{9nz$@P9J{!~cy84F8ugF#Maq!1%L} zf$^s&1H&&R28KUu3=FVhoL~rnw#e=Rg(n&Yl{M?3XQUA{WK;p}5cpqO%J3mQgW*YD zA?wTBe5r3)*;c=^vZB6c=M?|V&Tjsno!#{>JG<>mW@hD&%&g=u>FKVY)6!IL$Hj34 z7+W&%vU4!7FtczlFff9ao50gQ0|UcJ1_lODy^Y)QAteLy8ff*vUoHlQzt#*4f7=-t z{vT&x`2U81;r|~7hX4Nw;2#VO{~s_g{NKR9@IRe_;fE3f!*6K)fHfdMOYTpgrF)Pg zKwHZDphu};)i^3RARz!+(*ueh7zP!R76alEs#*&3&VMcjhW~yH4F5MVF#P{OAgvS1 z3x63H{@rC@_}k6E@JE}0;TI$1BoVm7LFZk9hVbE*L-PS>dwC%P0|V%IHMrtY#-IoR z(D|gGv1jBs%N-OhBij*RuYlV0X3%!cR|?ZTwtVq`f$?`01Jfrl2BtKSOF>72T!W@} zumhm%rwj~?p!svqv@b~AC>~rP06Mo9w8RQI&Op=eLo~j?-r#0n_@B?f@c#)7(mirs z0JWd?Ffb%$FfcG+N&8H985kJ6m>5ArprD;xgDc`kT>}mQ&=|x5XjuhLJPZsBpm`e5 z@c>{2LxTW09n{4BA{K;$d~c1m z3>UN%D`n_7AC&a|Npugd&?5#$VM_)^&?&P+Cj>{hOF?tK z*U{QQpyPpuK(K(a>K`5khJSPETjpcSSC9WQFkF7i!1yl!Jbxwz+QB;nf^gJ{pmkRp z(b7NYs4&nms)ILln4d8)@T_HIVE9+U!0`Xqz^8v$p7?v6f$@hf10$$EGI)b;)MY3k z(1w=&L1P!7ZK)`V1`p>y0|O&yUhB6F1H->(gD367^1%O93=H3S7#KiH+y+mmjk*M9 z2!M{a25mP5HKsurG$#N$z8$BAft3KYd;ao5*DDNAn;)LmvDyEFf#F9yxE%!E)jqJH zHEKJ-5CDz$fUfBPrF{?vB`{E)AgGc43P884ft>g!oPpv0_d!_pL-WJmlMKvn#Tl4D zw+YcdltyhNDFi^baxW;) z(k*~SEh0SxK7OeD!~f5NJN<+6z#ayMAH0wgt!Nr3 zqlQx!0-$sbTCWZ(`$075>YPC~1_W{zDE3Z!IMKgN77n9*3hAkRJl*>mWc^{ejLb zAm2o)>-@{W!1xnXo_-oS>Hq&$28KWEkXuHn9w4IzQZEDo(az=o4P1cMMo`ZNicJDl zO@A_mP}&EL1>I&~{4K)(-fBv50E`+yyAS}Sbx@iIB{C2Ot!oURopq#|3~~|}_YJA^ z&-nKP1M3HK1{ToCZlwBoR6QL-0CeaH`Z*oRbhMvGF#Ov;^wR(Tw9)iW$FxG8RghDM_drX2ST0z|z`!s_wy%OZ z{#prwD%&KOX>AkRdq>H?>Kkn>uGSo#N@wxvIs{;8T6Xki#=70_}} z0z<-}6XHNS#Asn38D@g>0H`hTcL?QyOAHME#YWRV8PQE&EueNjXw@e;kwNpo&cStZ z8Q5u{vh>vuO8?6l82&OdFo0I>(KkXzZ6hfJhCuoU_knpC82%j`Lg_yfQuc!m8Y3w_ zMitXH1VHT{(CKgBLKa&7Z)ad&03BgQUzgyt4Rmlm$a(*2hDiE<%E0hn8wRghAs$gQ)!v3Mf!J;t%Kq^A|%P4=jb=EC6!&C?4D)z{0@502=QB zrF$3#-Pi`o7lX*jyeTf2!KxLM!&gX$>5p) zf;qXAfq@Zp0?Ieg=_juTaUNL5zyLmh9p>;+dQgV|BQFC3<59Hs&x}FoWCeu>3=F*I znHd=WP8g&a;57roKYPfa&uHO4C=(Hiqd;RnjKT~IjObT544x%$DDDNNDNsY`k0JxZ zzjK2u5By_b_}#+5aD|D1;SGwbM>&H!1h6fE23-*|I33M$A6i%a_JLk$1Y2@E(DZf& zhX0^#%7Zi9MqPy~1hg3#7+ygW6{vs(;X$+o2Ai`%M+|-jUCC9z!0`XuAgBi}GBEtp z9c}kui>pB|2D-B03sU+Aow4aV=v~bK%BKH8TVom-82*EIv4=5N1 zf5?rx5G4e{85kIlj{yZ8iD-_ZV$gCxdEk!%1H*sNaU)*_pdJ7%^06H)_XlkPA;w{) zNM%1LK|Nw%U{EGT#;#;8`hqO(V%t{=*svT3=IF*GBDh0WMGJCWnkdi#=yYvj)8&k zDFXwGJ_ExLtRAt1(5MhY0s~_30+e7t=e2^4gd1i-1M=E`&?aIY28REt3=IDq85sUY zF);kkVqo}R#K7?$V!0;b*jh!h2!#_y|hX0_2;cg5J4Cv)UJ0l}#&EPN#qLJwx z(3Owi!~tys-DF^3kRF*qfz{Wd3=9k>pVVq}3=E@n0G%R$ zM%E;uwf(;_Ffh2&$gZJoEa*s7(85zt*$=}lL)}v&*!`f*U7$nWU~vheLF1sKZGY^E z0<=cE|ZM~`qZFfgDW91JQ~mN76e@X*m6qgD|P0Z& zz`Jf}y8twH25S3*(mo7>))jgTZEp=5*MsU}&>RgcZa_4s><=0?J{(w&fYytn@1p>P zF=$i1=D<2_)OH32hByWWh97ACG*Es3<%`h(qD}~a_EDpcD}(aD76t|eaq3txY6=-4 zAkV!@$6xgG%$4H%Ap4P-d|D~I9quX=`a|2i1X|LL3sewHcB56d0^Tu28POMj0}g~i!mJk8o+S+*EEK6f6p+S`}c<7+`r!p=l}o5 ziqHM~#c=NLYlgFbPBWbPIhEn~SAT~6uSFORzF}ZE11iGtMh56Ymjd+h9%e?C2h2?D zCd`bWX)wI{M#&HhW)P@81}vpVmSV_fZ^QV z8w}_F|06ryBj*7y``=%NGk>lyocxi=aQHnx!=?WW4Cg^rD%_(cSkCeUDF9t&rvO(q z%Aj`$fcDiL#8MA{*3N;>S{?dr(gSZ87@FoYFdX}0&T!_>4!rFfa9YOE_90I07sHu9 zn;4FNHDvhzpMil5T!etu;~vKn#-Ori^h^hOr+7^JK=HE|OZZ--pyI?+N6Qvw!a}ocs~QC?>1Oz{rGthB>I*2c7Q#suM6{V^m}yLI5-- z2bvEC&542H2N{Fv2h)LY6h(G|+B`G&vN0V0R>W}r-v_#-eONtl;r}bfl-e^4Oe~=D z8<4`2@dX0|gYW1$-4w+xEi{A9kN~a21zl8t6kp(eG3Z>;Xa)ub(0U?T*iBtCw>@BB zSbBzq;na^uzxpx+nA3)_8=v-o#K{-#*xr%|| z>VF1?<6mtU&i#8lplKhJ{y`X&2M)Y`%@C4N%Fe*R%Ekzq2B2BojT%gy5CGjT;K{(i zfPS_vC=Y<}YX$~}Rt5$J(ArY!*hmdi_Pt_Y*!e`1;qt6pe zFr4~X!EpZH-$9lBL3!Z(zuycef8>D2MSIub@yMvuAPWJ|`Eex-3=Ge)jFE!!18Dsm zXs@>IfUey=@{xgI{~Jw)vw!XmzO)Y=FFo_;D#N~43JeE9M>`C%SRQo_z7PPlhin)a z7|fd339zE?23{LjF! z`x(CQ9+exMAt1!SzyP|F9CQykdisa)LF3z?d3{T|?JGF`71ZaIV>tKk`ruCg|Nk?b z`+JVz$S3j9e!$>N;6%6zG=O2iz`y{yqXBd$AuQda)1XQobf%LZP46sEse_F9ocmGXSVI6bP6|4+Z#VjyUbKD!=7Y;h;5N|z{|u*oP6wYsvlO%qX0S%@sJn;^0ni$`7zPFg(7XWX zylM1$0LBLuc%Zh?MrJ0KOlD?g(D_7^uYCl~cb)kyH6+?Tu=IcS?>UBp?}Ql+z9TY- zN0kkM5CHWJK<5qZ!!dsh%Lo4%85luhr=am9&>jTPo=j)xTmtF)4nTe1Q$K7N&j0^B zWXgZgKAV%@4H-^>ii;r-v!hNVB?Lg@rJy!HsO=A$CxfSZw004)IA{zNbXFZ`V7!ij zfguojf0-Dxy#(rWV0Xg>(3tbnFZZzoWa1r0J=y?1$y0Q$GqAZvAIq@B>}jGxQ^S)CV|2091B^?y3jv{{!v42c6B1e&!jz zynrqTTAvEi4?5TJ3o`@5ImWeD*D`?D7|^`j#=b8YmZnLePyK9Vc>JG%!7UtTz>P`_ zyAS})qk+~8fyRwM`^!M%LZEUU)E_`k=S1)%V zKR|;gzZn^rUNFwue{9I5|I@!(8SejQVCV!56O4rD9rYubApkm)3v?DfsNDowj|$q0 z3A!%@v_9LHnSsHLantRIL$3XQ;(HOp-Tw>>p`b23nISW(Z8QX~{AXY|`8|1vjsN^% zIQuu00W@|y8iS*Nk`Ork3$*slf#KZ0FGD5|ocsHp;mAi*hGU>1EJ{2-s&~YP0B8-t zfw!^@XaC+FGU@;9pNkCpUyCw;_N$Hf2pkOnib7z=69$HL*I5|O{9ZL=(*K#?a~Y~; zFfoA6-lfR%qnby22;{aiFx>plz;NPQ>fqe>h&Cq&I$!t1x5&};&m%tNQZE2Nd&&=g zRAK;~yEfqST4?DV-92aiTxQt&LW*G@Xxfi@K`?642o8b8Cm9$d6&V@M{O%p}ZJ;y1 z8^L?vmw}c&N^*VJ!0JIO_)DQhZbM6m%xewx>`*(-o_*XTC6QHGl)b;+TX(K!Y zc0Xre2+w6?IQe6+t_A(eaO!6nxa{9Gy7zsAr&ii{A9M!F!MD;3Xa5`@bnT$CfA%pP zd?(6q6trcUHr^jKb%cjN+cL;GC?|ioF`WDNYS83?vwt5m9RF$yz89l)(Fl*EQSZ|< z1hzk7V3@KEbjMLH!?}Oo20Ra(`}c|A*q3CmpGU|0!ND*}&^QD@^QznKvooCi+sJVK zKl%52f%>eJ;B$Y!F&zI|!La8!%jkJOG>-F8<55H4_!kC-!ygnB6ciZF{AnDZs^Bxj ziEkAQt1hxLY`Tr=yiwk02+$)0wm)QG*mRqn;p7icbLb`AnnP#*K4UofBZcAkSJu(t zPI^T2s2w;%0MsQ|f1Qcp_*YkkGrvJYL$o{|8gv!pL58EB?HFoiF)}PahtpZ35~CqN zw-9Jv$iVRSKLf+wmr@KTe>5?i`}c(Eb-~#`4;W7UtYH9Ms|`BZq+>bVyfb|0Jq&06+@bK;&_4|4{@r9a^|PH}|7$IV>AM*jR)KB}A<8wQ zibg|VU_xNwF$RW-o0%96y;o#7{VSQ_%O0c-)Dw1e{M3I`Mr$c z^sjh^V_)PM_P=HvJ<9nXE^aaiQ&wjdWN%q zCNmuWI+x+Zw>b>w{!L;y^|OxQ+_to%QQ_yb66V~96EQVlr1kj;S@^B*jZ6pw!xAc+8+ z9+1R8K#chh7KbPSC!z;XH-p0zNxT877c7pHVIZ#j{~sKvh;;i0?AibS|D&bz9}Hky z{{IJsDvbUA|Njqgub}3m15j)Kg8~Sy=Kuc&xDyVb59(WpgP`X8 zVBr4)^)1Lz4UACE0|v$)P$eMo|Nk2yR09J;0o*wtA&5B0CkjX*_5XhZND|_6xHCaQ z5OHuwp%g_2K$0K_Ga!Y?|Ns9X;-DCUx(uuiCJu^eG;y%0;1mN@385Y!#KA0x=>Pu^ zac~O8C60*E|4{WH??RF?%$WZVK&l|(DEjuM_o>WLQz2QxJB zL#+J|HisJGAYWoL2cI~^zo_m9sV7eyq!a3^|NlRLcv!-p2yswMpqc|I=}^VNJj`?g z4o^_VM5|gM;-K`2nw}BjsOc6W4oc@J*$xt4;Cz9S?V#eI2tWx=s5mG-5y79@TC z2YC$a5x4{-{ewJ!lKi3h0#r0&6~|~vK+X9Nvj}Fv0ho4(2Vug{d<`n~P~8d1@Bjb* z|AFc+NV$PW{10mQK*}XhS%)eP5A}bj;fzZhp$e`74Kf3SkvJca0}e_22Xa7wizSf8|Ns9Wy!ikB0fZy|A$u6f(f^P= zjOT$A%`jc0~|u& zG6co^KVWBIivI_-zOj^P{~H*fjR&wsuv@^y0|t9Ym5UM%poR%MR`Guf4E3PkWI$8@ zzkz`ft2j7pKn?|m2-r?A0SM^}73^6EfnOXYejysb?!+aIl71lSG1Cu39GngzOcWBAI4I7*2@*v$Bt((3 z7fc!ym&n-*CJssvn8E)aOPvDM(STMZLWTc7K<~RiB|zl{l!;1xM&*w31}FqH7#JA9 zsgjX_fssK0oS;E0h7<+{FoueZQllX-8UmwWGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ON zU^E0qLtr!nMnhmU1V%$(Gz3ONU^E1%7Xr6pViXvzwGS0|FW~c{?E?-_dh%PKNSD_m!19nUv~DBzuDP4e`jYG{LIW!evqEQ z_$D)JK-@lRDfuDrKRcV@XLb(5m+V}g-`Uwof3vgC{L9Y%4UKD9oa3WG>EZ65?ChE^ znOPG5^Ya)UCnl2bpi!N43xNxTWeoqbvlxD4=cxV7&R+RHJNqXI@s6G*{{GF*KKeU5 z+wMVlD8sk(bh`Ot)FRSD;B86@!~en}hVPkK=6|!Z&r%TYDCyu|cJ|{xSy^E(l9Cv| zXJnA>uu&Z}4}l%ktqlLOG8uklWgGp?&b~~oxQChZFFX6?uk4)Q|0%Hyud}jg?w?T$ zNDF~)+1U(Vva+TBW@jIS#Tm8ff7#jheq?8x{m;#1m>fOY9w04-Xs7yhdIrP2)HLQl zS=qhRj&pSL{$yuw`;wg_@GU!sb{-lvnWPZ-o1Me(D?8ivUv~Bz^!TCy|M#z~tn~i{ z1q=`3;z)AmsAB4e!28@hhNpRjEPt}I=hGmrQQCnp^Z#aNpZJuOCGsh2bZmh7F-n?= zzp}C!er9E9{magN3X2~)&_90X_Cm3{#X|8sH}erIK;p^fKZ z&9~G{7k{#|r~Yq`Ww@U)+83Z&gpz2`tE?=B%%E_FKUrCGs2R7!nDs9^`}pUqY`)K< zeL)gq4X<(*1_lN`1_lNh1_lOQ1_lOO1_lOS1_p*`1_p*S1_p+F1_lPu#7r3j14A(b z17i+98%O%3h=|Mob8={&=aJI|XiVsPR<`nw(fG$3p*SQN85kJ285kIp85kIx85kHc z85kJ)7#JAVGcYimWME*p%fP_!nt_4g3j+hgF9rq%(5^Djra27E$jJDAzO&1}|GBwz zjsO4I+3){mXB+>^9xeZI#4)-sF9QRE9s>gdsM9osfq~&50|Ucj1_p+23=9ky@k*>1 z8xzz2Ej~W~=^g+7va>(`%+0g(iz^TrStyH$~ul*H`9atFFX7B z&#WvBQ2!IdVIxm~63 z<$qQdL)9Em5YW%D$wXGi?c&i+A7gZAHEA(iu>c)!NL zz);G-zyOj5jpqzl7<|pnX84?y&GR=qdp)h=AGEgUTUM6R2pIQcXJB9mgSOd0c^z&dmIOI4J0Up@G4FLou-{qFh{(GCZK{L8;3IrGC&n`(C7Y2c-c}S^%{H z2Bm|Eb<~TDEQW{anQY&(bL{?R=ga{0+yCL&5A-iP`_KRE?58i2lGZEo3EW|T%u9Y_ zWMFU`&<+lUhe2nKeq&}}_$SH0@ZXk!;eR{>!~aqShX1t;4FAg*82%y|Tr#@%rFuc#rXa16vE%zfQ$LCLWcIBV! zY*2bw@h>}jVHhJQ^@SO4GK!OFu z*$fQ-uQM?G|H8lkx-tQD;4B>fV_^9Im4V^^O$LVl3m6#w`Z6&5=7TJy;DALas67W- zX8|@8y5<5jrVDBV!qf~)dUs$L!wSzphM$?4Og}O*nC7^6FuqAnLiei=0|Nu-&|gsa zfDREJzzP851W-Qy@4&$De-kv$|AVfCAQFFLVEDg-f$@(!1LI@Rd|My`1H%)f^nDI8 zzBjl^IrIpjGaodr1*-!C;P~QDt!@%(W4UzGUD{Z`GV3@QFy6y&? zs2CU+4>2$>>VQ1P$cT5k9i(I=VNgB+txW)h6R5%sr@!YxR{s}aVE8wef#Lse3gaG{ zCXW4QV9)>?%)r2~n}LBz1>`R#Mo|zoicv$L2C0k$tzklyr6~`Tbr14H<82F5TG28ML-QROuC%Fs0&)CK_chd}uSH0KE#6Q{jz7{4+wFueh_ z+v^z^KsQv78_&4({rk_r@aqo)dl48Ljbr=e+sIs*g4GiVublYxOjiuT?B zB{k6W+CNtYhW~G=Q|9AJ2me1ZF#PcccXQc5%dTnfyCG(S00RR9XdDa_DWLfe(3&+G zdjXUefAKOf{NGOFxQEsme-AJ)eG+0|0xi9zvDb#4@u0EK70~<-s*poz>jhX<^AD6K zL47u89O2Br)R6zh!1yPMff1C%LBl_^_2CdS9W?d>3LFpytv{!s7eH40V`gCZzlc70 zAD*t(F);jQV_*O!aTGEt>dn zs2VEBz`y{S9|OhzOXxax>idNm)b|2e@xN(6;-B$fA2|NmZd2cHL)%2qyzfzH`44Ib z7}3r<;P}rSkoaf#R|$@Pc0byAZ)lnf+Viv@8vmfe(Sml~0LQ;K1H=EHbWQ_*7#RMC zf#aWco*SwrgBHx}hQ>c=ucI~XJOb+f|5s*U`2Ud3@&AH>;lK81{9}z-tpRR2di;Xg>;!WFdqJ%98)Y85sU=rcpB3!@%%Q1~T&pvI}DCD2WjQ<_ruBAE8wM zs3`y%6~a(NeF0FMfSQE=K&|sFR9*=RihDIkvk$bi=MkWR@K$b~@B4iDHPdWiv z30j%)M}mRj?{o%+|CFu_`oX~PZ$1OVUs*`L2c@rJ6jQYE3h2a1P(*+*=rl^&*n-zo zP(1zRU|{$k$-wacG#NVrelaloJI}!QJAr}m3l{@ul`&rXMkSF#0CXA#DDFXc0Jnkv zWnf_a!oa}zTbzOMZwdp$zwHbR|6bx*B@XJ^zh+=~cani&+7<=|-DwOA%z}_Zwvl`{ z%ET7}pcCw1GXS8|ia{q<<5N!uIZ!eLl?OjSgHw#Zbr=}_M?m-gPGMm9KZk+gKWIJ| zw0|Iqf#I431A`!F-xp~7@;d_qWANyv50WAYbWS;_;|B_M(D_HS?*NkIJfakX&hY@b zAB49+&tE4>cV1_n^t1;sz892lMGB|Zk*pgliO*n!x@ z>l{2PK_gjRNd5pFTuQ^WjDyGH6uJZyMu(u`2I?k*_5@OBKD*(U(|rP zWawB7TJP}=8fG`3chd|V*JHZ05$YFES$2Yffk6UO$*2fY2-G2k*IWh$hM|5w3g}#? zOHe<7x)9Y!ejR0^h5+cidf5I?Q2P&bJ`Spip~(a7X#)8NR0e>~fuX~`)*%^Cp!f#W z`JiwD-CY5?&t^z^1kKT)JJvQq%K(tyK>M2Lc4j-8TLvo^wC)RZ{}3pAK)7MBI+RRz zfzES>ooxVGcm_Jl8`MW2)3{-$%>`-34^&2i&Pg0jo&?1;Xg@P(Y6+CSL1$bIf|1QZ z5yX%M5e%!5ib&9PRztn%hb?SDM_j;;xdJ5vP;*BKo5B$$209`SG)x8x6HqZp<28Q+ z8oHos89`lJkY7P~B?AKkXwBPzI%)td2W{~L9eEB43lIiP_zuUsj~v9o3=9mgVGhtf z^j-!A22c|SS7-~L;%Y$?-IEv?7+`51 zRR4h%z6{QDK^GonVEFT&f#JX#ZibUTj2KS;PGUIoyNluM--Qh4|F2>=`*$hBnLm>l zPXEehIQheo;mAi(hE@=*U;l5f7ln*Z&zAm_Y46(C|N2#lut(RPQeV zZ9svxeL?$OKv(V#o~u7*AAqcdJ@H+F;qY7Fr54Mmf`&We+=jU|Hq2Y{rk#r z_V0OyGryY|j(yf<*nF3XVK?ZeVR$Tp;&}no&;J=27;mvKvFfpaj>P2xjh4XGjRXd0 zmpUDj8|9vGn-a&DXz<(Le{oT#5=LM)=m&d@s06PB=7Z_TWN+fzt`a{=MnLXy&gc8zwrM* z!-2Pt84PVfd-_0e|Br!z5j4dIsta(rW~7OO#)m-tU(j*^P#A&mRR#tI(9{Th`UaPXNSfeey5>N9seV_;Zyk(uGl?>2_>|Nl}w{z2uz#sB{qHs8L^m{6w6kk>(dKMW5O zLE{6UtDRx{U_ogB)HVQJ6$rW-f%(sy`78V!5_TKWuXG{f3UpmYFYgZc}0RJ|hd$VUc-L+^PQ&i>g-)3^u4 zKL~^R1xG*YGMoUd@uxw^4Id*x^#N$tbPP07Qev3_xW>7ZW41Gb1A-XzYNvv~cjltwI%o_Lw1f_{kusKnfx&=*fkB9YfdTtC%h^Am_%~-b z|NjGB;~!K8ocUeEaOFP(L-S(tLSkg=0+k1#^Kq&e7#Kiz%77YEpt1lNgT|9UWd$fb zfW{a>bqHuvEohTHsO@0P$iSe*xc){8!-fC9=^p=Q|4aq90cPwO*)d7E??L$+bcJ+3 z^ax$h@rlUk0G$mgbU}Ry&{)iKrlQ{64423l-v{MkO7PjgD;c)jWn$P1ng*sk5JpX4 z0M+@RrSzcdhD#V27*;VbFkEI}U;v$Y42pB~bO7T2X3TE6G$8SR{{MQ0v%gsw&Vrma z8hca=0Y*?V0GkG?^FjAb8bZgBiy0UgCNeNEfYzPuWME(b9k6qRF~4gU!^MHf|9_S; zOx?l6uxa!tY;bJRj{uD^g05KPW@2DaP*7mp{Vasx{QqC{Zvp)6Wq9%*)B~lT-$!ji z4FT}1&v!kBbN^n`J^oMqOk=qCpMl{Ts3Jmj)+mooApjbIIsAd2;q2dIbd7(|Ov8!q zW(=Sa2s(Ln)EX=yu=+9s!{`4D3}=3~&^7+g{yD&K;4Lr1LC|0}7I%#b(Jur*qi`pG zm@u6C_mWO&;4j1JUpWkS|1&U*_WtP?&$w&?t@=6io`vD;ANtG!ocni<;p7i#hEt%$ z@3XyG93OO%W(2L14B1xJ_^a&QRd(Z z0nnnbJe>G+} z`=^=V+}~r+;bQbDgFg)C{(WRP`}YdN*}t;m@-+;zvU_h||w3k>ONj*9r zw4an8NuCfNw7d`DV327HNFF!u*=d|-fx|NjAT#t#Vp0rLm2asU58_zesPpz-|h0n`~FLqHhB|HA-L2IK!= zKt%h8|NsAgKnk@F_5c4rK$geCKY(N&NIf$DM?KhA3?REe7=v$*CO-)HDDJlh#{x9i z|NsACj~X7>_{iZ0lK;WL0Ew>u5dD7`K#2zw8vp-;!s8!26+rm^8z89xqyiM4{~w_7 z!L9(w{{Ih_N9Kcte?XGM|Ns9%@%{h*A2j|yxO*V-|G=per1c<(`2QcB4@rLi|Nn1> z$TvbLOg>cM0hm10pa;-m17^_&X!`sEbqFN={*#9}6q3FlFv2|Y2W(CQLjhFZKaf5D z7#I|w=Kcpo5GcFC_zj@21|16vbqFX!d_YR459Cqf@dG=U4>AU<;}0Why#HgU2jxb% z{Qm}iP;P~%>;DHBQTZPjysJw!?7i>ENa&ZbOJdpXIY>8SN zqnQ66H6MYj1Qm!N8itY6HAoDdkDSjz@*j}$J(LfQ1`r#Bk<&GZkDRVSeB^Wu;{R_z z1Q(e902GU0)=(le7#J857#J8p!OO@1y;61p0|Nu-ymAl*#p@^@4S~@R7!85Z5Eu=C z(GVC7fzc2c4S~@R7!85Z5MX$nmCbM=C5_>GRu<3CtZc2nSy}GCva&;eXJ-ff$;z_( zk)0#|E+?1me|`bOudM9R5TjBEyvxjDxRQ~>@I5PA@ppE1>A&pk1OKwKU;NL`{_#IM z`|tnk>_7jrv%mby&c6RQJA3t?tnBdbS=s#mGcy^krjDfhw5Sx|NXGok$z}MKldGWc zCp$a!Uv~D5|Jm99{%2?ZCxCzc&CcHOD=XXPYeoj+&rHx^i%6!9GKmU-@0nQ)pEEKA z|7K@Th9-7G;f^l<^lw&H?3eU(reC9RKvbB+6#dN3VfdDv!~Hiqdm6NmK@VGE`EUPZ z=S2Najc53n1G>}#X2U3rD+FGpXEOZnX=V76on7!hJNp-bvW8$h{L9XM@H0ExIMEb)KY*_Wsh?#O2R&CZ_q zKP#T$MOF^Ts!@z31VHKYcUG4DzwGQU$YDnf_P^}x6W_A3`M-_4?iuL3e9)Q5>I@7F zpmUK!7#J8p&2rGW>Fo>*44{cX(0z-O85kJ4iVO_a{mRTFrln7?_C=1jf7#hjerIQ^ z{LUWM-DA+f!~6^k4EhWV44`{!IvE%kHZd?TfbOPv!N9-(I-?#mO%6Kl6D#&nRQ&%d zGn1sSMGj{??00{1at#0G47>6NbPoXNK4s9o@}N7VPe9N6Mh<7teUhMi zF;F}4S9T8H-|Xz3|Jm8!hzM&)9OdNv|DTidA2jm(Cp$a*O_v}I4=K{6GK@J36 zLIq0mw-^{0qCxvk=n>BN?D(9O&G0EJoBelocHCdcm>g)z7ZT1m`e@gpqW-s8Sp1LI z)T{%U{W3QfM9~3*f(}${{9|EY_|L<@@SmT7;V%~h%Nr&JhK~#kNuaJKhBJ`wya)AB zw=gg;=whfEK!HaYSq%UGA7=QHl`ZopJ0}}7y9??|{m;(+3Z0?(`7b;B82t z>>SX^v_G;oY`<9*|1#vbbsEG#h=Y0kS2GBLy5Mip5py2-x zO75-<4F5MVF#P`r+U|h^|6^eI|B`{>-#iA!{{{>U>@o}t4C|mt1#|~4=xQ!dQABlD z4uFBm3=9mQyWK%eH$U3g589pcmydzre-i`4|Mxh;{6BW_ANLs;Y)(PL{xt&wVn7@mD)VEh`xzyR863vw#e zTsasF0(Bxmp1RJ!z#u^#2Z6>IK}P=bB)!xDg*jGylY!y41_J}AM5B%?2Zoz0!C?d!bjv9y&E_>J*82)djLHPe?VEo$#4uAf= zlsk3sn9v6ef6&Anh0SYF+3=4a+OY+tSt{dyix^m*GBPlL`oe=J04a0{XpsvjQ*WiX z4a)S3fr06lFayKCi!@394F6X#F#cjsKMAkbk0R|1I{$yuh__u<};SZWPfI5`|=M6rspoPAm^naLv zfq|QH=iPvY(O)YDhX2o~69@m!Ffja;h0McH?!rN40<8VHjpFtv%wvq-85mgZFfuUw z%VS{p|DD1(c)`H%A2gQ7z`*bZX2WozL6dr*^uL(8W9%R!|FbhN{I6$V`2Us6IC#mx z@IQuu;RoX|P5Yq5kgK2t252EXmD3fdF8&Lu3(^=EKqo$e+mOVxOaC%3{5!|M02;&o zJ6yva6xN{j{&8sdgX#g07?m+-Oz0P=EYf3O_&1e-;r~NCb22{}7(P8@VA#8cf$_8o z1H)tn24>K_3zeNTc#H$h0zYP8VEDejVEVz#!0=aP`~b{PTP|zegKV!fCkt=gFF^gF>!Dj z0$Kq9YIA}TW7i;ZB4{Hbs4oI?*Fvhbi3gDfp-$0;ZWIBP{XS5!!9qncFff2NQG8-x zU>Iy}qgEJ!Hiv-H*cAo_2GBe(walf4IiN-S8=z$)Xkou7H4Gg@W`NqA`=I^<^}hy@ z)8I}4ZG?s`c&?>x-w)DIPDP`$ev z>aUFq;N^YL^R@@255yQ47(nywprR7AZ+bAhmDm6TRcW80d+q&+)ii+WKwUA=Vqs7m zfQG3+#peJzha#&%%{y3nT0e-I{e(P~1l{um>KcH~!~pHVBcx|A%9la+GrR`f+cFp( zh{H{weIhLkkTsy7^(W;FG~YXd#U=aSFfi;ALQ70PTGQjjMt-GJ%du zqu#P@d>%jYk%8gBTV{sSzl<5q{^?{m_wO{rxqq)2&j0_xaQ^=?B6wHhdnGWfZ|}|t%HmOJ)(@Ype^I%J8KZ?1hpYRU1Lz+ z3{)q8*1v$xt^!SMkk>}t^^}3(`hNz7)4x-pbt@u0!@`tQ`ojMo3}=4FG2Hmiz_9u5 zAPh!|odnvf3M!9aXV8Gk+GPw3436Y<@z4BbU^x9tgyGyj@{1m9QFQL_VTR*h1sP6& zTF?}Gc`zCPnlW=?U|;}klLvVcR4jp_0(8V4Xr-kz3llpN!6NwFKL&;~zrDfDN^Ids zw%EU~45xm&GMojq90p_XQO!-D5p2*Nb5KzPYNmqX0aP^IU}Rtfbs<1I4?#zFV6ONE zHG4qC&$)kX6c_#2;_2+4dT_I9;W4Utb`To`>au}GdqG_W&`JZ)atu&hfD+(S1_lOD zvuP^ysH^}61_o=;(Ugor;+hO6e{7;j_@Dc`oMHV9CLCSlK^&A6ISo`UfzmtZ=si$Z z2vi4xX8u5N0mGoU0p*qVj0y@oHyAhHy~A*kh`tuKM2=7F+`nB6C%&;WoB$oyMUkh5 zs%B8fA9Nn52Ll5GXdDT2#s_Ho31}-Q=Es<-u?kRM}9)rNh$@P!H+EH&O6F zGd%35P#K^5cZ}iK7a@jYpz-tJ8FW+u$GW4PNxqoLV$^&Qro?B>(30=0ucqd|rY=l&ffD-O>6J;-qKyB@gBv+}}74^h(n56%aF zL6eYj3}^mKW;p-vEm84s?%x}RvwtQqocJcgaN$402pe%AJ)8;YIQgA{;n){ehSR^S z8P5Kl%y91CC5H3=K0qf4{(|Xq|K2m4`*)Gy+~0`|r+!*8?0?P5a0IlBn2_P4@ovk#!Knc?CM z42)RW3`pXjy;%@N;Ik(nOwiIg&>=1$9;90ZV*UT$zyMl^3TFRf;0Mp-gE&7JKxd6W z?0UcsQU+lkut%^#UOK=4cH2J?33c-ikT0Nq`5^!Q|NjpRAd>w-{r~^}|A74agW&+k zj{l7O|NlRLIh_Chf2d*q|MP<^0U7iE|NnN7L!f~KX2Zq*F+eN_slmnm16OnS{~vfD z;bnuo^8f$;!~YRxVz8lZI1F+dG8^j7!yxxV+3g^IfdT;Hmj?`>V1TfHAhG{*Kodnf zD3qXVkP&~NdO_@e$ZU|n|Njsnh#QdE5N*xiaQ*`}?jV@`1I&H^W`6)X1r#rT7$NKr z_MqrW0L%W6N5sw_q}Tz)bptHq8IeMuq5eMuEZIJgM}*!7W(4~WgB?6ZK#2_&`~N|^ zgds`n|9{ZVZ5aE113cFLBgF`AHo^`6k=*tlDMtPy#mIkU^E0qLtr!n25Sg>$;@K-kdevs zD=S;;9xH1{t%JNaiu2FvfkeSsxWVGK&||8ufIm#g~z%g%m@&+kZbAOB`&C;ZRJVEB*) zx~GvSs|KYa(50Hcv$JLYWoMs5@*my{9YN~;WoO_0ot3Q(x*T{=dW;B1y~)gC_@A8( zx?DORcye4pRRwJNx9X z>}-KSdWkb=V=Cz24p5f`wD25s-X`e&GtkB5pdA6A9ZOqe80PVay!@$4*x}*m*g7T7qff00(Ea=_=(19SJ`!_aA zaB&^D5gq-FO#i?7lbxgWcVOEDpm+w|vI+7t=ztT@B6H9w1Q!_?7#1)vFn~79x-u{@ zXfiP|i!(7Xfp$uPHfn;>KlA>;K(W7BStONxur%^7JNw@6>})wu89D&*5AwGvv?~re z1RS*M4s_r`4FdxM=l~ng!3Cgvj?-yh)6*GVrKB=}E{TQti4^)@cJ{XKS=k&v=-(Cs zZTQf$p>erDd`#1eM)Cv$MJWWoIu#@*BZ$_?Ml%^H)~3 z@GttsJ?J8MP}vPy$h`x4m=4+gh6UFBxHyJ?+2Hn{^1tlt19-y#bdh*&F6bih^MA9m z4M1`KB0U3U6fJ4c*?6F`9JKMjh=Bn-22M5mK$m)h>aaiA*@}O&v*-QK&i;fg9G)g5 z{9o?o_F{KHfMv6p8N=t?d@|htO1hu{e$ZLr|5X?m{yQ)*{C8tu_-o3*@K=O^0d#&f zXdgHDz@7*O1_sd1Ns#|R2l{}@Ju>aZukBk_HpADfY>q$K*^Zz~=0R!ve|Glqf7#i4 z{^#aSNY~Ms%*4nDay$iR_<=@||Fc5R_+P`o@c%Ib!~bs#44_fu{~s6_{$F5V_}|FD z@I;A$fgu%|&q2GXL45okppxwKoIHj{Ir)s=v$EO$WM%Vy$;beeJ0R0Fq3sFK9SR^m zi5TSff4mF~|62*)Onvzd1A_!;bN@mH21e0Aavd*d&m8EiT+lg!piPY=_#d?Y=Qk+c z+Zh=C{{~HM;m5!KGcX+f$iT4Km4N|vWIqYk(N5_U=pCM*%i4*w1>|>d-1`uScPw`W z{M*dH_>-4`5p?bkk=D^x8K}SqrAq_itpSyF|3G(BfX;r$;z!WL4^I4zf#IJo3?_i@VPXNDAxXTgG*YR@z`y{Ce^5Fg z(h5-8|L;Qn-7x>QGBEyP8zgxE)D8okYyfJ1li>e<-V6-?e~}sgdl?x2a14@r7(oZD zf%?%T+}{e0e@hDPC|=3H_=}l=5p=2t5tIi0b&%ozCkzb#bs_#I-bNa#1f_q_!405$bV;xV zlm`BS+WMDB2m{c)Ky?fZKSB4|kYE>0m6B2Zflf1r9gg(hm4V^^eIml(4+F#h`3wyI z1qM;vg8~-R?g7<*X&?@17?cM+UG%U5B+0c`2B)`;ScB@F9Arq2c(ZQ zn`o#SbeJ`${R0|PCc~Pa3=E8*_V8bj-vSvJ{)6sl+R4BGT2b_W6$8W8RSXQ)Cm0w6 zH5nL4XkwG$5;C=b4sZdDDTBt<$ut6{4OAw8?&1O8+s4Jf@E_E-H-}z)2Re%$rf{&( zpz)@43=9mQaaPbtPqc6ss9y;h&jgKaQX!0~>O|0jFOa)PX#0?5zZnAq18A&k@*umD z33PWVDDQ&~dnC(aL}~)%1JI%8pm_{WBK6Q(8R+D=?FLtPC>XVQ0_^-{*`Bj7u3AnYadMEEY7kHywH@L@NUW z19|l?=t2R|Fw>bo77XY9?PNIr|0njd4gS4lxbVM+!94;r5dli~p!>=}^Q{bO0s~ep zgTerG=q>2fLeOEip!+PbMln`F(76Dh!{yKXc4RpB?+$iOP0B%SSY?$3XQt@jo&wk>00 zSPn`8n8pr(2&h~H%?W}|69uIO(A{OAaar8q06JR;bQtuxzx4#;9sT^R3;*6QocV1z z2+uGBjm3cKW6;1js9grCqmmgJ89{f8g4V(yq6T!r^O@hm4CnuyA=&?+@Bo#GH~%v* zfXWYqse^?DS`Pu5j0N>eK;ZzIF9o&LKz*n%=z0)MhK>~;4Cnt7ca{st&&U{5FC72M z&Hy?mY_Rwm(;c983aGpW)eoQ@I-mpcL2Wh=e#2<%eS_ib-|u9_|M`C>zzqaYI>NMf zFp7ZNGz{Q9I-sHgG>x9Y7?IP$aPIGCvV!2;zXJ>>e{eB?j-?rl9wy`_(9u$-f5{P5 z^&uyH82jAc*$f>k7>DdJFQ8*=4!mJzIQMS>8S(#<;q0F<@ZB^!K(jxD0(%h3&pXV( zaPBYYcriPM^WeLTh(G4@+`runr+*1DfSM44&}&3H33SXR=%}Kzf6ItI>h#>dy9{T3 zTQPu+NZki&brEgb;8X}Y;_u9F4hC=-5J(5-{+(wy^Vva|h;=QF=54MnhmU1O{CQkaP$E=%5CW2GCg^ zAR2bK0gQeCJ%KK}ox{{hna|4;o7(D5Jt|I|Z{Jwc{HCs=^^=rrh* z3xqzH{y$Lj{(-&lr~W_q+=f5({||tUwD?i~AAID+59sk2F#5xP@X;dwKSGY*`2X<# z|NqVZ{~vt(|NsAk|Nk>MgHD5hy8S~v_^6WqKjfiyvP0VhT1ZZ#ubW{Wg z54sdP>Y&jOpezLb7Z)>3h)ZYKoSMzBE-jZ~Ve-(}Dh}ET^dmcm8MMXyUv_rIzwGR} z|FW}Z{mstK`R+L{qKKv_Pu}E*(txW zvRMCQWzonL+1bCbZM}hteg2!Boe0{l{xy5>q$$w$slVCT zmjAP}UqjuAbNkr8?Ck4*va=Ni!M1VG)=1D40chrzfr0HuMrJ!9*F)w1{sV3Pr*8~^ zTnbuv=*Gan02=6@$iTp`f`Nenw22pV<=?IiUS3a#-NgGZJG=dVb{50kLePcc8Ja$DrhceADE-8N zHV-13Lj^V{MS~X4Er%}D=w@JGFd=5K05k_%{LjvQ1$84nKV1Eroh=XAFh~VA!;An; zTZEKnJYoz49(JG=5^q-r7;Xi1>Z7Ku9|6B%!e|`)M|6z-GKnpuS8!JJJ`6*s)`aLU~;d53t zs0E?_H#@uJUv~D)f7#hnzoe(%E0iKhk=3SF#xI|K{=Q}vjOb>AmX;^g4UA# zU%|leml=`^@P`bAvY?_9RJ)hqHwT=9Wf>U$UnL?AKwFU$=#v8&LE#T-{ehNr;`Ra~ zXebX92me61;5z|7Y+_*eCrF2|2e}uN{y|4iB!M{iG06SkZ4DI+4FBI^@dKz9+swcK zKK=ky)8RLcIF6&D1H(UC28REW85sT_WMKGzoPpu(4hDwWV+;(j zx(p1#^m02W1VAk

USYs{rwd#Qzx>7=JS|F#HE?G2vlgZ~&FWpmK~rJ|fb1;*~`) zFfbfsU|;|(<0M`ec9ozD-9hUSLA4@w71R|4l|6GA7#OIv^#gR2!deCf26}Dr03C}3 zN;^(e3>H%c1_n@%1r!%lG#0B-prw4985kIFFY&~xji4ZC8~bGj1_n^?mY`xX6@ZQ? z0@>kBrha_dKqDhv(5)i+_!MHufz~v9_z${jP>SK)-*|>||0Xh=`!}26+}|38vwy4@ zeu9o^VE~nxn;94wr!g=vg0^O17()dC(AJJg3=9mQo)~dQ%YfE$t-Z>~aQ3ekbUE`M zv^8Dt7*7A5$RMQvItC7O+#6^+4HcY<(+E(02Av8HT8ax=o{f8XKIob>(Di_4|2Q+8 z|MwKl-H0_==l=a?=vniGK~>j`!InUy8K<*Ik^qfZg)lHMoQ969f$B-j{03SAdgeDj z!?}NZaJn6~X7J+wZwwdz2ZOJ`1uefP$vI>z25pxDErA6c69#IpfZ_v`H$cl{nHWy} z1g(R!XE^_#xOJfC{w-xV{E?Z?*T{nE8_-N4sI3U{1L!!vBMb}-D;XFVCNfk^UBz(m zKj;cXM0$aRIVOFI0Tf@Lcq2PVDbfhqz5_Zs3Dj5uwK=*NN+)b!xbXiU!Sn<6%fEqk zKdR?J>j2OGabP(A{|iy>Klg7r!;w$)PCuw_M&^Oi!RcQD4CnssBf|aP8P5I<17Bx4 zSk^usdd~oM&)L6j4Cntp$Loi4|K>5A`pL}zTIYrw1_R9ox#!qtQ2pfxzP|K4Mqd8N zaPHp}hSR^qz*i`5dpOW8#GcYtSg787>q#>+Pax?_!0w61jKqDm}7ApM{Jl6UD{{ef5c!NBI zW`NQBVEzZt!I+a2;r67KH!*0gQ&I z`w!Lk{{u6~od5s-utVto_D~vBY=J@*>w!T1dQk=j23-aQ2GA^M zC<6mSjJup%-k0=rLRWr#`V3|b)p_Z)Q?pt%IlOeSdiGU#{@&~*f$V`UdJFff3Q zmg}>Xl3M&eHT4_5@cW;g{rg{bb|5G%P{&!|v1w5FEQPLW0L`-dGcYiKjy~XHW@2IC z;t^o@os|W;Hbd%vcJ_4w?tA|?JKKy3;RhOf1C7Uk#*9I8U_lHF44~CGSiSu_JBQ(2 zb`Ime?Cc(V?)#UWy$Q7Hg2MDC!N9-(I+g(xhk*<7jpZ`E_7%9 z%g+A)H!JJPzwB%iQ1s$IWgawj`Uf<{A;`e+Uy*^~ALz7276t~;s5b-X#8S`+0ia=0 zkW-PJfR_y_olbtR7FFU*CUv~ES|Jm7(|L5dfUhLv}!%tDEQk9E~;dwRa(gAEn zfv3ShQ$%764F9Vc82+DTVEF%tf#LrR28RE$85qi97#J8pQ!${Eu0c_Z%@BfOplaer zb`In3>};XG+1ZN!8ybWa6ckukKzrXo%ee3?$pN|WAINo}vw8o4nvtmZv|4bB zi$R2-OGs1zTIK}uJ&vU}pb0pT`~HLM2RGeN-3DpCF);k!!@%$lbeE4v72UnyAHqRz#p93vKY5 z2?`pJQ^DC8bVO+q1H;=$1_s0R3=C}c3=E*@9sDMcDXYl9zyKQGmnBm_X#NsZ#Dm&X zWEz228)(cPRK$>5If2?Jpdu7h72&mr3`tPC7Bs(yBYO~SB4|}PXpRt69TBaG2!)_) zh(Jg3P&)SrnhOHWn}F61;ixLHxCXq({Sza@nco5oXa6cPocSZkaPkK$gQhV^F)srH z1E?KIt-E_bEASw#Kv2sWzmMmGPNMh+nt2vsIQOrD;rzc-4CntpVmSZ*Cd2uE3mFc+ zvt(dktYu(eSj5P{#LEc22L2{|D_nBm;Nl?><6W~nax z|Icvv!y5+AC{T(~WMBf-!PpETTMRU#585yYT2Bp{Q3kDW0j(T^7zmoZy!@Ym;oLva z2^P3#(9Zrj#&GesFvDd~$p+C%WfGKIK&}2g3=9mQ*?UmSAJpC!U}#w+#&GW6F}yP~ z=l_3aIQQ2Re5wYO9fo8aDBM790?hz}N-~ z23pkuo}vA#4=GQO4)g=hCWHAu8P5GJW!U+caX@CMPX1tExb~lc;p|@<2FNTd=+qHV zS_G|PxzBL!Z!yEEpP&*IG*1W#clwi{`QA(5^+AFR=l(h~oco)^aPDsi!`VN|49C7O z(Rn=t{rx{`Bf$^=&!K>ZH$V(j`Uhyc9%zF;XweN4eE`%@{r~?1_zc(o{~I9mKL!V| z_z!-F{D=AnAbmd+|9=3P^MU#Q4-o$W|NsA>?NJZdA@l(x8niV6#7CxK>OVmB{g8*~ z``?eCLFWHI{{KIiKK#E4bjRBNhyNMC^oKgoloZ4NA94`-4+Dh$52GJI`QWK52GHDF zJtQ1JQ&OOrvPMv^0u(&hXwZBUXr>9Y4GDfe0BChCXub)A3C6&v0&+tDG^hv~_xzQW z%?cVc`r8?t@%&iP85ZCl_}0rsBWs?AiabvmZkD51$3i0sYC!W&;g8 z5@QxQYCvZH$<7x1mz}*Cec<(fcJ`Nl+1c@+U7bIuw@VP@Oi(`t)Vc)?tMNUKPl*1X zo&6Jw{h+y#qkpoq`9bjm(ntXYb=*PgZxb097@DCiRnYvxl!>;sNAc{qef%#wTZw{k zd(c2L$WD-5prflnx!IP1fl--(ff3Zo<9n8vnD;+B8?-kcuY=4uL4H{BL(MNy-T6qcT z6MzmY2aRmtR1Vs+3fhDHFFV`oe|Gkj|Jm7pLGJjJn)>%fbo93Wxp`v$azM>?bR)p! zDCjad4+e(+Jq!&0r!X-5k7Z!+uz+6r3|fs&)M&-`>}-ZR8Q{xL<^N@8NB+;vO`l|E zm(9z{3aVtVbnHNVsefz?4F8)L82*1j8gTo?z_8&V1A}b{1H(3S2b0PNT`n+-fq?;( zf)IMyKwIa)e)nQv`2QKnZsY;oTF5RkkUE4`5?P==6DY-^4tj%k*MZFa-;2X;kUI`D zP-iC_sP7Lt{2$?Tg#FX-*ngUV;U7O0c0++y-GXM=5cWgrb5Px$$iVO)bU6{ozvy@t z1H*qNDhy_V><5j!A=wXNg43Tc1H=CfSi%6b+sy!CHz=M!>dD5Sz38A(F*E}}_Jhm@ z?Vkqcfny8||Iaco96Zawu+^D?0Tg5)c{FXLaY1F(LkSxptQgUp6&x(rpm{_ zzz!2Q-)tO8=y41`)k5*{{It`hyBR=(n25m;(7*!> zA7nW5n}-3kM4U{E(X@doQP2QfGy?+zXfU&nVfyY93>U$JFfcoj=}Qb}{|eJ=&jfUr11+tliiK!~ z9RJ1uHvimT9fouNRzTC?D~9v`jxwD48_aOx8)SP7nl`F(k9`3Re1Qi4K}Qg)Fr59X z#c<}20K@XL3=AMSs#-RD3mjD0!01VszfBvBU|M3s?|BwH(|9}3!{Qv&{{15p5w;z!IfAE0){|5~6 z|H0=zI>>_-8vp}>Ym+1c!$a`G6y=Hybv^`H*h-|TFzf7#g?|Fg3}tD3g|%g&Dbn@xF#9@1sX1|7AO zgVu2db=X0jX@(y}9gTzHRZ#5&+E5Ov9YV#qxZVujrU=M*7U4jO6 z{$*!pAuUz~b*w=PL8Cx&Ki~^$LCsXqeT)C)7#RNBF);jBV_=X5jm3eExdH9A#M*>` zcAFLcWoM`T&(7ZdF*S8>x3%@WZd+UK&3+&+f}8?xCWk;*dVOPH`2U!JVd`cE24PbM z2K<@j8|Xm1>}-brd3o$zR#u?q9O#@fQ04;}4(ft{Flb5rb<}Ra-**fQ|E$UA{(zd! zpbe79=7W~%e?>JPRDFVvW`U|BngVsnkv8^%!(Ic+x+G9If$qsB+F+LZW!p^H-(A_v7eW2zwx*R$m)Jg*#(g4c8 z=yD9;R?+ESObloLC@@_7&&;3>stXuEQ`N-Y1q*H(gI4^4A`H}DfLH>$oC(x4KKmDR zJ4rXg`TzSFF8-g#klzVf*#Qn5uuh0-!Xzlmfbtlq(W}P5z{tX|^fc(+3m%5^|JJ}8 z-k?VC?q@Fc13+%Wya!*KrpS~T;o zFr53VLh&p;EQZhh1zkVZ$8i4tL5B1H7BigtYtC@}Kj>OA3@4E#0P47alIWS=pr*;` zUo7C=lc#@?Wf&!z{{R2a@c(~1!~g%x5PX1<;XeZ-!#{Rrh6DD@35@GyqK$fl32-a}1PL zKP^I-RJKO4ic6QUh?Cj8g+1c!%%94Olpn?XRwS`uBc$odm&b|q4 z6@XeaL7*BNhXEk3f(Euhw~~Suo_KUxSrz@w%K8q~532PmA^JfjHMquMWncgwX#pD8 z0aev5&<$QJ{|gF0tt#RF+1YFVXJ>zVpPG92OM1G=w+v9j1*}((f#Lsh28KTy85pXn z7#J8pi*n#j`<)GLm4VC<`k#|y>7$^K%*@Cr%EJmOu$$`d1Kx^_La^RxpFAKxD zf4U4;{~IzWX@UlxJs254X#=7JjRY0%pe-37Hwu8NPlogV5*W_^f5CA6|9ghHhn6r% z$_p{*g60FzbRxK*X(-UNP!NNFa4^H(7boCjA{YK&X1MfUgvcVBnSqOkkKyF^exzb^ zHN)Az3JT!6mJk+VXKlR?TFC{%q73K%U x_zy+~h69ca42=&N7?^)TaQjaNhW3vP43C=`7(OsFF#G_W uml diagrams + ] + +# note: breathe requires doxygen xml output -> must have GENERATE_XML = YES in Doxyfile.in +# match project name in Doxyfile.in +breathe_default_project = "xodoxxml" + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +pygments_style = 'sphinx' + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +#html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] +html_favicon = '_static/img/favicon.ico' diff --git a/docs/implementation.rst b/docs/implementation.rst new file mode 100644 index 00000000..e325f88c --- /dev/null +++ b/docs/implementation.rst @@ -0,0 +1,202 @@ +.. _implementation: + +.. toctree:: + :maxdepth: 2 + +Library +======= + +Library dependency tower for *xo-alloc*: + +.. ditaa:: + + +------------------------------------------+ + | xo_alloc | + +------------------------------------------+ + | xo_indentlog | + +------------------------------------------+ + +Install instructions :doc:`here` + +Components +========== + +Abstraction tower for *xo-alloc* components: + +.. ditaa:: + :--scale: 0.85 + + +----------------+-------------+ + | IAlloc | Object | + +----------------+-------------+ + + +-------------+ +-------------+ + | GC | | Forwarding1 | + +-------------+ +-------------+ + | ListAlloc | + +-------------+ + | ArenaAlloc | + +-------------+ + +* *IAlloc* + Allocator interface. + +* *Object* + Root Object Interface for types participating in garbage collection + +* *GC* + Incremental compacting garbage collector. + +* *ListAlloc* + Auto-expanding allocator. Contains a collection of ArenaAllocs + +* *ArenaAlloc* + Arena allocator (a.k.a bump allocator). + +* *Object* + Interface for types that participate in garbage collection + +* *Forwarding1* + Forwarding pointer. Supports the Object interface; + used internally by GC during evacuation. + +Key Points +---------- + +* Allocators can be reset, but do not support freeing of individual allocs. +* GC works with types that implement auxiliary GC-support methods. + Such types must inherit Object. +* A region may uses multiple arenas, but because of allocation activity + since the last GC. If necessary, GC will allocate a new to-space with a + single arena that's large enough to accomodate all objects that might survive + from a from-space that has acquired multiple arenas. + Intent is to scale up to find application's working set size, then stabilize + +Components +========== + +Allocators +---------- + +Inheritance +^^^^^^^^^^^ + +.. uml:: + :caption: allocators + :scale: 99% + :align: center + + class IAlloc { + + alloc() + + alloc_gc_copy() + + checkpoint() + + clear() + } + + class ArenaAlloc { + + free_ptr() + - lo_ : byte* + - checkpoint_ : byte* + - limit_ : byte* + } + + IAlloc <|-- ArenaAlloc + + class ListAlloc { + + expand() + + free_ptr() + - start_z_ + - hd_ + - full_l_ + } + + IAlloc <|-- ListAlloc + + class GC { + + add_gc_root() + + request_gc() + + gc_statistics() + - gc_root_v_[] : Object** + - nursery_[2] : ListAlloc* + - tenured_[2] : ListAlloc* + } + + IAlloc <|-- GC + + +Composition +^^^^^^^^^^^ + +.. uml:: + :caption: allocator composition + :scale: 99% + :align: center + + object gc<> + gc : nursery[from] = n0 + gc : nursery[to] = n1 + gc : tenured[from] = t0 + gc : tenured[to] = t1 + + object n0<> + + object n1<> + + object t0<> + + object t1<> + + gc o-- n0 + gc o-- n1 + gc o-- t0 + gc o-- t1 + + +Each ListAlloc composes like this: + +.. uml:: + :caption: ListAlloc composition + :scale: 99% + :align: center + + object x<> + x : hd_ = a0 + x : full_l = {a1, a2} + + object a0<> + a0 : lo_ = 0 + a0 : free_ = 12345 + a0 : hi_ = 1000000 + + object a1<> + + object a2<> + + x o-- a0 + x o-- a1 + x o-- a2 + +Here *a1* and *a2* are full, while *a0* can still allocate memory. + +Objects + +.. uml:: + :caption: objects + :scale: 99% + :align: center + + class Object { + + _is_forwarded() + + _offset_destination() + + _forward_to() + + _destination() + + _shallow_size() + + _shallow_copy() + + _forward_children() + } + + class Forwarding1 { + - dest_ : Object* + } + + Object <|-- Forwarding1 diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..198cf01c --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,14 @@ +# xo-alloc documentation master file + +xo-alloc documentation +====================== + +xo-alloc provides arena allocators and a generation garbage collector + +.. toctree:: + :maxdepth: 2 + :caption: xo-alloc contents + + install + introduction + implementation diff --git a/docs/install.rst b/docs/install.rst new file mode 100644 index 00000000..ab356be5 --- /dev/null +++ b/docs/install.rst @@ -0,0 +1,120 @@ +.. _install: + +.. toctree:: + :maxdepth: 2 + +Source +====== + +Source code lives on github `here`_ + +.. _here: https://github.com/rconybea/xo-alloc + +To clone from git: + +.. code-block:: bash + + git clone https://github.com/rconybea/xo-alloc + +Tested with gcc 13.3 + +Install +======= + +One-step Install +---------------- + +Install along with the reset of *XO* from `xo-umbrella2 source`_ + +.. _xo-umbrella2 source: https://github.com/rconybea/xo-umbrella2 + +Minimal Install +--------------- + +To build+install just required dependencies: +``xo-alloc`` uses several supporting libraries from the *XO* project: + +- `xo-indentlog source`_ (structured logging) +- `xo-cmake source`_ (shared cmake macros) + +.. _xo-indentlog source: https://github.com/rconybea/indentlog +.. _xo-cmake source: https://github.com/rconybea/xo-cmake + +Building from source +-------------------- + +Install scripts for XO libraries depend on helper scripts installed from `xo-cmake`. + +Preamble: + +.. code-block:: bash + + mkdir -p ~/proj/xo + cd ~/proj/xo + + git clone https://github.com/rconybea/xo-cmake + + PREFIX=/usr/local # ..or desired installation prefix + + # want PREFIX/bin in PATH to use xo-cmake helpers + PATH=$PREFIX/bin:$PATH + +Install `xo-cmake`: + +.. code-block:: bash + + cmake -B xo-cmake/.build -S xo-cmake + cmake --install xo-cmake/.build + +Install remaining dependencie(s) in topological order: + +.. code-block:: bash + + xo-build --clone --configure --build --install xo-indentlog + xo-build --clone --configure --build --install xo-alloc + +Directories under ``PREFIX`` will then contain: + +.. code-block:: + + PREFIX + +- bin + | +- xo-build + | +- xo-cmake-config + | \- xo-cmake-lcov-harness + +- include + | \- xo + | +- alloc/ + | \- indentlog/ + +- lib + | +- cmake + | | +- xo_alloc/ + | | \- indentlog/ + | +- lib*.so + +- share + +- cmake + | \- xo_macros + | +- code-coverage.cmake + | +- xo-project-macros.cmake + | \- xo_cxx.cmake + +- etc + | \- xo + | \- subsystem-list + \- xo-macros + +- Doxyfile.in + +- gen-ccov.in + \- xo-bootstrap-macros.cmake + +CMake Support +------------- + +To use built-in cmake support, when using ``xo-alloc`` from another project: + +Make sure ``PREFIX/lib/cmake`` is searched by cmake (for example include it in ``CMAKE_PREFIX_PATH``) + +Add to your ``CMakeLists.txt``: + +.. code-block:: cmake + + FindPackage(xo_alloc CONFIG REQUIRED) + target_link_libraries(mytarget INTERFACE xo_alloc) diff --git a/docs/introduction.rst b/docs/introduction.rst new file mode 100644 index 00000000..7b5333cf --- /dev/null +++ b/docs/introduction.rst @@ -0,0 +1,268 @@ +.. _introduction: + +.. toctree + :maxdepth: 2 + +Introduction +============ + +The ``xo-alloc`` library provides a in incremental, generational collector for c++ code. + +Features: + +* *incremental* - can reasonably expect short pause times. +* *generational* - focuses effort on collecting young objects, + on the basis that they're more likely to be garbage. +* *compacting* - each garbage collection cycle evacuates survivors to contiguous memory, + so effect is to defragment. +* *collects cycles* - collection algorithm naturally collects cyclic references + +Tradeoffs: + +* Application is responsible for spilling register values and protecting hardware stack, + since garbage collector cannot indepndently distinguish collectable object pointers from + non-pointer values. + +* GC will not spontaneously run without permission. Instead will set a pending bit, with GC + occurring only when application releases it (e.g. when stack+registers are known to be empty of values + subject to GC). + +* GC implementation is single-threaded. It cannot run in parallel with the mutator (i.e. application code) + In return this allows GC to be only lightly coupled with application. + +* GC divides each generation into separate from- and to- spaces. A collection cycle copies surviving + objects out of from-space. Once complete, the entire from-space is treated as empty, and available to + become to-space on a future cycle. This means that at any time only half of allocated memory is available + to the application; the rest is waiting to receive survivors from the next GC cycle. + +Design +------ + +Garbage Collector +^^^^^^^^^^^^^^^^^ + +The garbage collector supports two generations, labelled *nursery* and *tenured*. +Nursery objects that survive two collection cycles are promoted to tenured space. +Nursery and tenured objects are kept in separate memory areas, instead of being interspersed. + +Collection cycles come in two flavors: + +1. *incremental* collections - these collect only the nursery space. + +2. *full* collections - these collect both nursery and tenured spaces. + Full collection may incur noticeable GC pauses. + +Application Interaction +^^^^^^^^^^^^^^^^^^^^^^^ + +Application code that interacts with GC has several responsibilities. + +1. application must explicitly invoke GC, when convenient. Since in general any GC-eligible object + may get moved by the collector: once a collection cycle completes, + it's up to the application to re-load pointers from memory addresses + (GC roots) that have been shared with the collector. + +2. application must identify a set of GC roots. GC preserves everything reachable from any GC root + +3. The collector needs to know how to traverse GC-managed objects. + We teach it this by requiring that such objects inherit the ``xo::Object`` interface, + and implement auxiliary function detailed below. + +4. GC also needs to know when a mutation alters a pointer from one GC-managed object to another. + In particular, GC needs to track pointers from tenured space into nursery space, + and update them when an incremental collection moves nursery objects. + We do this by requiring application code use a GC-provided assignment primitive + on GC-eligible pointers. + + +Example GC Use +-------------- + +.. code-block:: cpp + :linenos: + + #include "xo/object/List.hpp" // polymorphic List with GC support + #include "xo/object/String.hpp" // string type with GC support + #include "xo/alloc/GC.hpp" + + int main() { + using xo::gc::Config; + using xo::obj::String; + using xo::obj::List; + using xo::gp; + + Config config = { .initial_nursery_z_ = 50*1000, + .initial_tenured_z_ = 10*1000*1000, + .debug_flag_ = false }; + + up gc = GC::make(config); + + Object::mm = gc; // use GC for allocation of Object (+ derived classes) + + gc->disable_gc(); // gc forbidden + + // tiny example data structure + gp s1 = String::copy("hello"); + gp s2 = String::copy(", "); + gp s3 = String::copy("world!"); + gp list = List::cons(s1, List::cons(s2, List::cons(s3, List::nil))); + + // tell GC what to preserve + gc->add_gc_root(reinterpret_cast(list.ptr_address()); + + gc->enable_gc(); // triggers immediate gc + + // s1, s2, s3 invalid. + // list at new address + + std::cout << "list.size=" << list->size << std::endl; + } + +GC-Eligible Types +----------------- + +Or, how to inherit ``xo::Object`` and provide GC support + +A type Foo that inherits ``xo::Object`` needs to provide overrides for Object methods ``_shallow_size()``, +``_shallow_copy()`` and ``_forward_children()``: + +Typical Pattern +^^^^^^^^^^^^^^^ + +GC support methods look something like this: + +* class definition + +.. code-block:: cpp + :linenos: + + #include "xo/alloc/Object.hpp" + + namespace xo { + class Foo : public xo::Object { + public: + ... + virtual std::size_t _shallow_size() const override; + virtual Object * _shallow_copy() const override; + virtual std::size_t _forward_children() override; + }; + } + +* use overloaded ``operator new`` + +A GC-eligible class will allocate instances using the ``MMPtr`` overload. +This allocates memory in GC-owned space + +.. code-block::cpp + :linenos: + + gp Foo::make(...) { + ... + return new MMPtr(mm) Foo(...); + } + +* ``_shallow_size()`` returns the amount of memory used by the subject: + +.. code-block:: cpp + :linenos: + + std::size_t Foo::_shallow_size() const { return sizeof(Foo); } + +* ``_shallow_copy()`` is invoked during GC to create a copy of the subject + + It should use the ``xo::Cpof`` argument to ``operator new``. + +.. code-block:: cpp + :linenos: + + Object * + Foo::_shallow_copy() const; + +* ``_forward_children()`` is invoked during GC to vist child ``xo::Object`` pointers + to make sure they survive + +.. code-block:: cpp + :linenos: + + std::size_t + Foo::_forward_children(); + +Atomic Types Without Object Pointers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Plain-old-data classes without embedded pointers + +.. code-block:: cpp + :linenos: + + Object * + Foo::_shallow_copy() const { + return new (Cpof(this)) Foo(*this); + } + +.. code-block:: cpp + :linenos: + + std::size_t + Foo::_forward_children() { return Foo::_shallow_size(); } + +For example see ``xo::obj::String`` in ``xo-object`` + +Non-GC Objects +^^^^^^^^^^^^^^ + +A class *Foo* that inherits ``xo::Object`` can opt-out of garbage collection by +omitting the ``MMptr(mm)`` overload. + +In that case `Foo::_shallow_size()`, `Foo::_shallow_copy()` and `Foo::_forward_children()` +will not be called: + +.. code-block:: cpp + :linenos: + + std::size_t Foo::_shallow_size() const { return sizeof(Foo); } + Object * Foo::_shallow_copy() const { assert(false); return nullptr; } + std::size_t Foo::_forward_children() { assert(false); return 0; } + +For example see ``xo::obj::Boolean`` in ``xo-object`` + +Structs Containing Object Pointers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A class with object pointers needs to tell GC how to traverse them + +.. code-block:: cpp + :linenos: + + #include "xo/alloc/Object.hpp" + + namespace xo { + class Foo : public xo::Object { + public: + ... + virtual std::size_t _shallow_size() const override; + virtual Object * _shallow_copy() const override; + virtual std::size_t _forward_children() override; + + private: + gp bar_; + gp quux_; + }; + } + +* ``_forward_children()`` is invoked during GC to fixup child pointers + that refer to forwarding objects: + +.. code-block:: cpp + :linenos: + + std::size_t + Foo::_forward_children() + { + Object::_forward_inplace(bar_); + Object::_forward_inplace(quux_); + + return Foo::_shallow_size(); + } + +For example see ``xo::obj::List`` in ``xo-object`` diff --git a/include/xo/alloc/AllocPolicy.hpp b/include/xo/alloc/AllocPolicy.hpp new file mode 100644 index 00000000..53f758ee --- /dev/null +++ b/include/xo/alloc/AllocPolicy.hpp @@ -0,0 +1,58 @@ +/* AllocPolicy.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include + +namespace xo { + /** Tag class, drives overload of operator new. + * See also: xoglobal, xocopy + **/ + struct xolib { + xolib() = default; + }; + + /** @brief opt-in allocator for XO libraries. + * + * By default delegates to vanilla operator new/delete, + * but can set alloc/free functions at runtime to + * adopt a different implementation. + * + * Intending this to op-in to garbage-collector? + * Not sure if we actually need this + * + * Use: + * struct Foo { .. }; + * auto p = new (xo) Foo(..); + **/ + class XoAllocPolicy { + public: + using AllocFn = void* (*)(std::size_t); + using FreeFn = void (*)(void *); + + public: + XoAllocPolicy() = default; + + static void * global_alloc(std::size_t z) { return ::operator new(z); } + static void global_free(void * x) { ::operator delete(x); } + + void * alloc(std::size_t z) { return (*alloc_)(z); } + void free(void * x) { (*free_)(x); } + + private: + AllocFn alloc_ = global_alloc; + FreeFn free_ = global_free; + }; + + /** singleton xolib instance **/ + static XoAllocPolicy xo; +} + +inline void * operator new(std::size_t z, xo::xolib) { + return xo::xo.alloc(z); +} + +void operator delete(void * ptr) noexcept; + +/* end AllocPolicy.hpp */ diff --git a/include/xo/alloc/LinearAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp similarity index 61% rename from include/xo/alloc/LinearAlloc.hpp rename to include/xo/alloc/ArenaAlloc.hpp index e1390dfc..e2e74e8f 100644 --- a/include/xo/alloc/LinearAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -1,4 +1,4 @@ -/* file LinearAlloc.hpp +/* file ArenaAlloc.hpp * * author: Roland Conybeare, Jul 2025 */ @@ -9,7 +9,7 @@ namespace xo { namespace gc { - /** @class LinearAlloc + /** @class ArenaAlloc * @brief Bump allocator with fixed capacity * * @text @@ -33,34 +33,39 @@ namespace xo { * * TODO: rename to ArenaAlloc **/ - class LinearAlloc : public IAlloc { + class ArenaAlloc : public IAlloc { public: - ~LinearAlloc(); + ~ArenaAlloc(); /** create allocator with capacity @p z, * with reserved capacity @p redline_z. **/ - static up make(std::size_t redline_z, std::size_t z); + static up make(const std::string & name, + std::size_t redline_z, + std::size_t z, + bool debug_flag); - std::uint8_t * free_ptr() const { return free_ptr_; } - void set_free_ptr(std::uint8_t * x); + const std::string & name() const { return name_; } + std::byte * free_ptr() const { return free_ptr_; } + void set_free_ptr(std::byte * x); // inherited from IAlloc... virtual std::size_t size() const override; virtual std::size_t available() const override; virtual std::size_t allocated() const override; - virtual bool is_before_checkpoint(const std::uint8_t * x) const override; + virtual bool contains(const void * x) const override; + virtual bool is_before_checkpoint(const void * x) const override; virtual std::size_t before_checkpoint() const override; virtual std::size_t after_checkpoint() const override; virtual void clear() override; virtual void checkpoint() override; - virtual std::uint8_t * alloc(std::size_t z) override; - + virtual std::byte * alloc(std::size_t z) override; + virtual void release_redline_memory() override; private: - LinearAlloc(std::size_t rz, std::size_t z); + ArenaAlloc(const std::string & name, std::size_t rz, std::size_t z, bool debug_flag); private: /** @@ -68,23 +73,30 @@ namespace xo { * - @ref free_ always a multiple of word size (assumed to be sizeof(void*)) **/ + /** optional instance name, for diagnostics **/ + std::string name_; + /** allocator owns memory in range [@ref lo_, @ref hi_) **/ - std::uint8_t * lo_ = nullptr; + std::byte * lo_ = nullptr; /** checkpoint (for GC support); divides objects into * older (addresses below checkpoint) * and younger (addresses above checkpoint) **/ - std::uint8_t * checkpoint_; + std::byte * checkpoint_; /** free pointer. memory in range [@ref free_, @ref limit_) available **/ - std::uint8_t * free_ptr_ = nullptr; + std::byte * free_ptr_ = nullptr; /** soft limit: end of released memory **/ - std::uint8_t * limit_ = nullptr; + std::byte * limit_ = nullptr; + /** amount of last-resort memory to reserve **/ + std::size_t redline_z_ = 0; /** hard limit: end of allocated memory **/ - std::uint8_t * hi_ = nullptr; + std::byte * hi_ = nullptr; + /** true to enable detailed debug logging **/ + bool debug_flag_ = false; }; } /*namespace gc*/ } /*namespace xo*/ -/* end LinearAlloc.hpp */ +/* end ArenaAlloc.hpp */ diff --git a/include/xo/alloc/Forwarding.hpp b/include/xo/alloc/Forwarding.hpp new file mode 100644 index 00000000..47a555da --- /dev/null +++ b/include/xo/alloc/Forwarding.hpp @@ -0,0 +1,28 @@ +/* Forwarding.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#pragma once + +#include "Object.hpp" + +namespace xo { + namespace gc { + class Forwarding : public Object { + public: + Forwarding() = default; + + // inherited from Object.. +#ifdef NOT_USING + virtual bool _is_forwarded() const override final { return true; } +#endif + virtual Object * _destination() override final { return destination_.ptr(); } + + private: + gp destination_; + }; + } /*namespace gc*/ +} /*namespace xo*/ + +/* end Forwarding.hpp */ diff --git a/include/xo/alloc/Forwarding1.hpp b/include/xo/alloc/Forwarding1.hpp new file mode 100644 index 00000000..62536651 --- /dev/null +++ b/include/xo/alloc/Forwarding1.hpp @@ -0,0 +1,40 @@ +/* file Forwarding1.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "Object.hpp" + +namespace xo { + namespace obj { + class Forwarding1 : public Object { + public: + explicit Forwarding1(gp dest); + + // inherited from Object.. + virtual bool _is_forwarded() const override { return true; } + virtual Object * _offset_destination(Object * src) const; + virtual std::size_t _shallow_size() const override; + virtual Object * _shallow_copy() const override; + virtual std::size_t _forward_children() override; + + private: + /** the object that used to be located at this address (i.e. @c this) + * has been moved to @ref destination_ , + * with original location overwritten by a forwarding pointer + * + * Require: + * - can only use Forwarding with types that have a single vtable. + * To forward a multiply-inheriting class with two vtables, use Forwarding2. + * - if you try to use Forwarding for an object with multiple vtables, + * one of the vtable pointers will be replaced by @ref destination_. + * UB revealed when GC traverses a pointer that relies on the 2nd + * vtable to index virtual methods. + **/ + gp dest_; + }; + + } /*namespace obj*/ +} /*namespace xo*/ + +/* end Forwarding1.hpp */ diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp new file mode 100644 index 00000000..f42864b7 --- /dev/null +++ b/include/xo/alloc/GC.hpp @@ -0,0 +1,310 @@ +/* GC.hpp + * + * author: Roland Conybeare, jul 2025 + */ + +#pragma once + +#include "ListAlloc.hpp" +#include "xo/indentlog/print/array.hpp" +#include +#include + +namespace xo { + /** types that can participate in GC inherit from this base class. See Object.hpp in this directory **/ + class Object; + + namespace gc { + enum class generation { + nursery, + tenured, + N + }; + + constexpr std::size_t gen2int(generation x) { return static_cast(x); } + + enum class role { + /** nursery: generation for new objects **/ + from_space, + /** tenured: generation for objects that have survived two collections **/ + to_space, + N, + }; + + constexpr std::size_t role2int(role x) { return static_cast(x); } + + /** @class Config + * @brief garbage collector configuration + **/ + struct Config { + /** initial size in bytes for youngest (Nursery) generation. + * GC allocates two nursery spaces of this size. + * Will allocate more space as needed + **/ + std::size_t initial_nursery_z_ = 0; + /** initial size in bytes for oldest (Tenured) generation. + * GC allocates two tenured spaces of this size + * Will allocate more space as needed + **/ + std::size_t initial_tenured_z_ = 0; + /** true to permit incremental garbage collection **/ + bool allow_incremental_gc_ = true; + /** true to enable debug logging **/ + bool debug_flag_ = false; + }; + + /** @class ObjectStatistics + * @brief placeholder for type-driven allocation statistics + * + * Passed to @ref Object::deep_move for example + **/ + class ObjectStatistics { + }; + + /** @class PerGenerationStatistics + * @brief garbage collection statistics for particular GC generation + **/ + class PerGenerationStatistics { + public: + /** update statistics after a GC cycle + * @param alloc_z. new allocations (since preceding GC) + * @param before_z. generation size (bytes allocated) before collection + * @param after_z. generation size after collection + * @param promote_z. bytes promoted to next generation + **/ + void include_gc(std::size_t alloc_z, std::size_t before_z, std::size_t after_z, + std::size_t promote_z); + /** update with current state (use at end of gc cycle) **/ + void update_snapshot(std::size_t after_z); + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** number of bytes currently in use **/ + std::size_t used_z_ = 0; + + /** number of collection cycles completed **/ + std::size_t n_gc_ = 0; + /** sum of new alloc bytes, sampled at start of each collection cycle **/ + std::size_t new_alloc_z_ = 0; + /** sum of allocated bytes sampled at beginning of each collection cycle **/ + std::size_t scanned_z_ = 0; + /** sum of bytes remaining after collection cycle **/ + std::size_t survive_z_ = 0; + /** sum of bytes promoted to next generation **/ + std::size_t promote_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const PerGenerationStatistics & x) { + x.display(os); + return os; + } + + /** @class GcStatistics + * @brief garbage collection statistics + **/ + class GcStatistics { + public: + /** update statistics after a GC cycle + * @param upto. nursery -> incremental collection; tenured -> full collection + * @param alloc_z. new allocations (since preceding GC) + * @param before_z. generation size (bytes allocated) before collection + * @param after_z. generation size after collection + * @param promote_z. bytes promoted to next generation + **/ + void include_gc(generation upto, std::size_t alloc_z, + std::size_t before_z, std::size_t after_z, std::size_t promote_z); + /** update snapshot for current state. + * Use with tenured stats after incremental gc + **/ + void update_snapshot(generation upto, std::size_t after_z); + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** statistics gathered across {incr, full} GCs respectively **/ + std::array(generation::N)> gen_v_; + /** total bytes allocated since inception **/ + std::size_t total_allocated_ = 0; + /** snapshot of total bytes promoted asof beginning of last gc cycle **/ + std::size_t total_promoted_sab_ = 0; + /** total bytes promoted from nursery->tenured since inception **/ + std::size_t total_promoted_ = 0; + + /** per-type statistics (placeholder) **/ + ObjectStatistics per_type_stats_; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatistics & x) { + x.display(os); + return os; + } + + /** @class GCRunstate + * @brief encapsulate state needed while GC is running + * + * state pertaining to a single GC invocation. + * We stash an instance of this in @ref GC as context, + * so that per-Object-derived-type auxiliary functions can be slightly streamlined + **/ + class GCRunstate { + public: + GCRunstate() = default; + explicit GCRunstate(bool in_progress, bool full_move) + : in_progress_{in_progress}, full_move_{full_move} {} + + bool in_progress() const { return in_progress_; } + bool full_move() const { return full_move_; } + + private: + /** true when GC begins; remains true until GC cycle complete **/ + bool in_progress_ = false; + /** true for full GC; false for incremental GC **/ + bool full_move_ = false; + }; + + /** @class GC + * @brief generational garbage collector + * + * Works with objects of type @ref xo::Object + **/ + class GC : public IAlloc { + public: + /** create new GC instance with configuration @p config **/ + explicit GC(const Config & config); + + /** create GC allocator. + * + * Initial memory consumption: + * approximately 2x @ref Config::nursery_size_ + 2x @ref Config::tenured_size_ + **/ + static up make(const Config & config); + + const GCRunstate & runstate() const { return runstate_; } + const GcStatistics & gc_statistics() const { return gc_statistics_; } + + /** true iff GC permitted in current state **/ + bool is_gc_enabled() const { return gc_enabled_ == 0; } + /** @return generation to which object at @p x belongs **/ + generation generation_of(const void * x) const; + /** @return generation that contains @p x, given it's in from-space **/ + generation fromspace_generation_of(const void * x) const; + /** true iff from-space contains @p x **/ + bool fromspace_contains(const void * x) const; + /** true during (and only during) a GC cycle **/ + bool gc_in_progress() const { return runstate_.in_progress(); } + /** return free pointer for generation @p gen, i.e. nursery or tenured space **/ + std::byte * free_ptr(generation gen); + + /** add gc root at address @p addr . Gc will keep alive anything reachable + * from @c *addr + **/ + void add_gc_root(Object ** addr); + /** request garbage collection. **/ + void request_gc(generation g); + /** disable garbage collection until matching call to @ref enable_gc. + * + * GC is disabled when number of calls to @ref disable_gc exceeds number of + * calls to @ref enable_gc. + **/ + void disable_gc(); + /** enable garbage collection + * + * GC is enabled when number of calls to @ref enable_gc is at least as large + * as number of calls to @ref disable_gc. + **/ + void enable_gc(); + + // inherited from IAlloc.. + + /** capacity in bytes (counting both free+allocated) for object storage. + * only counts one of {to-space, from-space}, + * since one role is always held empty between collections. + **/ + virtual std::size_t size() const override; + + virtual std::size_t allocated() const override; + virtual std::size_t available() const override; + /** only tests to-space **/ + virtual bool contains(const void * x) const override; + virtual bool is_before_checkpoint(const void * x) const override; + virtual std::size_t before_checkpoint() const override; + virtual std::size_t after_checkpoint() const override; + + virtual void clear() override; + virtual void checkpoint() override; + + virtual std::byte * alloc(std::size_t z) override; + virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) override; + + virtual void release_redline_memory() override; + + private: + /** begin GC now **/ + void execute_gc(generation g); + /** cleanup phase. aux function for @ref execute_gc **/ + void cleanup_phase(generation g); + /** swap roles of From/To spaces for nursery generation **/ + void swap_nursery(); + /** swap roles of From/To spaces for tenured generation **/ + void swap_tenured(); + /** swap roles of FromSpace/ToSpace **/ + void swap_spaces(generation g); + /** copy object **/ + void copy_object(Object ** addr, generation upto, ObjectStatistics * object_stats); + /** copy everything reachable from global gc roots **/ + void copy_globals(generation g); + + private: + /** garbage collector configuration **/ + Config config_; + + /** contains allocated objects, along with unreachable garbage to be collected. + * roles reverse after each incremental, or full, collection. + **/ + std::array, static_cast(role::N)> nursery_; + /** empty space, destination for objects that survive collection. + * roles reverse after each full collection. + **/ + std::array, static_cast(role::N)> tenured_; + + /** current state of GC activity. + * @text + * in_progress full_move descr + * ----------------------------------------- + * false * gc not running + * true false incremental gc + * true true full gc + * ----------------------------------------- + * @endtext + **/ + GCRunstate runstate_; + + /** root object handles: targets of handles in this vector are always preserved by GC. + * Application can introduce new root object pointers at any time provided GC not running, + * but cannot withdraw them. + **/ + std::vector gc_root_v_; + + /** allocation/collection counters **/ + GcStatistics gc_statistics_; + + /** trigger full GC whenever this much data arrives in tenured generation **/ + std::size_t full_gc_threshold_ = 0; + /** trigger incr GC whenever this much data arrives in nuresery generation **/ + std::size_t incr_gc_threshold_ = 0; + + /** true when GC requested, + * remains true until GC.. completes? begins? + **/ + bool incr_gc_pending_ = false; + bool full_gc_pending_ = false; + + /** enabled when 0. disabled when <0 **/ + int gc_enabled_ = 0; + }; + } /*namespace gc*/ + +} /*namespace xo*/ + +/* end GC.hpp */ diff --git a/include/xo/alloc/GCAlloc.hpp b/include/xo/alloc/GCAlloc.hpp deleted file mode 100644 index e0c6ab7a..00000000 --- a/include/xo/alloc/GCAlloc.hpp +++ /dev/null @@ -1,20 +0,0 @@ -/* file GCAlloc.hpp - * - * author: Roland Conybeare, Jul 2025 - */ - -#pragma once - -namespace xo { - namespace gc { - class GC : public IAlloc { - enum class Space { A, B, N_Space }; - enum class Gen { Nursery, Tenured }; - - }; - - } /*namespace mem */ -} /*namespace xo*/ - - -/* end GCAlloc.hpp */ diff --git a/include/xo/alloc/IAlloc.hpp b/include/xo/alloc/IAlloc.hpp index 848f182c..2f759c53 100644 --- a/include/xo/alloc/IAlloc.hpp +++ b/include/xo/alloc/IAlloc.hpp @@ -20,6 +20,13 @@ namespace xo { public: virtual ~IAlloc() {} + /** compute padding to add to an allocation of size z to bring it up to + * a multiple of word size (8 bytes on x86_64) + **/ + static std::uint32_t alloc_padding(std::size_t z); + /** z + alloc_padding(z) **/ + static std::size_t with_padding(std::size_t z); + /** allocator size in bytes (up to soft limit). * Includes unallocated mmeory **/ @@ -30,10 +37,12 @@ namespace xo { virtual std::size_t available() const = 0; /** number of bytes allocated from this allocator **/ virtual std::size_t allocated() const = 0; + /** true iff pointer x comes from this allocator **/ + virtual bool contains(const void * x) const = 0; /** true iff object at address @p x was allocated by this allocator, * and before checkpoint **/ - virtual bool is_before_checkpoint(const std::uint8_t * x) const = 0; + virtual bool is_before_checkpoint(const void * x) const = 0; /** number of bytes allocated before @ref checkpoint **/ virtual std::size_t before_checkpoint() const = 0; /** number of bytes allocated since @ref checkpoint **/ @@ -48,10 +57,39 @@ namespace xo { **/ virtual void checkpoint() = 0; /** allocate @p z bytes of memory. returns pointer to first address **/ - virtual std::uint8_t * alloc(std::size_t z) = 0; + virtual std::byte * alloc(std::size_t z) = 0; + /** allocate @p z bytes for copy of object at @p src. + * Only used in @ref GC. Default implementation asserts and returns nullptr + **/ + virtual std::byte * alloc_gc_copy(std::size_t z, const void * src); + /** release last-resort reserved memory **/ + virtual void release_redline_memory() = 0; }; } /*namespace gc*/ + + class MMPtr { + public: + explicit MMPtr(gc::IAlloc * mm) : mm_{mm} {} + + gc::IAlloc * mm_ = nullptr; + }; } /*namespace xo*/ +inline void * operator new (std::size_t z, const xo::MMPtr & mmp) { + return mmp.mm_->alloc(z); +} + +//inline void operator delete (void * p, const MMPtr & mmp) { +// mmp.mm_->free(reinterpret_cast(p)); +//} + +inline void * operator new[] (std::size_t z, const xo::MMPtr & mmp) { + return mmp.mm_->alloc(z); +} + +//inline void operator delete[] (void * p, const MMPtr & mmp) { +// mmp.mm_->free(reinterpret_cast(p)); +//} + /* end IAlloc.hpp */ diff --git a/include/xo/alloc/ListAlloc.hpp b/include/xo/alloc/ListAlloc.hpp index 780d2bd2..8d27e6b4 100644 --- a/include/xo/alloc/ListAlloc.hpp +++ b/include/xo/alloc/ListAlloc.hpp @@ -6,13 +6,17 @@ #pragma once #include "IAlloc.hpp" +#include #include #include namespace xo { namespace gc { + class ArenaAlloc; + /** GC-compatible allocator using a linked list of buckets. * + * - all allocs done from first allocator in list * GC Support: * - reserved memory, released after call to @ref release_redline_memory. * @@ -21,27 +25,60 @@ namespace xo { **/ class ListAlloc : public IAlloc { public: - ListAlloc(LinearAlloc* hd, - std::size_t cz, std::size_t nz; std::size_tz, - LinearAlloc* marked, bool use_redline, - bool redlined_flag, OnEmptyFn on_overflow); + ListAlloc(std::unique_ptr hd, + ArenaAlloc * marked, + std::size_t cz, std::size_t nz, std::size_t tz, + bool use_redline, + bool debug_flag); ~ListAlloc(); - static up make(std::size_t cz, std::size_t nz, - OnEmptyFn on_overflow); + static up make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag); + + /** reset to have at least @p z bytes of storage **/ + bool reset(std::size_t z); + + /** expand bucket list to accomodate a requrest of size @p z **/ + bool expand(std::size_t z); + + /** current free pointer **/ + std::byte * free_ptr() const; + + // inherited from IAlloc.. + + virtual std::size_t size() const override; + virtual std::size_t available() const override; + virtual std::size_t allocated() const override; + virtual bool contains(const void * x) const override; + virtual bool is_before_checkpoint(const void * x) const override; + virtual std::size_t before_checkpoint() const override; + virtual std::size_t after_checkpoint() const override; + + virtual void clear() override; + virtual void checkpoint() override; + virtual std::byte * alloc(std::size_t z) override; + virtual void release_redline_memory() override; private: + /** **/ std::size_t start_z_ = 0; - LinearAlloc* hd_ = nullptr; + /** all new allocs from this list **/ + std::unique_ptr hd_; + /** allocator that was in @ref hd_ when @ref checkpoint last called **/ + ArenaAlloc * marked_ = nullptr; + /** overflow allocs (expect list to be short); + * from trying to converge on app working set size + **/ + std::list> full_l_; std::size_t current_z_ = 0;; std::size_t next_z_ = 0;; std::size_t total_z_ = 0; bool use_redline_ = false; bool redlined_flag_ = false; + /** true to enable debug logging **/ + bool debug_flag_ = false; }; } /*namespace gc*/ } /*namespace xo*/ - /* end ListAlloc.hpp */ diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp new file mode 100644 index 00000000..f66e8e9a --- /dev/null +++ b/include/xo/alloc/Object.hpp @@ -0,0 +1,232 @@ +/* Object.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#pragma once + +#include "IAlloc.hpp" +#include +#include + +namespace xo { + namespace gc { + class GC; + class ObjectStatistics; + }; + + class Object; + + template + class gc_ptr; + + template + using gp = gc_ptr; + + /** wrapper for a pointer to garbage-collector-eligible T. + * Application code will usually use the alias template gp + **/ + template + class gc_ptr { + public: + using element_type = T; + + public: + gc_ptr() = default; + gc_ptr(T * p) : ptr_{p} {} + gc_ptr(const gc_ptr & x) : ptr_{x.ptr_} {} + + /** create from gc_ptr to some related type @tparam S **/ + template + gc_ptr(const gc_ptr & x) : ptr_{x.ptr()} {} + + static bool is_eq(gc_ptr x1, gc_ptr x2) { + std::uintptr_t u1 = reinterpret_cast(x1.ptr()); + std::uintptr_t u2 = reinterpret_cast(x2.ptr()); + + // multiple inheritance shenanigans. + // (allow interface pointers separated by one pointer) + + if (u1 >= u2) + return (u1 <= u2 + sizeof(std::uintptr_t)); + else + return (u2 <= u1 + sizeof(std::uintptr_t)); + } + + T * ptr() const { return ptr_; } + T ** ptr_address() { return &ptr_; } + + bool is_null() const { return ptr_ == nullptr; } + void make_null() { ptr_ = nullptr; } + + void assign_ptr(T * x) { ptr_ = x; } + + gc_ptr & operator=(const gc_ptr & x) { ptr_ = x.ptr(); return *this; } + T * operator->() const { return ptr_; } + + private: + T * ptr_ = nullptr; + }; + + /** Root class for all xo GC-collectable objects. + * + * Design note: + * + * relying on inheritance means we insist that GC traits + * for a type appear directly in that type's vtable, and at specific locations. + * This implies one level of indirection when GC traverses an instance. + * + * Would be feasible to relax the must-inherit-from-Object constraint, + * but cost would be an extra layer of indirection + **/ + class Object { + public: + virtual ~Object() = default; + + /** memory allocator for objects. Likely this will be a GC instance, + * but simple arena also supported. + **/ + static gc::IAlloc * mm; + + /** use from GC aux functions **/ + static gc::GC * _gc() { return reinterpret_cast(mm); } + + /** during GC + * 1. copy destination object @p *addr to (new) to-space. + * 2. overwrite existing object @p *addr with a forwarding pointer to + * copy made in step 1. + * 3. return the location of the copy make in step 1. + * + * @p src. source object to be forwarded + * @p gc. garbage collector + */ + static Object * _forward(Object * src, gc::GC * gc); + + template + static void _forward_inplace(T ** src_addr) { + Object * fwd = _forward(*src_addr, _gc()); + + *src_addr = reinterpret_cast(fwd); + } + + template + static void _forward_inplace(gp & src) { + _forward_inplace(src.ptr_address()); + } + + /** primary workhorse for garbage collection. + * + * we assign each object one of three colors: black|gray|white. + * + * color | location | children | action | + * ------+------------+------------+-------------------------+ + * black | from-space | any | move to to-space | + * gray | to-space | any | move remaining children | + * white | to-space | white/gray | done | + * + * initially all reachable objects are black. + * GC is complete when all reachable objects are white. + * GC needs a variable amount of temporary storage to keep track of all gray objects + **/ + static Object * _deep_move(Object * src, gc::GC * gc, gc::ObjectStatistics * stats); + + /** copy @p src to to-space, and replace original with forwarding pointer to new location. + * return the new location + **/ + static Object * _shallow_move(Object * src, gc::GC * gc); + + // GC support + + /** true iff this object represents a forwarding pointer. + * Forwarding pointers are exclusively created by the garbage collector; + * forwarding pointers (and only forwarding pointers) return true here. + **/ + virtual bool _is_forwarded() const { return false; } + + /** offset for uncommon situation where pointer address is offset from object + * base address + **/ + virtual Object * _offset_destination(Object * src) const { return src; }; + + /** replace this object with a forwarding pointer referring to @p dest. + **/ + virtual void _forward_to(Object * dest); + + /** if this object represents a forwarding pointer, return its new location. + * forwarding pointers belong to the garbage collector implementation. + * (if you have to ask -- no, your class is not a forwarding pointer) + * all other objects return nullptr here. + **/ + virtual Object * _destination() { return nullptr; } + + /** return amount of storage (including padding) consumed by this object, + * excluding immediate Object-pointer children + **/ + virtual std::size_t _shallow_size() const = 0; + + // TODO: _shallow_move() also overwrite *this with gc-only forwarding object point to C + + /** if subject is allocated by GC: + * - create copy C in to-space + * - destination C will be nursery|tenured depending on location of this. + * else + * - return this to disengage from GC + * + * Require: @ref mm is an instance of @ref gc::GC + **/ + virtual Object * _shallow_copy() const = 0; + + /** update child pointers that refer to forwarding pointers, + * replacing them with the correct destination. + * See @ref Object::deep_move + * + * this gray object, located in to-space. + * fwd1 forwarding objects. + * Located in from-space. Invalid at end of GC cycle. + * p1,p2 source pointers. + * D1,D2 already-forwarded objects. located in to-space. + * + * before: + * this fwd1 + * +----+ +-+ + * | p1 ----->|x|-------> D1 + * | | +-+ + * | | + * | p2 ----------------> D2 + * +----+ + * + * after: + * this + * +----+ + * | p1 ----------------> D1 + * | | + * | | + * | p2 ----------------> D2 + * +----+ + * + * this is now white + * + * @return shallow size of *this. Must exactly match the amount of memory in to-space + * allocated by @ref _shallow_move + * + **/ + virtual std::size_t _forward_children() = 0; + }; + + /** @class Cpof + * @brief argument to operator new used for garbage collector evacuation phase + * + * Tag overloaded operator new to activate allocation policy based on location + * in memory of source object. + **/ + class Cpof { + public: + explicit Cpof(const Object * src) : src_{src} {} + + const void * src_ = nullptr; + }; +} /*namespace xo*/ + +void * operator new (std::size_t z, const xo::Cpof & copy); + +/* end Object.hpp */ diff --git a/include/xo/alloc/Stack.hpp b/include/xo/alloc/Stack.hpp new file mode 100644 index 00000000..b894d853 --- /dev/null +++ b/include/xo/alloc/Stack.hpp @@ -0,0 +1,49 @@ +/* Stack.hpp + * + * author: Roland Conybeare, jul 2025 + */ + +#pragma once + +#include + +namespace xo { + namespace gc { + /** Simple stack implementation + **/ + template + class Stack { + public: + explicit Stack(std::size_t capacity) { + this->contents_.reserve(capacity); + } + + bool is_empty() const { return contents_.empty(); } + std::size_t available() const { return contents_.capacity() - contents_.size(); } + void drop() { contents_.resize(contents_.size() - 1); } + void push(const T & x) { contents_.push_back(x); } + T pop() { + T retval = contents_[contents_.size() - 1]; + this->drop(); + return retval; + } + const T & top() const { + return this->lookup(0); + } + const T & lookup(std::size_t i) const { + return contents_.at(contents_.size() - 1 - i); + } + void clear() { contents_.clear(); } + void reset_to(std::size_t z) { contents_.resize(z); } + + std::size_t n_elements() const { return contents_.size(); } + std::size_t capacity() const { return contents_.capacity(); } + + private: + std::vector contents_; + }; + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end Stack.hpp */ diff --git a/src/alloc/AllocPolicy.cpp b/src/alloc/AllocPolicy.cpp new file mode 100644 index 00000000..b1dc162f --- /dev/null +++ b/src/alloc/AllocPolicy.cpp @@ -0,0 +1,13 @@ +/* AllocPolicy.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "AllocPolicy.hpp" + +/* note: inline/.hpp definition not allowed for operator delete */ +void operator delete(void * ptr) noexcept { + xo::xo.free(ptr); +} + +/* end AllocPolicy.cpp */ diff --git a/src/alloc/LinearAlloc.cpp b/src/alloc/ArenaAlloc.cpp similarity index 52% rename from src/alloc/LinearAlloc.cpp rename to src/alloc/ArenaAlloc.cpp index 3ae57e70..227e2d63 100644 --- a/src/alloc/LinearAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -1,29 +1,33 @@ -/* file LinearAlloc.cpp +/* file ArenaAlloc.cpp * * author: Roland Conybeare */ -#include "LinearAlloc.hpp" +#include "ArenaAlloc.hpp" +#include "xo/indentlog/scope.hpp" #include "xo/indentlog/print/tag.hpp" #include namespace xo { namespace gc { - LinearAlloc::LinearAlloc(std::size_t rz, std::size_t z) + ArenaAlloc::ArenaAlloc(const std::string & name, std::size_t rz, std::size_t z, bool debug_flag) { - this->lo_ = (new std::uint8_t [rz + z]); + this->name_ = name; + this->lo_ = (new std::byte [rz + z]); this->checkpoint_ = lo_; this->free_ptr_ = lo_; this->limit_ = lo_ + z; + this->redline_z_ = rz; this->hi_ = limit_ + rz; + this->debug_flag_ = debug_flag; if (!lo_) { - throw std::runtime_error(tostr("LinearAlloc: allocation failed", + throw std::runtime_error(tostr("ArenaAlloc: allocation failed", xtag("size", rz + z))); } } - LinearAlloc::~LinearAlloc() + ArenaAlloc::~ArenaAlloc() { delete [] this->lo_; @@ -33,17 +37,19 @@ namespace xo { this->checkpoint_ = nullptr; this->free_ptr_ = nullptr; this->limit_ = nullptr; + this->redline_z_ = 0; this->hi_ = nullptr; + this->debug_flag_ = false; } - up - LinearAlloc::make(std::size_t rz, std::size_t z) + up + ArenaAlloc::make(const std::string & name, std::size_t rz, std::size_t z, bool debug_flag) { - return up(new LinearAlloc(rz, z)); + return up(new ArenaAlloc(name, rz, z, debug_flag)); } void - LinearAlloc::set_free_ptr(std::uint8_t * x) + ArenaAlloc::set_free_ptr(std::byte * x) { assert(lo_ <= x); assert(x < limit_); @@ -57,70 +63,79 @@ namespace xo { } std::size_t - LinearAlloc::size() const { + ArenaAlloc::size() const { return limit_ - lo_; } std::size_t - LinearAlloc::available() const { + ArenaAlloc::available() const { return limit_ - free_ptr_; } std::size_t - LinearAlloc::allocated() const { + ArenaAlloc::allocated() const { return free_ptr_ - lo_; } bool - LinearAlloc::is_before_checkpoint(const std::uint8_t * x) const { + ArenaAlloc::contains(const void * x) const { + return (lo_ <= x) && (x < hi_); + } + + bool + ArenaAlloc::is_before_checkpoint(const void * x) const { return (lo_ <= x) && (x < checkpoint_); } std::size_t - LinearAlloc::before_checkpoint() const + ArenaAlloc::before_checkpoint() const { return checkpoint_ - lo_; } std::size_t - LinearAlloc::after_checkpoint() const + ArenaAlloc::after_checkpoint() const { return free_ptr_ - checkpoint_; } void - LinearAlloc::clear() + ArenaAlloc::clear() { this->checkpoint_ = lo_; this->free_ptr_ = lo_; - this->limit_ = lo_; + this->limit_ = hi_ - redline_z_; } void - LinearAlloc::checkpoint() + ArenaAlloc::checkpoint() { this->checkpoint_ = this->free_ptr_; } - std::uint8_t * - LinearAlloc::alloc(std::size_t z) + std::byte * + ArenaAlloc::alloc(std::size_t z0) { + scope log(XO_DEBUG(debug_flag_)); + /* word size for alignment */ - constexpr uint32_t c_bpw = sizeof(void*); + constexpr uint32_t c_bpw = sizeof(std::uintptr_t); std::uintptr_t free_u64 = reinterpret_cast(free_ptr_); assert(free_u64 % c_bpw == 0ul); - /* round up to multiple of c_bpw */ - std::uint32_t dz = (c_bpw - (z % c_bpw)); - z += dz; + std::uint32_t dz = alloc_padding(z0); - assert(z % c_bpw == 0ul); + std::size_t z1 = z0 + dz; - std::uint8_t * retval = this->free_ptr_; + assert(z1 % c_bpw == 0ul); - this->free_ptr_ += z; + std::byte * retval = this->free_ptr_; + + this->free_ptr_ += z1; + + log && log(xtag("self", name_), xtag("z0", z0), xtag("+pad", dz), xtag("z1", z1)); if (free_ptr_ > limit_) { return nullptr; @@ -128,8 +143,14 @@ namespace xo { return retval; } + + void + ArenaAlloc::release_redline_memory() { + this->limit_ = this->hi_; + } + } /*namespace gc*/ } /*namespace xo*/ -/* end LinearAlloc.cpp */ +/* end ArenaAlloc.cpp */ diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index cc5768d8..bc0f919f 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -2,7 +2,12 @@ set(SELF_LIB xo_alloc) set(SELF_SRCS - LinearAlloc.cpp + IAlloc.cpp + ArenaAlloc.cpp + ListAlloc.cpp + GC.cpp + Object.cpp + Forwarding1.cpp ) xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp new file mode 100644 index 00000000..825115a2 --- /dev/null +++ b/src/alloc/Forwarding1.cpp @@ -0,0 +1,45 @@ +/* file Forwarding1.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "Forwarding1.hpp" +#include +#include + +namespace xo { + namespace obj { + Forwarding1::Forwarding1(gp dest) + : dest_{dest} + {} + + Object * + Forwarding1::_offset_destination(Object * src) const + { + intptr_t offset = src - static_cast(this); + + return dest_.ptr() + offset; + } + + std::size_t + Forwarding1::_shallow_size() const { + assert(false); + return 0; + } + + Object * + Forwarding1::_shallow_copy() const { + assert(false); + return nullptr; + } + + std::size_t + Forwarding1::_forward_children() { + assert(false); + return 0; + } + + } /*namespace obj*/ +} /*namespace xo*/ + +/* end Forwarding1.cpp */ diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp new file mode 100644 index 00000000..ec4aa647 --- /dev/null +++ b/src/alloc/GC.cpp @@ -0,0 +1,492 @@ +/* GC.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "GC.hpp" +#include "Object.hpp" +#include "xo/indentlog/scope.hpp" +#include +#include + +namespace xo { + namespace gc { + void + PerGenerationStatistics::include_gc(std::size_t alloc_z, + std::size_t before_z, + std::size_t after_z, + std::size_t promote_z) + { + this->update_snapshot(after_z); + + new_alloc_z_ += alloc_z; + scanned_z_ += before_z; + survive_z_ += after_z; + promote_z_ += promote_z; + } + + void + PerGenerationStatistics::update_snapshot(std::size_t after_z) + { + used_z_ = after_z; + } + + void + PerGenerationStatistics::display(std::ostream & os) const + { + os << ""; + } + + void + GcStatistics::include_gc(generation upto, + std::size_t alloc_z, + std::size_t before_z, + std::size_t after_z, + std::size_t promote_z) + { + gen_v_[static_cast(upto)].include_gc(alloc_z, before_z, after_z, promote_z); + } + + void + GcStatistics::update_snapshot(generation upto, + std::size_t after_z) + { + gen_v_[static_cast(upto)].update_snapshot(after_z); + } + + void + GcStatistics::display(std::ostream & os) const + { + os << ""; + } + + GC::GC(const Config & config) + : config_{config} + { + enum { NurseryFrom, NurseryTo, TenuredFrom, TenuredTo }; + + std::size_t nursery_size = config.initial_nursery_z_; + std::size_t tenured_size = config.initial_tenured_z_; + + nursery_[role2int(role::from_space)] + = ListAlloc::make("NA", nursery_size, 2 * nursery_size, config.debug_flag_); + nursery_[role2int(role::to_space) ] + = ListAlloc::make("NB", nursery_size, 2 * nursery_size, config.debug_flag_); + + tenured_[role2int(role::from_space)] + = ListAlloc::make("TA", tenured_size, 2 * tenured_size, config.debug_flag_); + tenured_[role2int(role::to_space) ] + = ListAlloc::make("TB", tenured_size, 2 * tenured_size, config.debug_flag_); + + this->checkpoint(); + } + + up + GC::make(const Config & config) + { + GC * gc = new GC(config); + + return up{gc}; + } + + std::size_t + GC::size() const + { + return nursery_[role2int(role::to_space)]->size() + tenured_[role2int(role::to_space)]->size(); + } + + std::size_t + GC::allocated() const + { + return (nursery_[role2int(role::to_space)]->allocated() + + tenured_[role2int(role::to_space)]->allocated()); + } + + std::size_t + GC::available() const + { + return nursery_[role2int(role::to_space)]->available(); + } + + bool + GC::fromspace_contains(const void * x) const + { + return (nursery_[role2int(role::from_space)]->contains(x) + || tenured_[role2int(role::from_space)]->contains(x)); + } + + bool + GC::contains(const void * x) const + { + return (nursery_[role2int(role::to_space)]->contains(x) + || tenured_[role2int(role::to_space)]->contains(x)); + } + + bool + GC::is_before_checkpoint(const void * x) const + { + return nursery_[role2int(role::to_space)]->is_before_checkpoint(x); + } + + std::size_t + GC::before_checkpoint() const + { + return nursery_[role2int(role::to_space)]->before_checkpoint(); + } + + std::size_t + GC::after_checkpoint() const + { + return nursery_[role2int(role::to_space)]->after_checkpoint(); + } + + generation + GC::fromspace_generation_of(const void * x) const + { + if (tenured_[role2int(role::from_space)]->contains(x)) + return generation::tenured; + + return generation::nursery; + } + + generation + GC::generation_of(const void * x) const + { + if (tenured_[role2int(role::to_space)]->contains(x)) + return generation::tenured; + + return generation::nursery; + } + + std::byte * + GC::free_ptr(generation gen) + { + switch(gen) { + case generation::nursery: + return nursery_[role2int(role::to_space)]->free_ptr(); + case generation::tenured: + return tenured_[role2int(role::to_space)]->free_ptr(); + case generation::N: + assert(false); + } + + return nullptr; + } + + void + GC::clear() + { + nursery_[role2int(role::from_space)]->clear(); + nursery_[role2int(role::to_space) ]->clear(); + + tenured_[role2int(role::from_space)]->clear(); + tenured_[role2int(role::to_space) ]->clear(); + } + + void + GC::add_gc_root(Object ** addr) + { + gc_root_v_.push_back(addr); + } + + void + GC::checkpoint() + { + nursery_[role2int(role::to_space) ]->checkpoint(); + } + + std::byte * + GC::alloc(std::size_t z) + { + std::byte * x = nursery_[role2int(role::to_space)]->alloc(z); + + if (!x) { + this->request_gc(generation::nursery); + + if (incr_gc_pending_ || full_gc_pending_) + nursery_[role2int(role::to_space)]->release_redline_memory(); + + /* try (just once) more, maybe request fits in redline space */ + x = nursery_[role2int(role::to_space)]->alloc(z); + + assert(x); + } + + return x; + } + + std::byte * + GC::alloc_gc_copy(std::size_t z, const void * src) + { + scope log(XO_DEBUG(config_.debug_flag_), xtag("z", z), xtag("+pad", IAlloc::alloc_padding(z))); + + generation g = this->fromspace_generation_of(src); + + std::byte * retval = nullptr; + + if (g == generation::tenured) + { + log && log("tenured"); + + retval = tenured_[role2int(role::to_space)]->alloc(z); + } else if (nursery_[role2int(role::from_space)]->is_before_checkpoint(src)) + { + log && log("promote"); + + /* nursery object has survived 2nd collection cycle + * -> promote into tenured generation + */ + retval = tenured_[role2int(role::to_space)]->alloc(z); + + this->gc_statistics_.total_promoted_ += IAlloc::with_padding(z); + } else { + log && log("nursery"); + + retval = nursery_[role2int(role::to_space)]->alloc(z); + + if (!retval) { + /* nursery space exhausted */ + + this->request_gc(generation::nursery); + + nursery_[role2int(role::to_space)]->release_redline_memory(); + + retval = nursery_[role2int(role::to_space)]->alloc(z); + } + } + + assert(retval); + + return retval; + } + + void + GC::release_redline_memory() + { + // not supported feature for GC + } + + void + GC::swap_nursery() + { + up tmp = std::move(nursery_[role2int(role::to_space)]); + nursery_[role2int(role::to_space)] = std::move(nursery_[role2int(role::from_space)]); + nursery_[role2int(role::from_space)] = std::move(tmp); + } + + void + GC::swap_tenured() + { + up tmp = std::move(tenured_[role2int(role::to_space)]); + tenured_[role2int(role::to_space)] = std::move(tenured_[role2int(role::from_space)]); + tenured_[role2int(role::from_space)] = std::move(tmp); + } + + void + GC::swap_spaces(generation target) + { + // will be copying into storage currently labelled FromSpace + + /* gc will copy some to-be-determined amount in [0..promote_z] + from nursery->tenured generation. + */ + std::size_t promote_z = nursery_[role2int(role::to_space)]->before_checkpoint(); + if (target == generation::tenured) { + /* gc on tenured generation may need this much space */ + std::size_t tenured_z = (tenured_[role2int(role::to_space)]->allocated() + + promote_z + + full_gc_threshold_); + + tenured_[role2int(role::from_space)]->reset(tenured_z); + + this->swap_tenured(); + } else { + if (tenured_[role2int(role::to_space)]->available() < promote_z) { + tenured_[role2int(role::to_space)]->expand(promote_z); + } + } + + nursery_[role2int(role::from_space)]->reset(nursery_[role2int(role::to_space)]->allocated() + - promote_z + + incr_gc_threshold_); + this->swap_nursery(); + } /*swap_spaces*/ + + void + GC::copy_object(Object ** pp_object, generation upto, ObjectStatistics * object_stats) + { + void * object_address = *pp_object; + + if (nursery_[role2int(role::to_space)]->contains(object_address) + || ((upto == generation::tenured) + && tenured_[role2int(role::to_space)]->contains(object_address))) + { + /* global is already in to-space */ + ; + } else if((upto == generation::nursery) && tenured_[role2int(role::to_space)]->contains(object_address)) + { + /* skip tenured objects when incremental collection */ + ; + } else { + *pp_object = Object::_deep_move(*pp_object, this, object_stats); + } + } + + void + GC::copy_globals(generation upto) + { + for (Object ** pp_root : gc_root_v_) { + this->copy_object(pp_root, upto, &gc_statistics_.per_type_stats_); + } + } + + void + GC::cleanup_phase(generation upto) + { + scope log(XO_DEBUG(config_.debug_flag_)); + + std::size_t N_allocated = nursery_[role2int(role::from_space)]->after_checkpoint(); + std::size_t T_allocated = tenured_[role2int(role::from_space)]->after_checkpoint(); + + std::size_t N_before_gc = nursery_[role2int(role::from_space)]->allocated(); + std::size_t T_before_gc = tenured_[role2int(role::from_space)]->allocated(); + + std::size_t N_after_gc = nursery_[role2int(role::to_space)]->allocated(); + std::size_t T_after_gc = tenured_[role2int(role::to_space)]->allocated(); + //std::byte * N_free_ptr = nursery_[role2int(role::to_space)]->free_ptr(); + + std::size_t promote_z = gc_statistics_.total_promoted_ - gc_statistics_.total_promoted_sab_; + + this->nursery_[role2int(role::from_space)]->reset(0); + this->tenured_[role2int(role::from_space)]->reset(0); + + /* objects currenty in to-space nursery have survived one collection */ + this->nursery_[role2int(role::to_space)]->checkpoint(); + + // nursery_[role2int(role::to_space)]->set_redline(nursery_[role2int(role::to_space)]->allocated() + incr_gc_threshold_) + + if (upto == generation::tenured) + this->tenured_[role2int(role::to_space)]->checkpoint(); + + if (log) { + log(xtag("N_allocated", N_allocated)); + log(xtag("N_before_gc", N_before_gc)); + log(xtag("N_after_gc", N_after_gc)); + log(xtag("T_allocated", T_allocated)); + log(xtag("T_before_gc", T_before_gc)); + log(xtag("T_after_gc", T_after_gc)); + } + + this->incr_gc_pending_ = false; + this->gc_statistics_.include_gc(generation::nursery, N_allocated, N_before_gc, N_after_gc, promote_z); + + if (upto == generation::tenured) { + this->full_gc_pending_ = false; + this->gc_statistics_.include_gc(generation::tenured, T_allocated, T_before_gc, T_after_gc, 0); + } else { + // still want to update tenured stats for current alloc size + this->gc_statistics_.update_snapshot(generation::tenured, T_after_gc); + } + } + + void + GC::execute_gc(generation target) + { + scope log(XO_DEBUG(config_.debug_flag_)); + + bool full_move = (target == generation::tenured); + + // TODO: RAII version in case of exceptions + this->runstate_ = GCRunstate(true /*in_progress*/, full_move); + + log && log("step 0: snapshot alloc stats"); + + /* new allocation since last GC */ + std::size_t new_alloc = this->after_checkpoint(); + + ++(gc_statistics_.gen_v_[static_cast(target)].n_gc_); + gc_statistics_.total_allocated_ += new_alloc; + gc_statistics_.total_promoted_sab_ = gc_statistics_.total_promoted_; + + log && log(xtag("new_alloc", new_alloc)); + + log && log("step 1: swap to/from roles"); + + this->swap_spaces(target); + + log && log("step 2a: copy globals"); + + this->copy_globals(target); + + log && log("step 2b: TODO: copy pinned"); + + log && log("step 3: TODO: forward mutation log"); + + log && log("step 4: TODO: notify destructor log"); + + log && log("step 5: TODO: keep reachable weak pointers"); + + log && log("step 6: cleanup"); + + this->cleanup_phase(target); + + this->runstate_ = GCRunstate(); + + log && log("statistics:"); + log && log(gc_statistics_); + } + + void + GC::request_gc(generation target) + { + if (!runstate_.in_progress() && (gc_enabled_ == 0)) { + if (!config_.allow_incremental_gc_) + target = generation::tenured; + + if ((target == generation::nursery) + && (tenured_[role2int(role::to_space)]->after_checkpoint() > full_gc_threshold_)) + { + /** full collection when >= @ref full_gc_threshold_ bytes added to tenured + * generation, since last full collection + **/ + target = generation::tenured; + } + + this->execute_gc(target); + } else { + this->incr_gc_pending_ = true; + if (target == generation::tenured) + this->full_gc_pending_ = true; + } + } + + void + GC::disable_gc() { + --gc_enabled_; + } + + void + GC::enable_gc() { + ++gc_enabled_; + + if (gc_enabled_ == 0) { + /* unblock gc */ + if (incr_gc_pending_) + this->request_gc(full_gc_pending_ ? generation::tenured : generation::nursery); + } + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* end GC.cpp */ diff --git a/src/alloc/IAlloc.cpp b/src/alloc/IAlloc.cpp new file mode 100644 index 00000000..4fbdd556 --- /dev/null +++ b/src/alloc/IAlloc.cpp @@ -0,0 +1,54 @@ +/* @file IAlloc.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "IAlloc.hpp" +#include +#include + +namespace xo { + namespace gc { + + std::uint32_t + IAlloc::alloc_padding(std::size_t z) + { + /* word size for alignment */ + constexpr uint32_t c_bpw = sizeof(std::uintptr_t); + + /* round up to multiple of c_bpw, but map 0 -> 0 + * (table assuming c_bpw==8) + * + * z%c_bpw dz + * ------------ + * 0 0 + * 1 7 + * 2 6 + * .. .. + * 7 1 + */ + std::uint32_t dz = (c_bpw - (z % c_bpw)) % c_bpw; + z += dz; + + assert(z % c_bpw == 0ul); + + return dz; + } + + std::size_t + IAlloc::with_padding(std::size_t z) + { + return z + alloc_padding(z); + } + + std::byte * + IAlloc::alloc_gc_copy(std::size_t /*z*/, const void * /*src*/) + { + assert(false); + return nullptr; + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end IAlloc.cpp */ diff --git a/src/alloc/ListAlloc.cpp b/src/alloc/ListAlloc.cpp new file mode 100644 index 00000000..76da8b19 --- /dev/null +++ b/src/alloc/ListAlloc.cpp @@ -0,0 +1,318 @@ +/* file ListAlloc.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "ListAlloc.hpp" +#include "ArenaAlloc.hpp" +#include +#include + +namespace xo { + namespace gc { + ListAlloc::ListAlloc(std::unique_ptr hd, + ArenaAlloc * marked, + std::size_t cz, std::size_t nz, std::size_t tz, + bool use_redline, + bool debug_flag) + : start_z_{cz}, + hd_{std::move(hd)}, + marked_{marked}, + full_l_{}, + current_z_{cz}, + next_z_{nz}, + total_z_{tz}, + use_redline_{use_redline}, + debug_flag_{debug_flag} + {} + + ListAlloc::~ListAlloc() + { + this->clear(); + } + + up + ListAlloc::make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag) + { + std::unique_ptr hd{ArenaAlloc::make(name, 0, cz, debug_flag)}; + + if (!hd) + return nullptr; + + ArenaAlloc * marked = nullptr; + + up retval{new ListAlloc(std::move(hd), + marked, + cz, nz, cz, + false /*!use_redline*/, + debug_flag)}; + + return retval; + } + + std::size_t + ListAlloc::size() const { + return total_z_; + } + + std::byte * + ListAlloc::free_ptr() const { + return hd_->free_ptr(); + } + + std::size_t + ListAlloc::available() const { + if (hd_) { + /* can only allocate from @ref hd_, + * so even if there were available space in @ref full_l_, + * it's not accessible to ListAlloc. + */ + + return hd_->available(); + } + + return 0; + } + + std::size_t + ListAlloc::allocated() const { + std::size_t total = 0; + + if (hd_) + total += hd_->allocated(); + + for (const auto & alloc : full_l_) + total += alloc->allocated(); + + return total; + } + + bool + ListAlloc::contains(const void * x) const { + if (hd_ && hd_->contains(x)) + return true; + + for (const auto & alloc : full_l_) { + if (alloc->contains(x)) + return true; + } + + return false; + } + + bool + ListAlloc::is_before_checkpoint(const void * x) const { + if (!marked_) + return false; + + if ((marked_ == hd_.get()) && hd_->contains(x)) + return hd_->is_before_checkpoint(x); + + /* + * 1. allocs in full_l_ appear in youngest-to-oldest order + * 2. allocators that appear before marked_ in full_l_ count as 'after checkpoint' + * 3. allocators that appear after marked_ in full_l_ count as 'before checkpoint' + */ + + bool younger_than_marked = true; + + for (const auto & alloc : full_l_) { + if (younger_than_marked) { + if (alloc.get() == marked_) { + /* nothing else to test on this iteration, + * already checked .marked_ specifically + */ + younger_than_marked = false; + } else { + /* after checkpoint */ + if (alloc->contains(x)) + return false; + } + } else { + if (alloc->contains(x)) + return true; + } + } + + return false; + } + + std::size_t + ListAlloc::before_checkpoint() const + { + if (marked_) { + if (full_l_.empty()) { + assert(marked_ == hd_.get()); + + return marked_->before_checkpoint(); + } + } else { + /* count everything allocated */ + return this->allocated(); + } + + std::size_t z = 0; + + /* control here: .marked & .full_l non-empty. */ + if (hd_.get() == marked_) { + z += hd_->before_checkpoint(); + + /* anything in .full_l older than marked .hd */ + for (const auto & alloc : full_l_) { + z += alloc->allocated(); + } + + return z; + } else { + /* messiest case: .marked is true, + * and not the youngest arena + */ + bool younger_than_marked = true; + + for (const auto & alloc : full_l_) { + if (younger_than_marked) { + if (alloc.get() == marked_) { + younger_than_marked = false; + z += marked_->before_checkpoint(); + } else { + ; + } + } else { + z += alloc->allocated(); + } + } + } + + return z; + } + + std::size_t + ListAlloc::after_checkpoint() const + { + if (!marked_) + return 0; + + if (full_l_.empty()) { + assert(marked_ == hd_.get()); + + return marked_->after_checkpoint(); + } + + bool younger_than_marked = true; + + std::size_t z = 0; + + for (const auto & alloc : full_l_) { + if (younger_than_marked) { + if (alloc.get() == marked_) { + younger_than_marked = false; + z += marked_->after_checkpoint(); + break; + } else { + z += alloc->allocated(); + } + } + } + + return z; + } + + void + ListAlloc::clear() { + // general hygiene + start_z_ = 0; + hd_.reset(); + marked_ = nullptr; + full_l_.clear(); + current_z_ = 0; + next_z_ = 0; + total_z_ = 0; + use_redline_ = false; + } + + bool + ListAlloc::reset(std::size_t z) + { + // warning: hd_->size() does not include redline memory + hd_->release_redline_memory(); + + bool recycle_head_bucket = hd_ && (z <= hd_->size()); + + this->full_l_.clear(); + this->marked_ = nullptr; + this->redlined_flag_ = false; + + if (recycle_head_bucket) { + this->hd_->clear(); + this->total_z_ = hd_->size(); + + return true; + } else { + this->hd_.reset(nullptr); + this->total_z_ = 0; + + return this->expand(z); + } + } + + bool + ListAlloc::expand(std::size_t z) + { + std::size_t cz = current_z_; + std::size_t nz = next_z_; + std::size_t tz; + + do { + tz = cz + nz; + cz = nz; + nz = tz; + } while (cz < z); + + std::string name = hd_->name() + "+exp"; + + std::unique_ptr new_alloc = ArenaAlloc::make(name, 0, cz, debug_flag_); + + if (!new_alloc) + return false; + + this->current_z_ = cz; + this->next_z_ = nz; + this->total_z_ += cz; + + this->hd_ = std::move(new_alloc); + + return true; + } + + void + ListAlloc::checkpoint() { + hd_->checkpoint(); + + this->marked_ = hd_.get(); + } + + std::byte * + ListAlloc::alloc(std::size_t z) { + std::byte * retval = hd_->alloc(z); + + if (retval) + return retval; + + if (this->expand(z)) + return hd_->alloc(z); + + return nullptr; + } + + void + ListAlloc::release_redline_memory() + { + if (use_redline_) + redlined_flag_ = true; + + this->hd_->release_redline_memory(); + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* end ListAlloc.cpp */ diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp new file mode 100644 index 00000000..7aa1a8d0 --- /dev/null +++ b/src/alloc/Object.cpp @@ -0,0 +1,196 @@ +/* Object.cpp + * + * author: Roalnd Conybeare, Jul 2025 + */ + +#include "Object.hpp" +#include "GC.hpp" +#include "Forwarding1.hpp" + +using xo::obj::Forwarding1; + +void * +operator new (std::size_t z, const xo::Cpof & cpof) +{ + using xo::gc::GC; + + GC * gc = reinterpret_cast(xo::Object::mm); + + return gc->alloc_gc_copy(z, cpof.src_); +} + +namespace xo { + gc::IAlloc * + Object::mm = nullptr; + + Object * + Object::_forward(Object * src, gc::GC * gc) + { + if (!src) + return src; + + if (src->_is_forwarded()) + return src->_offset_destination(src); + + bool full_move = gc->runstate().full_move(); + + if (!full_move && (gc->generation_of(src) == gc::generation::tenured)) { + /* don't move tenured objects during incremental collection */ + return src; + } + + Object::_shallow_move(src, gc); + + /* *src is now a forwarding pointer to copy in to-space */ + + return src->_offset_destination(src); + } + + Object * + Object::_deep_move(Object * from_src, gc::GC * gc, gc::ObjectStatistics * /*stats*/) + { + using gc::generation; + + if (!from_src) + return nullptr; + + Object * retval = from_src->_destination(); + + if (retval) + return retval; + + bool full_move = gc->runstate().full_move(); + + if (!full_move && gc->generation_of(from_src) == generation::tenured) { + /** incremental collection does not move already-tenured objects **/ + return from_src; + } + + /** + * To-space: + * + * to_lo = start of to-space + * w,W = white objects. An object x is white if x + all immediate children of x are in to-space + * (also implies this GC cycle put it there) + * g,G = grey objects. An object x is gray if it's in to-space, + * but possibly has >0 black children + * _ = free to-space memory + * N = nursery space + * T = tenured space + * + * wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________... + * ^ ^ ^ + * to_lo grey_lo(N) free_ptr(N) + * + * After moving children of one object, advancing {nursery_grey_lo, nursery_free_ptr} + * + * wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG______... + * ^ ^ ^ + * to_lo grey_lo(N) free_ptr(N) + * + * Invariant: + * + * objects in [to_lo, gray_lo) are white. + * all gray objects are in [gray_lo, free_ptr) + * memory starting at free_ptr is free. + * + * deep_move terminates when gray_lo catches up to free_ptr + * + * Above is simplified. Complication is that GC (including incremental) may + * promote objects from nursery (N) to tenured (T) + * + * So more accurate before/after picture + * + * N wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________... + * ^ ^ ^ + * to_lo(N) grey_lo(N) free_ptr(N) + * + * T wwwwwwwwwwwwwwgggggggggggg_______________________________... + * ^ ^ ^ + * to_lo(T) grey_lo(T) free_ptr(N) + * + * After moving children of one object, advancing {nursery_grey_lo, nursery_free_ptr} + * + * N wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG_____... + * ^ ^ ^ + * to_lo(N) grey_lo(N) free_ptr(N) + * + * T wwwwwwwwwwwwwwggggggggggggGGGGG_________________________... + * ^ ^ ^ + * to_lo(T) grey_lo(T) free_ptr(T) + * + * deep_move terminates when both: + * - gray_lo(N) catches up with free_ptr(N) + * - gray_lo(T) catches up with free_ptr(T) + * + **/ + + std::array gray_lo_v + = { gc->free_ptr(generation::nursery), gc->free_ptr(generation::tenured) }; + + Object * to_src = Object::_shallow_move(from_src, gc); + + std::size_t fixup_work = 0; + do { + fixup_work = 0; + + auto fixup_generation = [gc, &gray_lo_v](generation gen) { + std::size_t work = 0; + while(gray_lo_v[gen2int(gen)] < gc->free_ptr(gen)) { + Object * x = reinterpret_cast(gray_lo_v[gen2int(gen)]); + + // update per-class stats here + + std::size_t xz = x->_forward_children(); + + // must pad xz to multiple of word size, + // to match behavior of LinearAlloc::alloc() + // + xz += gc::IAlloc::alloc_padding(xz); + + gray_lo_v[gen2int(gen)] += xz; + ++work; + } + + return work; + }; + + fixup_work += fixup_generation(generation::nursery); + fixup_work += fixup_generation(generation::tenured); + } while (fixup_work > 0); + + return to_src; + } /*deep_move*/ + + Object * + Object::_shallow_move(Object * src, gc::GC * gc) + { + /* filter for source objects that are owned by GC. + * Care required though -- during GC from/to spaces have been swapped already + */ + if (gc->fromspace_contains(src)) + { + Object * dest = src->_shallow_copy(); + + if (dest != src) + src->_forward_to(dest); + + return dest; + } else { + return src; + } + } + + void + Object::_forward_to(Object * dest) + { + char * mem = reinterpret_cast(this); + + Forwarding1 * fwd = new (mem) Forwarding1(dest); + + (void)fwd; + } + +} /*namespace xo*/ + +/* end Object.cpp*/ diff --git a/utest/ArenaAlloc.test.cpp b/utest/ArenaAlloc.test.cpp new file mode 100644 index 00000000..aae2695b --- /dev/null +++ b/utest/ArenaAlloc.test.cpp @@ -0,0 +1,87 @@ +/* @file ArenaAlloc.test.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "xo/alloc/ArenaAlloc.hpp" +#include + +namespace xo { + using xo::gc::ArenaAlloc; + + namespace ut { + + namespace { + struct testcase_alloc { + testcase_alloc(std::size_t rz, std::size_t z) + : redline_z_{rz}, arena_z_{z} {} + + std::size_t redline_z_; + std::size_t arena_z_; + + }; + + std::vector + s_testcase_v = { + testcase_alloc(0, 4096) + }; + } + + + TEST_CASE("linearalloc", "[alloc]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + const testcase_alloc & tc = s_testcase_v[i_tc]; + + constexpr bool c_debug_flag = false; + + auto alloc = ArenaAlloc::make("linearalloc", tc.redline_z_, tc.arena_z_, c_debug_flag); + + REQUIRE(alloc.get()); + REQUIRE(alloc->name() == "linearalloc"); + REQUIRE(alloc->size() == tc.arena_z_); + REQUIRE(alloc->available() == tc.arena_z_); + REQUIRE(alloc->allocated() == 0); + REQUIRE(alloc->is_before_checkpoint(alloc->free_ptr()) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + auto free0 = alloc->free_ptr(); + + auto mem = alloc->alloc(tc.arena_z_); + + REQUIRE(mem != nullptr); + + REQUIRE(mem == free0); + + REQUIRE(alloc->size() == tc.arena_z_); + REQUIRE(alloc->available() == 0); + REQUIRE(alloc->allocated() == tc.arena_z_); + REQUIRE(alloc->is_before_checkpoint(mem) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == tc.arena_z_); + + alloc->clear(); + + REQUIRE(alloc->free_ptr() == free0); + REQUIRE(alloc->available() == tc.arena_z_); + REQUIRE(alloc->allocated() == 0); + REQUIRE(alloc->is_before_checkpoint(free0) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + mem = alloc->alloc(1); + + auto used = sizeof(void*); + REQUIRE(alloc->size() == tc.arena_z_); + REQUIRE(alloc->available() == tc.arena_z_ - used); + REQUIRE(alloc->allocated() == used); + REQUIRE(alloc->is_before_checkpoint(free0) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == used); + + } + } + + } /*namespace ut */ +} /*namespace xo*/ diff --git a/utest/CMakeLists.txt b/utest/CMakeLists.txt index e845f729..d37786e3 100644 --- a/utest/CMakeLists.txt +++ b/utest/CMakeLists.txt @@ -3,7 +3,8 @@ set(SELF_EXE utest.alloc) set(SELF_SRCS alloc_utest_main.cpp - LinearAlloc.test.cpp) + ArenaAlloc.test.cpp + GC.test.cpp) xo_add_utest_executable(${SELF_EXE} ${SELF_SRCS}) xo_self_dependency(${SELF_EXE} xo_alloc) diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp new file mode 100644 index 00000000..dc175615 --- /dev/null +++ b/utest/GC.test.cpp @@ -0,0 +1,69 @@ +/* @file GC.test.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "xo/alloc/GC.hpp" +#include + +namespace xo { + using xo::gc::GC; + using xo::gc::generation; + using xo::gc::Config; + + namespace ut { + + namespace { + struct testcase_gc { + testcase_gc(std::size_t nz, std::size_t tz) : nursery_z_{nz}, tenured_z_{tz} {} + + std::size_t nursery_z_; + std::size_t tenured_z_; + }; + + std::vector + s_testcase_v = { + testcase_gc(1024, 4096) + }; + } + + TEST_CASE("gc", "[alloc][gc]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + const testcase_gc & tc = s_testcase_v[i_tc]; + + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_}); + + REQUIRE(gc.get()); + REQUIRE(gc->size() == tc.nursery_z_ + tc.tenured_z_); + REQUIRE(gc->allocated() == 0); + REQUIRE(gc->available() == tc.nursery_z_); + REQUIRE(gc->before_checkpoint() == 0); + // ListAlloc model is that nothing is before checkpoint + // until it's first established + REQUIRE(gc->after_checkpoint() == 0); + + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->is_gc_enabled() == true); + REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 0); + REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + + /* gc with empty state */ + gc->request_gc(generation::nursery); + + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + + /* still empty state */ + gc->request_gc(generation::tenured); + + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 1); + } + } + } /*namespace ut*/ +} /*namespace xo*/ diff --git a/utest/LinearAlloc.test.cpp b/utest/LinearAlloc.test.cpp index 5d3a1fcd..b1909991 100644 --- a/utest/LinearAlloc.test.cpp +++ b/utest/LinearAlloc.test.cpp @@ -3,7 +3,7 @@ * author: Roland Conybeare, Jul 2025 */ -#include "xo/alloc/LinearAlloc.hpp" +#include "xo/alloc/ArenaAlloc.hpp" #include namespace xo { @@ -33,15 +33,53 @@ namespace xo { for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { const testcase_alloc & tc = s_testcase_v[i_tc]; - auto alloc = LinearAlloc::make(tc.redline_z_, tc.arena_z_); + constexpr bool c_debug_flag = false; + + auto alloc = LinearAlloc::make("linearalloc", tc.redline_z_, tc.arena_z_, c_debug_flag); REQUIRE(alloc.get()); + REQUIRE(alloc->name() == "linearalloc"); REQUIRE(alloc->size() == tc.arena_z_); REQUIRE(alloc->available() == tc.arena_z_); REQUIRE(alloc->allocated() == 0); REQUIRE(alloc->is_before_checkpoint(alloc->free_ptr()) == false); REQUIRE(alloc->before_checkpoint() == 0); REQUIRE(alloc->after_checkpoint() == 0); + + auto free0 = alloc->free_ptr(); + + auto mem = alloc->alloc(tc.arena_z_); + + REQUIRE(mem != nullptr); + + REQUIRE(mem == free0); + + REQUIRE(alloc->size() == tc.arena_z_); + REQUIRE(alloc->available() == 0); + REQUIRE(alloc->allocated() == tc.arena_z_); + REQUIRE(alloc->is_before_checkpoint(mem) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == tc.arena_z_); + + alloc->clear(); + + REQUIRE(alloc->free_ptr() == free0); + REQUIRE(alloc->available() == tc.arena_z_); + REQUIRE(alloc->allocated() == 0); + REQUIRE(alloc->is_before_checkpoint(free0) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + mem = alloc->alloc(1); + + auto used = sizeof(void*); + REQUIRE(alloc->size() == tc.arena_z_); + REQUIRE(alloc->available() == tc.arena_z_ - used); + REQUIRE(alloc->allocated() == used); + REQUIRE(alloc->is_before_checkpoint(free0) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == used); + } } From c7488cbfd5a649542e89e0a72dde1f5f47b64c7d Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 5 Aug 2025 11:08:36 -0500 Subject: [PATCH 02/69] xo-alloc: mutation log tracking in working state + unit test --- docs/index.rst | 3 + include/xo/alloc/ArenaAlloc.hpp | 24 +- include/xo/alloc/Forwarding1.hpp | 14 +- include/xo/alloc/GC.hpp | 136 +++++++-- include/xo/alloc/IAlloc.hpp | 16 +- include/xo/alloc/ListAlloc.hpp | 23 +- include/xo/alloc/Object.hpp | 25 +- src/alloc/Forwarding1.cpp | 5 + src/alloc/GC.cpp | 488 ++++++++++++++++++++++++++++--- src/alloc/IAlloc.cpp | 6 + src/alloc/ListAlloc.cpp | 10 + src/alloc/Object.cpp | 4 +- utest/CMakeLists.txt | 12 +- utest/GC.test.cpp | 3 + 14 files changed, 667 insertions(+), 102 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 198cf01c..de643009 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,3 +12,6 @@ xo-alloc provides arena allocators and a generation garbage collector install introduction implementation + glossary + genindex + search diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index e2e74e8f..12033646 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -45,24 +45,24 @@ namespace xo { std::size_t z, bool debug_flag); - const std::string & name() const { return name_; } std::byte * free_ptr() const { return free_ptr_; } void set_free_ptr(std::byte * x); // inherited from IAlloc... - virtual std::size_t size() const override; - virtual std::size_t available() const override; - virtual std::size_t allocated() const override; - virtual bool contains(const void * x) const override; - virtual bool is_before_checkpoint(const void * x) const override; - virtual std::size_t before_checkpoint() const override; - virtual std::size_t after_checkpoint() const override; + virtual const std::string & name() const final override { return name_; } + virtual std::size_t size() const final override; + virtual std::size_t available() const final override; + virtual std::size_t allocated() const final override; + virtual bool contains(const void * x) const final override; + virtual bool is_before_checkpoint(const void * x) const final override; + virtual std::size_t before_checkpoint() const final override; + virtual std::size_t after_checkpoint() const final override; - virtual void clear() override; - virtual void checkpoint() override; - virtual std::byte * alloc(std::size_t z) override; - virtual void release_redline_memory() override; + virtual void clear() final override; + virtual void checkpoint() final override; + virtual std::byte * alloc(std::size_t z) final override; + virtual void release_redline_memory() final override; private: ArenaAlloc(const std::string & name, std::size_t rz, std::size_t z, bool debug_flag); diff --git a/include/xo/alloc/Forwarding1.hpp b/include/xo/alloc/Forwarding1.hpp index 62536651..4fb61d08 100644 --- a/include/xo/alloc/Forwarding1.hpp +++ b/include/xo/alloc/Forwarding1.hpp @@ -12,11 +12,15 @@ namespace xo { explicit Forwarding1(gp dest); // inherited from Object.. - virtual bool _is_forwarded() const override { return true; } - virtual Object * _offset_destination(Object * src) const; - virtual std::size_t _shallow_size() const override; - virtual Object * _shallow_copy() const override; - virtual std::size_t _forward_children() override; + virtual bool _is_forwarded() const final override { return true; } + virtual Object * _offset_destination(Object * src) const final override; + virtual Object * _destination() final override; + /** never called on Forwarding1 **/ + virtual std::size_t _shallow_size() const final override; + /** never called on Forwarding1 **/ + virtual Object * _shallow_copy() const final override; + /** never called on Forwarding1 **/ + virtual std::size_t _forward_children() final override; private: /** the object that used to be located at this address (i.e. @c this) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index f42864b7..0c50a722 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -23,6 +23,12 @@ namespace xo { constexpr std::size_t gen2int(generation x) { return static_cast(x); } + enum class generation_result { + nursery, + tenured, + not_found + }; + enum class role { /** nursery: generation for new objects **/ from_space, @@ -131,6 +137,17 @@ namespace xo { /** total bytes promoted from nursery->tenured since inception **/ std::size_t total_promoted_ = 0; + /** total number of mutations to already-allocated objects, + * whether or not GC needs to log them. + **/ + std::size_t n_mutation_ = 0; + /** total number of mutation eligible for logging **/ + std::size_t n_logged_mutation_ = 0; + /** total number of cross-generation mutations (tenured->nursery when reported) **/ + std::size_t n_xgen_mutation_ = 0; + /** total number of cross-checkpoint mutations (N0 -> N1 when reported) **/ + std::size_t n_xckp_mutation_ = 0; + /** per-type statistics (placeholder) **/ ObjectStatistics per_type_stats_; }; @@ -163,6 +180,35 @@ namespace xo { bool full_move_ = false; }; + class MutationLogEntry { + public: + MutationLogEntry(Object * parent, Object ** lhs) : parent_{parent}, lhs_{lhs} {} + + Object * parent() const { return parent_; } + Object ** lhs() const { return lhs_; } + + Object * child() const { return *lhs_; } + + bool is_child_forwarded() const; + bool is_parent_forwarded() const; + + Object * parent_destination() const; + + /** Flag obsolete mutation. + * Future proofing, never happens for regular objects + **/ + bool is_dead() const { return false; } + + MutationLogEntry update_parent_moved(Object * parent_to) const; + void fixup_parent_child_moved(Object * child_to) { *lhs_ = child_to; } + + private: + Object * parent_; + Object ** lhs_; + }; + + using MutationLog = std::vector; + /** @class GC * @brief generational garbage collector * @@ -185,16 +231,18 @@ namespace xo { /** true iff GC permitted in current state **/ bool is_gc_enabled() const { return gc_enabled_ == 0; } - /** @return generation to which object at @p x belongs **/ - generation generation_of(const void * x) const; - /** @return generation that contains @p x, given it's in from-space **/ - generation fromspace_generation_of(const void * x) const; - /** true iff from-space contains @p x **/ - bool fromspace_contains(const void * x) const; /** true during (and only during) a GC cycle **/ bool gc_in_progress() const { return runstate_.in_progress(); } - /** return free pointer for generation @p gen, i.e. nursery or tenured space **/ + /** @return generation to which object at @p x belongs **/ + generation_result tospace_generation_of(const void * x) const; + /** @return generation that contains @p x, given it's in from-space **/ + generation_result fromspace_generation_of(const void * x) const; + /** true iff from-space contains @p x **/ + bool fromspace_contains(const void * x) const; + /** @return free pointer for generation @p gen, i.e. nursery or tenured space **/ std::byte * free_ptr(generation gen); + /** @return current size of (number of entries in) mutation log **/ + std::size_t mlog_size() const; /** add gc root at address @p addr . Gc will keep alive anything reachable * from @c *addr @@ -217,27 +265,43 @@ namespace xo { // inherited from IAlloc.. + virtual const std::string & name() const final override; /** capacity in bytes (counting both free+allocated) for object storage. * only counts one of {to-space, from-space}, * since one role is always held empty between collections. **/ - virtual std::size_t size() const override; + virtual std::size_t size() const final override; - virtual std::size_t allocated() const override; - virtual std::size_t available() const override; + virtual std::size_t allocated() const final override; + virtual std::size_t available() const final override; /** only tests to-space **/ - virtual bool contains(const void * x) const override; - virtual bool is_before_checkpoint(const void * x) const override; - virtual std::size_t before_checkpoint() const override; - virtual std::size_t after_checkpoint() const override; + virtual bool contains(const void * x) const final override; + virtual bool is_before_checkpoint(const void * x) const final override; + virtual std::size_t before_checkpoint() const final override; + virtual std::size_t after_checkpoint() const final override; + virtual bool debug_flag() const final override; - virtual void clear() override; - virtual void checkpoint() override; + virtual void clear() final override; + virtual void checkpoint() final override; - virtual std::byte * alloc(std::size_t z) override; - virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) override; + /** GC bookkeeping for an assignment that modifes an Object reference. + * Whenever an @ref Object instance P contains a member variable that can refer + * to another @ref Object, then we need to involve GC to perform the assignment. + * In particular a side-effect that changes the target of such reference to Q after P + * has been promoted, may lead to a tenured->nursery cross-generational pointer. + * GC needs to know about such pointers to it can update them as part of subsequent + * incremental collections. + * + * @param parent. object with member variable being modified + * @param lhs. address of a member variable within the allocation of @p parent. + * @param rhs. new target for @p *lhs + **/ + virtual void assign_member(Object * parent, Object ** lhs, Object* rhs) final override; - virtual void release_redline_memory() override; + virtual std::byte * alloc(std::size_t z) final override; + virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; + + virtual void release_redline_memory() final override; private: /** begin GC now **/ @@ -248,12 +312,35 @@ namespace xo { void swap_nursery(); /** swap roles of From/To spaces for tenured generation **/ void swap_tenured(); + /** swap roles of From/To spaces for mutation log **/ + void swap_mutation_log(); /** swap roles of FromSpace/ToSpace **/ void swap_spaces(generation g); /** copy object **/ void copy_object(Object ** addr, generation upto, ObjectStatistics * object_stats); /** copy everything reachable from global gc roots **/ void copy_globals(generation g); + /** review mutation log; may discover+rescue reachable objects. + **/ + void forward_mutation_log(generation upto); + /** Aux function for @ref execute_gc. Updates bookkeeping for cross-generational + * (T->N, aka xgen) and (N1->N0, aka xckp) pointers + **/ + void incremental_gc_forward_mlog(ObjectStatistics * per_type_stats); + + /** + * Aux function for @ref incremental_gc_forward_mlog. Calls this function until + * fixpoint. + * + * @param from_mlog incoming mutation log. Contains {xgen,xckp} pointers before GC. + * Contents of this log is consumed (+discarded) before method returns. + * @param to_mlog outgoing mutation log. Will contain {xgen,xckp} pointers after GC. + * @param defer_mlog contains log entries associated with possible garbage. + **/ + void incremental_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * per_type_stats); private: /** garbage collector configuration **/ @@ -262,11 +349,11 @@ namespace xo { /** contains allocated objects, along with unreachable garbage to be collected. * roles reverse after each incremental, or full, collection. **/ - std::array, static_cast(role::N)> nursery_; + std::array, role2int(role::N)> nursery_; /** empty space, destination for objects that survive collection. * roles reverse after each full collection. **/ - std::array, static_cast(role::N)> tenured_; + std::array, role2int(role::N)> tenured_; /** current state of GC activity. * @text @@ -286,6 +373,13 @@ namespace xo { **/ std::vector gc_root_v_; + /** log cross-generational and cross-checkpoint mutations. + * These need to be adjusted on next incremental collection + **/ + std::array, role2int(role::N)> mutation_log_; + /** temporary mutation log (for deferred entries) **/ + up defer_mutation_log_; + /** allocation/collection counters **/ GcStatistics gc_statistics_; diff --git a/include/xo/alloc/IAlloc.hpp b/include/xo/alloc/IAlloc.hpp index 2f759c53..272fe9ad 100644 --- a/include/xo/alloc/IAlloc.hpp +++ b/include/xo/alloc/IAlloc.hpp @@ -12,6 +12,8 @@ namespace xo { template using up = std::unique_ptr; + class Object; + namespace gc { /** @class IAllocator * @brief memory allocation interface with limited garbaga collector support @@ -27,6 +29,8 @@ namespace xo { /** z + alloc_padding(z) **/ static std::size_t with_padding(std::size_t z); + /** optional name for this allocator; labelling for diagnostics **/ + virtual const std::string & name() const = 0; /** allocator size in bytes (up to soft limit). * Includes unallocated mmeory **/ @@ -47,15 +51,25 @@ namespace xo { virtual std::size_t before_checkpoint() const = 0; /** number of bytes allocated since @ref checkpoint **/ virtual std::size_t after_checkpoint() const = 0; + /** @return true iff debug logging enabled **/ + virtual bool debug_flag() const { return false; } /** reset allocator to empty state. **/ virtual void clear() = 0; - /** remember allocator state. All currently-allocated addresses x + /** remember allocator state. All currently-allocated addresses xo * will satisfy is_before_checkpoint(x). Subsequent allocations x * will fail is_before_checkpoint(x), until checkpoint superseded * by @ref clear or another call to @ref checkpoint **/ virtual void checkpoint() = 0; + /** perform assignment + * @code + * *lhs = rhs + * @endcode + * plus additional book keeping if needed (e.g. in @ref GC) + * Default implementation just does the assignment. + **/ + virtual void assign_member(Object * parent, Object ** lhs, Object * rhs); /** allocate @p z bytes of memory. returns pointer to first address **/ virtual std::byte * alloc(std::size_t z) = 0; /** allocate @p z bytes for copy of object at @p src. diff --git a/include/xo/alloc/ListAlloc.hpp b/include/xo/alloc/ListAlloc.hpp index 8d27e6b4..db1a2df7 100644 --- a/include/xo/alloc/ListAlloc.hpp +++ b/include/xo/alloc/ListAlloc.hpp @@ -45,18 +45,19 @@ namespace xo { // inherited from IAlloc.. - virtual std::size_t size() const override; - virtual std::size_t available() const override; - virtual std::size_t allocated() const override; - virtual bool contains(const void * x) const override; - virtual bool is_before_checkpoint(const void * x) const override; - virtual std::size_t before_checkpoint() const override; - virtual std::size_t after_checkpoint() const override; + virtual const std::string & name() const final override; + virtual std::size_t size() const final override; + virtual std::size_t available() const final override; + virtual std::size_t allocated() const final override; + virtual bool contains(const void * x) const final override; + virtual bool is_before_checkpoint(const void * x) const final override; + virtual std::size_t before_checkpoint() const final override; + virtual std::size_t after_checkpoint() const final override; - virtual void clear() override; - virtual void checkpoint() override; - virtual std::byte * alloc(std::size_t z) override; - virtual void release_redline_memory() override; + virtual void clear() final override; + virtual void checkpoint() final override; + virtual std::byte * alloc(std::size_t z) final override; + virtual void release_redline_memory() final override; private: /** **/ diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index f66e8e9a..f8e45e84 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -15,8 +15,6 @@ namespace xo { class ObjectStatistics; }; - class Object; - template class gc_ptr; @@ -88,6 +86,13 @@ namespace xo { **/ static gc::IAlloc * mm; + /** assign value @p rhs to member @p *lhs of @p parent. + * if assignment creates a cross-generational or cross-checkpoint pointer, + * add mutation log entry + **/ + template + static void assign_member(gp parent, gp * lhs, gp rhs); + /** use from GC aux functions **/ static gc::GC * _gc() { return reinterpret_cast(mm); } @@ -127,6 +132,13 @@ namespace xo { * initially all reachable objects are black. * GC is complete when all reachable objects are white. * GC needs a variable amount of temporary storage to keep track of all gray objects + * + * Evacuate reachable object graph rooted at @p src to to-space. + * On return all objects reachable from @p src are white + * + * @param src address of object to evacuate + * @param gc garbage collector + * @param stats per-object-type GC statistics **/ static Object * _deep_move(Object * src, gc::GC * gc, gc::ObjectStatistics * stats); @@ -213,6 +225,15 @@ namespace xo { virtual std::size_t _forward_children() = 0; }; + template + void + Object::assign_member(gp parent, gp * lhs, gp rhs) + { + Object::mm->assign_member(parent.ptr(), + reinterpret_cast(lhs->ptr_address()), + rhs.ptr()); + } + /** @class Cpof * @brief argument to operator new used for garbage collector evacuation phase * diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp index 825115a2..ca4f051b 100644 --- a/src/alloc/Forwarding1.cpp +++ b/src/alloc/Forwarding1.cpp @@ -21,6 +21,11 @@ namespace xo { return dest_.ptr() + offset; } + Object * + Forwarding1::_destination() { + return dest_.ptr(); + } + std::size_t Forwarding1::_shallow_size() const { assert(false); diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index ec4aa647..6dd8a929 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -71,6 +71,51 @@ namespace xo { << ">"; } + bool + MutationLogEntry::is_child_forwarded() const + { + assert(!parent_->_is_forwarded()); + + return (*lhs_)->_is_forwarded(); + } + + bool + MutationLogEntry::is_parent_forwarded() const + { + return parent_->_is_forwarded(); + } + + Object * + MutationLogEntry::parent_destination() const + { + //const bool c_debug_flag = true; + //scope log(XO_DEBUG(c_debug_flag)); + + if (parent_->_is_forwarded()) { + //log && log("parent is forwarded", xtag("parent", (void*)parent_)); + + return parent_->_destination(); + } else { + //log && log("parent is ordinary", xtag("parent", (void*)parent_)); + + return parent_; + } + } + + MutationLogEntry + MutationLogEntry::update_parent_moved(Object * parent_to) const + { + std::byte * parent_from = reinterpret_cast(parent_); + std::byte * lhs_from = reinterpret_cast(lhs_); + + std::ptrdiff_t offset = (lhs_from - parent_from); + + std::byte * lhs_to = reinterpret_cast(parent_to) + offset; + + return MutationLogEntry(parent_to, + reinterpret_cast(lhs_to)); + } + GC::GC(const Config & config) : config_{config} { @@ -89,6 +134,10 @@ namespace xo { tenured_[role2int(role::to_space) ] = ListAlloc::make("TB", tenured_size, 2 * tenured_size, config.debug_flag_); + mutation_log_[role2int(role::from_space)] = std::make_unique(); + mutation_log_[role2int(role::to_space)] = std::make_unique(); + defer_mutation_log_ = std::make_unique(); + this->checkpoint(); } @@ -100,6 +149,13 @@ namespace xo { return up{gc}; } + const std::string & + GC::name() const + { + static std::string s_default_name = "GC"; + return s_default_name; + } + std::size_t GC::size() const { @@ -151,22 +207,34 @@ namespace xo { return nursery_[role2int(role::to_space)]->after_checkpoint(); } - generation + bool + GC::debug_flag() const + { + return config_.debug_flag_; + } + + generation_result GC::fromspace_generation_of(const void * x) const { if (tenured_[role2int(role::from_space)]->contains(x)) - return generation::tenured; + return generation_result::tenured; - return generation::nursery; + if (nursery_[role2int(role::from_space)]->contains(x)) + return generation_result::nursery; + + return generation_result::not_found; } - generation - GC::generation_of(const void * x) const + generation_result + GC::tospace_generation_of(const void * x) const { if (tenured_[role2int(role::to_space)]->contains(x)) - return generation::tenured; + return generation_result::tenured; - return generation::nursery; + if (nursery_[role2int(role::to_space)]->contains(x)) + return generation_result::nursery; + + return generation_result::not_found; } std::byte * @@ -184,6 +252,11 @@ namespace xo { return nullptr; } + std::size_t + GC::mlog_size() const { + return mutation_log_[role2int(role::to_space)]->size(); + } + void GC::clear() { @@ -231,39 +304,55 @@ namespace xo { { scope log(XO_DEBUG(config_.debug_flag_), xtag("z", z), xtag("+pad", IAlloc::alloc_padding(z))); - generation g = this->fromspace_generation_of(src); + generation_result gr = this->fromspace_generation_of(src); std::byte * retval = nullptr; - if (g == generation::tenured) - { - log && log("tenured"); + switch (gr) { + case generation_result::tenured: + { + log && log("tenured"); - retval = tenured_[role2int(role::to_space)]->alloc(z); - } else if (nursery_[role2int(role::from_space)]->is_before_checkpoint(src)) - { - log && log("promote"); - - /* nursery object has survived 2nd collection cycle - * -> promote into tenured generation - */ - retval = tenured_[role2int(role::to_space)]->alloc(z); - - this->gc_statistics_.total_promoted_ += IAlloc::with_padding(z); - } else { - log && log("nursery"); - - retval = nursery_[role2int(role::to_space)]->alloc(z); - - if (!retval) { - /* nursery space exhausted */ - - this->request_gc(generation::nursery); - - nursery_[role2int(role::to_space)]->release_redline_memory(); - - retval = nursery_[role2int(role::to_space)]->alloc(z); + retval = tenured_[role2int(role::to_space)]->alloc(z); } + break; + case generation_result::nursery: + { + if (nursery_[role2int(role::from_space)]->is_before_checkpoint(src)) + { + /* nursery object has survived 2nd collection cycle + * -> promote into tenured generation + */ + retval = tenured_[role2int(role::to_space)]->alloc(z); + + log && log("promote", xtag("addr", (void*)retval)); + + assert(this->tospace_generation_of(retval) == generation_result::tenured); + + this->gc_statistics_.total_promoted_ += IAlloc::with_padding(z); + } else { + log && log("nursery"); + + retval = nursery_[role2int(role::to_space)]->alloc(z); + + if (!retval) { + /* nursery space exhausted !? */ + + this->request_gc(generation::nursery); + + nursery_[role2int(role::to_space)]->release_redline_memory(); + + retval = nursery_[role2int(role::to_space)]->alloc(z); + } + } + } + break; + case generation_result::not_found: + /* something wrong -- we only copy objects that are known to be in from-space + */ + + assert(false); + break; } assert(retval); @@ -271,6 +360,63 @@ namespace xo { return retval; } + void + GC::assign_member(Object * parent, Object ** lhs, Object * rhs) + { + ++gc_statistics_.n_mutation_; + + *lhs = rhs; + + if (runstate_.in_progress()) { + /* don't log mutations (if any) during GC */ + return; + } + + if (!config_.allow_incremental_gc_) { + /* full GCs don't need mutation log, since no cross-generational pointers */ + return; + } + + switch (tospace_generation_of(rhs)) + { + case generation_result::tenured: + /* only need to log mutations that create tenured->nursery pointers */ + return; + + case generation_result::nursery: + switch (tospace_generation_of(parent)) { + case generation_result::nursery: + if (is_before_checkpoint(parent)) { + // N1->N0, so must log + this->mutation_log_[role2int(role::to_space)]->push_back(MutationLogEntry(parent, lhs)); + ++(this->gc_statistics_.n_logged_mutation_); + ++(this->gc_statistics_.n_xckp_mutation_); + } else { + // parent in N0, not an xckp mutation + return; + } + break; + case generation_result::tenured: + // T->N, so must log + this->mutation_log_[role2int(role::to_space)]->push_back(MutationLogEntry(parent, lhs)); + ++(this->gc_statistics_.n_logged_mutation_); + ++(this->gc_statistics_.n_xgen_mutation_); + break; + case generation_result::not_found: + // parent is global + // This may be ok (provided lhs is a gc root) + break; + } + break; + + case generation_result::not_found: + + // child is global; + // logging not required + break; + } + } + void GC::release_redline_memory() { @@ -293,10 +439,20 @@ namespace xo { tenured_[role2int(role::from_space)] = std::move(tmp); } + void + GC::swap_mutation_log() + { + up tmp = std::move(mutation_log_[role2int(role::to_space)]); + mutation_log_[role2int(role::to_space)] = std::move(mutation_log_[role2int(role::from_space)]); + mutation_log_[role2int(role::from_space)] = std::move(tmp); + } + void GC::swap_spaces(generation target) { - // will be copying into storage currently labelled FromSpace + scope log(XO_DEBUG(this->debug_flag())); + + // will be copying into the memory regions currently labelled FromSpace /* gc will copy some to-be-determined amount in [0..promote_z] from nursery->tenured generation. @@ -321,6 +477,14 @@ namespace xo { - promote_z + incr_gc_threshold_); this->swap_nursery(); + + this->swap_mutation_log(); + + log && log(xtag("nursery.from", nursery_[role2int(role::from_space)]->name())); + log && log(xtag("nursery.to", nursery_[role2int(role::to_space) ]->name())); + log && log(xtag("tenured.from", tenured_[role2int(role::from_space)]->name())); + log && log(xtag("tenured.to", tenured_[role2int(role::to_space) ]->name())); + } /*swap_spaces*/ void @@ -351,6 +515,242 @@ namespace xo { } } + void + GC::incremental_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * per_type_stats) + { + scope log(XO_DEBUG(config_.debug_flag_), xtag("from_mlog.size", from_mlog->size())); + + /* categorize pointers based on combination of {source address, destination address}, + * only care about the generation associated with an address. + * + * N0 : nursery(from), before checkpoint + * N0': nursery(to), before checkpoint + * N1 : nursery(from), after checkpoint + * N1': nursery(to), after checkpoint + * T : tenured(to) + * + * loc(P): parent region before GC + * loc(C): child region before GC + * + * | | forwarded | loc now post | loc after | + * | | already? | root copy | action | + * | loc(P) loc(C) | P C | P' C' | P' C' | defer | action + * ----|---------------+--------------+---------------+---------------+-------+--------------- + * (a) | T N0 | no no | T N0 | T N1' | | C->N1', +mlog + * (b) | | yes | N1' | N1' | | +mlog + * (c) | T N1 | no no | T N1 | T T | | C->T, -mlog + * (d) | | yes | T T | T T | | -mlog + * (e) | N1 N0 | no no | N1 N0 | N1 N0 | P ->C | defer + * (f) | | yes | N1 N1' | N1 N1' | P ->C'| defer + * (g) | | yes yes | T N1' | T N1' | | +mlog + * + * notes: + * (a) C survives due to xgen ptr {T -> N0}; after collection have xgen ptr {T -> N1}. + * (b) C already evac'd; after collection stil have xgen ptr {T -> N1} + * (c) C survives due to xgen ptr (T -> N1): promote to T, so no longer xgen + * (d) C already evac'd: after collection no longer xgen (T -> T) + * (e) P,C maybe garbage. don't move either, but defer mlog incase P saved by a subsequent mutation. + * in that case C saved alto, + will still have an xgen ptr, so still need an mlog entry + * (f) P maybe garbage, C survives. defer mlog incase P saved+promoted by a subsequent mutation; + * in that case will still have an xgen (T -> N) ptr, so still need an mlog entry. + */ + + std::size_t i_from = 0; + // number of rescued subgraphs via mutation log entries + std::size_t n_rescue = 0; + + for (MutationLogEntry & from_entry : *from_mlog) + { + if (log) { + if (i_from % 10000 == 0) + log(xtag("i_from", i_from)); + } + + void * parent = from_entry.parent(); + + if (tospace_generation_of(parent) == generation_result::tenured) + { + // cases (a)(b)(c)(d) + // loc(P) is T. T didn't move b/c incremental gc. + + if (from_entry.is_dead()) { + // obsolete mutation -- no longer belongs to parent, discard + } else { + // note: child obtained (as it must be) by reading from parent's memory _now_. + Object * child_from = from_entry.child(); + + if (child_from) { + if (!child_from->_is_forwarded()) { + // P->C*. + // either: + // - C*=C in from-space, so needs evac + // - C*=C' in to-space, P already updated b/c of another mutation + // + if (fromspace_generation_of(child_from) != generation_result::not_found) { + // C*=C in from-space. needs evac, along with reachable descendants + // + // Includes cases: + // (a) T->N0 + // (c) T->N1 + + ++n_rescue; + + Object::_deep_move(child_from, this, per_type_stats); + + // C forwards to C', fall thru to parent fixup below + // (a) T->N1' + // (c) T->T + } else { + // P updated via some other mutation + // so don't need this mlog + ; + } + } + + // re-test, state may have changed above + if (from_entry.is_child_forwarded()) { + // P->C, C moved to C' + // Includes cases (a),(c) from above + + Object * child_to = child_from->_destination(); + + from_entry.fixup_parent_child_moved(child_to); + + // P->C', loc(C') in {N1', T'} + + if (tospace_generation_of(child_to) == generation_result::nursery) { + // (b) loc(P)=T, loc(C')=N1'; also case (a) + + // still have xgen pointer, so need mlog for it + to_mlog->push_back(from_entry); + } else { + // (d) loc(P)=T, loc(C')=T; also case (c) + // no longer xgen, so does not require mlog + } + } + + } else { + // nullptr child, discard + } + } + } else if (from_entry.is_parent_forwarded()) { + // Must have: + // loc(P) = N1, because: + // loc(P)=N0 -> ineligible for mlog; + // loc(P)=T -> not moved on incr GC + // + // follows that loc(P') = T + // already have P'->C' when parent moved separately + + Object * parent_to = from_entry.parent_destination(); + + log(xtag("parent_to", (void*)parent_to)); + + assert(tospace_generation_of(parent_to) == generation_result::tenured); + + MutationLogEntry to_entry = from_entry.update_parent_moved(parent_to); + + Object * child_to = to_entry.child(); // after moving + + if (tospace_generation_of(child_to) == generation_result::nursery) { + if (to_entry.is_dead()) { + ; + } else { + // (g) loc(P)=N1, loc(C)=N0, loc(P')=T, loc(C')=N1 + to_mlog->push_back(to_entry); + } + + } + } else { + // loc(P) = N1, loc(C) = N0, P may be garbage + // Includes cases: + // (e) P->C, C not moved + // (f) P->C, C moved to C' + // + // P may yet be rescued by another mlog entry, so defer + + if (!from_entry.is_dead()) { + defer_mlog->push_back(from_entry); + } + } + + ++i_from; + } + + from_mlog->clear(); + + if (n_rescue == 0) { + // if we didn't rescue any objects + // then we now confirm that otherwise-unreachable parents in defer_mlog + // are garbage + + defer_mlog->clear(); + } + } + + void + GC::incremental_gc_forward_mlog(ObjectStatistics * per_type_stats) + { + /* control here: + * - incremental gc. + * - gc roots have been copied, along with everything reachable from them. + * + * plan: + * - forward mutation in *from_mutation_log, writing them to + * *to_mutationlog and/or *defer_mutation_log. + * Use defer when mutation P->C encountered, but P was not copied. + * P appears to be garbage, but may turn out to be live if encountered + * in another mutation. + * + */ + + MutationLog * to_mlog = mutation_log_[role2int(role::to_space)].get(); + + for (;;) { + MutationLog * from_mlog = mutation_log_[role2int(role::from_space)].get(); + MutationLog * defer_mlog = defer_mutation_log_.get(); + + this->incremental_gc_forward_mlog_phase(from_mlog, + to_mlog, + defer_mlog, + per_type_stats); + + assert(from_mlog->empty()); + + if (defer_mlog->empty()) { + /* fixpoint reached */ + break; + } + + /* control here: + * 1. at least one mlog triggered a rescue + * 2. at least one mlog was deferred (b/c otherwise-unreachable parent) + * + * it's conceivable deferred parent now reachable thanks to rescues; + * revisit entries in defer_mlog, + * + * using now-empty from_mlog as scratch for any remaining deferred entries + */ + + std::swap(mutation_log_[role2int(role::from_space)], defer_mutation_log_); + } + } + + void + GC::forward_mutation_log(generation upto) + { + scope log(XO_DEBUG(config_.debug_flag_)); + + if (upto == generation::tenured) { + log && log("TODO: forward mutation log for full GC"); + } else { + this->incremental_gc_forward_mlog(&gc_statistics_.per_type_stats_); + } + } + void GC::cleanup_phase(generation upto) { @@ -401,11 +801,11 @@ namespace xo { } void - GC::execute_gc(generation target) + GC::execute_gc(generation upto) { scope log(XO_DEBUG(config_.debug_flag_)); - bool full_move = (target == generation::tenured); + bool full_move = (upto == generation::tenured); // TODO: RAII version in case of exceptions this->runstate_ = GCRunstate(true /*in_progress*/, full_move); @@ -415,7 +815,7 @@ namespace xo { /* new allocation since last GC */ std::size_t new_alloc = this->after_checkpoint(); - ++(gc_statistics_.gen_v_[static_cast(target)].n_gc_); + ++(gc_statistics_.gen_v_[static_cast(upto)].n_gc_); gc_statistics_.total_allocated_ += new_alloc; gc_statistics_.total_promoted_sab_ = gc_statistics_.total_promoted_; @@ -423,15 +823,17 @@ namespace xo { log && log("step 1: swap to/from roles"); - this->swap_spaces(target); + this->swap_spaces(upto); log && log("step 2a: copy globals"); - this->copy_globals(target); + this->copy_globals(upto); log && log("step 2b: TODO: copy pinned"); - log && log("step 3: TODO: forward mutation log"); + log && log("step 3: forward mutation log"); + + this->forward_mutation_log(upto); log && log("step 4: TODO: notify destructor log"); @@ -439,7 +841,7 @@ namespace xo { log && log("step 6: cleanup"); - this->cleanup_phase(target); + this->cleanup_phase(upto); this->runstate_ = GCRunstate(); diff --git a/src/alloc/IAlloc.cpp b/src/alloc/IAlloc.cpp index 4fbdd556..6e06a644 100644 --- a/src/alloc/IAlloc.cpp +++ b/src/alloc/IAlloc.cpp @@ -41,6 +41,12 @@ namespace xo { return z + alloc_padding(z); } + void + IAlloc::assign_member(Object * /*parent*/, Object ** lhs, Object * rhs) + { + *lhs = rhs; + } + std::byte * IAlloc::alloc_gc_copy(std::size_t /*z*/, const void * /*src*/) { diff --git a/src/alloc/ListAlloc.cpp b/src/alloc/ListAlloc.cpp index 76da8b19..a5ae38e7 100644 --- a/src/alloc/ListAlloc.cpp +++ b/src/alloc/ListAlloc.cpp @@ -50,6 +50,16 @@ namespace xo { return retval; } + const std::string & + ListAlloc::name() const { + if (hd_) { + return hd_->name(); + } + + static std::string s_default_name = "ListAlloc"; + return s_default_name; + } + std::size_t ListAlloc::size() const { return total_z_; diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index 7aa1a8d0..d3772003 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -34,7 +34,7 @@ namespace xo { bool full_move = gc->runstate().full_move(); - if (!full_move && (gc->generation_of(src) == gc::generation::tenured)) { + if (!full_move && (gc->tospace_generation_of(src) == gc::generation_result::tenured)) { /* don't move tenured objects during incremental collection */ return src; } @@ -61,7 +61,7 @@ namespace xo { bool full_move = gc->runstate().full_move(); - if (!full_move && gc->generation_of(from_src) == generation::tenured) { + if (!full_move && gc->tospace_generation_of(from_src) == gc::generation_result::tenured) { /** incremental collection does not move already-tenured objects **/ return from_src; } diff --git a/utest/CMakeLists.txt b/utest/CMakeLists.txt index d37786e3..50882bba 100644 --- a/utest/CMakeLists.txt +++ b/utest/CMakeLists.txt @@ -1,11 +1,13 @@ # build unittest alloc/utest +# +# NOTE: more GC tests in xo-object/utest -set(SELF_EXE utest.alloc) -set(SELF_SRCS +set(UTEST_EXE utest.alloc) +set(UTEST_SRCS alloc_utest_main.cpp ArenaAlloc.test.cpp GC.test.cpp) -xo_add_utest_executable(${SELF_EXE} ${SELF_SRCS}) -xo_self_dependency(${SELF_EXE} xo_alloc) -xo_external_target_dependency(${SELF_EXE} Catch2 Catch2::Catch2) +xo_add_utest_executable(${UTEST_EXE} ${UTEST_SRCS}) +xo_self_dependency(${UTEST_EXE} xo_alloc) +xo_external_target_dependency(${UTEST_EXE} Catch2 Catch2::Catch2) diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index dc175615..12445624 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -65,5 +65,8 @@ namespace xo { REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 1); } } + } /*namespace ut*/ } /*namespace xo*/ + +/* GC.test.cpp */ From 19520f017b24329776af966d7920f9b5294f3f64 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 5 Aug 2025 11:09:26 -0500 Subject: [PATCH 03/69] xo-object: GC unit test --- docs/glossary.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/glossary.rst diff --git a/docs/glossary.rst b/docs/glossary.rst new file mode 100644 index 00000000..4e50a499 --- /dev/null +++ b/docs/glossary.rst @@ -0,0 +1,28 @@ +.. _glossary: + +Glossary +-------- + +.. glossary:: + GC + | garbage collector + + mlog + | mutation log. + | Remembers cross-generation and cross-checkpoint pointers + + nursery + | in garbage collector, memory region dedicated to young objects. + | These are objects that have survived less than 2 incremental collection cycles. + + tenured + | in garbage collector, memory region dedicated to older objects. + | These are defined as objects that have survived 2 or more incremental collection cycles. + + xgen + | cross-generation tenured->nursery pointer; requires special GC bookkeeping + + xckp + | cross-checkpoint pointer; requires special GC bookkeeping + +.. toctree:: From fc9180363d1a57510b190ca9bf85c118b284c2a1 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 6 Aug 2025 09:30:37 -0500 Subject: [PATCH 04/69] xo-object: generative GC utest + reinstate coverage build --- include/xo/alloc/GC.hpp | 1 + src/alloc/ArenaAlloc.cpp | 11 ++++++----- src/alloc/Forwarding1.cpp | 6 ++++++ src/alloc/GC.cpp | 19 ++++++++++++++++++- src/alloc/IAlloc.cpp | 2 ++ utest/CMakeLists.txt | 1 + 6 files changed, 34 insertions(+), 6 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 0c50a722..6f9573f3 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -218,6 +218,7 @@ namespace xo { public: /** create new GC instance with configuration @p config **/ explicit GC(const Config & config); + virtual ~GC(); /** create GC allocator. * diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index 227e2d63..c3f70d8c 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -56,6 +56,8 @@ namespace xo { if (lo_ <= x && x < limit_) { this->free_ptr_ = x; + if (this->checkpoint_ > free_ptr_) + this->checkpoint_ = free_ptr_; } else { throw std::runtime_error(tostr("LinearAllog::set_free_ptr(x): expected lo <= x < limit", xtag("lo", lo_), xtag("x", x), xtag("limit", limit_))); @@ -102,8 +104,7 @@ namespace xo { void ArenaAlloc::clear() { - this->checkpoint_ = lo_; - this->free_ptr_ = lo_; + this->set_free_ptr(lo_); this->limit_ = hi_ - redline_z_; } @@ -133,14 +134,14 @@ namespace xo { std::byte * retval = this->free_ptr_; - this->free_ptr_ += z1; - log && log(xtag("self", name_), xtag("z0", z0), xtag("+pad", dz), xtag("z1", z1)); - if (free_ptr_ > limit_) { + if (free_ptr_ + z1 > limit_) { return nullptr; } + this->free_ptr_ += z1; + return retval; } diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp index ca4f051b..32d95b60 100644 --- a/src/alloc/Forwarding1.cpp +++ b/src/alloc/Forwarding1.cpp @@ -26,23 +26,29 @@ namespace xo { return dest_.ptr(); } + // LCOV_EXCL_START std::size_t Forwarding1::_shallow_size() const { assert(false); return 0; } + // LCOV_EXCL_STOP + // LCOV_EXCL_START Object * Forwarding1::_shallow_copy() const { assert(false); return nullptr; } + // LCOV_EXCL_STOP + // LCOV_EXCL_START std::size_t Forwarding1::_forward_children() { assert(false); return 0; } + // LCOV_EXCL_STOP } /*namespace obj*/ } /*namespace xo*/ diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 6dd8a929..4f106779 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -141,6 +141,21 @@ namespace xo { this->checkpoint(); } + GC::~GC() { + /* hygiene */ + this->clear(); + + nursery_[role2int(role::from_space)].reset(); + nursery_[role2int(role::to_space) ].reset(); + + tenured_[role2int(role::from_space)].reset(); + tenured_[role2int(role::to_space) ].reset(); + + mutation_log_[role2int(role::from_space)].reset(); + mutation_log_[role2int(role::to_space) ].reset(); + defer_mutation_log_.reset(); + } + up GC::make(const Config & config) { @@ -245,8 +260,10 @@ namespace xo { return nursery_[role2int(role::to_space)]->free_ptr(); case generation::tenured: return tenured_[role2int(role::to_space)]->free_ptr(); + // LCOV_EXCL_START case generation::N: assert(false); + // LCOV_EXCL_STOP } return nullptr; @@ -647,7 +664,7 @@ namespace xo { Object * parent_to = from_entry.parent_destination(); - log(xtag("parent_to", (void*)parent_to)); + log && log(xtag("parent_to", (void*)parent_to)); assert(tospace_generation_of(parent_to) == generation_result::tenured); diff --git a/src/alloc/IAlloc.cpp b/src/alloc/IAlloc.cpp index 6e06a644..8fe4789a 100644 --- a/src/alloc/IAlloc.cpp +++ b/src/alloc/IAlloc.cpp @@ -47,12 +47,14 @@ namespace xo { *lhs = rhs; } + // LCOV_EXCL_START std::byte * IAlloc::alloc_gc_copy(std::size_t /*z*/, const void * /*src*/) { assert(false); return nullptr; } + // LCOV_EXCL_STOP } /*namespace gc*/ } /*namespace xo*/ diff --git a/utest/CMakeLists.txt b/utest/CMakeLists.txt index 50882bba..b38a442f 100644 --- a/utest/CMakeLists.txt +++ b/utest/CMakeLists.txt @@ -5,6 +5,7 @@ set(UTEST_EXE utest.alloc) set(UTEST_SRCS alloc_utest_main.cpp + IAlloc.test.cpp ArenaAlloc.test.cpp GC.test.cpp) From ff5b0cfb8aa478b5888858bed9587b4235052c8f Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 6 Aug 2025 09:32:09 -0500 Subject: [PATCH 05/69] xo-alloc xo-object; + utests --- utest/IAlloc.test.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 utest/IAlloc.test.cpp diff --git a/utest/IAlloc.test.cpp b/utest/IAlloc.test.cpp new file mode 100644 index 00000000..b0214749 --- /dev/null +++ b/utest/IAlloc.test.cpp @@ -0,0 +1,29 @@ +/* @file IAlloc.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/IAlloc.hpp" +#include + +namespace xo { + using xo::gc::IAlloc; + + namespace ut { + TEST_CASE("ialloc", "[alloc]") + { + REQUIRE(IAlloc::alloc_padding(0) == 0); + REQUIRE(IAlloc::alloc_padding(1) == 7); + REQUIRE(IAlloc::alloc_padding(2) == 6); + REQUIRE(IAlloc::alloc_padding(3) == 5); + REQUIRE(IAlloc::alloc_padding(4) == 4); + REQUIRE(IAlloc::alloc_padding(5) == 3); + REQUIRE(IAlloc::alloc_padding(6) == 2); + REQUIRE(IAlloc::alloc_padding(7) == 1); + REQUIRE(IAlloc::alloc_padding(8) == 0); + REQUIRE(IAlloc::alloc_padding(9) == 7); + } + } /*namespace ut*/ +} /*namespace xo*/ + +/* end IAlloc.test.cpp */ From 432e0efce2acfa5008a7a054d32d5e70eb4ab5d6 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 6 Aug 2025 13:53:31 -0500 Subject: [PATCH 06/69] xo-object: improve GC unittest + prep to integrate w/ xo::reflect --- include/xo/alloc/GC.hpp | 116 +-------------------- include/xo/alloc/GcStatistics.hpp | 161 ++++++++++++++++++++++++++++++ include/xo/alloc/Object.hpp | 1 + include/xo/alloc/generation.hpp | 26 +++++ src/alloc/CMakeLists.txt | 3 +- src/alloc/GC.cpp | 38 +++++++ src/alloc/GcStatistics.cpp | 68 +++++++++++++ utest/GC.test.cpp | 12 +-- 8 files changed, 305 insertions(+), 120 deletions(-) create mode 100644 include/xo/alloc/GcStatistics.hpp create mode 100644 include/xo/alloc/generation.hpp create mode 100644 src/alloc/GcStatistics.cpp diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 6f9573f3..4fb9f518 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -6,6 +6,7 @@ #pragma once #include "ListAlloc.hpp" +#include "GcStatistics.hpp" #include "xo/indentlog/print/array.hpp" #include #include @@ -15,20 +16,6 @@ namespace xo { class Object; namespace gc { - enum class generation { - nursery, - tenured, - N - }; - - constexpr std::size_t gen2int(generation x) { return static_cast(x); } - - enum class generation_result { - nursery, - tenured, - not_found - }; - enum class role { /** nursery: generation for new objects **/ from_space, @@ -59,104 +46,6 @@ namespace xo { bool debug_flag_ = false; }; - /** @class ObjectStatistics - * @brief placeholder for type-driven allocation statistics - * - * Passed to @ref Object::deep_move for example - **/ - class ObjectStatistics { - }; - - /** @class PerGenerationStatistics - * @brief garbage collection statistics for particular GC generation - **/ - class PerGenerationStatistics { - public: - /** update statistics after a GC cycle - * @param alloc_z. new allocations (since preceding GC) - * @param before_z. generation size (bytes allocated) before collection - * @param after_z. generation size after collection - * @param promote_z. bytes promoted to next generation - **/ - void include_gc(std::size_t alloc_z, std::size_t before_z, std::size_t after_z, - std::size_t promote_z); - /** update with current state (use at end of gc cycle) **/ - void update_snapshot(std::size_t after_z); - - /** @param os. write stats on this output stream **/ - void display(std::ostream & os) const; - - /** number of bytes currently in use **/ - std::size_t used_z_ = 0; - - /** number of collection cycles completed **/ - std::size_t n_gc_ = 0; - /** sum of new alloc bytes, sampled at start of each collection cycle **/ - std::size_t new_alloc_z_ = 0; - /** sum of allocated bytes sampled at beginning of each collection cycle **/ - std::size_t scanned_z_ = 0; - /** sum of bytes remaining after collection cycle **/ - std::size_t survive_z_ = 0; - /** sum of bytes promoted to next generation **/ - std::size_t promote_z_ = 0; - }; - - inline std::ostream & operator<< (std::ostream & os, const PerGenerationStatistics & x) { - x.display(os); - return os; - } - - /** @class GcStatistics - * @brief garbage collection statistics - **/ - class GcStatistics { - public: - /** update statistics after a GC cycle - * @param upto. nursery -> incremental collection; tenured -> full collection - * @param alloc_z. new allocations (since preceding GC) - * @param before_z. generation size (bytes allocated) before collection - * @param after_z. generation size after collection - * @param promote_z. bytes promoted to next generation - **/ - void include_gc(generation upto, std::size_t alloc_z, - std::size_t before_z, std::size_t after_z, std::size_t promote_z); - /** update snapshot for current state. - * Use with tenured stats after incremental gc - **/ - void update_snapshot(generation upto, std::size_t after_z); - - /** @param os. write stats on this output stream **/ - void display(std::ostream & os) const; - - /** statistics gathered across {incr, full} GCs respectively **/ - std::array(generation::N)> gen_v_; - /** total bytes allocated since inception **/ - std::size_t total_allocated_ = 0; - /** snapshot of total bytes promoted asof beginning of last gc cycle **/ - std::size_t total_promoted_sab_ = 0; - /** total bytes promoted from nursery->tenured since inception **/ - std::size_t total_promoted_ = 0; - - /** total number of mutations to already-allocated objects, - * whether or not GC needs to log them. - **/ - std::size_t n_mutation_ = 0; - /** total number of mutation eligible for logging **/ - std::size_t n_logged_mutation_ = 0; - /** total number of cross-generation mutations (tenured->nursery when reported) **/ - std::size_t n_xgen_mutation_ = 0; - /** total number of cross-checkpoint mutations (N0 -> N1 when reported) **/ - std::size_t n_xckp_mutation_ = 0; - - /** per-type statistics (placeholder) **/ - ObjectStatistics per_type_stats_; - }; - - inline std::ostream & operator<< (std::ostream & os, const GcStatistics & x) { - x.display(os); - return os; - } - /** @class GCRunstate * @brief encapsulate state needed while GC is running * @@ -228,7 +117,8 @@ namespace xo { static up make(const Config & config); const GCRunstate & runstate() const { return runstate_; } - const GcStatistics & gc_statistics() const { return gc_statistics_; } + const GcStatistics & native_gc_statistics() const { return gc_statistics_; } + GcStatisticsExt get_gc_statistics() const; /** true iff GC permitted in current state **/ bool is_gc_enabled() const { return gc_enabled_ == 0; } diff --git a/include/xo/alloc/GcStatistics.hpp b/include/xo/alloc/GcStatistics.hpp new file mode 100644 index 00000000..01cfc80b --- /dev/null +++ b/include/xo/alloc/GcStatistics.hpp @@ -0,0 +1,161 @@ +/* GcStatistics.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#pragma once + +#include "generation.hpp" +#include "xo/indentlog/print/pretty.hpp" +#include +#include + +namespace xo { + namespace gc { + /** @class ObjectStatistics + * @brief placeholder for type-driven allocation statistics + * + * Passed to @ref Object::deep_move for example + **/ + class ObjectStatistics { + }; + + /** @class PerGenerationStatistics + * @brief garbage collection statistics for particular GC generation + **/ + class PerGenerationStatistics { + public: + /** update statistics after a GC cycle + * @param alloc_z. new allocations (since preceding GC) + * @param before_z. generation size (bytes allocated) before collection + * @param after_z. generation size after collection + * @param promote_z. bytes promoted to next generation + **/ + void include_gc(std::size_t alloc_z, std::size_t before_z, std::size_t after_z, + std::size_t promote_z); + /** update with current state (use at end of gc cycle) **/ + void update_snapshot(std::size_t after_z); + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** number of bytes currently in use **/ + std::size_t used_z_ = 0; + + /** number of collection cycles completed **/ + std::size_t n_gc_ = 0; + /** sum of new alloc bytes, sampled at start of each collection cycle **/ + std::size_t new_alloc_z_ = 0; + /** sum of allocated bytes sampled at beginning of each collection cycle **/ + std::size_t scanned_z_ = 0; + /** sum of bytes remaining after collection cycle **/ + std::size_t survive_z_ = 0; + /** sum of bytes promoted to next generation **/ + std::size_t promote_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const PerGenerationStatistics & x) { + x.display(os); + return os; + } + + /** @class GcStatistics + * @brief garbage collection statistics + **/ + class GcStatistics { + public: + /** update statistics after a GC cycle + * @param upto. nursery -> incremental collection; tenured -> full collection + * @param alloc_z. new allocations (since preceding GC) + * @param before_z. generation size (bytes allocated) before collection + * @param after_z. generation size after collection + * @param promote_z. bytes promoted to next generation + **/ + void include_gc(generation upto, std::size_t alloc_z, + std::size_t before_z, std::size_t after_z, std::size_t promote_z); + /** update snapshot for current state. + * Use with tenured stats after incremental gc + **/ + void update_snapshot(generation upto, std::size_t after_z); + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** statistics gathered across {incr, full} GCs respectively **/ + std::array(generation::N)> gen_v_; + /** total bytes allocated since inception **/ + std::size_t total_allocated_ = 0; + /** snapshot of total bytes promoted asof beginning of last gc cycle **/ + std::size_t total_promoted_sab_ = 0; + /** total bytes promoted from nursery->tenured since inception **/ + std::size_t total_promoted_ = 0; + + /** total number of mutations to already-allocated objects, + * whether or not GC needs to log them. + **/ + std::size_t n_mutation_ = 0; + /** total number of mutation eligible for logging (cumulative across GCs) **/ + std::size_t n_logged_mutation_ = 0; + /** total number of cross-generation mutations + * (tenured->nursery when reported; cumulative across GCs) **/ + std::size_t n_xgen_mutation_ = 0; + /** total number of cross-checkpoint mutations + * (N0 -> N1 when reported; cumulative across GCs) + **/ + std::size_t n_xckp_mutation_ = 0; + + /** per-type statistics (placeholder) **/ + ObjectStatistics per_type_stats_; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatistics & x) { + x.display(os); + return os; + } + + /** @class GcStatisticsExt + * @brief extend GcStatistics for application convenience + **/ + class GcStatisticsExt : public GcStatistics { + public: + explicit GcStatisticsExt(const GcStatistics & x) : GcStatistics{x} {} + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** current capacity of nursery generation **/ + std::size_t nursery_z_ = 0; + /** current nursery survivor size **/ + std::size_t nursery_before_checkpoint_z_ = 0; + /** current nursery new alloc size **/ + std::size_t nursery_after_checkpoint_z_ = 0; + /** current capacity of tenured generation **/ + std::size_t tenured_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatisticsExt & x) { + x.display(os); + return os; + } + + } /*namespace gc*/ + + namespace print { + template <> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::PerGenerationStatistics &); + }; + + template<> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatistics &); + }; + + template<> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatisticsExt &); + }; + } /*namespace print*/ +} /*namespace xo*/ + +/* end GcStatistics.hpp */ diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index f8e45e84..347b5018 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -5,6 +5,7 @@ #pragma once +#include "xo/reflect/SelfTagging.hpp" #include "IAlloc.hpp" #include #include diff --git a/include/xo/alloc/generation.hpp b/include/xo/alloc/generation.hpp new file mode 100644 index 00000000..6a5a7c2e --- /dev/null +++ b/include/xo/alloc/generation.hpp @@ -0,0 +1,26 @@ +/* generation.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include + +namespace xo { + namespace gc { + enum class generation { + nursery, + tenured, + N + }; + + constexpr std::size_t gen2int(generation x) { return static_cast(x); } + + enum class generation_result { + nursery, + tenured, + not_found + }; + } /*namespace gc*/ +} /*namespace xo*/ + +/* end generation.hpp */ diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index bc0f919f..180b936f 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -6,11 +6,12 @@ set(SELF_SRCS ArenaAlloc.cpp ListAlloc.cpp GC.cpp + GcStatistics.cpp Object.cpp Forwarding1.cpp ) xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) -xo_dependency(${SELF_LIB} indentlog) +xo_dependency(${SELF_LIB} reflect) #end CMakeLists.txt diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 4f106779..d3b3c978 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -3,6 +3,7 @@ * author: Roland Conybeare, Jul 2025 */ +#include "GcStatistics.hpp" #include "GC.hpp" #include "Object.hpp" #include "xo/indentlog/scope.hpp" @@ -67,6 +68,31 @@ namespace xo { os << ""; + } + + void + GcStatisticsExt::display(std::ostream & os) const + { + os << ""; } @@ -228,6 +254,18 @@ namespace xo { return config_.debug_flag_; } + GcStatisticsExt + GC::get_gc_statistics() const + { + GcStatisticsExt retval = GcStatisticsExt(this->native_gc_statistics()); + + retval.nursery_z_ = nursery_[role2int(role::to_space)]->size(); + retval.nursery_before_checkpoint_z_ = nursery_[role2int(role::to_space)]->before_checkpoint(); + retval.tenured_z_ = tenured_[role2int(role::to_space)]->size(); + + return retval; + } + generation_result GC::fromspace_generation_of(const void * x) const { diff --git a/src/alloc/GcStatistics.cpp b/src/alloc/GcStatistics.cpp new file mode 100644 index 00000000..b78b693b --- /dev/null +++ b/src/alloc/GcStatistics.cpp @@ -0,0 +1,68 @@ +/* GcStatistics.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "GcStatistics.hpp" +#include "xo/indentlog/print/pretty_vector.hpp" + +namespace xo { + namespace print { + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::PerGenerationStatistics & x) + { + return ppii.pps()->pretty_struct(ppii, + "PerGenerationStatistics", + refrtag("used_z", x.used_z_), + refrtag("n_gc", x.n_gc_), + refrtag("new_alloc_z", x.new_alloc_z_), + refrtag("scanned_z", x.scanned_z_), + refrtag("survive_z", x.survive_z_), + refrtag("promote_z", x.promote_z_) + ); + } + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::GcStatistics & x) + { + return ppii.pps()->pretty_struct(ppii, + "GcStatistics", + refrtag("gen_v", x.gen_v_), + refrtag("total_allocated", x.total_allocated_), + refrtag("total_promoted_sab", x.total_promoted_sab_), + refrtag("total_promoted", x.total_promoted_), + refrtag("n_mutation", x.n_mutation_), + refrtag("n_logged_mutation", x.n_logged_mutation_), + refrtag("n_xgen_mutation", x.n_xgen_mutation_), + refrtag("n_xckp_mutation", x.n_xckp_mutation_) + ); + } + + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::GcStatisticsExt & x) + { + return ppii.pps()->pretty_struct(ppii, + "GcStatisticsExt", + refrtag("gen_v", x.gen_v_), + refrtag("total_allocated", x.total_allocated_), + refrtag("total_promoted_sab", x.total_promoted_sab_), + refrtag("total_promoted", x.total_promoted_), + refrtag("n_mutation", x.n_mutation_), + refrtag("n_logged_mutation", x.n_logged_mutation_), + refrtag("n_xgen_mutation", x.n_xgen_mutation_), + refrtag("n_xckp_mutation", x.n_xckp_mutation_), + refrtag("nursery_z", x.nursery_z_), + refrtag("nursery_before_checkpoint_z", x.nursery_before_checkpoint_z_), + refrtag("nursery_after_checkpoint_z", x.nursery_after_checkpoint_z_), + refrtag("tenured_z", x.tenured_z_)); + } + + + } /*namespace print*/ +} /*namespace xo*/ + +/* end GcStatistics.cpp */ diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index 12445624..c36deb2e 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -47,22 +47,22 @@ namespace xo { REQUIRE(gc->gc_in_progress() == false); REQUIRE(gc->is_gc_enabled() == true); - REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 0); - REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 0); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); /* gc with empty state */ gc->request_gc(generation::nursery); REQUIRE(gc->gc_in_progress() == false); - REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); - REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); /* still empty state */ gc->request_gc(generation::tenured); REQUIRE(gc->gc_in_progress() == false); - REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); - REQUIRE(gc->gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 1); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 1); } } From 227b2e5cf7155a10276a8e3b31767b36d130d518 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 6 Aug 2025 14:11:28 -0500 Subject: [PATCH 07/69] xo-alloc xo-object: + Object.self_tp --- include/xo/alloc/Forwarding1.hpp | 1 + include/xo/alloc/Object.hpp | 11 ++++++++++- src/alloc/Forwarding1.cpp | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/include/xo/alloc/Forwarding1.hpp b/include/xo/alloc/Forwarding1.hpp index 4fb61d08..17a38df2 100644 --- a/include/xo/alloc/Forwarding1.hpp +++ b/include/xo/alloc/Forwarding1.hpp @@ -12,6 +12,7 @@ namespace xo { explicit Forwarding1(gp dest); // inherited from Object.. + virtual TaggedPtr self_tp() const final override; virtual bool _is_forwarded() const final override { return true; } virtual Object * _offset_destination(Object * src) const final override; virtual Object * _destination() final override; diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 347b5018..47b26261 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -5,7 +5,7 @@ #pragma once -#include "xo/reflect/SelfTagging.hpp" +#include "xo/reflect/TaggedPtr.hpp" #include "IAlloc.hpp" #include #include @@ -79,6 +79,9 @@ namespace xo { * but cost would be an extra layer of indirection **/ class Object { + public: + using TaggedPtr = xo::reflect::TaggedPtr; + public: virtual ~Object() = default; @@ -148,6 +151,12 @@ namespace xo { **/ static Object * _shallow_move(Object * src, gc::GC * gc); + // Reflection support + + /** tagged pointer with runtime type information + **/ + virtual TaggedPtr self_tp() const = 0; + // GC support /** true iff this object represents a forwarding pointer. diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp index 32d95b60..2122d248 100644 --- a/src/alloc/Forwarding1.cpp +++ b/src/alloc/Forwarding1.cpp @@ -4,15 +4,25 @@ */ #include "Forwarding1.hpp" +#include "xo/reflect/Reflect.hpp" #include #include namespace xo { + using xo::reflect::Reflect; + using xo::reflect::TaggedPtr; + namespace obj { Forwarding1::Forwarding1(gp dest) : dest_{dest} {} + TaggedPtr + Forwarding1::self_tp() const + { + return Reflect::make_tp(const_cast(this)); + } + Object * Forwarding1::_offset_destination(Object * src) const { From 593dc064f9e8fb8c439d0c872314de3482a10f18 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 6 Aug 2025 22:34:20 -0500 Subject: [PATCH 08/69] xo-allod: + per-type stats + pretty printing --- include/xo/alloc/ArenaAlloc.hpp | 4 + include/xo/alloc/Forwarding1.hpp | 1 + include/xo/alloc/GC.hpp | 8 ++ include/xo/alloc/GcStatistics.hpp | 12 +-- include/xo/alloc/ListAlloc.hpp | 9 ++ include/xo/alloc/Object.hpp | 6 ++ include/xo/alloc/ObjectStatistics.hpp | 86 +++++++++++++++++ src/alloc/ArenaAlloc.cpp | 39 ++++++++ src/alloc/CMakeLists.txt | 1 + src/alloc/Forwarding1.cpp | 6 ++ src/alloc/GC.cpp | 128 +++++++------------------- src/alloc/GcStatistics.cpp | 87 +++++++++++++++++ src/alloc/ListAlloc.cpp | 11 +++ src/alloc/Object.cpp | 12 +++ src/alloc/ObjectStatistics.cpp | 73 +++++++++++++++ utest/LinearAlloc.test.cpp | 87 ----------------- 16 files changed, 379 insertions(+), 191 deletions(-) create mode 100644 include/xo/alloc/ObjectStatistics.hpp create mode 100644 src/alloc/ObjectStatistics.cpp delete mode 100644 utest/LinearAlloc.test.cpp diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 12033646..9f41a8f1 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -6,6 +6,7 @@ #pragma once #include "IAlloc.hpp" +#include "ObjectStatistics.hpp" namespace xo { namespace gc { @@ -48,6 +49,9 @@ namespace xo { std::byte * free_ptr() const { return free_ptr_; } void set_free_ptr(std::byte * x); + void capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const; + // inherited from IAlloc... virtual const std::string & name() const final override { return name_; } diff --git a/include/xo/alloc/Forwarding1.hpp b/include/xo/alloc/Forwarding1.hpp index 17a38df2..cf54597d 100644 --- a/include/xo/alloc/Forwarding1.hpp +++ b/include/xo/alloc/Forwarding1.hpp @@ -13,6 +13,7 @@ namespace xo { // inherited from Object.. virtual TaggedPtr self_tp() const final override; + virtual void display(std::ostream & os) const final override; virtual bool _is_forwarded() const final override { return true; } virtual Object * _offset_destination(Object * src) const final override; virtual Object * _destination() final override; diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 4fb9f518..daef1881 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -42,6 +42,8 @@ namespace xo { std::size_t initial_tenured_z_ = 0; /** true to permit incremental garbage collection **/ bool allow_incremental_gc_ = true; + /** true to report statistics **/ + bool stats_flag_ = false; /** true to enable debug logging **/ bool debug_flag_ = false; }; @@ -207,6 +209,8 @@ namespace xo { void swap_mutation_log(); /** swap roles of FromSpace/ToSpace **/ void swap_spaces(generation g); + /** scan to-space for object statistics before GC */ + void capture_object_statistics(generation upto, capture_phase phase); /** copy object **/ void copy_object(Object ** addr, generation upto, ObjectStatistics * object_stats); /** copy everything reachable from global gc roots **/ @@ -273,6 +277,10 @@ namespace xo { /** allocation/collection counters **/ GcStatistics gc_statistics_; + /** optional per-object-type counters. snapshot at beginning of collection cycle **/ + std::array object_statistics_sab_; + /** optional per-object-type counters. snapshot at end of collection cycle **/ + std::array object_statistics_sae_; /** trigger full GC whenever this much data arrives in tenured generation **/ std::size_t full_gc_threshold_ = 0; diff --git a/include/xo/alloc/GcStatistics.hpp b/include/xo/alloc/GcStatistics.hpp index 01cfc80b..24cac461 100644 --- a/include/xo/alloc/GcStatistics.hpp +++ b/include/xo/alloc/GcStatistics.hpp @@ -6,20 +6,13 @@ #pragma once #include "generation.hpp" +#include "xo/reflect/TypeDescr.hpp" #include "xo/indentlog/print/pretty.hpp" #include #include namespace xo { namespace gc { - /** @class ObjectStatistics - * @brief placeholder for type-driven allocation statistics - * - * Passed to @ref Object::deep_move for example - **/ - class ObjectStatistics { - }; - /** @class PerGenerationStatistics * @brief garbage collection statistics for particular GC generation **/ @@ -103,9 +96,6 @@ namespace xo { * (N0 -> N1 when reported; cumulative across GCs) **/ std::size_t n_xckp_mutation_ = 0; - - /** per-type statistics (placeholder) **/ - ObjectStatistics per_type_stats_; }; inline std::ostream & operator<< (std::ostream & os, const GcStatistics & x) { diff --git a/include/xo/alloc/ListAlloc.hpp b/include/xo/alloc/ListAlloc.hpp index db1a2df7..f9c20d00 100644 --- a/include/xo/alloc/ListAlloc.hpp +++ b/include/xo/alloc/ListAlloc.hpp @@ -6,6 +6,7 @@ #pragma once #include "IAlloc.hpp" +#include "ObjectStatistics.hpp" #include #include #include @@ -43,6 +44,14 @@ namespace xo { /** current free pointer **/ std::byte * free_ptr() const; + /** scan space (must not contain forwarding pointers, because loses size info) + * + gather stats by object type + * + * See @ref Object::self_tp + **/ + void capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const; + // inherited from IAlloc.. virtual const std::string & name() const final override; diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 47b26261..0f50bfed 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -157,6 +157,9 @@ namespace xo { **/ virtual TaggedPtr self_tp() const = 0; + /** print on stream @p os **/ + virtual void display(std::ostream & os) const = 0; + // GC support /** true iff this object represents a forwarding pointer. @@ -244,6 +247,9 @@ namespace xo { rhs.ptr()); } + std::ostream & + operator<< (std::ostream & os, gp x); + /** @class Cpof * @brief argument to operator new used for garbage collector evacuation phase * diff --git a/include/xo/alloc/ObjectStatistics.hpp b/include/xo/alloc/ObjectStatistics.hpp new file mode 100644 index 00000000..ced3b463 --- /dev/null +++ b/include/xo/alloc/ObjectStatistics.hpp @@ -0,0 +1,86 @@ +/* file ObjectStatistics.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#pragma once + +#include "xo/indentlog/print/pretty.hpp" +#include +#include + +namespace xo { + namespace reflect { class TypeDescrBase; } + + namespace gc { + enum class capture_phase { + /** snapshot-at-beginning **/ + sab, + /** snapshot-at-end **/ + sae, + }; + + /** @class PerObjectTypeStatistics + * @brief statistics for a particular object type + * + * Gathered for each leaf type descended from xo::obj::Object. + * See @ref xo::obj::Object::self_tp + * + * See @ref GC::capture_object_statistics + * (gathers @ref scanned_n_, @ref scanned_z_) + **/ + struct PerObjectTypeStatistics { + using TypeDescr = xo::reflect::TypeDescrBase const *; + + void display(std::ostream & os) const; + + /** stats here are for objects of this type **/ + TypeDescr td_ = nullptr; + /** number of objects scanned **/ + std::size_t scanned_n_ = 0; + /** number of bytes scanned **/ + std::size_t scanned_z_ = 0; + /** number of objects surviving **/ + std::size_t survive_n_ = 0; + /** number of bytes from surviving objects **/ + std::size_t survive_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const PerObjectTypeStatistics & x) { + x.display(os); + return os; + } + + /** @class ObjectStatistics + * @brief placeholder for type-driven allocation statistics + * + * Passed to @ref Object::deep_move for example + **/ + struct ObjectStatistics { + void display(std::ostream & os) const; + + /** per-object-type statistics, indexed by TypeId **/ + std::vector per_type_stats_v_; + }; + + inline std::ostream & operator<< (std::ostream & os, const ObjectStatistics & x) { + x.display(os); + return os; + } + + } /*namespace gc*/ + + namespace print { + template <> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::PerObjectTypeStatistics &); + }; + + template <> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::ObjectStatistics &); + }; + } /*namespace print*/ +} /*namespace xo*/ + +/* end ObjectStatistics.hpp */ diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index c3f70d8c..ae67365a 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -4,6 +4,8 @@ */ #include "ArenaAlloc.hpp" +#include "Object.hpp" +#include "ObjectStatistics.hpp" #include "xo/indentlog/scope.hpp" #include "xo/indentlog/print/tag.hpp" #include @@ -64,6 +66,43 @@ namespace xo { } } + void + ArenaAlloc::capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const + { + using xo::reflect::TaggedPtr; + + std::byte * p = lo_; + + while (p < free_ptr_) { + Object * obj = reinterpret_cast(p); + TaggedPtr tp = obj->self_tp(); + std::size_t z = obj->_shallow_size(); + std::uint32_t id = tp.td()->id().id(); + + if (p_dest->per_type_stats_v_.size() < id + 1) + p_dest->per_type_stats_v_.resize(id + 1); + + PerObjectTypeStatistics & dest = p_dest->per_type_stats_v_.at(id); + + dest.td_ = tp.td(); + switch (phase) { + case capture_phase::sab: + ++dest.scanned_n_; + dest.scanned_z_ += z; + break; + case capture_phase::sae: + ++dest.survive_n_; + dest.survive_z_ += z; + break; + } + + p += z; + } + + assert(p == free_ptr_); + } + std::size_t ArenaAlloc::size() const { return limit_ - lo_; diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index 180b936f..55b8641c 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -7,6 +7,7 @@ set(SELF_SRCS ListAlloc.cpp GC.cpp GcStatistics.cpp + ObjectStatistics.cpp Object.cpp Forwarding1.cpp ) diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp index 2122d248..5a941c82 100644 --- a/src/alloc/Forwarding1.cpp +++ b/src/alloc/Forwarding1.cpp @@ -23,6 +23,12 @@ namespace xo { return Reflect::make_tp(const_cast(this)); } + void + Forwarding1::display(std::ostream & os) const + { + os << ""; + } + Object * Forwarding1::_offset_destination(Object * src) const { diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index d3b3c978..afac26df 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -12,91 +12,6 @@ namespace xo { namespace gc { - void - PerGenerationStatistics::include_gc(std::size_t alloc_z, - std::size_t before_z, - std::size_t after_z, - std::size_t promote_z) - { - this->update_snapshot(after_z); - - new_alloc_z_ += alloc_z; - scanned_z_ += before_z; - survive_z_ += after_z; - promote_z_ += promote_z; - } - - void - PerGenerationStatistics::update_snapshot(std::size_t after_z) - { - used_z_ = after_z; - } - - void - PerGenerationStatistics::display(std::ostream & os) const - { - os << ""; - } - - void - GcStatistics::include_gc(generation upto, - std::size_t alloc_z, - std::size_t before_z, - std::size_t after_z, - std::size_t promote_z) - { - gen_v_[static_cast(upto)].include_gc(alloc_z, before_z, after_z, promote_z); - } - - void - GcStatistics::update_snapshot(generation upto, - std::size_t after_z) - { - gen_v_[static_cast(upto)].update_snapshot(after_z); - } - - void - GcStatistics::display(std::ostream & os) const - { - os << ""; - } - - void - GcStatisticsExt::display(std::ostream & os) const - { - os << ""; - } - bool MutationLogEntry::is_child_forwarded() const { @@ -542,6 +457,22 @@ namespace xo { } /*swap_spaces*/ + void + GC::capture_object_statistics(generation upto, capture_phase phase) + { + /* scan nursery */ + this->nursery_[role2int(role::to_space)]->capture_object_statistics + (phase, + &object_statistics_sab_[gen2int(generation::nursery)]); + + if (upto == generation::tenured) { + /* scan tenured */ + this->tenured_[role2int(role::to_space)]->capture_object_statistics + (phase, + &object_statistics_sab_[gen2int(generation::tenured)]); + } + } + void GC::copy_object(Object ** pp_object, generation upto, ObjectStatistics * object_stats) { @@ -566,7 +497,7 @@ namespace xo { GC::copy_globals(generation upto) { for (Object ** pp_root : gc_root_v_) { - this->copy_object(pp_root, upto, &gc_statistics_.per_type_stats_); + this->copy_object(pp_root, upto, &object_statistics_sae_[gen2int(upto)]); } } @@ -802,7 +733,7 @@ namespace xo { if (upto == generation::tenured) { log && log("TODO: forward mutation log for full GC"); } else { - this->incremental_gc_forward_mlog(&gc_statistics_.per_type_stats_); + this->incremental_gc_forward_mlog(&object_statistics_sae_[gen2int(generation::nursery)]); } } @@ -858,7 +789,7 @@ namespace xo { void GC::execute_gc(generation upto) { - scope log(XO_DEBUG(config_.debug_flag_)); + scope log(XO_DEBUG(config_.stats_flag_)); bool full_move = (upto == generation::tenured); @@ -876,7 +807,11 @@ namespace xo { log && log(xtag("new_alloc", new_alloc)); - log && log("step 1: swap to/from roles"); + log && log("step 0: (optional) scan for object statistics"); + + this->capture_object_statistics(upto, capture_phase::sab); + + log && log("step 1 : swap to/from roles"); this->swap_spaces(upto); @@ -886,18 +821,25 @@ namespace xo { log && log("step 2b: TODO: copy pinned"); - log && log("step 3: forward mutation log"); + log && log("step 3 : forward mutation log"); this->forward_mutation_log(upto); - log && log("step 4: TODO: notify destructor log"); + log && log("step 4 : TODO: notify destructor log"); - log && log("step 5: TODO: keep reachable weak pointers"); + log && log("step 5 : TODO: keep reachable weak pointers"); - log && log("step 6: cleanup"); + log && log("step 6 : cleanup"); this->cleanup_phase(upto); + this->capture_object_statistics(upto, capture_phase::sae); + + log && log("object statistics [nursery]:"); + log && log(refrtag("stats", object_statistics_sab_[gen2int(generation::nursery)])); + log && log("object statistics [tenured]:"); + log && log(refrtag("stats", object_statistics_sab_[gen2int(generation::tenured)])); + this->runstate_ = GCRunstate(); log && log("statistics:"); diff --git a/src/alloc/GcStatistics.cpp b/src/alloc/GcStatistics.cpp index b78b693b..d54c3cf2 100644 --- a/src/alloc/GcStatistics.cpp +++ b/src/alloc/GcStatistics.cpp @@ -7,6 +7,93 @@ #include "xo/indentlog/print/pretty_vector.hpp" namespace xo { + namespace gc { + void + PerGenerationStatistics::include_gc(std::size_t alloc_z, + std::size_t before_z, + std::size_t after_z, + std::size_t promote_z) + { + this->update_snapshot(after_z); + + new_alloc_z_ += alloc_z; + scanned_z_ += before_z; + survive_z_ += after_z; + promote_z_ += promote_z; + } + + void + PerGenerationStatistics::update_snapshot(std::size_t after_z) + { + used_z_ = after_z; + } + + void + PerGenerationStatistics::display(std::ostream & os) const + { + os << ""; + } + + void + GcStatistics::include_gc(generation upto, + std::size_t alloc_z, + std::size_t before_z, + std::size_t after_z, + std::size_t promote_z) + { + gen_v_[static_cast(upto)].include_gc(alloc_z, before_z, after_z, promote_z); + } + + void + GcStatistics::update_snapshot(generation upto, + std::size_t after_z) + { + gen_v_[static_cast(upto)].update_snapshot(after_z); + } + + void + GcStatistics::display(std::ostream & os) const + { + os << ""; + } + + void + GcStatisticsExt::display(std::ostream & os) const + { + os << ""; + } + } /*namespace gc*/ + namespace print { bool ppdetail::print_pretty(const ppindentinfo & ppii, diff --git a/src/alloc/ListAlloc.cpp b/src/alloc/ListAlloc.cpp index a5ae38e7..9d727591 100644 --- a/src/alloc/ListAlloc.cpp +++ b/src/alloc/ListAlloc.cpp @@ -50,6 +50,17 @@ namespace xo { return retval; } + void + ListAlloc::capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const + { + hd_->capture_object_statistics(phase, p_dest); + + for (const auto & arena : full_l_) + arena->capture_object_statistics(phase, p_dest); + + } + const std::string & ListAlloc::name() const { if (hd_) { diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index d3772003..b3db11ca 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -191,6 +191,18 @@ namespace xo { (void)fwd; } + std::ostream & + operator<< (std::ostream & os, gp x) + { + if (x.ptr()) { + x->display(os); + } else { + os << ""; + } + + return os; + } + } /*namespace xo*/ /* end Object.cpp*/ diff --git a/src/alloc/ObjectStatistics.cpp b/src/alloc/ObjectStatistics.cpp new file mode 100644 index 00000000..a69928dc --- /dev/null +++ b/src/alloc/ObjectStatistics.cpp @@ -0,0 +1,73 @@ +/* file ObjectStatistics.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "ObjectStatistics.hpp" +#include "xo/reflect/TypeDescr.hpp" +#include "xo/indentlog/print/pretty_vector.hpp" + +namespace xo { + namespace gc { + void + PerObjectTypeStatistics::display(std::ostream & os) const + { + os << "short_name()); + else + os << rtag("td", "nullptr"); + os << rtag("scanned_n", scanned_n_) + << rtag("scanned_z", scanned_z_) + << rtag("survive_n", survive_n_) + << rtag("survive_z", survive_z_) + << ">"; + } + + void + ObjectStatistics::display(std::ostream & os) const + { + os << ""; + } + } /*namespace gc*/ + + namespace print { + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::PerObjectTypeStatistics & x) + { + static constexpr std::string_view c_nullptr_str = "nullptr"; + + if (x.td_) { + return ppii.pps()->pretty_struct(ppii, + "PerObjectTypeStatistics", + refrtag("td", x.td_ ? x.td_->short_name() : c_nullptr_str), + refrtag("scanned_n", x.scanned_n_), + refrtag("scanned_z", x.scanned_z_), + refrtag("survive_n", x.survive_n_), + refrtag("survive_z", x.survive_z_)); + } else { + /* print nothing -- empty struct */ + return true; + } + } + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::ObjectStatistics & x) + { + return ppii.pps()->pretty_struct(ppii, + "ObjectTypeStatistics", + refrtag("per_type_stats_v", x.per_type_stats_v_)); + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* end ObjectStatistics.cpp */ diff --git a/utest/LinearAlloc.test.cpp b/utest/LinearAlloc.test.cpp deleted file mode 100644 index b1909991..00000000 --- a/utest/LinearAlloc.test.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* @file LinearAlloc.test.cpp - * - * author: Roland Conybeare, Jul 2025 - */ - -#include "xo/alloc/ArenaAlloc.hpp" -#include - -namespace xo { - using xo::gc::LinearAlloc; - - namespace ut { - - namespace { - struct testcase_alloc { - testcase_alloc(std::size_t rz, std::size_t z) - : redline_z_{rz}, arena_z_{z} {} - - std::size_t redline_z_; - std::size_t arena_z_; - - }; - - std::vector - s_testcase_v = { - testcase_alloc(0, 4096) - }; - } - - - TEST_CASE("linearalloc", "[alloc]") - { - for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { - const testcase_alloc & tc = s_testcase_v[i_tc]; - - constexpr bool c_debug_flag = false; - - auto alloc = LinearAlloc::make("linearalloc", tc.redline_z_, tc.arena_z_, c_debug_flag); - - REQUIRE(alloc.get()); - REQUIRE(alloc->name() == "linearalloc"); - REQUIRE(alloc->size() == tc.arena_z_); - REQUIRE(alloc->available() == tc.arena_z_); - REQUIRE(alloc->allocated() == 0); - REQUIRE(alloc->is_before_checkpoint(alloc->free_ptr()) == false); - REQUIRE(alloc->before_checkpoint() == 0); - REQUIRE(alloc->after_checkpoint() == 0); - - auto free0 = alloc->free_ptr(); - - auto mem = alloc->alloc(tc.arena_z_); - - REQUIRE(mem != nullptr); - - REQUIRE(mem == free0); - - REQUIRE(alloc->size() == tc.arena_z_); - REQUIRE(alloc->available() == 0); - REQUIRE(alloc->allocated() == tc.arena_z_); - REQUIRE(alloc->is_before_checkpoint(mem) == false); - REQUIRE(alloc->before_checkpoint() == 0); - REQUIRE(alloc->after_checkpoint() == tc.arena_z_); - - alloc->clear(); - - REQUIRE(alloc->free_ptr() == free0); - REQUIRE(alloc->available() == tc.arena_z_); - REQUIRE(alloc->allocated() == 0); - REQUIRE(alloc->is_before_checkpoint(free0) == false); - REQUIRE(alloc->before_checkpoint() == 0); - REQUIRE(alloc->after_checkpoint() == 0); - - mem = alloc->alloc(1); - - auto used = sizeof(void*); - REQUIRE(alloc->size() == tc.arena_z_); - REQUIRE(alloc->available() == tc.arena_z_ - used); - REQUIRE(alloc->allocated() == used); - REQUIRE(alloc->is_before_checkpoint(free0) == false); - REQUIRE(alloc->before_checkpoint() == 0); - REQUIRE(alloc->after_checkpoint() == used); - - } - } - - } /*namespace ut */ -} /*namespace xo*/ From 8b622e69995fb58f40d38931412fb4332e102b01 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 6 Aug 2025 22:50:29 -0500 Subject: [PATCH 09/69] xo-alloc: retire redline-memory feature --- include/xo/alloc/ArenaAlloc.hpp | 12 +++++++++- include/xo/alloc/GC.hpp | 2 ++ include/xo/alloc/IAlloc.hpp | 2 ++ include/xo/alloc/ListAlloc.hpp | 13 +++++++++-- src/alloc/ArenaAlloc.cpp | 39 ++++++++++++++++++++++++++++++--- src/alloc/GC.cpp | 8 +++++++ src/alloc/ListAlloc.cpp | 26 ++++++++++++++++++++-- utest/ArenaAlloc.test.cpp | 14 ++++++++++-- 8 files changed, 106 insertions(+), 10 deletions(-) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 9f41a8f1..51b72c9e 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -42,7 +42,9 @@ namespace xo { * with reserved capacity @p redline_z. **/ static up make(const std::string & name, +#ifdef REDLINE_MEMORY std::size_t redline_z, +#endif std::size_t z, bool debug_flag); @@ -66,10 +68,16 @@ namespace xo { virtual void clear() final override; virtual void checkpoint() final override; virtual std::byte * alloc(std::size_t z) final override; +#ifdef REDLINE_MEMORY virtual void release_redline_memory() final override; +#endif private: - ArenaAlloc(const std::string & name, std::size_t rz, std::size_t z, bool debug_flag); + ArenaAlloc(const std::string & name, +#ifdef REDLINE_MEMORY + std::size_t rz, +#endif + std::size_t z, bool debug_flag); private: /** @@ -91,8 +99,10 @@ namespace xo { std::byte * free_ptr_ = nullptr; /** soft limit: end of released memory **/ std::byte * limit_ = nullptr; +#ifdef REDLINE_MEMORY /** amount of last-resort memory to reserve **/ std::size_t redline_z_ = 0; +#endif /** hard limit: end of allocated memory **/ std::byte * hi_ = nullptr; /** true to enable detailed debug logging **/ diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index daef1881..679e5286 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -194,7 +194,9 @@ namespace xo { virtual std::byte * alloc(std::size_t z) final override; virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; +#ifdef REDLINE_MEMORY virtual void release_redline_memory() final override; +#endif private: /** begin GC now **/ diff --git a/include/xo/alloc/IAlloc.hpp b/include/xo/alloc/IAlloc.hpp index 272fe9ad..2eacd9cc 100644 --- a/include/xo/alloc/IAlloc.hpp +++ b/include/xo/alloc/IAlloc.hpp @@ -76,8 +76,10 @@ namespace xo { * Only used in @ref GC. Default implementation asserts and returns nullptr **/ virtual std::byte * alloc_gc_copy(std::size_t z, const void * src); +#ifdef REDLINE_MEMORY /** release last-resort reserved memory **/ virtual void release_redline_memory() = 0; +#endif }; } /*namespace gc*/ diff --git a/include/xo/alloc/ListAlloc.hpp b/include/xo/alloc/ListAlloc.hpp index f9c20d00..627bbf50 100644 --- a/include/xo/alloc/ListAlloc.hpp +++ b/include/xo/alloc/ListAlloc.hpp @@ -29,7 +29,9 @@ namespace xo { ListAlloc(std::unique_ptr hd, ArenaAlloc * marked, std::size_t cz, std::size_t nz, std::size_t tz, +#ifdef REDLINE_MEMORY bool use_redline, +#endif bool debug_flag); ~ListAlloc(); @@ -66,7 +68,9 @@ namespace xo { virtual void clear() final override; virtual void checkpoint() final override; virtual std::byte * alloc(std::size_t z) final override; +#ifdef REDLINE_MEMORY virtual void release_redline_memory() final override; +#endif private: /** **/ @@ -79,11 +83,16 @@ namespace xo { * from trying to converge on app working set size **/ std::list> full_l_; - std::size_t current_z_ = 0;; - std::size_t next_z_ = 0;; + /** size of current arena @ref hd_ **/ + std::size_t current_z_ = 0; + /** if @ref hd_ fills, size of next arena to allocate **/ + std::size_t next_z_ = 0; + /** total size of @ref hd_ + contents of @ref full_l_ **/ std::size_t total_z_ = 0; +#ifdef REDLINE_MEMORY bool use_redline_ = false; bool redlined_flag_ = false; +#endif /** true to enable debug logging **/ bool debug_flag_ = false; diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index ae67365a..94053560 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -12,20 +12,37 @@ namespace xo { namespace gc { - ArenaAlloc::ArenaAlloc(const std::string & name, std::size_t rz, std::size_t z, bool debug_flag) + ArenaAlloc::ArenaAlloc(const std::string & name, +#ifdef REDLINE_MEMORY + std::size_t rz, +#endif + std::size_t z, bool debug_flag) { this->name_ = name; +#ifdef REDLINE_MEMORY this->lo_ = (new std::byte [rz + z]); +#else + this->lo_ = (new std::byte [z]); +#endif this->checkpoint_ = lo_; this->free_ptr_ = lo_; this->limit_ = lo_ + z; +#ifdef REDLINE_MEMORY this->redline_z_ = rz; this->hi_ = limit_ + rz; +#else + this->hi_ = limit_; +#endif this->debug_flag_ = debug_flag; if (!lo_) { +#ifdef REDLINE_MEMORY throw std::runtime_error(tostr("ArenaAlloc: allocation failed", xtag("size", rz + z))); +#else + throw std::runtime_error(tostr("ArenaAlloc: allocation failed", + xtag("size", z))); +#endif } } @@ -39,15 +56,25 @@ namespace xo { this->checkpoint_ = nullptr; this->free_ptr_ = nullptr; this->limit_ = nullptr; +#ifdef REDLINE_MEMORY this->redline_z_ = 0; +#endif this->hi_ = nullptr; this->debug_flag_ = false; } up - ArenaAlloc::make(const std::string & name, std::size_t rz, std::size_t z, bool debug_flag) + ArenaAlloc::make(const std::string & name, +#ifdef REDLINE_MEMORY + std::size_t rz, +#endif + std::size_t z, bool debug_flag) { - return up(new ArenaAlloc(name, rz, z, debug_flag)); + return up(new ArenaAlloc(name, +#ifdef REDLINE_MEMORY + rz, +#endif + z, debug_flag)); } void @@ -144,7 +171,11 @@ namespace xo { ArenaAlloc::clear() { this->set_free_ptr(lo_); +#ifdef REDLINE_MEMORY this->limit_ = hi_ - redline_z_; +#else + this->limit_ = hi_; +#endif } void @@ -184,10 +215,12 @@ namespace xo { return retval; } +#ifdef REDLINE_MEMORY void ArenaAlloc::release_redline_memory() { this->limit_ = this->hi_; } +#endif } /*namespace gc*/ } /*namespace xo*/ diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index afac26df..460ac770 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -255,13 +255,17 @@ namespace xo { std::byte * x = nursery_[role2int(role::to_space)]->alloc(z); if (!x) { + /* ListAlloc won't fail -- instead will increase heap size */ + this->request_gc(generation::nursery); +#ifdef REDLINE_MEMORY if (incr_gc_pending_ || full_gc_pending_) nursery_[role2int(role::to_space)]->release_redline_memory(); /* try (just once) more, maybe request fits in redline space */ x = nursery_[role2int(role::to_space)]->alloc(z); +#endif assert(x); } @@ -310,7 +314,9 @@ namespace xo { this->request_gc(generation::nursery); +#ifdef REDLINE_MEMORY nursery_[role2int(role::to_space)]->release_redline_memory(); +#endif retval = nursery_[role2int(role::to_space)]->alloc(z); } @@ -387,11 +393,13 @@ namespace xo { } } +#ifdef REDLINE_MEMORY void GC::release_redline_memory() { // not supported feature for GC } +#endif void GC::swap_nursery() diff --git a/src/alloc/ListAlloc.cpp b/src/alloc/ListAlloc.cpp index 9d727591..1f84d5c6 100644 --- a/src/alloc/ListAlloc.cpp +++ b/src/alloc/ListAlloc.cpp @@ -13,7 +13,9 @@ namespace xo { ListAlloc::ListAlloc(std::unique_ptr hd, ArenaAlloc * marked, std::size_t cz, std::size_t nz, std::size_t tz, +#ifdef REDLINE_MEMORY bool use_redline, +#endif bool debug_flag) : start_z_{cz}, hd_{std::move(hd)}, @@ -22,7 +24,9 @@ namespace xo { current_z_{cz}, next_z_{nz}, total_z_{tz}, +#ifdef REDLINE_MEMOORY use_redline_{use_redline}, +#endif debug_flag_{debug_flag} {} @@ -34,7 +38,11 @@ namespace xo { up ListAlloc::make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag) { - std::unique_ptr hd{ArenaAlloc::make(name, 0, cz, debug_flag)}; + std::unique_ptr hd{ArenaAlloc::make(name, +#ifdef REDLINE_MEMORY + 0, +#endif + cz, debug_flag)}; if (!hd) return nullptr; @@ -44,7 +52,9 @@ namespace xo { up retval{new ListAlloc(std::move(hd), marked, cz, nz, cz, +#ifdef REDLINE_MEMORY false /*!use_redline*/, +#endif debug_flag)}; return retval; @@ -248,20 +258,26 @@ namespace xo { current_z_ = 0; next_z_ = 0; total_z_ = 0; +#ifdef REDLINE_MEMORY use_redline_ = false; +#endif } bool ListAlloc::reset(std::size_t z) { +#ifdef REDLINE_MEMORY // warning: hd_->size() does not include redline memory hd_->release_redline_memory(); +#endif bool recycle_head_bucket = hd_ && (z <= hd_->size()); this->full_l_.clear(); this->marked_ = nullptr; +#ifdef REDLINE_MEMORY this->redlined_flag_ = false; +#endif if (recycle_head_bucket) { this->hd_->clear(); @@ -291,7 +307,11 @@ namespace xo { std::string name = hd_->name() + "+exp"; - std::unique_ptr new_alloc = ArenaAlloc::make(name, 0, cz, debug_flag_); + std::unique_ptr new_alloc = ArenaAlloc::make(name, +#ifdef REDLINE_MEMORY + 0, +#endif + cz, debug_flag_); if (!new_alloc) return false; @@ -325,6 +345,7 @@ namespace xo { return nullptr; } +#ifdef REDLINE_MEMORY void ListAlloc::release_redline_memory() { @@ -333,6 +354,7 @@ namespace xo { this->hd_->release_redline_memory(); } +#endif } /*namespace gc*/ } /*namespace xo*/ diff --git a/utest/ArenaAlloc.test.cpp b/utest/ArenaAlloc.test.cpp index aae2695b..0e4582e9 100644 --- a/utest/ArenaAlloc.test.cpp +++ b/utest/ArenaAlloc.test.cpp @@ -14,9 +14,15 @@ namespace xo { namespace { struct testcase_alloc { testcase_alloc(std::size_t rz, std::size_t z) - : redline_z_{rz}, arena_z_{z} {} + : +#ifdef REDLINE_MEMORY + redline_z_{rz}, +#endif + arena_z_{z} {} +#ifdef REDLINE_MEMORY std::size_t redline_z_; +#endif std::size_t arena_z_; }; @@ -35,7 +41,11 @@ namespace xo { constexpr bool c_debug_flag = false; - auto alloc = ArenaAlloc::make("linearalloc", tc.redline_z_, tc.arena_z_, c_debug_flag); + auto alloc = ArenaAlloc::make("linearalloc", +#ifdef REDLINE_MEMORY + tc.redline_z_, +#endif + tc.arena_z_, c_debug_flag); REQUIRE(alloc.get()); REQUIRE(alloc->name() == "linearalloc"); From a6e44308257817f6ce085e52cb7d4666365b4b9a Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 7 Aug 2025 18:32:14 -0500 Subject: [PATCH 10/69] xo-alloc / xo-object: utest coverage + assorted bugfixes --- include/xo/alloc/ArenaAlloc.hpp | 29 +---- include/xo/alloc/GC.hpp | 9 ++ include/xo/alloc/IAlloc.hpp | 6 +- include/xo/alloc/ListAlloc.hpp | 16 +-- include/xo/alloc/generation.hpp | 10 ++ src/alloc/ArenaAlloc.cpp | 70 +++++------ src/alloc/CMakeLists.txt | 1 + src/alloc/Forwarding1.cpp | 2 +- src/alloc/GC.cpp | 137 ++++++++++---------- src/alloc/GcStatistics.cpp | 40 +++--- src/alloc/ListAlloc.cpp | 199 +++++++++++++++-------------- src/alloc/ObjectStatistics.cpp | 12 +- src/alloc/generation.cpp | 22 ++++ utest/ArenaAlloc.test.cpp | 9 -- utest/CMakeLists.txt | 8 +- utest/Forwarding1.test.cpp | 85 +++++++++++++ utest/GC.test.cpp | 1 + utest/GcStatistics.test.cpp | 216 ++++++++++++++++++++++++++++++++ utest/ListAlloc.test.cpp | 58 +++++++++ utest/ObjectStatistics.test.cpp | 185 +++++++++++++++++++++++++++ utest/generation.test.cpp | 38 ++++++ 21 files changed, 865 insertions(+), 288 deletions(-) create mode 100644 src/alloc/generation.cpp create mode 100644 utest/Forwarding1.test.cpp create mode 100644 utest/GcStatistics.test.cpp create mode 100644 utest/ListAlloc.test.cpp create mode 100644 utest/ObjectStatistics.test.cpp create mode 100644 utest/generation.test.cpp diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 51b72c9e..8e162a95 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -15,36 +15,20 @@ namespace xo { * * @text * - * before @ref release_redline_memory - * - * <-----allocated----> <-free-> <-reserved-> - * XXXXXXXXXXXXXXXXXXXX______________________ - * ^ ^ ^ ^ - * lo free redline hi - * limit - * - * after @ref release_redline_memory - * * <-----allocated----> <--------free-------> * XXXXXXXXXXXXXXXXXXXX______________________ * ^ ^ ^ * lo free hi * limit * @endtext - * - * TODO: rename to ArenaAlloc **/ class ArenaAlloc : public IAlloc { public: ~ArenaAlloc(); /** create allocator with capacity @p z, - * with reserved capacity @p redline_z. **/ static up make(const std::string & name, -#ifdef REDLINE_MEMORY - std::size_t redline_z, -#endif std::size_t z, bool debug_flag); @@ -56,7 +40,7 @@ namespace xo { // inherited from IAlloc... - virtual const std::string & name() const final override { return name_; } + virtual const std::string & name() const final override; virtual std::size_t size() const final override; virtual std::size_t available() const final override; virtual std::size_t allocated() const final override; @@ -64,19 +48,14 @@ namespace xo { virtual bool is_before_checkpoint(const void * x) const final override; virtual std::size_t before_checkpoint() const final override; virtual std::size_t after_checkpoint() const final override; + virtual bool debug_flag() const final override; virtual void clear() final override; virtual void checkpoint() final override; virtual std::byte * alloc(std::size_t z) final override; -#ifdef REDLINE_MEMORY - virtual void release_redline_memory() final override; -#endif private: ArenaAlloc(const std::string & name, -#ifdef REDLINE_MEMORY - std::size_t rz, -#endif std::size_t z, bool debug_flag); private: @@ -99,10 +78,6 @@ namespace xo { std::byte * free_ptr_ = nullptr; /** soft limit: end of released memory **/ std::byte * limit_ = nullptr; -#ifdef REDLINE_MEMORY - /** amount of last-resort memory to reserve **/ - std::size_t redline_z_ = 0; -#endif /** hard limit: end of allocated memory **/ std::byte * hi_ = nullptr; /** true to enable detailed debug logging **/ diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 679e5286..4560f3ac 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -199,6 +199,15 @@ namespace xo { #endif private: + ListAlloc * nursery_to() const { return nursery(role::to_space); } + ListAlloc * nursery_from() const { return nursery(role::from_space); } + + ListAlloc * tenured_to() const { return tenured(role::to_space); } + ListAlloc * tenured_from() const { return tenured(role::from_space); } + + ListAlloc * nursery(role r) const { return nursery_[role2int(r)].get(); } + ListAlloc * tenured(role r) const { return tenured_[role2int(r)].get(); } + /** begin GC now **/ void execute_gc(generation g); /** cleanup phase. aux function for @ref execute_gc **/ diff --git a/include/xo/alloc/IAlloc.hpp b/include/xo/alloc/IAlloc.hpp index 2eacd9cc..b8270f53 100644 --- a/include/xo/alloc/IAlloc.hpp +++ b/include/xo/alloc/IAlloc.hpp @@ -52,7 +52,7 @@ namespace xo { /** number of bytes allocated since @ref checkpoint **/ virtual std::size_t after_checkpoint() const = 0; /** @return true iff debug logging enabled **/ - virtual bool debug_flag() const { return false; } + virtual bool debug_flag() const = 0; /** reset allocator to empty state. **/ virtual void clear() = 0; @@ -76,10 +76,6 @@ namespace xo { * Only used in @ref GC. Default implementation asserts and returns nullptr **/ virtual std::byte * alloc_gc_copy(std::size_t z, const void * src); -#ifdef REDLINE_MEMORY - /** release last-resort reserved memory **/ - virtual void release_redline_memory() = 0; -#endif }; } /*namespace gc*/ diff --git a/include/xo/alloc/ListAlloc.hpp b/include/xo/alloc/ListAlloc.hpp index 627bbf50..ac2f9894 100644 --- a/include/xo/alloc/ListAlloc.hpp +++ b/include/xo/alloc/ListAlloc.hpp @@ -29,9 +29,6 @@ namespace xo { ListAlloc(std::unique_ptr hd, ArenaAlloc * marked, std::size_t cz, std::size_t nz, std::size_t tz, -#ifdef REDLINE_MEMORY - bool use_redline, -#endif bool debug_flag); ~ListAlloc(); @@ -40,8 +37,8 @@ namespace xo { /** reset to have at least @p z bytes of storage **/ bool reset(std::size_t z); - /** expand bucket list to accomodate a requrest of size @p z **/ - bool expand(std::size_t z); + /** expand bucket list to accomodate a request of size @p z **/ + bool expand(std::size_t z, const std::string & name); /** current free pointer **/ std::byte * free_ptr() const; @@ -64,13 +61,11 @@ namespace xo { virtual bool is_before_checkpoint(const void * x) const final override; virtual std::size_t before_checkpoint() const final override; virtual std::size_t after_checkpoint() const final override; + virtual bool debug_flag() const final override; virtual void clear() final override; virtual void checkpoint() final override; virtual std::byte * alloc(std::size_t z) final override; -#ifdef REDLINE_MEMORY - virtual void release_redline_memory() final override; -#endif private: /** **/ @@ -89,11 +84,6 @@ namespace xo { std::size_t next_z_ = 0; /** total size of @ref hd_ + contents of @ref full_l_ **/ std::size_t total_z_ = 0; -#ifdef REDLINE_MEMORY - bool use_redline_ = false; - bool redlined_flag_ = false; -#endif - /** true to enable debug logging **/ bool debug_flag_ = false; }; diff --git a/include/xo/alloc/generation.hpp b/include/xo/alloc/generation.hpp index 6a5a7c2e..82c48808 100644 --- a/include/xo/alloc/generation.hpp +++ b/include/xo/alloc/generation.hpp @@ -3,6 +3,9 @@ * author: Roland Conybeare, Aug 2025 */ +#pragma once + +#include #include namespace xo { @@ -15,6 +18,13 @@ namespace xo { constexpr std::size_t gen2int(generation x) { return static_cast(x); } + const char * gen2str(generation x); + + inline std::ostream & operator<<(std::ostream & os, generation x) { + os << gen2str(x); + return os; + } + enum class generation_result { nursery, tenured, diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index 94053560..e978f566 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -13,36 +13,19 @@ namespace xo { namespace gc { ArenaAlloc::ArenaAlloc(const std::string & name, -#ifdef REDLINE_MEMORY - std::size_t rz, -#endif std::size_t z, bool debug_flag) { this->name_ = name; -#ifdef REDLINE_MEMORY - this->lo_ = (new std::byte [rz + z]); -#else this->lo_ = (new std::byte [z]); -#endif this->checkpoint_ = lo_; this->free_ptr_ = lo_; this->limit_ = lo_ + z; -#ifdef REDLINE_MEMORY - this->redline_z_ = rz; - this->hi_ = limit_ + rz; -#else this->hi_ = limit_; -#endif this->debug_flag_ = debug_flag; if (!lo_) { -#ifdef REDLINE_MEMORY - throw std::runtime_error(tostr("ArenaAlloc: allocation failed", - xtag("size", rz + z))); -#else throw std::runtime_error(tostr("ArenaAlloc: allocation failed", xtag("size", z))); -#endif } } @@ -56,24 +39,15 @@ namespace xo { this->checkpoint_ = nullptr; this->free_ptr_ = nullptr; this->limit_ = nullptr; -#ifdef REDLINE_MEMORY - this->redline_z_ = 0; -#endif this->hi_ = nullptr; this->debug_flag_ = false; } up ArenaAlloc::make(const std::string & name, -#ifdef REDLINE_MEMORY - std::size_t rz, -#endif std::size_t z, bool debug_flag) { return up(new ArenaAlloc(name, -#ifdef REDLINE_MEMORY - rz, -#endif z, debug_flag)); } @@ -97,22 +71,36 @@ namespace xo { ArenaAlloc::capture_object_statistics(capture_phase phase, ObjectStatistics * p_dest) const { + scope log(XO_DEBUG(debug_flag_), + xtag("name", name_), + xtag("capacity", limit_ - lo_), + xtag("alloc", free_ptr_ - lo_), + xtag("lo", (void*)lo_), + xtag("free_ptr", (void*)free_ptr_)); + using xo::reflect::TaggedPtr; std::byte * p = lo_; while (p < free_ptr_) { - Object * obj = reinterpret_cast(p); - TaggedPtr tp = obj->self_tp(); - std::size_t z = obj->_shallow_size(); + Object * obj = reinterpret_cast(p); + TaggedPtr tp = obj->self_tp(); + std::size_t z = obj->_shallow_size(); std::uint32_t id = tp.td()->id().id(); + log && log(xtag("obj", (void*)obj), + xtag("z", z), + xtag("typeid", id)); + if (p_dest->per_type_stats_v_.size() < id + 1) p_dest->per_type_stats_v_.resize(id + 1); PerObjectTypeStatistics & dest = p_dest->per_type_stats_v_.at(id); dest.td_ = tp.td(); + + log && log(xtag("td", tp.td()->short_name())); + switch (phase) { case capture_phase::sab: ++dest.scanned_n_; @@ -130,6 +118,11 @@ namespace xo { assert(p == free_ptr_); } + const std::string & + ArenaAlloc::name() const { + return name_; + } + std::size_t ArenaAlloc::size() const { return limit_ - lo_; @@ -167,15 +160,17 @@ namespace xo { return free_ptr_ - checkpoint_; } + bool + ArenaAlloc::debug_flag() const + { + return debug_flag_; + } + void ArenaAlloc::clear() { this->set_free_ptr(lo_); -#ifdef REDLINE_MEMORY - this->limit_ = hi_ - redline_z_; -#else this->limit_ = hi_; -#endif } void @@ -204,7 +199,7 @@ namespace xo { std::byte * retval = this->free_ptr_; - log && log(xtag("self", name_), xtag("z0", z0), xtag("+pad", dz), xtag("z1", z1)); + log && log(xtag("self", name_), xtag("z0", z0), xtag("+pad", dz), xtag("z1", z1), xtag("avail", this->available())); if (free_ptr_ + z1 > limit_) { return nullptr; @@ -215,13 +210,6 @@ namespace xo { return retval; } -#ifdef REDLINE_MEMORY - void - ArenaAlloc::release_redline_memory() { - this->limit_ = this->hi_; - } -#endif - } /*namespace gc*/ } /*namespace xo*/ diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index 55b8641c..589f3b90 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -10,6 +10,7 @@ set(SELF_SRCS ObjectStatistics.cpp Object.cpp Forwarding1.cpp + generation.cpp ) xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp index 5a941c82..4b47e4f2 100644 --- a/src/alloc/Forwarding1.cpp +++ b/src/alloc/Forwarding1.cpp @@ -26,7 +26,7 @@ namespace xo { void Forwarding1::display(std::ostream & os) const { - os << ""; + os << "self_tp().td()->short_name()) << ">"; } Object * diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 460ac770..2990adf6 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -175,7 +175,8 @@ namespace xo { GcStatisticsExt retval = GcStatisticsExt(this->native_gc_statistics()); retval.nursery_z_ = nursery_[role2int(role::to_space)]->size(); - retval.nursery_before_checkpoint_z_ = nursery_[role2int(role::to_space)]->before_checkpoint(); + retval.nursery_before_checkpoint_z_ = this->nursery_to()->before_checkpoint(); + retval.nursery_after_checkpoint_z_ = this->nursery_to()->after_checkpoint(); retval.tenured_z_ = tenured_[role2int(role::to_space)]->size(); return retval; @@ -254,21 +255,9 @@ namespace xo { { std::byte * x = nursery_[role2int(role::to_space)]->alloc(z); - if (!x) { - /* ListAlloc won't fail -- instead will increase heap size */ + /* ListAlloc won't fail unless we exhaust memory -- instead will increase heap size */ - this->request_gc(generation::nursery); - -#ifdef REDLINE_MEMORY - if (incr_gc_pending_ || full_gc_pending_) - nursery_[role2int(role::to_space)]->release_redline_memory(); - - /* try (just once) more, maybe request fits in redline space */ - x = nursery_[role2int(role::to_space)]->alloc(z); -#endif - - assert(x); - } + assert(x); return x; } @@ -308,18 +297,6 @@ namespace xo { log && log("nursery"); retval = nursery_[role2int(role::to_space)]->alloc(z); - - if (!retval) { - /* nursery space exhausted !? */ - - this->request_gc(generation::nursery); - -#ifdef REDLINE_MEMORY - nursery_[role2int(role::to_space)]->release_redline_memory(); -#endif - - retval = nursery_[role2int(role::to_space)]->alloc(z); - } } } break; @@ -393,14 +370,6 @@ namespace xo { } } -#ifdef REDLINE_MEMORY - void - GC::release_redline_memory() - { - // not supported feature for GC - } -#endif - void GC::swap_nursery() { @@ -428,40 +397,62 @@ namespace xo { void GC::swap_spaces(generation target) { - scope log(XO_DEBUG(this->debug_flag())); + scope log(XO_DEBUG(this->debug_flag()), xtag("upto", target)); // will be copying into the memory regions currently labelled FromSpace /* gc will copy some to-be-determined amount in [0..promote_z] from nursery->tenured generation. */ - std::size_t promote_z = nursery_[role2int(role::to_space)]->before_checkpoint(); + std::size_t max_promote_z = nursery_[role2int(role::to_space)]->before_checkpoint(); + + log && log(xtag("max_promote_z", max_promote_z)); + if (target == generation::tenured) { /* gc on tenured generation may need this much space */ - std::size_t tenured_z = (tenured_[role2int(role::to_space)]->allocated() - + promote_z - + full_gc_threshold_); + std::size_t need_tenured_z = (tenured_[role2int(role::to_space)]->allocated() + + max_promote_z + + full_gc_threshold_); - tenured_[role2int(role::from_space)]->reset(tenured_z); + log && log("need_tenured_z", need_tenured_z); + + tenured_from()->reset(need_tenured_z); this->swap_tenured(); } else { - if (tenured_[role2int(role::to_space)]->available() < promote_z) { - tenured_[role2int(role::to_space)]->expand(promote_z); + std::size_t avail_tenured_z = tenured_[role2int(role::to_space)]->available(); + + log && log(xtag("avail_tenured_z", avail_tenured_z)); + + if (avail_tenured_z < max_promote_z) { + ListAlloc * tenured_to = this->tenured_to(); + + tenured_to->expand(max_promote_z, tenured_to->name() + "+"); } } - nursery_[role2int(role::from_space)]->reset(nursery_[role2int(role::to_space)]->allocated() - - promote_z - + incr_gc_threshold_); + /* subtracting max_promote_z is correct here, since anything not promoted is garbage */ + std::size_t need_nursery_z = (nursery(role::to_space)->allocated() + - max_promote_z + + incr_gc_threshold_); + + log && log(xtag("need_nursery_z", need_nursery_z)); + + /* (from-space is about to become to-space, to receive surviving nursery objects) */ + nursery(role::from_space)->reset(need_nursery_z); + this->swap_nursery(); this->swap_mutation_log(); - log && log(xtag("nursery.from", nursery_[role2int(role::from_space)]->name())); - log && log(xtag("nursery.to", nursery_[role2int(role::to_space) ]->name())); - log && log(xtag("tenured.from", tenured_[role2int(role::from_space)]->name())); - log && log(xtag("tenured.to", tenured_[role2int(role::to_space) ]->name())); + ListAlloc * N_from = nursery(role::from_space); + log && log(xtag("nursery.from", N_from->name()), xtag("size", N_from->size())); + ListAlloc * N_to = nursery(role::to_space); + log && log(xtag("nursery.to", N_to->name()), xtag("size", N_to->size())); + ListAlloc * T_from = tenured(role::from_space); + log && log(xtag("tenured.from", T_from->name()), xtag("size", T_from->size())); + ListAlloc * T_to = tenured(role::to_space); + log && log(xtag("tenured.to", T_to->name()), xtag("size", T_to->size())); } /*swap_spaces*/ @@ -504,8 +495,12 @@ namespace xo { void GC::copy_globals(generation upto) { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("roots", gc_root_v_.size())); + for (Object ** pp_root : gc_root_v_) { - this->copy_object(pp_root, upto, &object_statistics_sae_[gen2int(upto)]); + this->copy_object(pp_root, upto, + &object_statistics_sae_[gen2int(upto)]); } } @@ -750,28 +745,30 @@ namespace xo { { scope log(XO_DEBUG(config_.debug_flag_)); - std::size_t N_allocated = nursery_[role2int(role::from_space)]->after_checkpoint(); - std::size_t T_allocated = tenured_[role2int(role::from_space)]->after_checkpoint(); + std::size_t N_allocated = nursery_from()->after_checkpoint(); + std::size_t T_allocated = tenured_from()->after_checkpoint(); - std::size_t N_before_gc = nursery_[role2int(role::from_space)]->allocated(); - std::size_t T_before_gc = tenured_[role2int(role::from_space)]->allocated(); + std::size_t N_before_gc = nursery_from()->allocated(); + std::size_t T_before_gc = tenured_from()->allocated(); - std::size_t N_after_gc = nursery_[role2int(role::to_space)]->allocated(); - std::size_t T_after_gc = tenured_[role2int(role::to_space)]->allocated(); + std::size_t N_after_gc = nursery_to()->allocated(); + std::size_t T_after_gc = tenured_to()->allocated(); //std::byte * N_free_ptr = nursery_[role2int(role::to_space)]->free_ptr(); - std::size_t promote_z = gc_statistics_.total_promoted_ - gc_statistics_.total_promoted_sab_; + std::size_t promote_z = (gc_statistics_.total_promoted_ + - gc_statistics_.total_promoted_sab_); - this->nursery_[role2int(role::from_space)]->reset(0); - this->tenured_[role2int(role::from_space)]->reset(0); + /* Don't reset from-space here, it's unnecessary. + * Would be permissible, but interferes with GC object modelling in + * xo-object/utest/GC.test.cpp + */ + //this->nursery_[role2int(role::from_space)]->reset(0); + //this->tenured_[role2int(role::from_space)]->reset(0); /* objects currenty in to-space nursery have survived one collection */ - this->nursery_[role2int(role::to_space)]->checkpoint(); - - // nursery_[role2int(role::to_space)]->set_redline(nursery_[role2int(role::to_space)]->allocated() + incr_gc_threshold_) - + this->nursery_to()->checkpoint(); if (upto == generation::tenured) - this->tenured_[role2int(role::to_space)]->checkpoint(); + this->tenured_to()->checkpoint(); if (log) { log(xtag("N_allocated", N_allocated)); @@ -819,7 +816,7 @@ namespace xo { this->capture_object_statistics(upto, capture_phase::sab); - log && log("step 1 : swap to/from roles"); + log && log("step 1 : swap to/from roles"); this->swap_spaces(upto); @@ -829,15 +826,15 @@ namespace xo { log && log("step 2b: TODO: copy pinned"); - log && log("step 3 : forward mutation log"); + log && log("step 3 : forward mutation log"); this->forward_mutation_log(upto); - log && log("step 4 : TODO: notify destructor log"); + log && log("step 4 : TODO: notify destructor log"); - log && log("step 5 : TODO: keep reachable weak pointers"); + log && log("step 5 : TODO: keep reachable weak pointers"); - log && log("step 6 : cleanup"); + log && log("step 6 : cleanup"); this->cleanup_phase(upto); diff --git a/src/alloc/GcStatistics.cpp b/src/alloc/GcStatistics.cpp index d54c3cf2..7b7a6ff9 100644 --- a/src/alloc/GcStatistics.cpp +++ b/src/alloc/GcStatistics.cpp @@ -32,12 +32,12 @@ namespace xo { PerGenerationStatistics::display(std::ostream & os) const { os << ""; } @@ -62,9 +62,9 @@ namespace xo { GcStatistics::display(std::ostream & os) const { os << ""; } diff --git a/src/alloc/ListAlloc.cpp b/src/alloc/ListAlloc.cpp index 1f84d5c6..6e37a325 100644 --- a/src/alloc/ListAlloc.cpp +++ b/src/alloc/ListAlloc.cpp @@ -5,6 +5,7 @@ #include "ListAlloc.hpp" #include "ArenaAlloc.hpp" +#include "xo/indentlog/scope.hpp" #include #include @@ -13,9 +14,6 @@ namespace xo { ListAlloc::ListAlloc(std::unique_ptr hd, ArenaAlloc * marked, std::size_t cz, std::size_t nz, std::size_t tz, -#ifdef REDLINE_MEMORY - bool use_redline, -#endif bool debug_flag) : start_z_{cz}, hd_{std::move(hd)}, @@ -24,9 +22,6 @@ namespace xo { current_z_{cz}, next_z_{nz}, total_z_{tz}, -#ifdef REDLINE_MEMOORY - use_redline_{use_redline}, -#endif debug_flag_{debug_flag} {} @@ -39,9 +34,6 @@ namespace xo { ListAlloc::make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag) { std::unique_ptr hd{ArenaAlloc::make(name, -#ifdef REDLINE_MEMORY - 0, -#endif cz, debug_flag)}; if (!hd) @@ -52,9 +44,6 @@ namespace xo { up retval{new ListAlloc(std::move(hd), marked, cz, nz, cz, -#ifdef REDLINE_MEMORY - false /*!use_redline*/, -#endif debug_flag)}; return retval; @@ -134,34 +123,31 @@ namespace xo { bool ListAlloc::is_before_checkpoint(const void * x) const { if (!marked_) - return false; + return true; - if ((marked_ == hd_.get()) && hd_->contains(x)) - return hd_->is_before_checkpoint(x); + if (marked_ && marked_->contains(x)) + return marked_->is_before_checkpoint(x); /* - * 1. allocs in full_l_ appear in youngest-to-oldest order - * 2. allocators that appear before marked_ in full_l_ count as 'after checkpoint' - * 3. allocators that appear after marked_ in full_l_ count as 'before checkpoint' + * 1. allocs in full_l_ appear in oldest-to-youngest order + * 2. allocators that appear before marked_ in full_l_ count as 'before checkpoint' + * 3. allocators that appear after marked_ in full_l_ count as 'after checkpoint' */ - bool younger_than_marked = true; + bool older_than_marked = true; for (const auto & alloc : full_l_) { - if (younger_than_marked) { + if (older_than_marked) { if (alloc.get() == marked_) { /* nothing else to test on this iteration, * already checked .marked_ specifically */ - younger_than_marked = false; + break; } else { - /* after checkpoint */ + /* before checkpoint */ if (alloc->contains(x)) - return false; + return true; } - } else { - if (alloc->contains(x)) - return true; } } @@ -171,55 +157,63 @@ namespace xo { std::size_t ListAlloc::before_checkpoint() const { + scope log(XO_DEBUG(false && debug_flag_), xtag("marked", marked_ ? marked_->name() : "")); + if (marked_) { if (full_l_.empty()) { assert(marked_ == hd_.get()); return marked_->before_checkpoint(); + } else { + std::size_t z = 0; + + /* control here: .marked & .full_l non-empty. */ + if (hd_.get() == marked_) { + z += hd_->before_checkpoint(); + + /* anything in .full_l is older than marked .hd */ + for (const auto & alloc : full_l_) { + z += alloc->allocated(); + } + + return z; + } else { + /* messiest case: .marked is true, + * and not the youngest arena + */ + + /* full_l always in increasing time order: oldest-to-youngest order */ + size_t i_alloc = 0; + for (const auto & alloc : full_l_) { + log && log(xtag("i_alloc", i_alloc), + xtag("alloc", alloc->name()), + xtag("z", z)); + + if (alloc.get() == marked_) { + log && log("marked", xtag("+z", marked_->before_checkpoint())); + z += marked_->before_checkpoint(); + break; + } else { + log && log("older than marked", xtag("+z", alloc->allocated())); + z += alloc->allocated(); + } + ++i_alloc; + } + } + + return z; } } else { - /* count everything allocated */ + /* count *everything* allocated */ return this->allocated(); } - - std::size_t z = 0; - - /* control here: .marked & .full_l non-empty. */ - if (hd_.get() == marked_) { - z += hd_->before_checkpoint(); - - /* anything in .full_l older than marked .hd */ - for (const auto & alloc : full_l_) { - z += alloc->allocated(); - } - - return z; - } else { - /* messiest case: .marked is true, - * and not the youngest arena - */ - bool younger_than_marked = true; - - for (const auto & alloc : full_l_) { - if (younger_than_marked) { - if (alloc.get() == marked_) { - younger_than_marked = false; - z += marked_->before_checkpoint(); - } else { - ; - } - } else { - z += alloc->allocated(); - } - } - } - - return z; } std::size_t ListAlloc::after_checkpoint() const { + scope log(XO_DEBUG(false && debug_flag_), xtag("marked", marked_ ? marked_->name() : "")); + if (!marked_) return 0; @@ -229,25 +223,44 @@ namespace xo { return marked_->after_checkpoint(); } - bool younger_than_marked = true; + bool older_than_marked = true; std::size_t z = 0; + std::size_t i_alloc = 0; for (const auto & alloc : full_l_) { - if (younger_than_marked) { + log && log(xtag("i_alloc", i_alloc), + xtag("alloc", alloc->name()), + xtag("z", z)); + + if (older_than_marked) { if (alloc.get() == marked_) { - younger_than_marked = false; + log && log("marked", xtag("+z", marked_->after_checkpoint())); + older_than_marked = false; z += marked_->after_checkpoint(); - break; - } else { - z += alloc->allocated(); } + } else { + /* younger than marked */ + log && log("younger", xtag("+z", alloc->allocated())); + z += alloc->allocated(); } + + ++i_alloc; } + /** head must be included, since it's always the youngest bucket **/ + z += hd_->after_checkpoint(); + + log && log("z", z); + return z; } + bool + ListAlloc::debug_flag() const { + return debug_flag_; + } + void ListAlloc::clear() { // general hygiene @@ -258,26 +271,17 @@ namespace xo { current_z_ = 0; next_z_ = 0; total_z_ = 0; -#ifdef REDLINE_MEMORY - use_redline_ = false; -#endif } bool ListAlloc::reset(std::size_t z) { -#ifdef REDLINE_MEMORY - // warning: hd_->size() does not include redline memory - hd_->release_redline_memory(); -#endif + scope log(XO_DEBUG(debug_flag_), xtag("z", z)); bool recycle_head_bucket = hd_ && (z <= hd_->size()); this->full_l_.clear(); this->marked_ = nullptr; -#ifdef REDLINE_MEMORY - this->redlined_flag_ = false; -#endif if (recycle_head_bucket) { this->hd_->clear(); @@ -285,16 +289,22 @@ namespace xo { return true; } else { + std::string old_name = this->hd_->name(); + this->hd_.reset(nullptr); this->total_z_ = 0; - return this->expand(z); + return this->expand(z, old_name + "+"); } } bool - ListAlloc::expand(std::size_t z) + ListAlloc::expand(std::size_t z, const std::string & name) { + scope log(XO_DEBUG(debug_flag_), xtag("name", name)); + + //log && log("before", xtag("before_ckp", this->before_checkpoint())); + std::size_t cz = current_z_; std::size_t nz = next_z_; std::size_t tz; @@ -305,12 +315,9 @@ namespace xo { nz = tz; } while (cz < z); - std::string name = hd_->name() + "+exp"; + log && log("expand to", xtag("cz", cz)); std::unique_ptr new_alloc = ArenaAlloc::make(name, -#ifdef REDLINE_MEMORY - 0, -#endif cz, debug_flag_); if (!new_alloc) @@ -320,41 +327,43 @@ namespace xo { this->next_z_ = nz; this->total_z_ += cz; + if (hd_) + this->full_l_.push_back(std::move(hd_)); + this->hd_ = std::move(new_alloc); + //log && log("after", xtag("before_ckp", this->before_checkpoint())); + return true; } void ListAlloc::checkpoint() { + scope log(XO_DEBUG(debug_flag_)); + hd_->checkpoint(); this->marked_ = hd_.get(); + + log && log(xtag("hd", (void*)hd_.get()), xtag("marked", (void*)marked_)); } std::byte * ListAlloc::alloc(std::size_t z) { + scope log(XO_DEBUG(debug_flag_)); + std::byte * retval = hd_->alloc(z); if (retval) return retval; - if (this->expand(z)) + log && log("space exhausted -> expand"); + + if (this->expand(z, hd_->name() + "+")) return hd_->alloc(z); return nullptr; } - -#ifdef REDLINE_MEMORY - void - ListAlloc::release_redline_memory() - { - if (use_redline_) - redlined_flag_ = true; - - this->hd_->release_redline_memory(); - } -#endif } /*namespace gc*/ } /*namespace xo*/ diff --git a/src/alloc/ObjectStatistics.cpp b/src/alloc/ObjectStatistics.cpp index a69928dc..863a02cb 100644 --- a/src/alloc/ObjectStatistics.cpp +++ b/src/alloc/ObjectStatistics.cpp @@ -14,13 +14,13 @@ namespace xo { { os << "short_name()); + os << xrtag("td", td_->short_name()); else - os << rtag("td", "nullptr"); - os << rtag("scanned_n", scanned_n_) - << rtag("scanned_z", scanned_z_) - << rtag("survive_n", survive_n_) - << rtag("survive_z", survive_z_) + os << xrtag("td", "nullptr"); + os << xrtag("scanned_n", scanned_n_) + << xrtag("scanned_z", scanned_z_) + << xrtag("survive_n", survive_n_) + << xrtag("survive_z", survive_z_) << ">"; } diff --git a/src/alloc/generation.cpp b/src/alloc/generation.cpp new file mode 100644 index 00000000..54c151ac --- /dev/null +++ b/src/alloc/generation.cpp @@ -0,0 +1,22 @@ +/* generation.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "generation.hpp" + +namespace xo { + namespace gc { + const char * gen2str(generation x) { + switch (x) { + case generation::nursery: return "nursery"; + case generation::tenured: return "tenured"; + case generation::N: break; + } + return "?generation"; + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* generation.cpp */ diff --git a/utest/ArenaAlloc.test.cpp b/utest/ArenaAlloc.test.cpp index 0e4582e9..52f135f3 100644 --- a/utest/ArenaAlloc.test.cpp +++ b/utest/ArenaAlloc.test.cpp @@ -15,14 +15,8 @@ namespace xo { struct testcase_alloc { testcase_alloc(std::size_t rz, std::size_t z) : -#ifdef REDLINE_MEMORY - redline_z_{rz}, -#endif arena_z_{z} {} -#ifdef REDLINE_MEMORY - std::size_t redline_z_; -#endif std::size_t arena_z_; }; @@ -42,9 +36,6 @@ namespace xo { constexpr bool c_debug_flag = false; auto alloc = ArenaAlloc::make("linearalloc", -#ifdef REDLINE_MEMORY - tc.redline_z_, -#endif tc.arena_z_, c_debug_flag); REQUIRE(alloc.get()); diff --git a/utest/CMakeLists.txt b/utest/CMakeLists.txt index b38a442f..907bafcb 100644 --- a/utest/CMakeLists.txt +++ b/utest/CMakeLists.txt @@ -7,7 +7,13 @@ set(UTEST_SRCS alloc_utest_main.cpp IAlloc.test.cpp ArenaAlloc.test.cpp - GC.test.cpp) + ListAlloc.test.cpp + GC.test.cpp + GcStatistics.test.cpp + ObjectStatistics.test.cpp + Forwarding1.test.cpp + generation.test.cpp +) xo_add_utest_executable(${UTEST_EXE} ${UTEST_SRCS}) xo_self_dependency(${UTEST_EXE} xo_alloc) diff --git a/utest/Forwarding1.test.cpp b/utest/Forwarding1.test.cpp new file mode 100644 index 00000000..e5dfb1fe --- /dev/null +++ b/utest/Forwarding1.test.cpp @@ -0,0 +1,85 @@ +/* Forwarding1.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "Forwarding1.hpp" +#include "ArenaAlloc.hpp" +#include "xo/reflect/Reflect.hpp" +#include +#include + +namespace xo { + using xo::reflect::Reflect; + using xo::obj::Forwarding1; + + namespace gc { + namespace { + class DummyObject : public Object { + public: + explicit DummyObject(const char * data) { + ::strncpy(data_.data(), data, 128); + } + + gp member() const { return member_; } + void assign_member(Object * x) { + Object::mm->assign_member(this, member_.ptr_address(), x); + } + + TaggedPtr self_tp() const final override { + return Reflect::make_tp(const_cast(this)); + } + + void display(std::ostream & os) const final override { os << data_; } + + virtual std::size_t _shallow_size() const final override { return sizeof(*this); } + virtual Object * _shallow_copy() const final override { return new (Cpof(this)) DummyObject(*this); } + virtual std::size_t _forward_children() final override { return _shallow_size(); } + + private: + std::array data_; + gp member_; + }; + } + + TEST_CASE("Forwarding1", "[gc][alloc]") + { + bool saved = tag_config::tag_color_enabled; + tag_config::tag_color_enabled = false; + + gp obj = new DummyObject("Well, I wasn't expecting that!"); + gp fwd = new Forwarding1(obj); + + REQUIRE(fwd->_destination() == obj.ptr()); + REQUIRE(fwd->_offset_destination(fwd.ptr()) == obj.ptr()); + + REQUIRE(fwd->self_tp().td()->short_name() == "Forwarding1"); + + std::stringstream ss; + ss << fwd; + + REQUIRE(ss.str() == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("IAlloc.assign_member", "[gc][alloc]") + { + /* not giving this nit it's own translation unit. + */ + + gp obj = new DummyObject("This also a surprise.."); + + up arena = ArenaAlloc::make("test", 1024, false); + + Object::mm = arena.get(); + + obj->assign_member(obj.ptr()); + + REQUIRE(obj->member().ptr() == obj.ptr()); + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end Forwarding1.test.cpp */ diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index c36deb2e..d961e3dd 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -37,6 +37,7 @@ namespace xo { .initial_tenured_z_ = tc.tenured_z_}); REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); REQUIRE(gc->size() == tc.nursery_z_ + tc.tenured_z_); REQUIRE(gc->allocated() == 0); REQUIRE(gc->available() == tc.nursery_z_); diff --git a/utest/GcStatistics.test.cpp b/utest/GcStatistics.test.cpp new file mode 100644 index 00000000..5f2fbe96 --- /dev/null +++ b/utest/GcStatistics.test.cpp @@ -0,0 +1,216 @@ +/* @file GcStatistics.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/GcStatistics.hpp" +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/tostr.hpp" +#include "xo/indentlog/print/hex.hpp" +#include +#include + +namespace xo { + using xo::gc::GcStatistics; + using xo::gc::GcStatisticsExt; + using xo::gc::PerGenerationStatistics; + using xo::print::ppconfig; + + namespace ut { + TEST_CASE("PerGenerationStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerGenerationStatistics stats; + + std::string s = tostr(stats); + + //std::cerr << hex_view(s.c_str(), s.c_str() + s.length(), true /*as_text*/) << std::endl; + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + GcStatistics stats; + + std::string s = tostr(stats); + + REQUIRE(s == + "" + /***/ " ]" + /**/ " :total_allocated 0" + /**/ " :total_promoted_sab 0" + ">"); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatisticsExt", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + GcStatisticsExt stats({}); + + std::string s = tostr(stats); + + REQUIRE(s == " ] :total_allocated 0 :total_promoted_sab 0 :nursery_z 0 :nursery_before_ckp_z 0 :nursery_after_ckp_z 0 :tenured_z 0 :n_mutation 0 :n_logged_mutation 0 :n_xgen_mutation 0 :n_xckp_mutation 0>"); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatistics-pretty", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + std::stringstream ss; + ppconfig ppc; + GcStatistics stats; + + std::string actual = toppstr2(ppc, stats); + std::string expected + = (",\n" + " ]\n" + " :total_allocated 0\n" + " :total_promoted_sab 0\n" + " :total_promoted 0\n" + " :n_mutation 0\n" + " :n_logged_mutation 0\n" + " :n_xgen_mutation 0\n" + " :n_xckp_mutation 0>"); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + REQUIRE(actual == expected); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatisticsExt-pretty", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + std::stringstream ss; + ppconfig ppc; + GcStatisticsExt stats({}); + + std::string actual = toppstr2(ppc, stats); + std::string expected + = (",\n" + " ]\n" + " :total_allocated 0\n" + " :total_promoted_sab 0\n" + " :total_promoted 0\n" + " :n_mutation 0\n" + " :n_logged_mutation 0\n" + " :n_xgen_mutation 0\n" + " :n_xckp_mutation 0\n" + " :nursery_z 0\n" + " :nursery_before_checkpoint_z 0\n" + " :nursery_after_checkpoint_z 0\n" + " :tenured_z 0>"); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + REQUIRE(actual == expected); + + tag_config::tag_color_enabled = saved; + } + } +} /*namespace xo*/ + +/* GcStatistics.test.cpp */ diff --git a/utest/ListAlloc.test.cpp b/utest/ListAlloc.test.cpp new file mode 100644 index 00000000..2db22bf2 --- /dev/null +++ b/utest/ListAlloc.test.cpp @@ -0,0 +1,58 @@ +/* ListAlloc.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/ListAlloc.hpp" +#include + +namespace xo { + using xo::gc::ListAlloc; + + namespace ut { + TEST_CASE("ListAlloc", "[alloc][gc]") + { + /** teeny weeny allocator **/ + up alloc = ListAlloc::make("test", 16, 32, false); + + REQUIRE(alloc->name() == "test"); + REQUIRE(alloc->size() == 16); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + /* will expand */ + std::byte * mem1 = alloc->alloc(20); + + REQUIRE(mem1); + REQUIRE(alloc->size() == 16 + 32); + /* round up to multiple of 8 */ + REQUIRE(alloc->before_checkpoint() == 24); + REQUIRE(alloc->after_checkpoint() == 0); + + alloc->checkpoint(); + + std::byte * mem2 = alloc->alloc(30); + + REQUIRE(mem2); + REQUIRE(alloc->size() == 16 + 32 + 48); + REQUIRE(alloc->before_checkpoint() == 24); + /* round up to multiple of 8 */ + REQUIRE(alloc->after_checkpoint() == 32); + + std::byte * mem3 = alloc->alloc(40); + + REQUIRE(mem3); + REQUIRE(alloc->size() == 16 + 32 + 48 + 80); + REQUIRE(alloc->before_checkpoint() == 24); + /* already multiple of 8 */ + REQUIRE(alloc->after_checkpoint() == 32 + 40); + + REQUIRE(alloc->is_before_checkpoint(mem1) == true); + REQUIRE(alloc->is_before_checkpoint(mem2) == false); + REQUIRE(alloc->is_before_checkpoint(mem3) == false); + } + + } /*namespace ut*/ +} /*namespace xo*/ + +/* ListAlloc.test.cpp */ diff --git a/utest/ObjectStatistics.test.cpp b/utest/ObjectStatistics.test.cpp new file mode 100644 index 00000000..7530450d --- /dev/null +++ b/utest/ObjectStatistics.test.cpp @@ -0,0 +1,185 @@ +/* @file ObjectStatistics.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/ObjectStatistics.hpp" +#include "xo/reflect/Reflect.hpp" +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/ppstr.hpp" +#include "xo/indentlog/print/tostr.hpp" +#include "xo/indentlog/print/hex.hpp" +#include + +namespace xo { + using xo::gc::ObjectStatistics; + using xo::gc::PerObjectTypeStatistics; + using xo::reflect::Reflect; + using xo::print::ppconfig; + + namespace ut { + TEST_CASE("PerObjectTypeStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerObjectTypeStatistics stats; + + std::string s = tostr(stats); + + //std::cerr << hex_view(s.c_str(), s.c_str() + s.length(), true /*as_text*/) << std::endl; + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("PerObjectTypeStatistics-1", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerObjectTypeStatistics stats; + stats.td_ = Reflect::require(); + stats.scanned_n_ = 4; + stats.scanned_z_ = 16; + stats.survive_n_ = 2; + stats.survive_z_ = 8; + + std::string s = tostr(stats); + + //std::cerr << hex_view(s.c_str(), s.c_str() + s.length(), true /*as_text*/) << std::endl; + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + ObjectStatistics stats; + + std::string s = tostr(stats); + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics-1", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + ObjectStatistics stats; + stats.per_type_stats_v_.push_back(PerObjectTypeStatistics()); + + std::string s = tostr(stats); + + REQUIRE(s == ">"); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics-pretty", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + std::stringstream ss; + ppconfig ppc; + ObjectStatistics stats; + + std::string actual = toppstr2(ppc, stats); + std::string expected + = (""); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + REQUIRE(actual == expected); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics-pretty-1", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerObjectTypeStatistics objstats; + objstats.td_ = Reflect::require(); + objstats.scanned_n_ = 4; + objstats.scanned_z_ = 16; + objstats.survive_n_ = 2; + objstats.survive_z_ = 8; + + std::stringstream ss; + ppconfig ppc; + ObjectStatistics stats; + stats.per_type_stats_v_.push_back(objstats); + + std::string actual = toppstr2(ppc, stats); + + std::string expected + = (" ]>"); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + tag_config::tag_color_enabled = saved; + } + } +} /*namespace xo*/ + +/* ObjectStatistics.test.cpp */ diff --git a/utest/generation.test.cpp b/utest/generation.test.cpp new file mode 100644 index 00000000..415d6258 --- /dev/null +++ b/utest/generation.test.cpp @@ -0,0 +1,38 @@ +/* generation.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/generation.hpp" +#include +#include + +namespace xo { + namespace gc { + TEST_CASE("generation", "[gc]") { + REQUIRE(::strcmp(gen2str(generation::nursery), "nursery") == 0); + REQUIRE(::strcmp(gen2str(generation::tenured), "tenured") == 0); + REQUIRE(::strcmp(gen2str(generation::N), "?generation") == 0); + + { + std::stringstream ss; + ss << generation::nursery; + REQUIRE(ss.str() == "nursery"); + } + + { + std::stringstream ss; + ss << generation::tenured; + REQUIRE(ss.str() == "tenured"); + } + + { + std::stringstream ss; + ss << generation::N; + REQUIRE(ss.str() == "?generation"); + } + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* generation.test.cpp */ From d8b3d7a148595b265f97c85b47a8d0b6183199b6 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 12 Aug 2025 00:14:30 -0500 Subject: [PATCH 11/69] xo-alloc: reserve virtual memory, commit pages on demand --- CMakeLists.txt | 2 +- include/xo/alloc/ArenaAlloc.hpp | 20 ++++++- include/xo/alloc/GC.hpp | 9 ++- include/xo/alloc/IAlloc.hpp | 4 +- include/xo/alloc/ListAlloc.hpp | 1 + src/alloc/ArenaAlloc.cpp | 99 ++++++++++++++++++++++++++++----- src/alloc/GC.cpp | 17 +++++- src/alloc/ListAlloc.cpp | 11 ++++ 8 files changed, 140 insertions(+), 23 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eebf3aff..0e9de5c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# alloc/CMakeLists.txt +# xo-alloc/CMakeLists.txt cmake_minimum_required(VERSION 3.10) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 8e162a95..63de43e8 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -24,6 +24,8 @@ namespace xo { **/ class ArenaAlloc : public IAlloc { public: + ArenaAlloc(const ArenaAlloc &) = delete; + ArenaAlloc(ArenaAlloc &&) = delete; ~ArenaAlloc(); /** create allocator with capacity @p z, @@ -38,10 +40,14 @@ namespace xo { void capture_object_statistics(capture_phase phase, ObjectStatistics * p_dest) const; + /** expand available (i.e. committed) space to size @p z **/ + bool expand(std::size_t z); + // inherited from IAlloc... virtual const std::string & name() const final override; virtual std::size_t size() const final override; + virtual std::size_t committed() const final override; virtual std::size_t available() const final override; virtual std::size_t allocated() const final override; virtual bool contains(const void * x) const final override; @@ -54,6 +60,9 @@ namespace xo { virtual void checkpoint() final override; virtual std::byte * alloc(std::size_t z) final override; + ArenaAlloc & operator=(const ArenaAlloc &) = delete; + ArenaAlloc & operator=(ArenaAlloc &&) = delete; + private: ArenaAlloc(const std::string & name, std::size_t z, bool debug_flag); @@ -67,8 +76,15 @@ namespace xo { /** optional instance name, for diagnostics **/ std::string name_; + /** size of a VM page **/ + std::size_t page_z_; + /** allocator owns memory in range [@ref lo_, @ref hi_) **/ std::byte * lo_ = nullptr; + /** prefix of this size is actually committed. + * Remainder uses uncommitted virtual address space + **/ + std::size_t committed_z_ = 0; /** checkpoint (for GC support); divides objects into * older (addresses below checkpoint) * and younger (addresses above checkpoint) @@ -76,9 +92,9 @@ namespace xo { std::byte * checkpoint_; /** free pointer. memory in range [@ref free_, @ref limit_) available **/ std::byte * free_ptr_ = nullptr; - /** soft limit: end of released memory **/ + /** soft limit: end of committed virtual memory **/ std::byte * limit_ = nullptr; - /** hard limit: end of allocated memory **/ + /** hard limit: end of reserved virtual memory **/ std::byte * hi_ = nullptr; /** true to enable detailed debug logging **/ bool debug_flag_ = false; diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 4560f3ac..c5432935 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -164,7 +164,8 @@ namespace xo { * since one role is always held empty between collections. **/ virtual std::size_t size() const final override; - + /** for committed count both to-space and from-space **/ + virtual std::size_t committed() const final override; virtual std::size_t allocated() const final override; virtual std::size_t available() const final override; /** only tests to-space **/ @@ -194,10 +195,6 @@ namespace xo { virtual std::byte * alloc(std::size_t z) final override; virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; -#ifdef REDLINE_MEMORY - virtual void release_redline_memory() final override; -#endif - private: ListAlloc * nursery_to() const { return nursery(role::to_space); } ListAlloc * nursery_from() const { return nursery(role::from_space); } @@ -208,6 +205,8 @@ namespace xo { ListAlloc * nursery(role r) const { return nursery_[role2int(r)].get(); } ListAlloc * tenured(role r) const { return tenured_[role2int(r)].get(); } + MutationLog * mutation_log(role r) const { return mutation_log_[role2int(r)].get(); } + /** begin GC now **/ void execute_gc(generation g); /** cleanup phase. aux function for @ref execute_gc **/ diff --git a/include/xo/alloc/IAlloc.hpp b/include/xo/alloc/IAlloc.hpp index b8270f53..0b6791f0 100644 --- a/include/xo/alloc/IAlloc.hpp +++ b/include/xo/alloc/IAlloc.hpp @@ -31,10 +31,12 @@ namespace xo { /** optional name for this allocator; labelling for diagnostics **/ virtual const std::string & name() const = 0; - /** allocator size in bytes (up to soft limit). + /** allocator size in bytes (up to reserved limit) * Includes unallocated mmeory **/ virtual std::size_t size() const = 0; + /** committed size in bytes **/ + virtual std::size_t committed() const = 0; /** number of unallocated bytes available (up to soft limit) * from this allocator **/ diff --git a/include/xo/alloc/ListAlloc.hpp b/include/xo/alloc/ListAlloc.hpp index ac2f9894..7b592b4e 100644 --- a/include/xo/alloc/ListAlloc.hpp +++ b/include/xo/alloc/ListAlloc.hpp @@ -55,6 +55,7 @@ namespace xo { virtual const std::string & name() const final override; virtual std::size_t size() const final override; + virtual std::size_t committed() const final override; virtual std::size_t available() const final override; virtual std::size_t allocated() const final override; virtual bool contains(const void * x) const final override; diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index e978f566..49a2d16d 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -8,6 +8,7 @@ #include "ObjectStatistics.hpp" #include "xo/indentlog/scope.hpp" #include "xo/indentlog/print/tag.hpp" +#include #include namespace xo { @@ -15,27 +16,52 @@ namespace xo { ArenaAlloc::ArenaAlloc(const std::string & name, std::size_t z, bool debug_flag) { + scope log(XO_DEBUG(debug_flag), xtag("name", name)); + this->name_ = name; - this->lo_ = (new std::byte [z]); - this->checkpoint_ = lo_; - this->free_ptr_ = lo_; - this->limit_ = lo_ + z; - this->hi_ = limit_; - this->debug_flag_ = debug_flag; + this->page_z_ = getpagesize(); + + // reserve virtual memory + + void * base = mmap(nullptr, z, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + // could use this as fallback.. + //base = (new std::byte [z]); + + if (base == MAP_FAILED) { + throw std::runtime_error(tostr("ArenaAlloc: uncommitted allocation failed", + xtag("size", z))); + } + + this->lo_ = reinterpret_cast(base); + this->committed_z_ = 0; + this->checkpoint_ = lo_; + this->free_ptr_ = lo_; + this->limit_ = lo_ + z; + this->hi_ = limit_; + this->debug_flag_ = debug_flag; if (!lo_) { throw std::runtime_error(tostr("ArenaAlloc: allocation failed", xtag("size", z))); } + + log && log(xtag("lo", (void*)lo_), xtag("page_z", page_z_)); } ArenaAlloc::~ArenaAlloc() { - delete [] this->lo_; // hygiene.. + if (lo_) { + munmap(lo_, hi_ - lo_); + } + // could use this as fallback if not using uncommitted technique + //delete [] this->lo_; + this->lo_ = nullptr; + this->committed_z_ = 0; this->checkpoint_ = nullptr; this->free_ptr_ = nullptr; this->limit_ = nullptr; @@ -51,6 +77,46 @@ namespace xo { z, debug_flag)); } + namespace { + /* alignment better be a power of 2 */ + std::size_t + align_lub(std::size_t x, std::size_t align) + { + /* e.g: + * align = 4096, x%align = 100 -> dx = 3996 + * align = 4096, x%align = 0 -> dx = 0 + */ + std::size_t dx = (align - (x % align)) % align; + + return x + dx; + } + } + + bool + ArenaAlloc::expand(size_t offset_z) { + scope log(XO_DEBUG(debug_flag_), xtag("offset_z", offset_z), xtag("committed_z", committed_z_)); + + if (offset_z <= committed_z_) + return true; + + std::size_t align_offset_z = align_lub(offset_z, page_z_); + std::byte * commit_start = lo_ + committed_z_; + std::size_t new_commit_z = align_offset_z - committed_z_; + + log && log(xtag("align_offset_z", align_offset_z), + xtag("new_commit_z", new_commit_z)); + + if (mprotect(commit_start, new_commit_z, PROT_READ | PROT_WRITE) != 0) { + throw std::runtime_error(tostr("ArenaAlloc::expand: commit failure", + xtag("committed_z", committed_z_), + xtag("new_commit_z", new_commit_z))); + } + + this->committed_z_ = align_offset_z; + + return true; + } + void ArenaAlloc::set_free_ptr(std::byte * x) { @@ -59,7 +125,7 @@ namespace xo { if (lo_ <= x && x < limit_) { this->free_ptr_ = x; - if (this->checkpoint_ > free_ptr_) + if (checkpoint_ > free_ptr_) this->checkpoint_ = free_ptr_; } else { throw std::runtime_error(tostr("LinearAllog::set_free_ptr(x): expected lo <= x < limit", @@ -128,6 +194,11 @@ namespace xo { return limit_ - lo_; } + std::size_t + ArenaAlloc::committed() const { + return committed_z_; + } + std::size_t ArenaAlloc::available() const { return limit_ - free_ptr_; @@ -197,13 +268,15 @@ namespace xo { assert(z1 % c_bpw == 0ul); + this->expand(this->allocated() + z1); + std::byte * retval = this->free_ptr_; - log && log(xtag("self", name_), xtag("z0", z0), xtag("+pad", dz), xtag("z1", z1), xtag("avail", this->available())); - - if (free_ptr_ + z1 > limit_) { - return nullptr; - } + log && log(xtag("self", name_), + xtag("z0", z0), + xtag("+pad", dz), + xtag("z1", z1), + xtag("avail", this->available())); this->free_ptr_ += z1; diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 2990adf6..b72ac721 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -115,7 +115,16 @@ namespace xo { std::size_t GC::size() const { - return nursery_[role2int(role::to_space)]->size() + tenured_[role2int(role::to_space)]->size(); + return nursery_to()->size() + tenured_to()->size(); + } + + std::size_t + GC::committed() const + { + return (nursery_to()->committed() + + nursery_from()->committed() + + tenured_to()->committed() + + tenured_from()->committed()); } std::size_t @@ -182,6 +191,12 @@ namespace xo { return retval; } + std::size_t + GC::nursery_to_committed() const + { + return nursery_to()->committed(); + } + generation_result GC::fromspace_generation_of(const void * x) const { diff --git a/src/alloc/ListAlloc.cpp b/src/alloc/ListAlloc.cpp index 6e37a325..29cdb477 100644 --- a/src/alloc/ListAlloc.cpp +++ b/src/alloc/ListAlloc.cpp @@ -75,6 +75,17 @@ namespace xo { return total_z_; } + std::size_t + ListAlloc::committed() const { + std::size_t z = 0; + if (hd_) + z += hd_->committed(); + for (const auto & a : full_l_) + z += a->committed(); + + return z; + } + std::byte * ListAlloc::free_ptr() const { return hd_->free_ptr(); From 258555e9ebf691cca61d70d4ec139f4c51d7b209 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 12 Aug 2025 00:16:00 -0500 Subject: [PATCH 12/69] xo-alloc: GC mutation log works for full GC --- include/xo/alloc/GC.hpp | 42 ++++++- src/alloc/GC.cpp | 254 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 269 insertions(+), 27 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index c5432935..a0bcef2b 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -40,6 +40,11 @@ namespace xo { * Will allocate more space as needed **/ std::size_t initial_tenured_z_ = 0; + /** trigger incremental GC after this many bytes allocated in nursery **/ + std::size_t incr_gc_threshold_ = 64*1024; + /** trigger full GC after this many bytes promoted to tenured **/ + std::size_t full_gc_threshold_ = 512*1024; + /** true to permit incremental garbage collection **/ bool allow_incremental_gc_ = true; /** true to report statistics **/ @@ -118,6 +123,7 @@ namespace xo { **/ static up make(const Config & config); + const Config & config() const { return config_; } const GCRunstate & runstate() const { return runstate_; } const GcStatistics & native_gc_statistics() const { return gc_statistics_; } GcStatisticsExt get_gc_statistics() const; @@ -126,6 +132,19 @@ namespace xo { bool is_gc_enabled() const { return gc_enabled_ == 0; } /** true during (and only during) a GC cycle **/ bool gc_in_progress() const { return runstate_.in_progress(); } + /** @return committed size of Nursery to-space **/ + std::size_t nursery_to_committed() const; + /** @return nursery bytes used before checkpoint **/ + std::size_t nursery_before_checkpoint() const; + /** @return nursery bytes used after checkpoint **/ + std::size_t nursery_after_checkpoint() const; + /** @return committed size of Tenured to-space **/ + std::size_t tenured_to_committed() const; + /** @return tenured bytes used before checkpoint **/ + std::size_t tenured_before_checkpoint() const; + /** @return tenured bytes used after checkpoint = promoted since last GC **/ + std::size_t tenured_after_checkpoint() const; + /** @return generation to which object at @p x belongs **/ generation_result tospace_generation_of(const void * x) const; /** @return generation that contains @p x, given it's in from-space **/ @@ -232,7 +251,6 @@ namespace xo { * (T->N, aka xgen) and (N1->N0, aka xckp) pointers **/ void incremental_gc_forward_mlog(ObjectStatistics * per_type_stats); - /** * Aux function for @ref incremental_gc_forward_mlog. Calls this function until * fixpoint. @@ -246,6 +264,23 @@ namespace xo { MutationLog * to_mlog, MutationLog * defer_mlog, ObjectStatistics * per_type_stats); + /** Aux function for @ref execute_gc. Updates bookkeeping for cross-generational + * (T->N, aka xgen) and (N1->N0, aka xckcp) pointers on full gc + **/ + void full_gc_forward_mlog(ObjectStatistics * per_type_stats); + /** + * Aux function for @ref full_gc_forward_mlog. Calls this function until fixpoint. + * + * @param from_mlog incoming mutation log. Contains {xgen,xckp} pointers before GC. + * Contents of this log is consumed (+discarded) before method returns. + * @param to_mlog outgoing mutation log. Will contain {xgen,xckp} pointers after GC. + * @param defer_mlog contains log entries associated with possible garbage. + * + **/ + void full_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * per_type_stats); private: /** garbage collector configuration **/ @@ -292,11 +327,6 @@ namespace xo { /** optional per-object-type counters. snapshot at end of collection cycle **/ std::array object_statistics_sae_; - /** trigger full GC whenever this much data arrives in tenured generation **/ - std::size_t full_gc_threshold_ = 0; - /** trigger incr GC whenever this much data arrives in nuresery generation **/ - std::size_t incr_gc_threshold_ = 0; - /** true when GC requested, * remains true until GC.. completes? begins? **/ diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index b72ac721..bcb8118b 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -163,13 +163,13 @@ namespace xo { std::size_t GC::before_checkpoint() const { - return nursery_[role2int(role::to_space)]->before_checkpoint(); + return this->nursery_to()->before_checkpoint(); } std::size_t GC::after_checkpoint() const { - return nursery_[role2int(role::to_space)]->after_checkpoint(); + return this->nursery_to()->after_checkpoint(); } bool @@ -197,16 +197,34 @@ namespace xo { return nursery_to()->committed(); } - generation_result - GC::fromspace_generation_of(const void * x) const + std::size_t + GC::nursery_before_checkpoint() const { - if (tenured_[role2int(role::from_space)]->contains(x)) - return generation_result::tenured; + return nursery_to()->before_checkpoint(); + } - if (nursery_[role2int(role::from_space)]->contains(x)) - return generation_result::nursery; + std::size_t + GC::nursery_after_checkpoint() const + { + return nursery_to()->after_checkpoint(); + } - return generation_result::not_found; + std::size_t + GC::tenured_to_committed() const + { + return tenured_to()->committed(); + } + + std::size_t + GC::tenured_before_checkpoint() const + { + return tenured_to()->before_checkpoint(); + } + + std::size_t + GC::tenured_after_checkpoint() const + { + return tenured_to()->after_checkpoint(); } generation_result @@ -221,6 +239,18 @@ namespace xo { return generation_result::not_found; } + generation_result + GC::fromspace_generation_of(const void * x) const + { + if (tenured_[role2int(role::from_space)]->contains(x)) + return generation_result::tenured; + + if (nursery_[role2int(role::from_space)]->contains(x)) + return generation_result::nursery; + + return generation_result::not_found; + } + std::byte * GC::free_ptr(generation gen) { @@ -262,16 +292,26 @@ namespace xo { void GC::checkpoint() { - nursery_[role2int(role::to_space) ]->checkpoint(); + nursery_to()->checkpoint(); + /* checkpoint T generation so we can trigger GC based on new T objects rather than + * overall T size + */ + tenured_to()->checkpoint(); } std::byte * GC::alloc(std::size_t z) { - std::byte * x = nursery_[role2int(role::to_space)]->alloc(z); + auto N_to = this->nursery_to(); + + if (!incr_gc_pending_ && (N_to->after_checkpoint() > config_.incr_gc_threshold_)) { + /* automatically ups to generation::tenured */ + this->request_gc(generation::nursery); + } + + std::byte * x = N_to->alloc(z); /* ListAlloc won't fail unless we exhaust memory -- instead will increase heap size */ - assert(x); return x; @@ -291,17 +331,17 @@ namespace xo { { log && log("tenured"); - retval = tenured_[role2int(role::to_space)]->alloc(z); + retval = this->tenured_to()->alloc(z); } break; case generation_result::nursery: { - if (nursery_[role2int(role::from_space)]->is_before_checkpoint(src)) + if (this->nursery_from()->is_before_checkpoint(src)) { /* nursery object has survived 2nd collection cycle * -> promote into tenured generation */ - retval = tenured_[role2int(role::to_space)]->alloc(z); + retval = this->tenured_to()->alloc(z); log && log("promote", xtag("addr", (void*)retval)); @@ -311,7 +351,7 @@ namespace xo { } else { log && log("nursery"); - retval = nursery_[role2int(role::to_space)]->alloc(z); + retval = this->nursery_to()->alloc(z); } } break; @@ -427,7 +467,7 @@ namespace xo { /* gc on tenured generation may need this much space */ std::size_t need_tenured_z = (tenured_[role2int(role::to_space)]->allocated() + max_promote_z - + full_gc_threshold_); + + config_.full_gc_threshold_); log && log("need_tenured_z", need_tenured_z); @@ -449,7 +489,7 @@ namespace xo { /* subtracting max_promote_z is correct here, since anything not promoted is garbage */ std::size_t need_nursery_z = (nursery(role::to_space)->allocated() - max_promote_z - + incr_gc_threshold_); + + config_.incr_gc_threshold_); log && log(xtag("need_nursery_z", need_nursery_z)); @@ -695,6 +735,139 @@ namespace xo { } } + void + GC::full_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * /*per_type_stats*/) + { + scope log(XO_DEBUG(config_.debug_flag_), xtag("from_mlog.size", from_mlog->size())); + + /* categorize pointers based on combination of {source address, destination address}, + * only care about the generation associated with an address. + * + * N0 : nursery(from), before checkpoint + * N0': nursery(to), before checkpoint + * N1 : nursery(from), after checkpoint + * N1': nursery(to), after checkpoint + * T : tenured(from) + * T': tenured(to) + * + * loc(P): parent region before GC + * loc(C): child region before GC + * + * | | forwarded | loc now post | loc after | + * | | already? | root copy | action | + * | loc(P) loc(C) | P C | P' C' | P' C' | defer | action + * ----|---------------+--------------+---------------+---------------+-------+--------------- + * (a) | T N0 | no no | T N0 | T N0 | P ->C | defer + * (b) | | yes | N1' | N1' | P ->C'| defer + * | | yes no | impossible + * (b2)| | yes | T' N1' | T' N1' | | +mlog + * (c) | T N1 | no no | T N1 | T T | P ->C | defer + * (d) | | yes | T T' | T T' | P ->C'| defer + * | | yes no | impossible + * (d2)| | yes | T' T' | T' T' | | -mlog + * (e) | N1 N0 | no no | N1 N0 | N1 N0 | P ->C | defer + * (f) | | yes | N1 N1' | N1 N1' | P ->C'| defer + * | | yes no | impossible + * (g) | | yes yes | T' N1' | T' N1' | | +mlog + * + * notes: + * (a) P,C maybe garbage. don't move either, but defer mlog incase P saved by a subsequent mutation; + * in that case C saved also, + will still have an xgen ptr, and still need an mlog entry. + * (b) C already evac'd, but P maybe garbage. defer mlog incase P rescued by a subsequent mutation; + * in that case will still have an xgen (T -> N) ptre, and still need an mlog entry. + * (b2) P,C already evac'd. Must update+rembexember xgen ptr {T -> N1} + * (c) P,C maybe garbage. don't move either, but defer mlog in case P saved by a subsequent mutation; + * in that case C promoted, no longer xgen + * (d) P maybe garbage. defer in case P saved by a subsequent mutation. + * C now tenured, so will no longer have an xgen pointer. + * (d2) P,C already evac'd. After collection no longer have xgen pointer, so no mlog. + * (e) P,C maybe garbage. don't move either, but defer mlog incase P saved by a subsequent mutation. + * in that case C saved alto, + will still have an xgen ptr, so still need an mlog entry + * (f) P maybe garbage, C survives. defer mlog incase P saved+promoted by a subsequent mutation; + * in that case will still have an xgen (T -> N) ptr, so still need an mlog entry. + * (g) P,C already evac'd. Still have xgen pointer, must mlog + */ + + std::size_t i_from = 0; + // number of rescued subgraphs via mutation log entries + std::size_t n_rescue = 0; + + for (MutationLogEntry & from_entry : *from_mlog) + { + log && (i_from % 10000 == 0) && log(xtag("i_from", i_from)); + + if (from_entry.is_parent_forwarded()) { + Object * parent_to = from_entry.parent_destination(); + + log && log(xtag("parent_to", (void*)parent_to)); + + assert(tospace_generation_of(parent_to) == generation_result::tenured); + + MutationLogEntry to_entry = from_entry.update_parent_moved(parent_to); + + // note: child obtained (as it must be) by reading from prarent's memory _now_. + // Since parent has moved, child has too + Object * child_to = to_entry.child(); // after moveing + + if (tospace_generation_of(parent_to) == generation_result::tenured) + { + // cases (b2)(d2)(g), loc(P) is T' + // In all these cases parent has already been moved; + // therefore child has also been moved. + // Just need to decide whether to keep mlog entry + + if (from_entry.is_dead()) { + // obsolete mutation -- no longer belongs to parent, discard + } else if (child_to) { + assert(!child_to->_is_forwarded()); + + if (tospace_generation_of(child_to) == generation_result::nursery) { + // case + // (b2) loc(P')=T', loc(C')=N1' --> +mlog + // (g) loc(P')=T', loc(C')=N1' --> +mlog + // + to_mlog->push_back(to_entry); + } else { + // case + // (d2) loc(P')=T', loc(C')=T' --> -mlog + } + } + } else { + // impossible - wouldn't have made mlog entry + + + assert(false); + } + } else { + // case + // (a) defer + // (b) defer + // (c) defer + // (d) defer + // (e) defer + // (f) defer + + defer_mlog->push_back(from_entry); + } + + ++i_from; + } + + from_mlog->clear(); + + if (n_rescue == 0) { + // if we didn't rescue any objects + // then we now confirm that otherwise-unreachable parents in defer_mlog + // are garbage + + defer_mlog->clear(); + } + } + + void GC::incremental_gc_forward_mlog(ObjectStatistics * per_type_stats) { @@ -703,7 +876,7 @@ namespace xo { * - gc roots have been copied, along with everything reachable from them. * * plan: - * - forward mutation in *from_mutation_log, writing them to + * - forward mutations in *from_mutation_log, writing them to * *to_mutationlog and/or *defer_mutation_log. * Use defer when mutation P->C encountered, but P was not copied. * P appears to be garbage, but may turn out to be live if encountered @@ -743,13 +916,52 @@ namespace xo { } } + void + GC::full_gc_forward_mlog(ObjectStatistics * per_type_stats) + { + /* control here: + * - full gc. + * - gc roots have been copied, along with everything reachable + * from them. + * + * plan: + * - forward mutations in *from_mutation_log, writing them to + * *to_mutation_log and/or *defer_mutation_log. + */ + + MutationLog * to_mlog = this->mutation_log(role::to_space); + + for (;;) { + MutationLog * from_mlog = this->mutation_log(role::from_space); + MutationLog * defer_mlog = defer_mutation_log_.get(); + + this->full_gc_forward_mlog_phase(from_mlog, + to_mlog, + defer_mlog, + per_type_stats); + assert(from_mlog->empty()); + + if (defer_mlog->empty()) + break; + + /* control here: + * 1. at least one mlog triggered a rescue + * 2. at least one mlog was deferred (had otherwise-unreachable parent) + * + * possible that deferred parent is now reachable thanks to a rescue; + * to confirm/refute this need to revisit entries in defer_mlog. + */ + std::swap(mutation_log_[role2int(role::from_space)], defer_mutation_log_); + } + } + void GC::forward_mutation_log(generation upto) { scope log(XO_DEBUG(config_.debug_flag_)); if (upto == generation::tenured) { - log && log("TODO: forward mutation log for full GC"); + this->full_gc_forward_mlog(&object_statistics_sae_[gen2int(generation::tenured)]); } else { this->incremental_gc_forward_mlog(&object_statistics_sae_[gen2int(generation::nursery)]); } @@ -874,7 +1086,7 @@ namespace xo { target = generation::tenured; if ((target == generation::nursery) - && (tenured_[role2int(role::to_space)]->after_checkpoint() > full_gc_threshold_)) + && (this->tenured_to()->after_checkpoint() > config_.full_gc_threshold_)) { /** full collection when >= @ref full_gc_threshold_ bytes added to tenured * generation, since last full collection From 279a1a040c6bdccf36553a5bc3b98236de56fd91 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 12 Aug 2025 12:53:06 -0500 Subject: [PATCH 13/69] xo-callback xo-alloc: + GC copy callbacks + unique_ptr cbset support --- include/xo/alloc/GC.hpp | 28 ++++++++++++++++++++++++++++ src/alloc/CMakeLists.txt | 1 + src/alloc/GC.cpp | 20 ++++++++++++++++++-- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index a0bcef2b..64cf198d 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -7,6 +7,7 @@ #include "ListAlloc.hpp" #include "GcStatistics.hpp" +#include "xo/callback/UpCallbackSet.hpp" #include "xo/indentlog/print/array.hpp" #include #include @@ -105,12 +106,31 @@ namespace xo { using MutationLog = std::vector; + /** @class GcCopyCallback + * @brief optional callback to observe individual copy operations during GC + * + * For viz + **/ + class GcCopyCallback { + public: + virtual void notify_gc_copy(std::size_t z, const void * src_addr, const void * dest_addr, + generation src_gen, generation dest_gen) = 0; + /** invoked when added to callback set (i.e. @ref GC::GcCopyCallbackSet) **/ + void notify_add_callback() {} + /** invoked when removed from callback set **/ + void notify_remove_callback() {} + }; + /** @class GC * @brief generational garbage collector * * Works with objects of type @ref xo::Object **/ class GC : public IAlloc { + public: + using CallbackId = xo::fn::CallbackId; + using GcCopyCallbackSet = xo::fn::UpCallbackSet; + public: /** create new GC instance with configuration @p config **/ explicit GC(const Config & config); @@ -160,6 +180,11 @@ namespace xo { * from @c *addr **/ void add_gc_root(Object ** addr); + /** may optionally use this to observe GC copy phase. + * Will be invoked once _per surviving object_, so not cheap. + * Intended for GC visualization. + **/ + CallbackId add_gc_copy_callback(up fn); /** request garbage collection. **/ void request_gc(generation g); /** disable garbage collection until matching call to @ref enable_gc. @@ -335,6 +360,9 @@ namespace xo { /** enabled when 0. disabled when <0 **/ int gc_enabled_ = 0; + + /** for (optional) viz: invoke when copying individual objects **/ + GcCopyCallbackSet gc_copy_cbset_; }; } /*namespace gc*/ diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index 589f3b90..07f87784 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -15,5 +15,6 @@ set(SELF_SRCS xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) xo_dependency(${SELF_LIB} reflect) +xo_dependency(${SELF_LIB} callback) #end CMakeLists.txt diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index bcb8118b..d0aaa660 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -289,6 +289,12 @@ namespace xo { gc_root_v_.push_back(addr); } + auto + GC::add_gc_copy_callback(up fn) -> CallbackId + { + return gc_copy_cbset_.add_callback(std::move(fn)); + } + void GC::checkpoint() { @@ -322,16 +328,19 @@ namespace xo { { scope log(XO_DEBUG(config_.debug_flag_), xtag("z", z), xtag("+pad", IAlloc::alloc_padding(z))); - generation_result gr = this->fromspace_generation_of(src); + generation_result src_gr = this->fromspace_generation_of(src); std::byte * retval = nullptr; - switch (gr) { + switch (src_gr) { case generation_result::tenured: { log && log("tenured"); retval = this->tenured_to()->alloc(z); + + gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, + z, src, retval, generation::tenured, generation::tenured); } break; case generation_result::nursery: @@ -347,11 +356,18 @@ namespace xo { assert(this->tospace_generation_of(retval) == generation_result::tenured); + gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, + z, src, retval, generation::nursery, generation::tenured); + this->gc_statistics_.total_promoted_ += IAlloc::with_padding(z); + } else { log && log("nursery"); retval = this->nursery_to()->alloc(z); + + gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, + z, src, retval, generation::nursery, generation::nursery); } } break; From 2ec50720922c6153656ef5914763c7cf0839b461 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 12 Aug 2025 13:14:01 -0500 Subject: [PATCH 14/69] xo-alloc: alter GC to use LinearAlloc directly - retire ListAlloc --- include/xo/alloc/ArenaAlloc.hpp | 11 ++++++++--- include/xo/alloc/GC.hpp | 18 +++++++++--------- src/alloc/GC.cpp | 24 ++++++++++++------------ 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 63de43e8..64546cf3 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -28,15 +28,20 @@ namespace xo { ArenaAlloc(ArenaAlloc &&) = delete; ~ArenaAlloc(); - /** create allocator with capacity @p z, + /** Create allocator with capacity @p z, + * Reserve memory addresses for @p z bytes, + * but don't commit them until needed **/ static up make(const std::string & name, - std::size_t z, - bool debug_flag); + std::size_t z, + bool debug_flag); std::byte * free_ptr() const { return free_ptr_; } void set_free_ptr(std::byte * x); + /** Reset to empty state **/ + void reset(std::size_t /*z_ignored*/) { this->clear(); } + void capture_object_statistics(capture_phase phase, ObjectStatistics * p_dest) const; diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 64cf198d..a653b54b 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -5,7 +5,7 @@ #pragma once -#include "ListAlloc.hpp" +#include "ArenaAlloc.hpp" #include "GcStatistics.hpp" #include "xo/callback/UpCallbackSet.hpp" #include "xo/indentlog/print/array.hpp" @@ -240,14 +240,14 @@ namespace xo { virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; private: - ListAlloc * nursery_to() const { return nursery(role::to_space); } - ListAlloc * nursery_from() const { return nursery(role::from_space); } + ArenaAlloc * nursery_to() const { return nursery(role::to_space); } + ArenaAlloc * nursery_from() const { return nursery(role::from_space); } - ListAlloc * tenured_to() const { return tenured(role::to_space); } - ListAlloc * tenured_from() const { return tenured(role::from_space); } + ArenaAlloc * tenured_to() const { return tenured(role::to_space); } + ArenaAlloc * tenured_from() const { return tenured(role::from_space); } - ListAlloc * nursery(role r) const { return nursery_[role2int(r)].get(); } - ListAlloc * tenured(role r) const { return tenured_[role2int(r)].get(); } + ArenaAlloc * nursery(role r) const { return nursery_[role2int(r)].get(); } + ArenaAlloc * tenured(role r) const { return tenured_[role2int(r)].get(); } MutationLog * mutation_log(role r) const { return mutation_log_[role2int(r)].get(); } @@ -314,11 +314,11 @@ namespace xo { /** contains allocated objects, along with unreachable garbage to be collected. * roles reverse after each incremental, or full, collection. **/ - std::array, role2int(role::N)> nursery_; + std::array, role2int(role::N)> nursery_; /** empty space, destination for objects that survive collection. * roles reverse after each full collection. **/ - std::array, role2int(role::N)> tenured_; + std::array, role2int(role::N)> tenured_; /** current state of GC activity. * @text diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index d0aaa660..6ccc4d15 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -66,14 +66,14 @@ namespace xo { std::size_t tenured_size = config.initial_tenured_z_; nursery_[role2int(role::from_space)] - = ListAlloc::make("NA", nursery_size, 2 * nursery_size, config.debug_flag_); + = ArenaAlloc::make("NA", nursery_size, config.debug_flag_); nursery_[role2int(role::to_space) ] - = ListAlloc::make("NB", nursery_size, 2 * nursery_size, config.debug_flag_); + = ArenaAlloc::make("NB", nursery_size, config.debug_flag_); tenured_[role2int(role::from_space)] - = ListAlloc::make("TA", tenured_size, 2 * tenured_size, config.debug_flag_); + = ArenaAlloc::make("TA", tenured_size, config.debug_flag_); tenured_[role2int(role::to_space) ] - = ListAlloc::make("TB", tenured_size, 2 * tenured_size, config.debug_flag_); + = ArenaAlloc::make("TB", tenured_size, config.debug_flag_); mutation_log_[role2int(role::from_space)] = std::make_unique(); mutation_log_[role2int(role::to_space)] = std::make_unique(); @@ -444,7 +444,7 @@ namespace xo { void GC::swap_nursery() { - up tmp = std::move(nursery_[role2int(role::to_space)]); + up tmp = std::move(nursery_[role2int(role::to_space)]); nursery_[role2int(role::to_space)] = std::move(nursery_[role2int(role::from_space)]); nursery_[role2int(role::from_space)] = std::move(tmp); } @@ -452,7 +452,7 @@ namespace xo { void GC::swap_tenured() { - up tmp = std::move(tenured_[role2int(role::to_space)]); + up tmp = std::move(tenured_[role2int(role::to_space)]); tenured_[role2int(role::to_space)] = std::move(tenured_[role2int(role::from_space)]); tenured_[role2int(role::from_space)] = std::move(tmp); } @@ -496,9 +496,9 @@ namespace xo { log && log(xtag("avail_tenured_z", avail_tenured_z)); if (avail_tenured_z < max_promote_z) { - ListAlloc * tenured_to = this->tenured_to(); + ArenaAlloc * tenured_to = this->tenured_to(); - tenured_to->expand(max_promote_z, tenured_to->name() + "+"); + tenured_to->expand(max_promote_z); } } @@ -516,13 +516,13 @@ namespace xo { this->swap_mutation_log(); - ListAlloc * N_from = nursery(role::from_space); + ArenaAlloc * N_from = nursery(role::from_space); log && log(xtag("nursery.from", N_from->name()), xtag("size", N_from->size())); - ListAlloc * N_to = nursery(role::to_space); + ArenaAlloc * N_to = nursery(role::to_space); log && log(xtag("nursery.to", N_to->name()), xtag("size", N_to->size())); - ListAlloc * T_from = tenured(role::from_space); + ArenaAlloc * T_from = tenured(role::from_space); log && log(xtag("tenured.from", T_from->name()), xtag("size", T_from->size())); - ListAlloc * T_to = tenured(role::to_space); + ArenaAlloc * T_to = tenured(role::to_space); log && log(xtag("tenured.to", T_to->name()), xtag("size", T_to->size())); } /*swap_spaces*/ From 8fa254418a4c801f1d8d2b995279cb31d788db1e Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 14 Aug 2025 09:50:59 -0500 Subject: [PATCH 15/69] xo-alloc: + gc history xo-imgui: gui examples --- include/xo/alloc/ArenaAlloc.hpp | 15 ++++ include/xo/alloc/GC.hpp | 28 +++++++ include/xo/alloc/GcStatistics.hpp | 66 +++++++++++++++++ include/xo/alloc/ListAlloc.hpp | 3 + include/xo/alloc/generation.hpp | 1 + src/alloc/ArenaAlloc.cpp | 12 +++ src/alloc/GC.cpp | 118 +++++++++++++++++++++++++++--- src/alloc/GcStatistics.cpp | 32 +++++++- src/alloc/ListAlloc.cpp | 17 ++++- utest/CMakeLists.txt | 1 + utest/GcStatistics.test.cpp | 4 +- utest/ListAlloc.test.cpp | 11 ++- 12 files changed, 290 insertions(+), 18 deletions(-) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 64546cf3..66dd6a70 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -36,9 +36,24 @@ namespace xo { std::size_t z, bool debug_flag); + /** size of virtual address range reserved for this allocator **/ + std::size_t reserved() const { return this->size(); } + + std::size_t page_size() const { return page_z_; } std::byte * free_ptr() const { return free_ptr_; } void set_free_ptr(std::byte * x); + /** if address @p x is allocated from this arena, + * return true along with offset relative to base address @ref lo_ + * otherwise return false with 0 + **/ + std::pair location_of(const void * x) const; + + /** allocated span **/ + std::pair allocated_span() const { + return std::make_pair(lo_, free_ptr_); + } + /** Reset to empty state **/ void reset(std::size_t /*z_ignored*/) { this->clear(); } diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index a653b54b..ab58b902 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -50,6 +50,8 @@ namespace xo { bool allow_incremental_gc_ = true; /** true to report statistics **/ bool stats_flag_ = false; + /** remember basic gc statistics for this many GC's; separately for incremental + full GCs **/ + std::size_t stats_history_z_ = 256; /** true to enable debug logging **/ bool debug_flag_ = false; }; @@ -147,17 +149,24 @@ namespace xo { const GCRunstate & runstate() const { return runstate_; } const GcStatistics & native_gc_statistics() const { return gc_statistics_; } GcStatisticsExt get_gc_statistics() const; + const GcStatisticsHistory & gc_history() const { return gc_history_; } /** true iff GC permitted in current state **/ bool is_gc_enabled() const { return gc_enabled_ == 0; } /** true during (and only during) a GC cycle **/ bool gc_in_progress() const { return runstate_.in_progress(); } + /** @return reserved size of Nursery to-space **/ + std::size_t nursery_to_reserved() const; /** @return committed size of Nursery to-space **/ std::size_t nursery_to_committed() const; /** @return nursery bytes used before checkpoint **/ std::size_t nursery_before_checkpoint() const; /** @return nursery bytes used after checkpoint **/ std::size_t nursery_after_checkpoint() const; + /** @return allocated memory range for nursery **/ + std::pair nursery_span(role role) const; + /** @return reserved size of Tenured to-space **/ + std::size_t tenured_to_reserved() const; /** @return committed size of Tenured to-space **/ std::size_t tenured_to_committed() const; /** @return tenured bytes used before checkpoint **/ @@ -167,8 +176,24 @@ namespace xo { /** @return generation to which object at @p x belongs **/ generation_result tospace_generation_of(const void * x) const; + /** @return generation to which object at @p x belongs, + * location relative to base address for that generation, + * and allocated size of that generation + * @p role chooses between to-space and from-space + **/ + std::tuple location_of(role role, const void * x) const; + /** @return generation to which object at @p x belongs, + * location relative to base address for @p x, + * and allocated size of generation + **/ + std::tuple tospace_location_of(const void * x) const; /** @return generation that contains @p x, given it's in from-space **/ generation_result fromspace_generation_of(const void * x) const; + /** @return generation to which object at @p x belongs, + * location relative to base address for @p x, + * and allocated size of generation + **/ + std::tuple fromspace_location_of(const void * x) const; /** true iff from-space contains @p x **/ bool fromspace_contains(const void * x) const; /** @return free pointer for generation @p gen, i.e. nursery or tenured space **/ @@ -361,6 +386,9 @@ namespace xo { /** enabled when 0. disabled when <0 **/ int gc_enabled_ = 0; + /** rotating per-gc statistics history **/ + GcStatisticsHistory gc_history_; + /** for (optional) viz: invoke when copying individual objects **/ GcCopyCallbackSet gc_copy_cbset_; }; diff --git a/include/xo/alloc/GcStatistics.hpp b/include/xo/alloc/GcStatistics.hpp index 24cac461..d73f8947 100644 --- a/include/xo/alloc/GcStatistics.hpp +++ b/include/xo/alloc/GcStatistics.hpp @@ -6,6 +6,7 @@ #pragma once #include "generation.hpp" +#include "CircularBuffer.hpp" #include "xo/reflect/TypeDescr.hpp" #include "xo/indentlog/print/pretty.hpp" #include @@ -57,6 +58,8 @@ namespace xo { **/ class GcStatistics { public: + GcStatistics() = default; + /** update statistics after a GC cycle * @param upto. nursery -> incremental collection; tenured -> full collection * @param alloc_z. new allocations (since preceding GC) @@ -108,6 +111,7 @@ namespace xo { **/ class GcStatisticsExt : public GcStatistics { public: + GcStatisticsExt() = default; explicit GcStatisticsExt(const GcStatistics & x) : GcStatistics{x} {} /** @param os. write stats on this output stream **/ @@ -128,6 +132,63 @@ namespace xo { return os; } + /** @class GcStatisticsHistoryItem + * @brief info we want to record over time (won't have cumulative things in it) + **/ + class GcStatisticsHistoryItem { + public: + GcStatisticsHistoryItem() = default; + GcStatisticsHistoryItem(generation upto, + std::size_t new_alloc_z, + std::size_t survive_z, + std::size_t promote_z, + std::size_t persist_z, + std::size_t effort_z, + std::size_t garbage0_z, + std::size_t garbage1_z, + std::size_t garbageN_z) + : upto_{upto}, + new_alloc_z_{new_alloc_z}, + survive_z_{survive_z}, + promote_z_{promote_z}, + persist_z_{persist_z}, + effort_z_{effort_z}, + garbage0_z_{garbage0_z}, + garbage1_z_{garbage1_z}, + garbageN_z_{garbageN_z} + {} + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** type of GC that generated this record **/ + generation upto_; + /** #of bytes new allocation **/ + std::size_t new_alloc_z_ = 0; + /** #of bytes surviving their first collection (i.e. N0->N1) **/ + std::size_t survive_z_ = 0; + /** #of bytes promoted to tenured. + * Comprises all objects surviving their 2nd collection (i.e. N1->T) + **/ + std::size_t promote_z_ = 0; + /** #of bytes surviving 3rd of later collection **/ + std::size_t persist_z_ = 0; + /** #of bytes copied **/ + std::size_t effort_z_ = 0; + /** #of bytes garbage from N0 (i.e. survived 0 GCs) **/ + std::size_t garbage0_z_ = 0; + /** #of bytes garbage from N1 (i.e. survived 1 GCs) **/ + std::size_t garbage1_z_ = 0; + /** #of bytes garbage from T (i.e. survived 2+ GCs) **/ + std::size_t garbageN_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatisticsHistoryItem & x) { + x.display(os); + return os; + } + + using GcStatisticsHistory = CircularBuffer; } /*namespace gc*/ namespace print { @@ -145,6 +206,11 @@ namespace xo { struct ppdetail { static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatisticsExt &); }; + + template<> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatisticsHistoryItem &); + }; } /*namespace print*/ } /*namespace xo*/ diff --git a/include/xo/alloc/ListAlloc.hpp b/include/xo/alloc/ListAlloc.hpp index 7b592b4e..75b148ac 100644 --- a/include/xo/alloc/ListAlloc.hpp +++ b/include/xo/alloc/ListAlloc.hpp @@ -34,6 +34,9 @@ namespace xo { static up make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag); + /** page size used by underlying ArenaAlloc **/ + std::size_t page_size() const; + /** reset to have at least @p z bytes of storage **/ bool reset(std::size_t z); diff --git a/include/xo/alloc/generation.hpp b/include/xo/alloc/generation.hpp index 82c48808..2ed89276 100644 --- a/include/xo/alloc/generation.hpp +++ b/include/xo/alloc/generation.hpp @@ -30,6 +30,7 @@ namespace xo { tenured, not_found }; + } /*namespace gc*/ } /*namespace xo*/ diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index 49a2d16d..f3145bb2 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -113,6 +113,7 @@ namespace xo { } this->committed_z_ = align_offset_z; + this->limit_ = this->lo_ + committed_z_; return true; } @@ -133,6 +134,16 @@ namespace xo { } } + std::pair + ArenaAlloc::location_of(const void * x) const + { + if ((lo_ <= x) && (x < hi_)) { + return std::make_pair(true, reinterpret_cast(x) - lo_); + } else { + return std::make_pair(false, 0); + } + } + void ArenaAlloc::capture_object_statistics(capture_phase phase, ObjectStatistics * p_dest) const @@ -276,6 +287,7 @@ namespace xo { xtag("z0", z0), xtag("+pad", dz), xtag("z1", z1), + xtag("size", this->size()), xtag("avail", this->available())); this->free_ptr_ += z1; diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 6ccc4d15..a552ae1e 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -79,6 +79,8 @@ namespace xo { mutation_log_[role2int(role::to_space)] = std::make_unique(); defer_mutation_log_ = std::make_unique(); + this->gc_history_ = CircularBuffer(config.stats_history_z_); + this->checkpoint(); } @@ -191,6 +193,12 @@ namespace xo { return retval; } + std::size_t + GC::nursery_to_reserved() const + { + return nursery_to()->reserved(); + } + std::size_t GC::nursery_to_committed() const { @@ -209,6 +217,17 @@ namespace xo { return nursery_to()->after_checkpoint(); } + std::pair + GC::nursery_span(role role) const { + return nursery(role)->allocated_span(); + } + + std::size_t + GC::tenured_to_reserved() const + { + return tenured_to()->reserved(); + } + std::size_t GC::tenured_to_committed() const { @@ -239,6 +258,40 @@ namespace xo { return generation_result::not_found; } + std::tuple + GC::location_of(role role, const void *x) const + { + { + auto space = this->tenured(role); + auto [is_tenured, offset] = space->location_of(x); + + if (is_tenured) + return std::make_tuple(generation_result::tenured, offset, space->allocated()); + } + + { + auto space = this->nursery(role); + auto [is_nursery, offset] = nursery(role)->location_of(x); + + if (is_nursery) + return std::make_tuple(generation_result::nursery, offset, space->allocated()); + } + + return std::make_tuple(generation_result::not_found, 0, 0); + } + + std::tuple + GC::tospace_location_of(const void * x) const + { + return location_of(role::to_space, x); + } + + std::tuple + GC::fromspace_location_of(const void * x) const + { + return location_of(role::from_space, x); + } + generation_result GC::fromspace_generation_of(const void * x) const { @@ -988,8 +1041,11 @@ namespace xo { { scope log(XO_DEBUG(config_.debug_flag_)); - std::size_t N_allocated = nursery_from()->after_checkpoint(); - std::size_t T_allocated = tenured_from()->after_checkpoint(); + std::size_t N0_before_gc = nursery_from()->after_checkpoint(); + std::size_t N1_before_gc = nursery_from()->before_checkpoint(); + + std::size_t T0_before_gc = tenured_from()->after_checkpoint(); + std::size_t T1_before_gc = tenured_from()->before_checkpoint(); std::size_t N_before_gc = nursery_from()->allocated(); std::size_t T_before_gc = tenured_from()->allocated(); @@ -998,9 +1054,37 @@ namespace xo { std::size_t T_after_gc = tenured_to()->allocated(); //std::byte * N_free_ptr = nursery_[role2int(role::to_space)]->free_ptr(); + std::size_t new_alloc_z = N0_before_gc; + /* survive_z: bytes surviving first collection */ + std::size_t survive_z = N_after_gc; + /* promote_z: bytes surviving 2nd collection */ std::size_t promote_z = (gc_statistics_.total_promoted_ - gc_statistics_.total_promoted_sab_); + /* #of bytes copied by this collection cycle */ + std::size_t effort_z = 0; + if (upto == generation::nursery) { + effort_z = N_after_gc + promote_z; + } else { + effort_z += N_after_gc + T_after_gc; + } + + /* persist_z: bytes surviving 3rd or later collection */ + std::size_t persist_z = 0; + if (upto == generation::tenured) + persist_z = T_after_gc - promote_z; + /* #of bytes found to be garbage on first collection + * (reminder: N_after_gc consists *entirely* of survives from N0_before_gc; + * + all such survivors are in N_after_gc) + */ + std::size_t garbage0_z = (N0_before_gc - N_after_gc); + /* #of bytes found to be garbage on 2nd collection */ + std::size_t garbage1_z = (N1_before_gc - promote_z); + /* #of bytes found to be garbage on 3rd or later collection */ + std::size_t garbageN_z = 0; + if (upto == generation::tenured) + garbageN_z = (T_before_gc - T_after_gc + promote_z); + /* Don't reset from-space here, it's unnecessary. * Would be permissible, but interferes with GC object modelling in * xo-object/utest/GC.test.cpp @@ -1014,25 +1098,38 @@ namespace xo { this->tenured_to()->checkpoint(); if (log) { - log(xtag("N_allocated", N_allocated)); - log(xtag("N_before_gc", N_before_gc)); + log(xtag("N0_before_gc", N0_before_gc)); + log(xtag("N1_before_gc", N1_before_gc)); log(xtag("N_after_gc", N_after_gc)); - log(xtag("T_allocated", T_allocated)); - log(xtag("T_before_gc", T_before_gc)); + + log(xtag("T0_before_gc", T0_before_gc)); + log(xtag("T1_before_gc", T1_before_gc)); log(xtag("T_after_gc", T_after_gc)); } + GcStatisticsHistoryItem item(upto, + new_alloc_z, + survive_z, + promote_z, + persist_z, + effort_z, + garbage0_z, + garbage1_z, + garbageN_z); + + this->gc_history_.push_back(item); + this->incr_gc_pending_ = false; - this->gc_statistics_.include_gc(generation::nursery, N_allocated, N_before_gc, N_after_gc, promote_z); + this->gc_statistics_.include_gc(generation::nursery, N0_before_gc, N_before_gc, N_after_gc, promote_z); if (upto == generation::tenured) { this->full_gc_pending_ = false; - this->gc_statistics_.include_gc(generation::tenured, T_allocated, T_before_gc, T_after_gc, 0); + this->gc_statistics_.include_gc(generation::tenured, T0_before_gc, T_before_gc, T_after_gc, 0); } else { // still want to update tenured stats for current alloc size this->gc_statistics_.update_snapshot(generation::tenured, T_after_gc); } - } + } /*cleanup_phase*/ void GC::execute_gc(generation upto) @@ -1090,6 +1187,9 @@ namespace xo { this->runstate_ = GCRunstate(); + // not this way.. reports cumulative stats + // this->gc_history_.push_back(this->get_gc_statistics()); + log && log("statistics:"); log && log(gc_statistics_); } diff --git a/src/alloc/GcStatistics.cpp b/src/alloc/GcStatistics.cpp index 7b7a6ff9..cb7ebebf 100644 --- a/src/alloc/GcStatistics.cpp +++ b/src/alloc/GcStatistics.cpp @@ -92,6 +92,22 @@ namespace xo { // << xtag("per_type_stats", per_type_stats_) << ">"; } + + void + GcStatisticsHistoryItem::display(std::ostream & os) const + { + os << ""; + } + } /*namespace gc*/ namespace print { @@ -148,7 +164,21 @@ namespace xo { refrtag("tenured_z", x.tenured_z_)); } - + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::GcStatisticsHistoryItem & x) + { + return ppii.pps()->pretty_struct(ppii, + "GcStatisticsHistoryItem", + refrtag("upto", gen2str(x.upto_)), + refrtag("survive_z", x.survive_z_), + refrtag("promote_z", x.promote_z_), + refrtag("persist_z", x.persist_z_), + refrtag("effort_z", x.effort_z_), + refrtag("garbage0_z", x.garbage0_z_), + refrtag("garbage1_z", x.garbage1_z_), + refrtag("garbageN_z", x.garbageN_z_)); + } } /*namespace print*/ } /*namespace xo*/ diff --git a/src/alloc/ListAlloc.cpp b/src/alloc/ListAlloc.cpp index 29cdb477..8b0a2dbb 100644 --- a/src/alloc/ListAlloc.cpp +++ b/src/alloc/ListAlloc.cpp @@ -70,6 +70,11 @@ namespace xo { return s_default_name; } + std::size_t + ListAlloc::page_size() const { + return hd_->page_size(); + } + std::size_t ListAlloc::size() const { return total_z_; @@ -109,8 +114,9 @@ namespace xo { ListAlloc::allocated() const { std::size_t total = 0; - if (hd_) + if (hd_) { total += hd_->allocated(); + } for (const auto & alloc : full_l_) total += alloc->allocated(); @@ -363,10 +369,17 @@ namespace xo { ListAlloc::alloc(std::size_t z) { scope log(XO_DEBUG(debug_flag_)); + /* ArenaAlloc::alloc() may modify its own size */ + + std::size_t z_pre = hd_->size(); std::byte * retval = hd_->alloc(z); - if (retval) + if (retval) { + std::size_t z_post = hd_->size(); + this->total_z_ += (z_post - z_pre); + return retval; + } log && log("space exhausted -> expand"); diff --git a/utest/CMakeLists.txt b/utest/CMakeLists.txt index 907bafcb..379ed925 100644 --- a/utest/CMakeLists.txt +++ b/utest/CMakeLists.txt @@ -12,6 +12,7 @@ set(UTEST_SRCS GcStatistics.test.cpp ObjectStatistics.test.cpp Forwarding1.test.cpp + CircularBuffer.test.cpp generation.test.cpp ) diff --git a/utest/GcStatistics.test.cpp b/utest/GcStatistics.test.cpp index 5f2fbe96..f5700f25 100644 --- a/utest/GcStatistics.test.cpp +++ b/utest/GcStatistics.test.cpp @@ -76,7 +76,7 @@ namespace xo { tag_config::tag_color_enabled = false; - GcStatisticsExt stats({}); + GcStatisticsExt stats; std::string s = tostr(stats); @@ -154,7 +154,7 @@ namespace xo { std::stringstream ss; ppconfig ppc; - GcStatisticsExt stats({}); + GcStatisticsExt stats; std::string actual = toppstr2(ppc, stats); std::string expected diff --git a/utest/ListAlloc.test.cpp b/utest/ListAlloc.test.cpp index 2db22bf2..0a62fd0b 100644 --- a/utest/ListAlloc.test.cpp +++ b/utest/ListAlloc.test.cpp @@ -12,7 +12,10 @@ namespace xo { namespace ut { TEST_CASE("ListAlloc", "[alloc][gc]") { - /** teeny weeny allocator **/ + /** teeny weeny allocator. + * but underlying ArenaAlloc works in multiples of VM page size + * (most likely 4k) + **/ up alloc = ListAlloc::make("test", 16, 32, false); REQUIRE(alloc->name() == "test"); @@ -24,7 +27,7 @@ namespace xo { std::byte * mem1 = alloc->alloc(20); REQUIRE(mem1); - REQUIRE(alloc->size() == 16 + 32); + REQUIRE(alloc->size() == alloc->page_size()); /* round up to multiple of 8 */ REQUIRE(alloc->before_checkpoint() == 24); REQUIRE(alloc->after_checkpoint() == 0); @@ -34,7 +37,7 @@ namespace xo { std::byte * mem2 = alloc->alloc(30); REQUIRE(mem2); - REQUIRE(alloc->size() == 16 + 32 + 48); + REQUIRE(alloc->size() == alloc->page_size()); REQUIRE(alloc->before_checkpoint() == 24); /* round up to multiple of 8 */ REQUIRE(alloc->after_checkpoint() == 32); @@ -42,7 +45,7 @@ namespace xo { std::byte * mem3 = alloc->alloc(40); REQUIRE(mem3); - REQUIRE(alloc->size() == 16 + 32 + 48 + 80); + REQUIRE(alloc->size() == alloc->page_size()); REQUIRE(alloc->before_checkpoint() == 24); /* already multiple of 8 */ REQUIRE(alloc->after_checkpoint() == 32 + 40); From df9ad3b8551c2d7528b07d011c3c3265fd60cf31 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 20 Aug 2025 19:53:37 -0500 Subject: [PATCH 16/69] missed files: xo-alloc/CircularBuffer --- include/xo/alloc/CircularBuffer.hpp | 245 ++++++++++++++++++++++++++++ utest/CircularBuffer.test.cpp | 174 ++++++++++++++++++++ 2 files changed, 419 insertions(+) create mode 100644 include/xo/alloc/CircularBuffer.hpp create mode 100644 utest/CircularBuffer.test.cpp diff --git a/include/xo/alloc/CircularBuffer.hpp b/include/xo/alloc/CircularBuffer.hpp new file mode 100644 index 00000000..cf1729ce --- /dev/null +++ b/include/xo/alloc/CircularBuffer.hpp @@ -0,0 +1,245 @@ +/* CircularBuffer.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#pragma once + +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/tostr.hpp" +#include "xo/indentlog/print/tag.hpp" +#include +#include +#include +//#include + +namespace xo { + namespace gc { + /** @class CircularBuffer + * @brief A circular buffer + * + * push operations may overwrite prior contents, + * i.e. buffer behavior on overflow + * old + * + * @tparam T is type for buffer elements. + **/ + template + class CircularBuffer { + public: + using value_type = T; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type &; + using const_reference = const value_type &; + using pointer = value_type *; + using const_pointer = const value_type *; + + template + class _iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = TT; + using difference_type = std::ptrdiff_t; + using pointer = value_type *; + using reference = value_type &; + + _iterator(Parent * p, std::int64_t ix) : parent_{p}, index_{ix} {} + + reference operator* () const { return (*parent_)[index_]; } + pointer operator->() const { return &(*parent_)[index_]; } + _iterator & operator++() { ++index_; return *this; } + _iterator operator++(int) { _iterator retval(parent_, index_); ++index_; return retval; } + + auto operator<=>(const _iterator & other) { + if (parent_ == other.parent_) + return index_ <=> other.index_; + else + return std::partial_ordering::unordered; + } + + bool operator==(const _iterator & other) const = default; + + private: + Parent * parent_ = nullptr; + /** index position + * (-1 = just before front = rend, 0 = front, z-1 = back, z = just after back = end) + **/ + std::int64_t index_ = 0; + }; + + using iterator = _iterator, T>; + using const_iterator = _iterator, const T>; + + public: + explicit CircularBuffer(std::size_t capacity = 0, bool debug_flag = false); + CircularBuffer(const CircularBuffer& other) = default; + CircularBuffer(CircularBuffer&& other) noexcept = default; + ~CircularBuffer() = default; + + static constexpr std::int64_t npos = -1; + + /** @return location of i'th element. i: 0=front, 1=second etc **/ + std::size_t location_of(std::size_t i) const; + /** @return ordinal index (relative to front) of location @p loc; + * npos if not used + **/ + //std::int64_t index_of(std::size_t loc) const; // not implemented yet + + // standard container methods + bool empty() const noexcept { return size_ == 0; } + size_type size() const noexcept { return size_; } + size_type max_size() const noexcept { return contents_.size(); } + // void reserve(size_type new_capacity); // not implemented + size_type capacity() const noexcept { return contents_.size(); } + // void shrink_to_fit(); // not implemented + + reference at(size_type pos) { + if ((pos < 0) || (pos >= size_)) { + throw std::out_of_range(tostr("CircularBuffer::at: index out of range", + xtag("pos", pos), xtag("size", size_))); + } + + return contents_[this->location_of(pos)]; + } + + const_reference at(size_type pos) const { + reference retval = const_cast(this)->at(pos); + return retval; + } + + reference operator[](size_type pos) { + return contents_[this->location_of(pos)]; + } + + const_reference operator[](size_type pos) const { + return contents_[this->location_of(pos)]; + } + + reference front() { return contents_[front_ix_]; } + const_reference front() const { + reference retval = const_cast(this)->front(); + return retval; + } + + reference back() { return contents_[location_of(size_ - 1)]; } + const_reference back() const { + reference retval = const_cast(this)->back(); + return retval; + } + + iterator begin() { return iterator(this, 0); } + iterator end() { return iterator(this, size_); } + const_iterator begin() const { return const_iterator(this, 0); } + const_iterator end() const { return const_iterator(this, size_); } + + // reverse_iterator rbegin(); + // reverse_iterator rend(); + // const_reverse_iterator rbegin() const; + // const_reverse_iterator rend() const; + + // General Methods + + void clear() { + size_ = 0; + front_ix_ = 0; + std::size_t capacity = contents_.size(); + contents_.clear(); + contents_.resize(capacity); + } + + /** push @p x on to the end of this buffer. + * If buffer is at capacity, overwrites the oldest element + **/ + CircularBuffer & push_back(const T & x); + + // template + //reference emplace_back(Args&&... args); + + CircularBuffer & pop_back(); + + // push_front(); + // pop_front(); + + CircularBuffer& operator=(const CircularBuffer& other) = default; + CircularBuffer& operator=(CircularBuffer&& other) noexcept = default; + + private: + /** number of elements in buffer. Not the same as @code contents_.size(); + * the latter represents buffer capacity. + * + * Promise: + * size_ <= contents_.size() + **/ + std::size_t size_ = 0; + /** first element is @code contents_.at(front_ix_) **/ + std::size_t front_ix_ = 0; + /** buffer contents. contents_.size() represents buffer capacity + * first element stored in @code contents_.at(front_) + * last element stored in @code contents_.at((front_ + size_ - 1) % contents_.size()) + **/ + std::vector contents_; + /** true to enable debug logging **/ + bool debug_flag_ = false; + }; + + template + CircularBuffer::CircularBuffer(std::size_t capacity, bool debug_flag) + : size_{0}, front_ix_{0}, contents_{capacity}, debug_flag_{debug_flag} + { + } + + template + std::size_t + CircularBuffer::location_of(std::size_t i) const + { + if (size_ == 0) + return 0; + else + return (front_ix_ + i) % size_; + } + + template + CircularBuffer & + CircularBuffer::push_back(const T & x) { + scope log(XO_DEBUG(debug_flag_), rtag("x", x), xrtag("size", size_)); + + if (size_ < contents_.size()) { + ++size_; + /* _after_ incr .size_ */ + std::size_t back_ix = location_of(size_ - 1); + + this->contents_[back_ix] = x; + + log && log(xtag("back_ix", back_ix), xtag("+size", size_)); + } else { + std::size_t back_ix = location_of(size_); + + this->contents_[back_ix] = x; + /* buffer was full, so oldest element replaced */ + this->front_ix_ = (this->front_ix_ + 1) % contents_.size(); + + log && log(xtag("back_ix", back_ix), xtag("+front", front_ix_)); + } + + return *this; + } + + template + CircularBuffer & + CircularBuffer::pop_back() { + if (size_ > 0) { + std::size_t back_ix = location_of(size_ - 1); + + this->contents_[back_ix] = T(); + --(this->size_); + } else { + assert(false); + } + + return *this; + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* CircularBuffer.hpp */ diff --git a/utest/CircularBuffer.test.cpp b/utest/CircularBuffer.test.cpp new file mode 100644 index 00000000..f3d63c7e --- /dev/null +++ b/utest/CircularBuffer.test.cpp @@ -0,0 +1,174 @@ +/* CircularBuffer.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/CircularBuffer.hpp" +#include "xo/indentlog/print/vector.hpp" +#include + +namespace xo { + using xo::gc::CircularBuffer; + + namespace ut { + TEST_CASE("circular_buffer_0", "[circular_buffer]") + { + CircularBuffer q(10, false /*debug_flag*/); + q.push_back("a"); + REQUIRE(q.back() == "a"); + q.push_back("b"); + REQUIRE(q.back() == "b"); + q.push_back("c"); + REQUIRE(q.back() == "c"); + REQUIRE(q.location_of(0) == 0); + REQUIRE(q.location_of(1) == 1); + REQUIRE(q.location_of(2) == 2); + //REQUIRE(q.index_of(0) == 0); + + REQUIRE(q.size() == 3); + REQUIRE(q.front() == "a"); + REQUIRE(q.at(0) == "a"); + REQUIRE(q.at(1) == "b"); + REQUIRE(q.at(2) == "c"); + + CircularBuffer q2; + + q2 = q; + + q.clear(); + + REQUIRE(q2.size() == 3); + REQUIRE(q2.front() == "a"); + REQUIRE(q2.at(0) == "a"); + REQUIRE(q2.at(1) == "b"); + REQUIRE(q2.at(2) == "c"); + } + + TEST_CASE("circular_buffer_1", "[circular_buffer]") + { + CircularBuffer q(2, false /*debug_flag*/); + q.push_back("a"); + REQUIRE(q.back() == "a"); + q.push_back("b"); + REQUIRE(q.back() == "b"); + q.push_back("c"); + REQUIRE(q.back() == "c"); + REQUIRE(q.location_of(0) == 1); + REQUIRE(q.location_of(1) == 0); + //REQUIRE(q.index_of(0) == 0); + + REQUIRE(q.size() == 2); + REQUIRE(q.front() == "b"); + REQUIRE(q.at(0) == "b"); + REQUIRE(q.at(1) == "c"); + + { + std::size_t i = 0; + for (const auto & qi : q) { + REQUIRE(qi == q.at(i)); + ++i; + } + } + + CircularBuffer q2 = q; + + q.clear(); + + REQUIRE(q2.size() == 2); + REQUIRE(q2.front() == "b"); + REQUIRE(q2.at(0) == "b"); + REQUIRE(q2.at(1) == "c"); + + { + std::size_t i = 0; + for (const auto & qi : q) { + REQUIRE(qi == q2.at(i)); + ++i; + } + } + } + } + + namespace { + struct Testcase_CircularBuffer { + explicit Testcase_CircularBuffer(std::size_t capacity, + const std::vector & contents) + : capacity_{capacity}, + contents_{contents} {} + + std::size_t capacity_ = 0; + std::vector contents_; + }; + + std::vector + s_testcase_v = { + Testcase_CircularBuffer(0, {}), + Testcase_CircularBuffer(1, {"a"}), + Testcase_CircularBuffer(2, {"a", "b"}), + Testcase_CircularBuffer(2, {"a", "b", "c", "d"}) + }; + } + + namespace ut { + TEST_CASE("circular_buffer_2", "[circular_buffer]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + const Testcase_CircularBuffer & tc = s_testcase_v[i_tc]; + + INFO(tostr(xtag("i_tc", i_tc), + xtag("capacity", tc.capacity_), + xrtag("contents", tc.contents_))); + + for (std::size_t j_phase = 0; j_phase < 2; ++j_phase) { + constexpr bool c_debug_flag = false; + + CircularBuffer q(tc.capacity_, false /*debug_flag*/); + + REQUIRE(q.empty()); + REQUIRE(q.size() == 0); + REQUIRE(q.begin() == q.end()); + REQUIRE(q.capacity() == tc.capacity_); + + std::size_t n = 0; + for (const auto & s : tc.contents_) { + INFO(tostr(xtag("n0", n), xtag("s", s))); + ++n; + INFO(xtag("n1", n)); + + q.push_back(s); + + REQUIRE(q.back() == s); + REQUIRE(q.capacity() == tc.capacity_); + REQUIRE(q.size() == std::min(n, tc.capacity_)); + + std::size_t i = 0; + for (const auto & qi : q) { + INFO(tostr(xtag("i", i), xtag("qi", qi))); + + if (n <= tc.capacity_) { + REQUIRE(qi == tc.contents_.at(i)); + REQUIRE(qi == tc.contents_[i]); + } else { + REQUIRE(qi == tc.contents_.at(n - tc.capacity_ + i)); + REQUIRE(qi == tc.contents_[n - tc.capacity_ + i]); + } + ++i; + } + + REQUIRE(i == std::min(n, tc.capacity_)); + + if (tc.contents_.size() <= tc.capacity_) + REQUIRE(q.front() == tc.contents_.at(0)); + } + + q.clear(); + + REQUIRE(q.size() == 0); + REQUIRE(q.capacity() == tc.capacity_); + } + } + } + } +} + +/* CircularBuffer.test.cpp */ From 78195b02185d80ac0e8b5c705d44fd7f35b670ce Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 22 Aug 2025 15:10:56 -0400 Subject: [PATCH 17/69] xo-alloc: + timing stats + timeseries tooltips --- include/xo/alloc/GC.hpp | 5 ++- include/xo/alloc/GcStatistics.hpp | 75 +++++++++++++++++++++++-------- src/alloc/CMakeLists.txt | 1 + src/alloc/GC.cpp | 55 +++++++++++++++-------- src/alloc/GcStatistics.cpp | 14 +++++- 5 files changed, 110 insertions(+), 40 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index ab58b902..7c4f42c6 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -132,6 +132,7 @@ namespace xo { public: using CallbackId = xo::fn::CallbackId; using GcCopyCallbackSet = xo::fn::UpCallbackSet; + using nanos = decltype(xo::qty::qty::nanosecond); public: /** create new GC instance with configuration @p config **/ @@ -224,6 +225,8 @@ namespace xo { * as number of calls to @ref disable_gc. **/ void enable_gc(); + /** same as @c this->enable_gc() followed by @c this->disable_gc() **/ + void enable_gc_once(); // inherited from IAlloc.. @@ -279,7 +282,7 @@ namespace xo { /** begin GC now **/ void execute_gc(generation g); /** cleanup phase. aux function for @ref execute_gc **/ - void cleanup_phase(generation g); + void cleanup_phase(generation g, nanos dt); /** swap roles of From/To spaces for nursery generation **/ void swap_nursery(); /** swap roles of From/To spaces for tenured generation **/ diff --git a/include/xo/alloc/GcStatistics.hpp b/include/xo/alloc/GcStatistics.hpp index d73f8947..d0f54084 100644 --- a/include/xo/alloc/GcStatistics.hpp +++ b/include/xo/alloc/GcStatistics.hpp @@ -8,6 +8,8 @@ #include "generation.hpp" #include "CircularBuffer.hpp" #include "xo/reflect/TypeDescr.hpp" +#include "xo/unit/quantity.hpp" +#include "xo/unit/quantity_iostream.hpp" #include "xo/indentlog/print/pretty.hpp" #include #include @@ -60,6 +62,13 @@ namespace xo { public: GcStatistics() = default; + /** update statistics at beginning of a GC cycle + * @param upto. nursery -> incremental collection; tenured -> full collection + * @param alloc_z. new allocations (since preceding GC) + **/ + void begin_gc(generation upto, + std::size_t alloc_z); + /** update statistics after a GC cycle * @param upto. nursery -> incremental collection; tenured -> full collection * @param alloc_z. new allocations (since preceding GC) @@ -74,6 +83,9 @@ namespace xo { **/ void update_snapshot(generation upto, std::size_t after_z); + /** number of collection cycles, whether full or incremental **/ + std::size_t n_gc() const { return gen_v_[gen2int(generation::nursery)].n_gc_; } + /** @param os. write stats on this output stream **/ void display(std::ostream & os) const; @@ -136,31 +148,54 @@ namespace xo { * @brief info we want to record over time (won't have cumulative things in it) **/ class GcStatisticsHistoryItem { + public: + using nanos = xo::qty::type::nanoseconds; + public: GcStatisticsHistoryItem() = default; - GcStatisticsHistoryItem(generation upto, - std::size_t new_alloc_z, - std::size_t survive_z, - std::size_t promote_z, - std::size_t persist_z, - std::size_t effort_z, - std::size_t garbage0_z, - std::size_t garbage1_z, - std::size_t garbageN_z) - : upto_{upto}, - new_alloc_z_{new_alloc_z}, - survive_z_{survive_z}, - promote_z_{promote_z}, - persist_z_{persist_z}, - effort_z_{effort_z}, - garbage0_z_{garbage0_z}, - garbage1_z_{garbage1_z}, - garbageN_z_{garbageN_z} - {} + constexpr GcStatisticsHistoryItem(std::size_t gc_seq, + generation upto, + std::size_t new_alloc_z, + std::size_t survive_z, + std::size_t promote_z, + std::size_t persist_z, + std::size_t effort_z, + std::size_t garbage0_z, + std::size_t garbage1_z, + std::size_t garbageN_z, + nanos dt) : gc_seq_{gc_seq}, + upto_{upto}, + new_alloc_z_{new_alloc_z}, + survive_z_{survive_z}, + promote_z_{promote_z}, + persist_z_{persist_z}, + effort_z_{effort_z}, + garbage0_z_{garbage0_z}, + garbage1_z_{garbage1_z}, + garbageN_z_{garbageN_z}, + dt_{dt} {} + constexpr GcStatisticsHistoryItem(const GcStatisticsHistoryItem &) = default; + + GcStatisticsHistoryItem & operator=(const GcStatisticsHistoryItem & x) { + gc_seq_ = x.gc_seq_; + upto_ = x.upto_; + new_alloc_z_ = x.new_alloc_z_; + survive_z_ = x.survive_z_; + promote_z_ = x.promote_z_; + persist_z_ = x.persist_z_; + effort_z_ = x.effort_z_; + garbage0_z_ = x.garbage0_z_; + garbage1_z_ = x.garbage1_z_; + garbageN_z_ = x.garbageN_z_; + this->dt_.scale_ = x.dt_.scale_; + return *this; + } /** @param os. write stats on this output stream **/ void display(std::ostream & os) const; + /** sequence number for collection being reported **/ + std::size_t gc_seq_ = 0; /** type of GC that generated this record **/ generation upto_; /** #of bytes new allocation **/ @@ -181,6 +216,8 @@ namespace xo { std::size_t garbage1_z_ = 0; /** #of bytes garbage from T (i.e. survived 2+ GCs) **/ std::size_t garbageN_z_ = 0; + /** elapsed time for this GC (see @ref GC::execute_gc) **/ + nanos dt_; }; inline std::ostream & operator<< (std::ostream & os, const GcStatisticsHistoryItem & x) { diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index 07f87784..b83e2618 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -14,6 +14,7 @@ set(SELF_SRCS ) xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) +xo_dependency(${SELF_LIB} xo_unit) xo_dependency(${SELF_LIB} reflect) xo_dependency(${SELF_LIB} callback) diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index a552ae1e..e6fb188f 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -7,6 +7,7 @@ #include "GC.hpp" #include "Object.hpp" #include "xo/indentlog/scope.hpp" +#include #include #include @@ -1037,9 +1038,9 @@ namespace xo { } void - GC::cleanup_phase(generation upto) + GC::cleanup_phase(generation upto, nanos dt) { - scope log(XO_DEBUG(config_.debug_flag_)); + scope log(XO_DEBUG(config_.stats_flag_)); std::size_t N0_before_gc = nursery_from()->after_checkpoint(); std::size_t N1_before_gc = nursery_from()->before_checkpoint(); @@ -1098,6 +1099,7 @@ namespace xo { this->tenured_to()->checkpoint(); if (log) { + log(xtag("gcseq_before_gc", gc_statistics_.n_gc())); log(xtag("N0_before_gc", N0_before_gc)); log(xtag("N1_before_gc", N1_before_gc)); log(xtag("N_after_gc", N_after_gc)); @@ -1107,18 +1109,6 @@ namespace xo { log(xtag("T_after_gc", T_after_gc)); } - GcStatisticsHistoryItem item(upto, - new_alloc_z, - survive_z, - promote_z, - persist_z, - effort_z, - garbage0_z, - garbage1_z, - garbageN_z); - - this->gc_history_.push_back(item); - this->incr_gc_pending_ = false; this->gc_statistics_.include_gc(generation::nursery, N0_before_gc, N_before_gc, N_after_gc, promote_z); @@ -1129,6 +1119,23 @@ namespace xo { // still want to update tenured stats for current alloc size this->gc_statistics_.update_snapshot(generation::tenured, T_after_gc); } + GcStatisticsHistoryItem item(gc_statistics_.n_gc(), + upto, + new_alloc_z, + survive_z, + promote_z, + persist_z, + effort_z, + garbage0_z, + garbage1_z, + garbageN_z, + dt); + + log && log(xtag("gcseq_after_gc", gc_statistics_.n_gc()), + xtag("item", item)); + + this->gc_history_.push_back(item); + } /*cleanup_phase*/ void @@ -1136,6 +1143,8 @@ namespace xo { { scope log(XO_DEBUG(config_.stats_flag_)); + auto t0 = std::chrono::steady_clock::now(); + bool full_move = (upto == generation::tenured); // TODO: RAII version in case of exceptions @@ -1146,9 +1155,7 @@ namespace xo { /* new allocation since last GC */ std::size_t new_alloc = this->after_checkpoint(); - ++(gc_statistics_.gen_v_[static_cast(upto)].n_gc_); - gc_statistics_.total_allocated_ += new_alloc; - gc_statistics_.total_promoted_sab_ = gc_statistics_.total_promoted_; + gc_statistics_.begin_gc(upto, new_alloc); log && log(xtag("new_alloc", new_alloc)); @@ -1176,10 +1183,13 @@ namespace xo { log && log("step 6 : cleanup"); - this->cleanup_phase(upto); - this->capture_object_statistics(upto, capture_phase::sae); + auto t1 = std::chrono::steady_clock::now(); + auto dt = std::chrono::duration_cast(t1 - t0); + + this->cleanup_phase(upto, xo::qty::qty::nanoseconds(dt.count())); + log && log("object statistics [nursery]:"); log && log(refrtag("stats", object_statistics_sab_[gen2int(generation::nursery)])); log && log("object statistics [tenured]:"); @@ -1233,6 +1243,13 @@ namespace xo { this->request_gc(full_gc_pending_ ? generation::tenured : generation::nursery); } } + + void + GC::enable_gc_once() { + this->enable_gc(); + this->disable_gc(); + } + } /*namespace gc*/ } /*namespace xo*/ diff --git a/src/alloc/GcStatistics.cpp b/src/alloc/GcStatistics.cpp index cb7ebebf..72624403 100644 --- a/src/alloc/GcStatistics.cpp +++ b/src/alloc/GcStatistics.cpp @@ -16,6 +16,7 @@ namespace xo { { this->update_snapshot(after_z); + //++n_gc_; new_alloc_z_ += alloc_z; scanned_z_ += before_z; survive_z_ += after_z; @@ -41,6 +42,15 @@ namespace xo { << ">"; } + void + GcStatistics::begin_gc(generation upto, + std::size_t new_alloc) + { + ++(this->gen_v_[static_cast(upto)].n_gc_); + this->total_allocated_ += new_alloc; + this->total_promoted_sab_ = total_promoted_; + } + void GcStatistics::include_gc(generation upto, std::size_t alloc_z, @@ -105,6 +115,7 @@ namespace xo { << xrtag("garbage0_z", garbage0_z_) << xrtag("garbage1_z", garbage1_z_) << xrtag("garbageN_z", garbageN_z_) + << xrtag("dt", dt_) << ">"; } @@ -177,7 +188,8 @@ namespace xo { refrtag("effort_z", x.effort_z_), refrtag("garbage0_z", x.garbage0_z_), refrtag("garbage1_z", x.garbage1_z_), - refrtag("garbageN_z", x.garbageN_z_)); + refrtag("garbageN_z", x.garbageN_z_), + refrtag("dt", x.dt_)); } } /*namespace print*/ } /*namespace xo*/ From d751093a87021e99ec655d0e29dd46d1801229d9 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 23 Aug 2025 10:47:52 -0400 Subject: [PATCH 18/69] xo-imgui: ex2: animate GC copy step --- include/xo/alloc/GC.hpp | 12 +++++++++--- include/xo/alloc/generation.hpp | 10 ++++++++++ src/alloc/GC.cpp | 13 +++++++++---- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 7c4f42c6..8ddb3e23 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -154,6 +154,8 @@ namespace xo { /** true iff GC permitted in current state **/ bool is_gc_enabled() const { return gc_enabled_ == 0; } + /** true iff GC has been requested **/ + bool is_gc_pending() const { return incr_gc_pending_ || full_gc_pending_; } /** true during (and only during) a GC cycle **/ bool gc_in_progress() const { return runstate_.in_progress(); } /** @return reserved size of Nursery to-space **/ @@ -223,10 +225,14 @@ namespace xo { * * GC is enabled when number of calls to @ref enable_gc is at least as large * as number of calls to @ref disable_gc. + * + * @return true iff GC performed **/ - void enable_gc(); - /** same as @c this->enable_gc() followed by @c this->disable_gc() **/ - void enable_gc_once(); + bool enable_gc(); + /** same as @c this->enable_gc() followed by @c this->disable_gc() + * @return true iff GC performed + **/ + bool enable_gc_once(); // inherited from IAlloc.. diff --git a/include/xo/alloc/generation.hpp b/include/xo/alloc/generation.hpp index 2ed89276..0acd943a 100644 --- a/include/xo/alloc/generation.hpp +++ b/include/xo/alloc/generation.hpp @@ -7,6 +7,7 @@ #include #include +#include namespace xo { namespace gc { @@ -31,6 +32,15 @@ namespace xo { not_found }; + inline generation valid_genresult2gen(generation_result x) { + assert(x != generation_result::not_found); + + if (x == generation_result::nursery) + return generation::nursery; + else + return generation::tenured; + } + } /*namespace gc*/ } /*namespace xo*/ diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index e6fb188f..200ac218 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -1233,21 +1233,26 @@ namespace xo { --gc_enabled_; } - void + bool GC::enable_gc() { ++gc_enabled_; if (gc_enabled_ == 0) { /* unblock gc */ - if (incr_gc_pending_) + if (incr_gc_pending_) { this->request_gc(full_gc_pending_ ? generation::tenured : generation::nursery); + return true; + } } + + return false; } - void + bool GC::enable_gc_once() { - this->enable_gc(); + bool retval = this->enable_gc(); this->disable_gc(); + return retval; } } /*namespace gc*/ From 0a19c8b043a07a435f33a4d7b3c0bf904f354fe1 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 23 Aug 2025 13:09:59 -0400 Subject: [PATCH 19/69] xo-alloc: track GC efficiency --- include/xo/alloc/GcStatistics.hpp | 11 +++++++++++ src/alloc/GcStatistics.cpp | 17 +++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/include/xo/alloc/GcStatistics.hpp b/include/xo/alloc/GcStatistics.hpp index d0f54084..c0058b5f 100644 --- a/include/xo/alloc/GcStatistics.hpp +++ b/include/xo/alloc/GcStatistics.hpp @@ -176,6 +176,17 @@ namespace xo { dt_{dt} {} constexpr GcStatisticsHistoryItem(const GcStatisticsHistoryItem &) = default; + std::size_t garbage_z() const { return garbage0_z_ + garbage1_z_ + garbageN_z_; } + + float efficiency() const { + std::size_t gz = this->garbage_z(); + + return gz / static_cast(effort_z_ + gz); + } + + /** collection rate, in bytes/sec **/ + float collection_rate() const; + GcStatisticsHistoryItem & operator=(const GcStatisticsHistoryItem & x) { gc_seq_ = x.gc_seq_; upto_ = x.upto_; diff --git a/src/alloc/GcStatistics.cpp b/src/alloc/GcStatistics.cpp index 72624403..deb30685 100644 --- a/src/alloc/GcStatistics.cpp +++ b/src/alloc/GcStatistics.cpp @@ -103,6 +103,23 @@ namespace xo { << ">"; } + float + GcStatisticsHistoryItem::collection_rate() const { + using namespace xo::qty::qty; + + float gz = this->garbage_z(); + + auto dt_nanos = this->dt_.with_repr(); + auto dt_sec = dt_nanos.rescale_ext(); + auto rate = gz / dt_sec; + float retval = rate.scale(); + + //scope log(XO_DEBUG(true)); + //log && log(xtag("gz", gz), xtag("dt_sec", dt_sec), xtag("rate", rate), xtag("rate/sec", retval)); + + return retval; + } + void GcStatisticsHistoryItem::display(std::ostream & os) const { From 4c824edbe46ed29678a5f5c63b1fc95036b98042 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 24 Aug 2025 17:03:04 -0400 Subject: [PATCH 20/69] xo-imgui: ex2: + average efficiency + plot --- include/xo/alloc/GcStatistics.hpp | 44 +++++++++++++++++++++++-------- src/alloc/GC.cpp | 13 ++++++++- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/include/xo/alloc/GcStatistics.hpp b/include/xo/alloc/GcStatistics.hpp index c0058b5f..b591bf22 100644 --- a/include/xo/alloc/GcStatistics.hpp +++ b/include/xo/alloc/GcStatistics.hpp @@ -163,17 +163,23 @@ namespace xo { std::size_t garbage0_z, std::size_t garbage1_z, std::size_t garbageN_z, - nanos dt) : gc_seq_{gc_seq}, - upto_{upto}, - new_alloc_z_{new_alloc_z}, - survive_z_{survive_z}, - promote_z_{promote_z}, - persist_z_{persist_z}, - effort_z_{effort_z}, - garbage0_z_{garbage0_z}, - garbage1_z_{garbage1_z}, - garbageN_z_{garbageN_z}, - dt_{dt} {} + nanos dt, + std::size_t sum_effort_z, + std::size_t sum_garbage_z) + : gc_seq_{gc_seq}, + upto_{upto}, + new_alloc_z_{new_alloc_z}, + survive_z_{survive_z}, + promote_z_{promote_z}, + persist_z_{persist_z}, + effort_z_{effort_z}, + garbage0_z_{garbage0_z}, + garbage1_z_{garbage1_z}, + garbageN_z_{garbageN_z}, + dt_{dt}, + sum_effort_z_{sum_effort_z}, + sum_garbage_z_{sum_garbage_z} + {} constexpr GcStatisticsHistoryItem(const GcStatisticsHistoryItem &) = default; std::size_t garbage_z() const { return garbage0_z_ + garbage1_z_ + garbageN_z_; } @@ -184,6 +190,11 @@ namespace xo { return gz / static_cast(effort_z_ + gz); } + /** lifetime byte-weighted average collection efficiency. Always in [0.0, 1.0] **/ + float average_efficiency() const { + return sum_garbage_z_ / static_cast(sum_effort_z_ + sum_garbage_z_); + } + /** collection rate, in bytes/sec **/ float collection_rate() const; @@ -199,6 +210,10 @@ namespace xo { garbage1_z_ = x.garbage1_z_; garbageN_z_ = x.garbageN_z_; this->dt_.scale_ = x.dt_.scale_; + + sum_effort_z_ = x.sum_effort_z_; + sum_garbage_z_ = x.sum_garbage_z_; + return *this; } @@ -229,6 +244,13 @@ namespace xo { std::size_t garbageN_z_ = 0; /** elapsed time for this GC (see @ref GC::execute_gc) **/ nanos dt_; + + // ----- cumulative statistics ----- + + /** sum (in bytes) copied by collections since inception **/ + std::size_t sum_effort_z_ = 0; + /** sum (in bytes) of garbage collected since inception **/ + std::size_t sum_garbage_z_ = 0; }; inline std::ostream & operator<< (std::ostream & os, const GcStatisticsHistoryItem & x) { diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 200ac218..1b5481cb 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -1119,6 +1119,15 @@ namespace xo { // still want to update tenured stats for current alloc size this->gc_statistics_.update_snapshot(generation::tenured, T_after_gc); } + + std::size_t sum_effort_z = effort_z; + std::size_t sum_garbage_z = garbage0_z + garbage1_z + garbageN_z; + + if (gc_history_.size() > 0) { + sum_effort_z += gc_history_.back().sum_effort_z_; + sum_garbage_z += gc_history_.back().sum_garbage_z_; + } + GcStatisticsHistoryItem item(gc_statistics_.n_gc(), upto, new_alloc_z, @@ -1129,7 +1138,9 @@ namespace xo { garbage0_z, garbage1_z, garbageN_z, - dt); + dt, + sum_effort_z, + sum_garbage_z); log && log(xtag("gcseq_after_gc", gc_statistics_.n_gc()), xtag("item", item)); From a9563204706635cc7fe3dee9014c1991899a2e03 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 25 Aug 2025 16:09:18 -0400 Subject: [PATCH 21/69] xo-imgui: ex: display both from+to spaces + refactor&streamline --- include/xo/alloc/GC.hpp | 9 +++++++++ src/alloc/GC.cpp | 2 ++ 2 files changed, 11 insertions(+) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 8ddb3e23..1e55c497 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -147,6 +147,8 @@ namespace xo { static up make(const Config & config); const Config & config() const { return config_; } + std::uint8_t nursery_polarity() const { return nursery_polarity_; } + std::uint8_t tenured_polarity() const { return tenured_polarity_; } const GCRunstate & runstate() const { return runstate_; } const GcStatistics & native_gc_statistics() const { return gc_statistics_; } GcStatisticsExt get_gc_statistics() const; @@ -345,6 +347,13 @@ namespace xo { /** garbage collector configuration **/ Config config_; + /** keep track of the identity of from-space and to-space. + * assist for animation (see xo-imgui/example/ex2). + * polarity alternates between 0 and 1 on each GC + **/ + std::uint8_t nursery_polarity_ = 0; + std::uint8_t tenured_polarity_ = 0; + /** contains allocated objects, along with unreachable garbage to be collected. * roles reverse after each incremental, or full, collection. **/ diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 1b5481cb..5f75c2a0 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -501,6 +501,7 @@ namespace xo { up tmp = std::move(nursery_[role2int(role::to_space)]); nursery_[role2int(role::to_space)] = std::move(nursery_[role2int(role::from_space)]); nursery_[role2int(role::from_space)] = std::move(tmp); + nursery_polarity_ = 1 - nursery_polarity_; } void @@ -509,6 +510,7 @@ namespace xo { up tmp = std::move(tenured_[role2int(role::to_space)]); tenured_[role2int(role::to_space)] = std::move(tenured_[role2int(role::from_space)]); tenured_[role2int(role::from_space)] = std::move(tmp); + tenured_polarity_ = 1 - tenured_polarity_; } void From 6c7b13e44327c668e6aa9000f551d03315576a66 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 26 Aug 2025 13:36:18 -0400 Subject: [PATCH 22/69] xo-alloc: try to make commit happen at start of GC cycle --- include/xo/alloc/ArenaAlloc.hpp | 8 +-- include/xo/alloc/GC.hpp | 27 +++++++--- src/alloc/ArenaAlloc.cpp | 6 +++ src/alloc/GC.cpp | 88 ++++++++++++++++++--------------- 4 files changed, 79 insertions(+), 50 deletions(-) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 66dd6a70..11fcdb72 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -54,13 +54,15 @@ namespace xo { return std::make_pair(lo_, free_ptr_); } - /** Reset to empty state **/ - void reset(std::size_t /*z_ignored*/) { this->clear(); } + /** Reset to empty state; provision at least @p need_z bytes of (committed) space **/ + void reset(std::size_t need_z); void capture_object_statistics(capture_phase phase, ObjectStatistics * p_dest) const; - /** expand available (i.e. committed) space to size @p z **/ + /** expand available (i.e. committed) space to size at least @p z + * In practice will round up to a multiple of @ref page_z_ + **/ bool expand(std::size_t z); // inherited from IAlloc... diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 1e55c497..d145e2c7 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -33,12 +33,16 @@ namespace xo { struct Config { /** initial size in bytes for youngest (Nursery) generation. * GC allocates two nursery spaces of this size. - * Will allocate more space as needed + * This number represents reserved address space. + * pages are committed on demand. + * Initial committment will be up to @ref incr_gc_threshold_ **/ std::size_t initial_nursery_z_ = 0; /** initial size in bytes for oldest (Tenured) generation. - * GC allocates two tenured spaces of this size - * Will allocate more space as needed + * GC allocates two tenured spaces of this size. + * This number represents reserved address space. + * pages are committed on demand. + * Initial committment will be up to @ref full_gc_threshold_ **/ std::size_t initial_tenured_z_ = 0; /** trigger incremental GC after this many bytes allocated in nursery **/ @@ -158,6 +162,8 @@ namespace xo { bool is_gc_enabled() const { return gc_enabled_ == 0; } /** true iff GC has been requested **/ bool is_gc_pending() const { return incr_gc_pending_ || full_gc_pending_; } + /** true iff full GC pending **/ + bool is_full_gc_pending() const { return full_gc_pending_; } /** true during (and only during) a GC cycle **/ bool gc_in_progress() const { return runstate_.in_progress(); } /** @return reserved size of Nursery to-space **/ @@ -170,6 +176,10 @@ namespace xo { std::size_t nursery_after_checkpoint() const; /** @return allocated memory range for nursery **/ std::pair nursery_span(role role) const; + /** @return nursery bytes used in from-space + * (only interesting during GC copy phase, e.g. during scope of a GcCopyCallback call) + **/ + std::size_t nursery_from_allocated() const; /** @return reserved size of Tenured to-space **/ std::size_t tenured_to_reserved() const; /** @return committed size of Tenured to-space **/ @@ -186,19 +196,19 @@ namespace xo { * and allocated size of that generation * @p role chooses between to-space and from-space **/ - std::tuple location_of(role role, const void * x) const; + std::tuple location_of(role role, const void * x) const; /** @return generation to which object at @p x belongs, * location relative to base address for @p x, * and allocated size of generation **/ - std::tuple tospace_location_of(const void * x) const; + std::tuple tospace_location_of(const void * x) const; /** @return generation that contains @p x, given it's in from-space **/ generation_result fromspace_generation_of(const void * x) const; /** @return generation to which object at @p x belongs, * location relative to base address for @p x, * and allocated size of generation **/ - std::tuple fromspace_location_of(const void * x) const; + std::tuple fromspace_location_of(const void * x) const; /** true iff from-space contains @p x **/ bool fromspace_contains(const void * x) const; /** @return free pointer for generation @p gen, i.e. nursery or tenured space **/ @@ -215,7 +225,10 @@ namespace xo { * Intended for GC visualization. **/ CallbackId add_gc_copy_callback(up fn); - /** request garbage collection. **/ + /** request garbage collection. + * If GC currently disabled, collection will be deferred until the next time GC + * is in an enabled state. See @ref disable_gc and @ref enable_gc + **/ void request_gc(generation g); /** disable garbage collection until matching call to @ref enable_gc. * diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index f3145bb2..9703d103 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -144,6 +144,12 @@ namespace xo { } } + void + ArenaAlloc::reset(std::size_t need_z) { + this->clear(); + this->expand(need_z); + } + void ArenaAlloc::capture_object_statistics(capture_phase phase, ObjectStatistics * p_dest) const diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 5f75c2a0..7d18da57 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -68,16 +68,23 @@ namespace xo { nursery_[role2int(role::from_space)] = ArenaAlloc::make("NA", nursery_size, config.debug_flag_); + nursery_[role2int(role::to_space) ] = ArenaAlloc::make("NB", nursery_size, config.debug_flag_); tenured_[role2int(role::from_space)] = ArenaAlloc::make("TA", tenured_size, config.debug_flag_); + tenured_[role2int(role::to_space) ] = ArenaAlloc::make("TB", tenured_size, config.debug_flag_); + nursery_[role2int(role::from_space)]->expand(config.incr_gc_threshold_); + nursery_[role2int(role::to_space) ]->expand(config.incr_gc_threshold_); + tenured_[role2int(role::from_space)]->expand(config.full_gc_threshold_); + tenured_[role2int(role::to_space) ]->expand(config.full_gc_threshold_); + mutation_log_[role2int(role::from_space)] = std::make_unique(); - mutation_log_[role2int(role::to_space)] = std::make_unique(); + mutation_log_[role2int(role::to_space )] = std::make_unique(); defer_mutation_log_ = std::make_unique(); this->gc_history_ = CircularBuffer(config.stats_history_z_); @@ -194,6 +201,12 @@ namespace xo { return retval; } + std::size_t + GC::nursery_from_allocated() const + { + return nursery_from()->allocated(); + } + std::size_t GC::nursery_to_reserved() const { @@ -259,7 +272,7 @@ namespace xo { return generation_result::not_found; } - std::tuple + std::tuple GC::location_of(role role, const void *x) const { { @@ -267,7 +280,7 @@ namespace xo { auto [is_tenured, offset] = space->location_of(x); if (is_tenured) - return std::make_tuple(generation_result::tenured, offset, space->allocated()); + return std::make_tuple(generation_result::tenured, offset, space->allocated(), space->committed()); } { @@ -275,19 +288,19 @@ namespace xo { auto [is_nursery, offset] = nursery(role)->location_of(x); if (is_nursery) - return std::make_tuple(generation_result::nursery, offset, space->allocated()); + return std::make_tuple(generation_result::nursery, offset, space->allocated(), space->committed()); } - return std::make_tuple(generation_result::not_found, 0, 0); + return std::make_tuple(generation_result::not_found, 0, 0, 0); } - std::tuple + std::tuple GC::tospace_location_of(const void * x) const { return location_of(role::to_space, x); } - std::tuple + std::tuple GC::fromspace_location_of(const void * x) const { return location_of(role::from_space, x); @@ -533,29 +546,26 @@ namespace xo { */ std::size_t max_promote_z = nursery_[role2int(role::to_space)]->before_checkpoint(); - log && log(xtag("max_promote_z", max_promote_z)); + ArenaAlloc * tenured_to = this->tenured_to(); + + /* tenured generation may need this much space */ + std::size_t need_tenured_z = (tenured_to->allocated() + + max_promote_z + + config_.full_gc_threshold_); + + log && log(xtag("alloc_tenured_z", tenured_to->allocated()), + xtag("max_promote_z", max_promote_z), + xtag("full_gc_threshold", config_.full_gc_threshold_), + xtag("need_tenured_z", need_tenured_z)); + + tenured_to->expand(tenured_to->allocated() + + max_promote_z + + config_.full_gc_threshold_); if (target == generation::tenured) { - /* gc on tenured generation may need this much space */ - std::size_t need_tenured_z = (tenured_[role2int(role::to_space)]->allocated() - + max_promote_z - + config_.full_gc_threshold_); - - log && log("need_tenured_z", need_tenured_z); - - tenured_from()->reset(need_tenured_z); + tenured_from()->clear(); this->swap_tenured(); - } else { - std::size_t avail_tenured_z = tenured_[role2int(role::to_space)]->available(); - - log && log(xtag("avail_tenured_z", avail_tenured_z)); - - if (avail_tenured_z < max_promote_z) { - ArenaAlloc * tenured_to = this->tenured_to(); - - tenured_to->expand(max_promote_z); - } } /* subtracting max_promote_z is correct here, since anything not promoted is garbage */ @@ -1220,24 +1230,22 @@ namespace xo { void GC::request_gc(generation target) { + /** full collection when >= @ref full_gc_threshold_ bytes added to tenured + * generation, since last full collection + **/ + bool need_full_gc + = ((target == generation::tenured) + || (this->tenured_to()->after_checkpoint() > config_.full_gc_threshold_) + || !config_.allow_incremental_gc_); + + if (need_full_gc) + target = generation::tenured; + if (!runstate_.in_progress() && (gc_enabled_ == 0)) { - if (!config_.allow_incremental_gc_) - target = generation::tenured; - - if ((target == generation::nursery) - && (this->tenured_to()->after_checkpoint() > config_.full_gc_threshold_)) - { - /** full collection when >= @ref full_gc_threshold_ bytes added to tenured - * generation, since last full collection - **/ - target = generation::tenured; - } - this->execute_gc(target); } else { this->incr_gc_pending_ = true; - if (target == generation::tenured) - this->full_gc_pending_ = true; + this->full_gc_pending_ |= need_full_gc; } } From f46c1d613e0a2984479efea252bf41b36cb88e5b Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 29 Aug 2025 19:33:09 -0400 Subject: [PATCH 23/69] xo-imgui: clang compiler nits --- include/xo/alloc/GC.hpp | 2 ++ include/xo/alloc/ObjectStatistics.hpp | 3 ++- src/alloc/ArenaAlloc.cpp | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index d145e2c7..313306bc 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -119,6 +119,8 @@ namespace xo { **/ class GcCopyCallback { public: + virtual ~GcCopyCallback() = default; + virtual void notify_gc_copy(std::size_t z, const void * src_addr, const void * dest_addr, generation src_gen, generation dest_gen) = 0; /** invoked when added to callback set (i.e. @ref GC::GcCopyCallbackSet) **/ diff --git a/include/xo/alloc/ObjectStatistics.hpp b/include/xo/alloc/ObjectStatistics.hpp index ced3b463..43ddd70e 100644 --- a/include/xo/alloc/ObjectStatistics.hpp +++ b/include/xo/alloc/ObjectStatistics.hpp @@ -56,7 +56,8 @@ namespace xo { * * Passed to @ref Object::deep_move for example **/ - struct ObjectStatistics { + class ObjectStatistics { + public: void display(std::ostream & os) const; /** per-object-type statistics, indexed by TypeId **/ diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index 9703d103..4fdcefe8 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -9,6 +9,7 @@ #include "xo/indentlog/scope.hpp" #include "xo/indentlog/print/tag.hpp" #include +#include // for getpagesize() on OSX #include namespace xo { From 954921c641a5c31714bbdec9e33774a67f8a775e Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 15 Nov 2025 12:25:03 -0500 Subject: [PATCH 24/69] xo-alloc: doc improvements --- docs/install.rst | 2 +- include/xo/alloc/ArenaAlloc.hpp | 44 ++++++++++++++++++++++++++------ include/xo/alloc/Forwarding.hpp | 28 -------------------- include/xo/alloc/Forwarding1.hpp | 16 +++++++++--- include/xo/alloc/Object.hpp | 4 +-- 5 files changed, 52 insertions(+), 42 deletions(-) delete mode 100644 include/xo/alloc/Forwarding.hpp diff --git a/docs/install.rst b/docs/install.rst index ab356be5..a61d9eea 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -24,7 +24,7 @@ Install One-step Install ---------------- -Install along with the reset of *XO* from `xo-umbrella2 source`_ +Install along with the rest of *XO* from `xo-umbrella2 source`_ .. _xo-umbrella2 source: https://github.com/rconybea/xo-umbrella2 diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 11fcdb72..da67f8f2 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -11,16 +11,41 @@ namespace xo { namespace gc { /** @class ArenaAlloc - * @brief Bump allocator with fixed capacity + * @brief Bump allocator with fixed capacity with dynamic virtual memory commitment. * * @text * - * <-----allocated----> <--------free-------> - * XXXXXXXXXXXXXXXXXXXX______________________ - * ^ ^ ^ - * lo free hi - * limit + * allocation order: + * -----------------------> + * + * <----------------- .size() ------------------> + * <----------------- .committed() ---------------> + * + * <-------allocated------><--------free--------> <---uncommitted----> + * XXXXXXXXXXXXXXXXXXXXXXXX______________________ .................... + * ^ ^ ^ ^ ^ + * lo checkpoint free limit hi + * + * +- .alloc() -> + * +-- .expand() --> + * > < .before_checkpoint() + * > < .after_checkpoint() + * * @endtext + * + * Design Notes: + * - non-copyable, non-moveable + * - always heap-allocated + * - @ref lo_ <= @ref checkpoint_ <= @ref free_ <= @ref limit_ <= @ref hi_ + * - memory obtained from mmap(), not heap + * - memory addresses are stable. Expand storage by committing VM pages. + * - @ref lo_ is aligned on VM page size (guaranteed by mmap()) + * - @ref lo_ + @ref committed_z_ <= @ref hi_ + * - @ref limit_ <= @ref lO_ + @ref committed_z_ + * - @ref committed_z_ is always a multiple of VM page size + * - @ref limit_ is not guaranteed to be aligned with VM page size. + * - @ref expand increases @ref limit_ and @ref committed_z_ as needed. + * **/ class ArenaAlloc : public IAlloc { public: @@ -57,11 +82,14 @@ namespace xo { /** Reset to empty state; provision at least @p need_z bytes of (committed) space **/ void reset(std::size_t need_z); + /** gc support: If used for storing xo::Object instances, scan allocated memory + * to populate @p *p_dest. + **/ void capture_object_statistics(capture_phase phase, ObjectStatistics * p_dest) const; /** expand available (i.e. committed) space to size at least @p z - * In practice will round up to a multiple of @ref page_z_ + * In practice will round up to a multiple of @ref page_z_. **/ bool expand(std::size_t z); @@ -98,7 +126,7 @@ namespace xo { /** optional instance name, for diagnostics **/ std::string name_; - /** size of a VM page **/ + /** size of a VM page (from getpagesize()) **/ std::size_t page_z_; /** allocator owns memory in range [@ref lo_, @ref hi_) **/ diff --git a/include/xo/alloc/Forwarding.hpp b/include/xo/alloc/Forwarding.hpp deleted file mode 100644 index 47a555da..00000000 --- a/include/xo/alloc/Forwarding.hpp +++ /dev/null @@ -1,28 +0,0 @@ -/* Forwarding.hpp - * - * author: Roland Conybeare, Jul 2025 - */ - -#pragma once - -#include "Object.hpp" - -namespace xo { - namespace gc { - class Forwarding : public Object { - public: - Forwarding() = default; - - // inherited from Object.. -#ifdef NOT_USING - virtual bool _is_forwarded() const override final { return true; } -#endif - virtual Object * _destination() override final { return destination_.ptr(); } - - private: - gp destination_; - }; - } /*namespace gc*/ -} /*namespace xo*/ - -/* end Forwarding.hpp */ diff --git a/include/xo/alloc/Forwarding1.hpp b/include/xo/alloc/Forwarding1.hpp index cf54597d..6276c1ad 100644 --- a/include/xo/alloc/Forwarding1.hpp +++ b/include/xo/alloc/Forwarding1.hpp @@ -7,6 +7,16 @@ namespace xo { namespace obj { + /** @class Forwarding1 + * @brief forwarding pointer for garbage collector. + * + * Used internally by garbage collector (see @ref GC). + * During evacuate phase overwrite from-space objects in-place + * with an instance of this class. + * + * This class suitable only for singly-inheriting objects, + * i.e. those that have exactly one vtable. + **/ class Forwarding1 : public Object { public: explicit Forwarding1(gp dest); @@ -17,11 +27,11 @@ namespace xo { virtual bool _is_forwarded() const final override { return true; } virtual Object * _offset_destination(Object * src) const final override; virtual Object * _destination() final override; - /** never called on Forwarding1 **/ + /** required by Object i/face, but never called on Forwarding1 **/ virtual std::size_t _shallow_size() const final override; - /** never called on Forwarding1 **/ + /** required by Object i/face, but never called on Forwarding1 **/ virtual Object * _shallow_copy() const final override; - /** never called on Forwarding1 **/ + /** required by Object i/face, but never called on Forwarding1 **/ virtual std::size_t _forward_children() final override; private: diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 0f50bfed..8d74f5af 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -75,8 +75,8 @@ namespace xo { * for a type appear directly in that type's vtable, and at specific locations. * This implies one level of indirection when GC traverses an instance. * - * Would be feasible to relax the must-inherit-from-Object constraint, - * but cost would be an extra layer of indirection + * Would be feasible to relax the must-inherit-from-Object constraint + * by having GC use its own wrapper, at cost of an extra layer of indirection **/ class Object { public: From 1c97e2aa9300688ac111457ff280c5a331ff4cef Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 15 Nov 2025 16:38:18 -0500 Subject: [PATCH 25/69] xo-alloc: utest: fix broken alloc utests --- include/xo/alloc/GC.hpp | 6 ++++ src/alloc/CMakeLists.txt | 2 ++ src/alloc/GC.cpp | 12 +++++++ utest/CMakeLists.txt | 13 ++++--- utest/GC.test.cpp | 75 ++++++++++++++++++++++++---------------- 5 files changed, 75 insertions(+), 33 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 313306bc..1e73362e 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -168,6 +168,12 @@ namespace xo { bool is_full_gc_pending() const { return full_gc_pending_; } /** true during (and only during) a GC cycle **/ bool gc_in_progress() const { return runstate_.in_progress(); } + + /** @return pagesize (will be the same for {nursery, tenured} spaces) **/ + std::size_t pagesize() const; + + /** @return allocation portion of Nursery to-space **/ + std::size_t nursery_to_allocated() const; /** @return reserved size of Nursery to-space **/ std::size_t nursery_to_reserved() const; /** @return committed size of Nursery to-space **/ diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index b83e2618..5645b717 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -14,7 +14,9 @@ set(SELF_SRCS ) xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) +# xo-unit used for time measurement xo_dependency(${SELF_LIB} xo_unit) +xo_dependency(${SELF_LIB} indentlog) xo_dependency(${SELF_LIB} reflect) xo_dependency(${SELF_LIB} callback) diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 7d18da57..051a68b3 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -201,12 +201,24 @@ namespace xo { return retval; } + std::size_t + GC::pagesize() const + { + return nursery_to()->page_size(); + } + std::size_t GC::nursery_from_allocated() const { return nursery_from()->allocated(); } + std::size_t + GC::nursery_to_allocated() const + { + return nursery_to()->allocated(); + } + std::size_t GC::nursery_to_reserved() const { diff --git a/utest/CMakeLists.txt b/utest/CMakeLists.txt index 379ed925..366cf664 100644 --- a/utest/CMakeLists.txt +++ b/utest/CMakeLists.txt @@ -1,4 +1,4 @@ -# build unittest alloc/utest +# xo-alloc/utest/CMakeLists.txt # # NOTE: more GC tests in xo-object/utest @@ -16,6 +16,11 @@ set(UTEST_SRCS generation.test.cpp ) -xo_add_utest_executable(${UTEST_EXE} ${UTEST_SRCS}) -xo_self_dependency(${UTEST_EXE} xo_alloc) -xo_external_target_dependency(${UTEST_EXE} Catch2 Catch2::Catch2) +if (ENABLE_TESTING) + xo_add_utest_executable(${UTEST_EXE} ${UTEST_SRCS}) + xo_self_dependency(${UTEST_EXE} xo_alloc) + xo_dependency(${UTEST_EXE} reflect) + xo_external_target_dependency(${UTEST_EXE} Catch2 Catch2::Catch2) +endif() + +# end CmakeLists.txt diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index d961e3dd..67a0fdc0 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -15,55 +15,72 @@ namespace xo { namespace { struct testcase_gc { - testcase_gc(std::size_t nz, std::size_t tz) : nursery_z_{nz}, tenured_z_{tz} {} + testcase_gc(std::size_t nz, std::size_t tz, std::size_t n_gct, std::size_t t_gct) + : nursery_z_{nz}, tenured_z_{tz}, incr_gc_threshold_{n_gct}, full_gc_threshold_{t_gct} {} std::size_t nursery_z_; std::size_t tenured_z_; + std::size_t incr_gc_threshold_; + std::size_t full_gc_threshold_; }; std::vector s_testcase_v = { - testcase_gc(1024, 4096) + // nz tz n_gct t_gct + testcase_gc(1024, 4096, 1024, 4096) }; } TEST_CASE("gc", "[alloc][gc]") { for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { - const testcase_gc & tc = s_testcase_v[i_tc]; + try { + const testcase_gc & tc = s_testcase_v[i_tc]; - up gc = GC::make( - {.initial_nursery_z_ = tc.nursery_z_, - .initial_tenured_z_ = tc.tenured_z_}); + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_, + .incr_gc_threshold_ = tc.incr_gc_threshold_, + .full_gc_threshold_ = tc.full_gc_threshold_, + }); - REQUIRE(gc.get()); - REQUIRE(gc->name() == "GC"); - REQUIRE(gc->size() == tc.nursery_z_ + tc.tenured_z_); - REQUIRE(gc->allocated() == 0); - REQUIRE(gc->available() == tc.nursery_z_); - REQUIRE(gc->before_checkpoint() == 0); - // ListAlloc model is that nothing is before checkpoint - // until it's first established - REQUIRE(gc->after_checkpoint() == 0); + REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); + REQUIRE(gc->nursery_to_allocated() == 0); + REQUIRE(gc->nursery_to_committed() >= tc.nursery_z_); + REQUIRE(gc->nursery_to_reserved() >= tc.nursery_z_); + REQUIRE(gc->nursery_to_reserved() < tc.nursery_z_ + gc->pagesize()); + REQUIRE(gc->size() >= tc.nursery_z_ + tc.tenured_z_); + REQUIRE(gc->size() < tc.nursery_z_ + gc->pagesize() + tc.tenured_z_ + gc->pagesize()); + REQUIRE(gc->allocated() == 0); + REQUIRE(gc->available() == gc->nursery_to_reserved()); + REQUIRE(gc->before_checkpoint() == 0); + // ListAlloc model is that nothing is before checkpoint + // until it's first established + REQUIRE(gc->after_checkpoint() == 0); - REQUIRE(gc->gc_in_progress() == false); - REQUIRE(gc->is_gc_enabled() == true); - REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 0); - REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->is_gc_enabled() == true); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 0); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); - /* gc with empty state */ - gc->request_gc(generation::nursery); + /* gc with empty state */ + gc->request_gc(generation::nursery); - REQUIRE(gc->gc_in_progress() == false); - REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); - REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); - /* still empty state */ - gc->request_gc(generation::tenured); + /* still empty state */ + gc->request_gc(generation::tenured); - REQUIRE(gc->gc_in_progress() == false); - REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); - REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 1); + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 1); + } catch (std::exception &ex) { + std::cerr << "caught exception: " << ex.what() << std::endl; + REQUIRE(false); + } } } From b4c89d8624e4a1050d86b7eaade93202aa46738e Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 16 Nov 2025 20:10:23 -0500 Subject: [PATCH 26/69] xo-interpreter adds + explict mm arg to ctors (retiring Object::mm) --- include/xo/alloc/GC.hpp | 2 + include/xo/alloc/Object.hpp | 3 +- src/alloc/ArenaAlloc.cpp | 71 +++++++++++++++++++++++------------ src/alloc/CMakeLists.txt | 4 +- src/alloc/GC.cpp | 74 ++++++++++++++++++++++++------------- src/alloc/Object.cpp | 2 +- utest/Forwarding1.test.cpp | 2 +- 7 files changed, 105 insertions(+), 53 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 1e73362e..abbb9657 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -54,6 +54,8 @@ namespace xo { bool allow_incremental_gc_ = true; /** true to report statistics **/ bool stats_flag_ = false; + /** true to capture per-type object statistics **/ + bool object_stats_flag_ = false; /** remember basic gc statistics for this many GC's; separately for incremental + full GCs **/ std::size_t stats_history_z_ = 256; /** true to enable debug logging **/ diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 8d74f5af..5a981998 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -258,8 +258,9 @@ namespace xo { **/ class Cpof { public: - explicit Cpof(const Object * src) : src_{src} {} + explicit Cpof(gc::IAlloc * mm, const Object * src) : mm_{mm}, src_{src} {} + gc::IAlloc * mm_ = nullptr; const void * src_ = nullptr; }; } /*namespace xo*/ diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index 4fdcefe8..febbcb61 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -26,6 +26,11 @@ namespace xo { void * base = mmap(nullptr, z, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + log && log("acquired memory [lo,hi) using mmap", + xtag("lo", base), + xtag("z", z), + xtag("hi", reinterpret_cast(base) + z)); + // could use this as fallback.. //base = (new std::byte [z]); @@ -39,7 +44,7 @@ namespace xo { this->checkpoint_ = lo_; this->free_ptr_ = lo_; this->limit_ = lo_ + z; - this->hi_ = limit_; + this->hi_ = lo_ + z; this->debug_flag_ = debug_flag; if (!lo_) { @@ -52,22 +57,25 @@ namespace xo { ArenaAlloc::~ArenaAlloc() { + scope log(XO_DEBUG(debug_flag_)); // hygiene.. if (lo_) { + log && log("unmap [lo,hi)", xtag("lo", lo_), xtag("z", hi_ - lo_), xtag("hi", hi_)); + munmap(lo_, hi_ - lo_); } - // could use this as fallback if not using uncommitted technique + // could use this as fallback if we dropped the uncommitted technique //delete [] this->lo_; - this->lo_ = nullptr; + this->lo_ = nullptr; this->committed_z_ = 0; - this->checkpoint_ = nullptr; - this->free_ptr_ = nullptr; - this->limit_ = nullptr; - this->hi_ = nullptr; - this->debug_flag_ = false; + this->checkpoint_ = nullptr; + this->free_ptr_ = nullptr; + this->limit_ = nullptr; + this->hi_ = nullptr; + this->debug_flag_ = false; } up @@ -94,26 +102,41 @@ namespace xo { } bool - ArenaAlloc::expand(size_t offset_z) { + ArenaAlloc::expand(size_t offset_z) + { scope log(XO_DEBUG(debug_flag_), xtag("offset_z", offset_z), xtag("committed_z", committed_z_)); - if (offset_z <= committed_z_) + if (offset_z <= committed_z_) { + log && log("trivial success, offset within committed range", + xtag("offset_z", offset_z), + xtag("committed_z", committed_z_)); return true; - - std::size_t align_offset_z = align_lub(offset_z, page_z_); - std::byte * commit_start = lo_ + committed_z_; - std::size_t new_commit_z = align_offset_z - committed_z_; - - log && log(xtag("align_offset_z", align_offset_z), - xtag("new_commit_z", new_commit_z)); - - if (mprotect(commit_start, new_commit_z, PROT_READ | PROT_WRITE) != 0) { - throw std::runtime_error(tostr("ArenaAlloc::expand: commit failure", - xtag("committed_z", committed_z_), - xtag("new_commit_z", new_commit_z))); } - this->committed_z_ = align_offset_z; + if (lo_ + offset_z > limit_) { + throw std::runtime_error(tostr("ArenaAlloc::expand: requested size exceeds reserved size", + xtag("requested", offset_z), xtag("reserved", reserved()))); + } + + std::size_t aligned_offset_z = align_lub(offset_z, page_z_); + std::byte * commit_start = lo_ + committed_z_; + std::size_t add_commit_z = aligned_offset_z - committed_z_; + + log && log(xtag("aligned_offset_z", aligned_offset_z), + xtag("add_commit_z", add_commit_z)); + + log && log("expand committed range", + xtag("commit_start", commit_start), + xtag("add_commit_z", add_commit_z), + xtag("commit_end", commit_start + add_commit_z)); + + if (mprotect(commit_start, add_commit_z, PROT_READ | PROT_WRITE) != 0) { + throw std::runtime_error(tostr("ArenaAlloc::expand: commit failure", + xtag("committed_z", committed_z_), + xtag("add_commit_z", add_commit_z))); + } + + this->committed_z_ = aligned_offset_z; this->limit_ = this->lo_ + committed_z_; return true; @@ -167,6 +190,8 @@ namespace xo { std::byte * p = lo_; while (p < free_ptr_) { + log && log(xtag("p", (void *)p)); + Object * obj = reinterpret_cast(p); TaggedPtr tp = obj->self_tp(); std::size_t z = obj->_shallow_size(); diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index 5645b717..6bbc2f32 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -15,9 +15,9 @@ set(SELF_SRCS xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) # xo-unit used for time measurement -xo_dependency(${SELF_LIB} xo_unit) +xo_headeronly_dependency(${SELF_LIB} xo_unit) xo_dependency(${SELF_LIB} indentlog) xo_dependency(${SELF_LIB} reflect) -xo_dependency(${SELF_LIB} callback) +xo_headeronly_dependency(${SELF_LIB} callback) #end CMakeLists.txt diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 051a68b3..8181cbb1 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -66,16 +66,36 @@ namespace xo { std::size_t nursery_size = config.initial_nursery_z_; std::size_t tenured_size = config.initial_tenured_z_; - nursery_[role2int(role::from_space)] + if (config_.incr_gc_threshold_ > nursery_size) { + throw std::runtime_error(tostr("GC::ctor: expected nursery gc threshold < nursery size", + xtag("nursery-gc-threshold", config_.incr_gc_threshold_), + xtag("nursery-size", nursery_size))); + } + + if (nursery_size + config_.full_gc_threshold_ > tenured_size) { + throw std::runtime_error(tostr("GC::ctor: expected nursery size + tennured gc threshold < tenured size", + xtag("nursery-size", nursery_size), + xtag("tenured-size", tenured_size), + xtag("full-gc-threshold", config_.full_gc_threshold_) + )); + } + + if (config_.incr_gc_threshold_ > nursery_size) + this->config_.incr_gc_threshold_ = nursery_size; + + if (config_.full_gc_threshold_ > tenured_size) + this->config_.full_gc_threshold_ = tenured_size; + + this->nursery_[role2int(role::from_space)] = ArenaAlloc::make("NA", nursery_size, config.debug_flag_); - nursery_[role2int(role::to_space) ] + this->nursery_[role2int(role::to_space) ] = ArenaAlloc::make("NB", nursery_size, config.debug_flag_); - tenured_[role2int(role::from_space)] + this->tenured_[role2int(role::from_space)] = ArenaAlloc::make("TA", tenured_size, config.debug_flag_); - tenured_[role2int(role::to_space) ] + this->tenured_[role2int(role::to_space) ] = ArenaAlloc::make("TB", tenured_size, config.debug_flag_); nursery_[role2int(role::from_space)]->expand(config.incr_gc_threshold_); @@ -83,9 +103,9 @@ namespace xo { tenured_[role2int(role::from_space)]->expand(config.full_gc_threshold_); tenured_[role2int(role::to_space) ]->expand(config.full_gc_threshold_); - mutation_log_[role2int(role::from_space)] = std::make_unique(); - mutation_log_[role2int(role::to_space )] = std::make_unique(); - defer_mutation_log_ = std::make_unique(); + this->mutation_log_[role2int(role::from_space)] = std::make_unique(); + this->mutation_log_[role2int(role::to_space )] = std::make_unique(); + this->defer_mutation_log_ = std::make_unique(); this->gc_history_ = CircularBuffer(config.stats_history_z_); @@ -96,23 +116,25 @@ namespace xo { /* hygiene */ this->clear(); - nursery_[role2int(role::from_space)].reset(); - nursery_[role2int(role::to_space) ].reset(); + this->nursery_[role2int(role::from_space)].reset(); + this->nursery_[role2int(role::to_space) ].reset(); - tenured_[role2int(role::from_space)].reset(); - tenured_[role2int(role::to_space) ].reset(); + this->tenured_[role2int(role::from_space)].reset(); + this->tenured_[role2int(role::to_space) ].reset(); - mutation_log_[role2int(role::from_space)].reset(); - mutation_log_[role2int(role::to_space) ].reset(); - defer_mutation_log_.reset(); + this->gc_root_v_.clear(); + + this->mutation_log_[role2int(role::from_space)].reset(); + this->mutation_log_[role2int(role::to_space) ].reset(); + this->defer_mutation_log_.reset(); } up GC::make(const Config & config) { - GC * gc = new GC(config); + //GC * gc = new GC(config); - return up{gc}; + return std::make_unique(config); } const std::string & @@ -608,16 +630,18 @@ namespace xo { void GC::capture_object_statistics(generation upto, capture_phase phase) { - /* scan nursery */ - this->nursery_[role2int(role::to_space)]->capture_object_statistics - (phase, - &object_statistics_sab_[gen2int(generation::nursery)]); - - if (upto == generation::tenured) { - /* scan tenured */ - this->tenured_[role2int(role::to_space)]->capture_object_statistics + if (config_.object_stats_flag_) { + /* scan nursery */ + this->nursery_[role2int(role::to_space)]->capture_object_statistics (phase, - &object_statistics_sab_[gen2int(generation::tenured)]); + &object_statistics_sab_[gen2int(generation::nursery)]); + + if (upto == generation::tenured) { + /* scan tenured */ + this->tenured_[role2int(role::to_space)]->capture_object_statistics + (phase, + &object_statistics_sab_[gen2int(generation::tenured)]); + } } } diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index b3db11ca..475d84ad 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -14,7 +14,7 @@ operator new (std::size_t z, const xo::Cpof & cpof) { using xo::gc::GC; - GC * gc = reinterpret_cast(xo::Object::mm); + GC * gc = reinterpret_cast(cpof.mm_); return gc->alloc_gc_copy(z, cpof.src_); } diff --git a/utest/Forwarding1.test.cpp b/utest/Forwarding1.test.cpp index e5dfb1fe..b39c35a1 100644 --- a/utest/Forwarding1.test.cpp +++ b/utest/Forwarding1.test.cpp @@ -33,7 +33,7 @@ namespace xo { void display(std::ostream & os) const final override { os << data_; } virtual std::size_t _shallow_size() const final override { return sizeof(*this); } - virtual Object * _shallow_copy() const final override { return new (Cpof(this)) DummyObject(*this); } + virtual Object * _shallow_copy() const final override { return new (Cpof(Object::mm, this)) DummyObject(*this); } virtual std::size_t _forward_children() final override { return _shallow_size(); } private: From 78c6c5cde98e4bf42b03a790c9036aec1b419a0a Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 17 Nov 2025 10:41:35 -0500 Subject: [PATCH 27/69] xo-interpreter CVector for StackFrame reflection + OSX imgui edits --- include/xo/alloc/Object.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 5a981998..81f0611c 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -61,7 +61,9 @@ namespace xo { void assign_ptr(T * x) { ptr_ = x; } gc_ptr & operator=(const gc_ptr & x) { ptr_ = x.ptr(); return *this; } + T * operator->() const { return ptr_; } + T & operator*() const { return *ptr_; } private: T * ptr_ = nullptr; From 7b82ace806fd05912ae6e9847714c33567c2268d Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 17 Nov 2025 22:31:10 -0500 Subject: [PATCH 28/69] xo-interpreter: prep for xo-symboltable --- README.md | 17 +++++++++++++++-- include/xo/alloc/Object.hpp | 2 -- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index eeef6ca5..5a42f922 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,16 @@ -# xo-alloc -- arena allocator with rudimentary GC support +# xo-alloc -- arena allocator and incremental garbage collector -Xo-alloc is a lightweight arena allocator +# Rules for writing garbage-collected classes. + +Topics +* allocation - allocate Objects (inheriting xo::Object) before owned scratch space. + Can relax this if/when abandon the bad-for-locality use of two pointers + into to-space to keep track of grey objects. Want to use stack anyway + so we can do depth-first search. +* destructors - can omit except for finalization +* assignment - MUST USE Object::assign_member() to assign pointers to gc-owned memory. + Only necessary for old->new pointers, so don't need to worry about this + for initialization. +* finalization - not supported (yet) + +- padding - use IAlloc::with_padding(z) for hand-allocated objects. diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 81f0611c..a29b8de5 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -191,8 +191,6 @@ namespace xo { **/ virtual std::size_t _shallow_size() const = 0; - // TODO: _shallow_move() also overwrite *this with gc-only forwarding object point to C - /** if subject is allocated by GC: * - create copy C in to-space * - destination C will be nursery|tenured depending on location of this. From f3887debcaefe4ae2a7aecc7a65815452c16d2f6 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 19 Nov 2025 12:38:54 -0500 Subject: [PATCH 29/69] xo-alloc / xo-refcnt: feature flags for easy tests. --- include/xo/alloc/Object.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index a29b8de5..5727331e 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -39,6 +39,11 @@ namespace xo { template gc_ptr(const gc_ptr & x) : ptr_{x.ptr()} {} + /** convenience for static asserts **/ + static constexpr bool is_gc_ptr = true; + /** see also: xo/refcnt/Refcounted.hpp **/ + static constexpr bool is_rc_ptr = false; + static bool is_eq(gc_ptr x1, gc_ptr x2) { std::uintptr_t u1 = reinterpret_cast(x1.ptr()); std::uintptr_t u2 = reinterpret_cast(x2.ptr()); From 2c21eede1fcd1082463a877740d60c383c542fff Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 20 Nov 2025 21:26:18 -0500 Subject: [PATCH 30/69] xo-interpreter: setting up for gc in interactive interpreter --- include/xo/alloc/GC.hpp | 4 ++-- src/alloc/GC.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index abbb9657..ce7b0f85 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -37,14 +37,14 @@ namespace xo { * pages are committed on demand. * Initial committment will be up to @ref incr_gc_threshold_ **/ - std::size_t initial_nursery_z_ = 0; + std::size_t initial_nursery_z_ = 64*1024*1024; /** initial size in bytes for oldest (Tenured) generation. * GC allocates two tenured spaces of this size. * This number represents reserved address space. * pages are committed on demand. * Initial committment will be up to @ref full_gc_threshold_ **/ - std::size_t initial_tenured_z_ = 0; + std::size_t initial_tenured_z_ = 128*1024*1024; /** trigger incremental GC after this many bytes allocated in nursery **/ std::size_t incr_gc_threshold_ = 64*1024; /** trigger full GC after this many bytes promoted to tenured **/ diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 8181cbb1..22654700 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -73,7 +73,7 @@ namespace xo { } if (nursery_size + config_.full_gc_threshold_ > tenured_size) { - throw std::runtime_error(tostr("GC::ctor: expected nursery size + tennured gc threshold < tenured size", + throw std::runtime_error(tostr("GC::ctor: expected nursery size + tenured gc threshold < tenured size", xtag("nursery-size", nursery_size), xtag("tenured-size", tenured_size), xtag("full-gc-threshold", config_.full_gc_threshold_) From dd41635a5652d267f0878bb04b03eb85572be97f Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 22 Nov 2025 20:13:33 -0500 Subject: [PATCH 31/69] xo-tokenizer: refactor to correct accounting for line/consume/errpos --- include/xo/alloc/ArenaAlloc.hpp | 86 ++++++++++++++++++++++--- src/alloc/ArenaAlloc.cpp | 107 +++++++++++++++++++++++--------- 2 files changed, 157 insertions(+), 36 deletions(-) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index da67f8f2..e0bfed2f 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -18,11 +18,11 @@ namespace xo { * allocation order: * -----------------------> * - * <----------------- .size() ------------------> - * <----------------- .committed() ---------------> + * <----------------- .size(), .reserved() ---------------------------> + * <----------------- .committed() -------------> * - * <-------allocated------><--------free--------> <---uncommitted----> - * XXXXXXXXXXXXXXXXXXXXXXXX______________________ .................... + * <-------allocated------><--------free--------><-----uncommitted----> + * XXXXXXXXXXXXXXXXXXXXXXXX______________________...................... * ^ ^ ^ ^ ^ * lo checkpoint free limit hi * @@ -31,12 +31,77 @@ namespace xo { * > < .before_checkpoint() * > < .after_checkpoint() * + * lifetime: + * + * 1. initial state after ctor + * + * >< committed()=0 + * <---------------------------uncommitted----------------------------> + * .................................................................... + * ^ ^ + * lo hi + * checkpoint + * free + * limit + * + * 1a. one call to ::mmap() + * 1b. vm address space [lo,hi) is reserved + * 1c. address space [lo,hi) is inaccessible. no read|write|execute permission + * + * 2. after first allocation of n bytes + * + * <--committed---> + * <--free--><--------------------uncommitted--------------------> + * > <- allocated + * XXXXXX__________..................................................... + * ^ ^ ^ ^ + * lo lo+n limit hi + * ^ free + * checkpoint + * + * 2a. committed just enough hugepages (2mb each) to accomodate n, + * i.e. expand-on-demand: + * - one call to ::mprotect() + * - .limit = .lo + (k+1) * .hugepage_z for some integer k>=0 + * - k * .page_z <= n < (k+1) * .hugepage_z + * 2b. expect immediate cost 1-5us, includes: + * - TLB flush + * invalidate TLB entries for committed range on all cores that this + * process' threads have run on since process inception. + * Also, if a kernel thread has run on one of said cores, it may + * have borrowed our TLB entries + * - page table update + * write to entry for each vm page + * - kernel overhead 100-1000 cycles (< 1us) + * 2c. expect deferred cost 1us-2us per hugepage: + * - committed pages aren't backed by physical memory until + * first touched; minor page fault on first access for each page. + * - so about 256-512us for 1MB + * 3. after .expand(z) + * + * <-------------committed------------> + * <------------free------------><----------uncomitted-----------> + * > <- allocated + * XXXXXX______________________________................................. + * ^ ^ ^ ^ + * lo lo+n limit hi + * ^ free + * checkpoint + * + * 3a. same as case 2. but without advancing .free pointer. + * + * 4. after dtor + * + * 4a. all memory returned to o/s, no longer reserved. + * - one call to ::munmap() + * * @endtext * * Design Notes: * - non-copyable, non-moveable - * - always heap-allocated * - @ref lo_ <= @ref checkpoint_ <= @ref free_ <= @ref limit_ <= @ref hi_ + * - memory for ArenaAlloc itself (not the memory it allocates), ~100 bytes + * always heap allocated. Use ArenaAlloc::make() * - memory obtained from mmap(), not heap * - memory addresses are stable. Expand storage by committing VM pages. * - @ref lo_ is aligned on VM page size (guaranteed by mmap()) @@ -55,7 +120,7 @@ namespace xo { /** Create allocator with capacity @p z, * Reserve memory addresses for @p z bytes, - * but don't commit them until needed + * (but don't commit them until needed) **/ static up make(const std::string & name, std::size_t z, @@ -127,7 +192,12 @@ namespace xo { std::string name_; /** size of a VM page (from getpagesize()) **/ - std::size_t page_z_; + std::size_t page_z_ = 0; + + /** size of a huge VM page. hardwiring this in ctor (to 2MB). + * larger pages relieve pressure on TLB, but suboptimal if use << 2MB + **/ + std::size_t hugepage_z_ = 0; /** allocator owns memory in range [@ref lo_, @ref hi_) **/ std::byte * lo_ = nullptr; @@ -139,7 +209,7 @@ namespace xo { * older (addresses below checkpoint) * and younger (addresses above checkpoint) **/ - std::byte * checkpoint_; + std::byte * checkpoint_ = nullptr; /** free pointer. memory in range [@ref free_, @ref limit_) available **/ std::byte * free_ptr_ = nullptr; /** soft limit: end of committed virtual memory **/ diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index febbcb61..0a0365e2 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -13,37 +13,101 @@ #include namespace xo { + using std::byte; + namespace gc { + namespace { + /* alignment better be a power of 2 */ + std::size_t + align_lub(std::size_t x, std::size_t align) + { + /* e.g: + * align = 4096, x%align = 100 -> dx = 3996 + * align = 4096, x%align = 0 -> dx = 0 + */ + std::size_t dx = (align - (x % align)) % align; + + return x + dx; + } + } + ArenaAlloc::ArenaAlloc(const std::string & name, - std::size_t z, bool debug_flag) + std::size_t z, + bool debug_flag) { scope log(XO_DEBUG(debug_flag), xtag("name", name)); + constexpr size_t c_hugepage_z = 2 * 1024 * 1024; + this->name_ = name; this->page_z_ = getpagesize(); + this->hugepage_z_ = c_hugepage_z; - // reserve virtual memory + // 1. need k pagetable entries where k is lub {k | k * .page_z >= z} + // 2. base will be aligned with .page_z but likely not with .hugepage_z + // 3. bad to have misalignment, because misaligned {prefix, suffix} of [base, base+z) + // will use 4k pages instead of 2mb pages + // + // strategy: + // 4. round up z to multiple of c_hugepage_z + // 5. over-request so reserved range contains an aligned subrange of size z + // 6. unmap misaligned prefix + // 7. unmap misaligned suffix. + // 8. enable huge pages for now-aligned remainder of reserved range + // + // Z. note: rejecting inferior MAP_HUGETLB|MAP_HUGE_2MB flags on ::mmap here: + // Za. requires previously-reserved memory in /proc/sys/vm/nr_hugepages + // Zb. reserved pages permenently resident in RAM, never swapped + // Zc. memory cost incurred even if no application is using said pages - void * base = mmap(nullptr, z, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + z = align_lub(z, c_hugepage_z); // 4. + + // 5. + byte * base = reinterpret_cast(::mmap(nullptr, + z + c_hugepage_z, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0)); log && log("acquired memory [lo,hi) using mmap", xtag("lo", base), xtag("z", z), - xtag("hi", reinterpret_cast(base) + z)); - - // could use this as fallback.. - //base = (new std::byte [z]); + xtag("hi", reinterpret_cast(base) + z)); if (base == MAP_FAILED) { throw std::runtime_error(tostr("ArenaAlloc: uncommitted allocation failed", xtag("size", z))); } - this->lo_ = reinterpret_cast(base); + byte * aligned_base = reinterpret_cast(align_lub(reinterpret_cast(base), + c_hugepage_z)); + + assert(reinterpret_cast(aligned_base) % c_hugepage_z == 0); + assert(aligned_base >= base); + assert(aligned_base < base + c_hugepage_z); + + if (base < aligned_base) { + size_t prefix = aligned_base - base; + + ::munmap(base, prefix); // 6. + } + + byte * aligned_hi = aligned_base + z; + byte * hi = base + z + c_hugepage_z; + + if (aligned_hi < hi) { + size_t suffix = hi - aligned_hi; + + ::munmap(aligned_hi, suffix); // 7. + } + + ::madvise(aligned_base, z, MADV_HUGEPAGE); // 8. + + this->lo_ = aligned_base; this->committed_z_ = 0; this->checkpoint_ = lo_; this->free_ptr_ = lo_; - this->limit_ = lo_ + z; + this->limit_ = lo_; this->hi_ = lo_ + z; this->debug_flag_ = debug_flag; @@ -52,7 +116,9 @@ namespace xo { xtag("size", z))); } - log && log(xtag("lo", (void*)lo_), xtag("page_z", page_z_)); + log && log(xtag("lo", (void*)lo_), + xtag("page_z", page_z_), + xtag("hugepage_z", hugepage_z_)); } ArenaAlloc::~ArenaAlloc() @@ -64,7 +130,7 @@ namespace xo { if (lo_) { log && log("unmap [lo,hi)", xtag("lo", lo_), xtag("z", hi_ - lo_), xtag("hi", hi_)); - munmap(lo_, hi_ - lo_); + ::munmap(lo_, hi_ - lo_); } // could use this as fallback if we dropped the uncommitted technique //delete [] this->lo_; @@ -86,21 +152,6 @@ namespace xo { z, debug_flag)); } - namespace { - /* alignment better be a power of 2 */ - std::size_t - align_lub(std::size_t x, std::size_t align) - { - /* e.g: - * align = 4096, x%align = 100 -> dx = 3996 - * align = 4096, x%align = 0 -> dx = 0 - */ - std::size_t dx = (align - (x % align)) % align; - - return x + dx; - } - } - bool ArenaAlloc::expand(size_t offset_z) { @@ -118,7 +169,7 @@ namespace xo { xtag("requested", offset_z), xtag("reserved", reserved()))); } - std::size_t aligned_offset_z = align_lub(offset_z, page_z_); + std::size_t aligned_offset_z = align_lub(offset_z, hugepage_z_); std::byte * commit_start = lo_ + committed_z_; std::size_t add_commit_z = aligned_offset_z - committed_z_; @@ -130,7 +181,7 @@ namespace xo { xtag("add_commit_z", add_commit_z), xtag("commit_end", commit_start + add_commit_z)); - if (mprotect(commit_start, add_commit_z, PROT_READ | PROT_WRITE) != 0) { + if (::mprotect(commit_start, add_commit_z, PROT_READ | PROT_WRITE) != 0) { throw std::runtime_error(tostr("ArenaAlloc::expand: commit failure", xtag("committed_z", committed_z_), xtag("add_commit_z", add_commit_z))); From 54dbbf69440e7dd32b4402104d12741031049779 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 22 Nov 2025 23:06:51 -0500 Subject: [PATCH 32/69] xo-tokenizer: streamline error path during tokenization --- src/alloc/ArenaAlloc.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index 0a0365e2..e5609297 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -64,10 +64,10 @@ namespace xo { // 5. byte * base = reinterpret_cast(::mmap(nullptr, - z + c_hugepage_z, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0)); + z + c_hugepage_z, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0)); log && log("acquired memory [lo,hi) using mmap", xtag("lo", base), @@ -101,7 +101,13 @@ namespace xo { ::munmap(aligned_hi, suffix); // 7. } +#ifdef __linux__ ::madvise(aligned_base, z, MADV_HUGEPAGE); // 8. +#endif + // TODO: for OSX -> need something else here. + // MAP_ALIGNED_SUPER with mmap() and/or + // use mach_vm_allocate() + // this->lo_ = aligned_base; this->committed_z_ = 0; From 40128c423c44d42994744f6e8b2f27a474a611a6 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 23 Nov 2025 11:35:05 -0500 Subject: [PATCH 33/69] xo-alloc: bugfix expand: limit_ is soft, hi_ is hard. + docs --- include/xo/alloc/ArenaAlloc.hpp | 6 ++++-- src/alloc/ArenaAlloc.cpp | 27 ++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index e0bfed2f..302d9c0a 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -127,7 +127,7 @@ namespace xo { bool debug_flag); /** size of virtual address range reserved for this allocator **/ - std::size_t reserved() const { return this->size(); } + std::size_t reserved() const { return hi_ - lo_; }; std::size_t page_size() const { return page_z_; } std::byte * free_ptr() const { return free_ptr_; } @@ -212,7 +212,9 @@ namespace xo { std::byte * checkpoint_ = nullptr; /** free pointer. memory in range [@ref free_, @ref limit_) available **/ std::byte * free_ptr_ = nullptr; - /** soft limit: end of committed virtual memory **/ + /** soft limit: end of committed virtual memory + * invariant: @ref limit_ = @ref lo_ + @ref committed_z_ + **/ std::byte * limit_ = nullptr; /** hard limit: end of reserved virtual memory **/ std::byte * hi_ = nullptr; diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index e5609297..c06b9518 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -170,15 +170,37 @@ namespace xo { return true; } - if (lo_ + offset_z > limit_) { + if (lo_ + offset_z > hi_) { throw std::runtime_error(tostr("ArenaAlloc::expand: requested size exceeds reserved size", xtag("requested", offset_z), xtag("reserved", reserved()))); } + /* + * pre: + * + * _______________................................... + * ^ ^ ^ + * lo limit hi + * + * < committed_z > + * <----------offset_z-----------> + * > <- z: 0 <= z < hugepage_z + * <---------aligned_offset_z---------> + * <--- add_commit_z --> + * + * post: + * ____________________________________.............. + * ^ ^ ^ + * lo limit hi + * + */ + std::size_t aligned_offset_z = align_lub(offset_z, hugepage_z_); std::byte * commit_start = lo_ + committed_z_; std::size_t add_commit_z = aligned_offset_z - committed_z_; + assert(limit_ == lo_ + committed_z_); + log && log(xtag("aligned_offset_z", aligned_offset_z), xtag("add_commit_z", add_commit_z)); @@ -196,6 +218,9 @@ namespace xo { this->committed_z_ = aligned_offset_z; this->limit_ = this->lo_ + committed_z_; + assert(committed_z_ % hugepage_z_ == 0); + assert(reinterpret_cast(limit_) % hugepage_z_ == 0); + return true; } From eec5bc098186f9bea236d0d0be438cc763897283 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 23 Nov 2025 21:41:14 -0500 Subject: [PATCH 34/69] xo-interpreter: + toplevel env in VSM --- include/xo/alloc/GC.hpp | 13 +++++++++++++ src/alloc/GC.cpp | 21 +++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index ce7b0f85..8e12990e 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -7,6 +7,7 @@ #include "ArenaAlloc.hpp" #include "GcStatistics.hpp" +#include "Object.hpp" #include "xo/callback/UpCallbackSet.hpp" #include "xo/indentlog/print/array.hpp" #include @@ -154,6 +155,9 @@ namespace xo { **/ static up make(const Config & config); + /** runtime downcast **/ + static GC * from(IAlloc * mm); + const Config & config() const { return config_; } std::uint8_t nursery_polarity() const { return nursery_polarity_; } std::uint8_t tenured_polarity() const { return tenured_polarity_; } @@ -230,6 +234,15 @@ namespace xo { * from @c *addr **/ void add_gc_root(Object ** addr); + /** reverse the effect of previous call to @ref add_gc_root **/ + void remove_gc_root(Object ** addr); + + /** convenience wrapper **/ + template + void add_gc_root_dwim(gp * p) { this->add_gc_root(reinterpret_cast(p->ptr_address())); } + template + void remove_gc_root_dwim(gp * p) { this->remove_gc_root(reinterpret_cast(p->ptr_address())); } + /** may optionally use this to observe GC copy phase. * Will be invoked once _per surviving object_, so not cheap. * Intended for GC visualization. diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 22654700..c78768fb 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -132,11 +132,15 @@ namespace xo { up GC::make(const Config & config) { - //GC * gc = new GC(config); - return std::make_unique(config); } + GC * + GC::from(IAlloc * mm) + { + return dynamic_cast(mm); + } + const std::string & GC::name() const { @@ -390,6 +394,19 @@ namespace xo { gc_root_v_.push_back(addr); } + void + GC::remove_gc_root(Object ** addr) + { + /* Multithreaded GC not supported */ + + assert(!this->gc_in_progress()); + + auto new_end_ix = std::remove(gc_root_v_.begin(), gc_root_v_.end(), addr); + + /* erase now-unused slots */ + gc_root_v_.erase(new_end_ix, gc_root_v_.end()); + } + auto GC::add_gc_copy_callback(up fn) -> CallbackId { From 760bb556b24fa86124d1c61c882a94d927f839f7 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 23 Nov 2025 22:57:52 -0500 Subject: [PATCH 35/69] xo-interpreter/xo-alloc: GlobalEnv + mm -> shallow_copy() --- include/xo/alloc/Forwarding1.hpp | 2 +- include/xo/alloc/Object.hpp | 5 ++++- src/alloc/Forwarding1.cpp | 2 +- src/alloc/Object.cpp | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/include/xo/alloc/Forwarding1.hpp b/include/xo/alloc/Forwarding1.hpp index 6276c1ad..77e081f9 100644 --- a/include/xo/alloc/Forwarding1.hpp +++ b/include/xo/alloc/Forwarding1.hpp @@ -30,7 +30,7 @@ namespace xo { /** required by Object i/face, but never called on Forwarding1 **/ virtual std::size_t _shallow_size() const final override; /** required by Object i/face, but never called on Forwarding1 **/ - virtual Object * _shallow_copy() const final override; + virtual Object * _shallow_copy(gc::IAlloc * mm) const final override; /** required by Object i/face, but never called on Forwarding1 **/ virtual std::size_t _forward_children() final override; diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 5727331e..849cf9bb 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -57,6 +57,9 @@ namespace xo { return (u2 <= u1 + sizeof(std::uintptr_t)); } + /** (for consistency's sake) **/ + T * get() const { return ptr_; } + T * ptr() const { return ptr_; } T ** ptr_address() { return &ptr_; } @@ -204,7 +207,7 @@ namespace xo { * * Require: @ref mm is an instance of @ref gc::GC **/ - virtual Object * _shallow_copy() const = 0; + virtual Object * _shallow_copy(gc::IAlloc * mm) const = 0; /** update child pointers that refer to forwarding pointers, * replacing them with the correct destination. diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp index 4b47e4f2..b4a44ff6 100644 --- a/src/alloc/Forwarding1.cpp +++ b/src/alloc/Forwarding1.cpp @@ -52,7 +52,7 @@ namespace xo { // LCOV_EXCL_START Object * - Forwarding1::_shallow_copy() const { + Forwarding1::_shallow_copy(gc::IAlloc *) const { assert(false); return nullptr; } diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index 475d84ad..309e0886 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -170,7 +170,7 @@ namespace xo { */ if (gc->fromspace_contains(src)) { - Object * dest = src->_shallow_copy(); + Object * dest = src->_shallow_copy(gc); if (dest != src) src->_forward_to(dest); From 2f2cb735f3a413f854dee8d84947430ed0b1a8b5 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 24 Nov 2025 09:55:43 -0500 Subject: [PATCH 36/69] xo-interpreter: refactor for explicit gc::GC* dep --- include/xo/alloc/ArenaAlloc.hpp | 2 ++ include/xo/alloc/Forwarding1.hpp | 2 +- include/xo/alloc/GC.hpp | 9 +++++++++ include/xo/alloc/IAlloc.hpp | 27 ++++++++++++++++++++++----- include/xo/alloc/Object.hpp | 16 ++++++++-------- src/alloc/ArenaAlloc.cpp | 8 ++++++++ src/alloc/Forwarding1.cpp | 2 +- src/alloc/GC.cpp | 13 +++++++++++++ src/alloc/IAlloc.cpp | 12 ++++++++++++ src/alloc/Object.cpp | 30 +++++++++++++----------------- 10 files changed, 89 insertions(+), 32 deletions(-) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 302d9c0a..ed3fd59c 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -175,6 +175,8 @@ namespace xo { virtual void checkpoint() final override; virtual std::byte * alloc(std::size_t z) final override; + virtual bool check_owned(Object * src) const final override; + ArenaAlloc & operator=(const ArenaAlloc &) = delete; ArenaAlloc & operator=(ArenaAlloc &&) = delete; diff --git a/include/xo/alloc/Forwarding1.hpp b/include/xo/alloc/Forwarding1.hpp index 77e081f9..88ca849a 100644 --- a/include/xo/alloc/Forwarding1.hpp +++ b/include/xo/alloc/Forwarding1.hpp @@ -32,7 +32,7 @@ namespace xo { /** required by Object i/face, but never called on Forwarding1 **/ virtual Object * _shallow_copy(gc::IAlloc * mm) const final override; /** required by Object i/face, but never called on Forwarding1 **/ - virtual std::size_t _forward_children() final override; + virtual std::size_t _forward_children(gc::GC * mm) final override; private: /** the object that used to be located at this address (i.e. @c this) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 8e12990e..423dd2f8 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -308,6 +308,15 @@ namespace xo { **/ virtual void assign_member(Object * parent, Object ** lhs, Object* rhs) final override; + /** during GC check for source objects owned by GC. + * See Object::_shallow_move. + **/ + virtual bool check_owned(Object * src) const final override; + /** queries during GC to determine if object at address @p src should move: + * - full GC -> always + * - incr GC -> if not tenured + **/ + virtual bool check_move(Object * src) const final override; virtual std::byte * alloc(std::size_t z) final override; virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; diff --git a/include/xo/alloc/IAlloc.hpp b/include/xo/alloc/IAlloc.hpp index 0b6791f0..fceaca94 100644 --- a/include/xo/alloc/IAlloc.hpp +++ b/include/xo/alloc/IAlloc.hpp @@ -16,7 +16,12 @@ namespace xo { namespace gc { /** @class IAllocator - * @brief memory allocation interface with limited garbaga collector support + * @brief memory allocation interface with limited garbage collector support + * + * Garbage collector support methods: + * - checkpoint() + * - assign_member() + * - alloc_gc_copy() **/ class IAlloc { public: @@ -56,14 +61,28 @@ namespace xo { /** @return true iff debug logging enabled **/ virtual bool debug_flag() const = 0; - /** reset allocator to empty state. **/ - virtual void clear() = 0; /** remember allocator state. All currently-allocated addresses xo * will satisfy is_before_checkpoint(x). Subsequent allocations x * will fail is_before_checkpoint(x), until checkpoint superseded * by @ref clear or another call to @ref checkpoint **/ virtual void checkpoint() = 0; + + /** allocate @p z bytes of memory. returns pointer to first address **/ + virtual std::byte * alloc(std::size_t z) = 0; + /** reset allocator to empty state. **/ + virtual void clear() = 0; + + // ----- GC-specific methods ----- + + /** true iff this allocator owns object at address @p src. + * Use to assist Object::_shallow_move + **/ + virtual bool check_owned(Object * src) const; + /** true iff object at address @p src must move as part of + * in-progress collection phase + **/ + virtual bool check_move(Object * src) const; /** perform assignment * @code * *lhs = rhs @@ -72,8 +91,6 @@ namespace xo { * Default implementation just does the assignment. **/ virtual void assign_member(Object * parent, Object ** lhs, Object * rhs); - /** allocate @p z bytes of memory. returns pointer to first address **/ - virtual std::byte * alloc(std::size_t z) = 0; /** allocate @p z bytes for copy of object at @p src. * Only used in @ref GC. Default implementation asserts and returns nullptr **/ diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 849cf9bb..f1d03225 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -122,15 +122,15 @@ namespace xo { static Object * _forward(Object * src, gc::GC * gc); template - static void _forward_inplace(T ** src_addr) { - Object * fwd = _forward(*src_addr, _gc()); + static void _forward_inplace(T ** src_addr, gc::GC * gc) { + Object * fwd = _forward(*src_addr, gc); *src_addr = reinterpret_cast(fwd); } template - static void _forward_inplace(gp & src) { - _forward_inplace(src.ptr_address()); + static void _forward_inplace(gp & src, gc::GC * gc) { + _forward_inplace(src.ptr_address(), gc); } /** primary workhorse for garbage collection. @@ -156,10 +156,10 @@ namespace xo { **/ static Object * _deep_move(Object * src, gc::GC * gc, gc::ObjectStatistics * stats); - /** copy @p src to to-space, and replace original with forwarding pointer to new location. + /** copy @p src to to-space. Overwrite original with forwarding pointer to new location. * return the new location **/ - static Object * _shallow_move(Object * src, gc::GC * gc); + static Object * _shallow_move(Object * src, gc::IAlloc * gc); // Reflection support @@ -207,7 +207,7 @@ namespace xo { * * Require: @ref mm is an instance of @ref gc::GC **/ - virtual Object * _shallow_copy(gc::IAlloc * mm) const = 0; + virtual Object * _shallow_copy(gc::IAlloc * gc) const = 0; /** update child pointers that refer to forwarding pointers, * replacing them with the correct destination. @@ -243,7 +243,7 @@ namespace xo { * allocated by @ref _shallow_move * **/ - virtual std::size_t _forward_children() = 0; + virtual std::size_t _forward_children(gc::GC * gc) = 0; }; template diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index c06b9518..15078a0e 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -356,6 +356,14 @@ namespace xo { return free_ptr_ - checkpoint_; } + bool + ArenaAlloc::check_owned(Object * src) const + { + byte * addr = reinterpret_cast(src); + + return (lo_ <= addr) && (addr < hi_); + } + bool ArenaAlloc::debug_flag() const { diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp index b4a44ff6..2cc183d5 100644 --- a/src/alloc/Forwarding1.cpp +++ b/src/alloc/Forwarding1.cpp @@ -60,7 +60,7 @@ namespace xo { // LCOV_EXCL_START std::size_t - Forwarding1::_forward_children() { + Forwarding1::_forward_children(gc::GC *) { assert(false); return 0; } diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index c78768fb..bb556620 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -559,6 +559,19 @@ namespace xo { } } + bool + GC::check_owned(Object * src) const + { + return this->fromspace_contains(src); + } + + bool + GC::check_move(Object * src) const + { + return (this->runstate().full_move() + || (this->tospace_generation_of(src) != gc::generation_result::tenured)); + } + void GC::swap_nursery() { diff --git a/src/alloc/IAlloc.cpp b/src/alloc/IAlloc.cpp index 8fe4789a..9754fb47 100644 --- a/src/alloc/IAlloc.cpp +++ b/src/alloc/IAlloc.cpp @@ -47,6 +47,18 @@ namespace xo { *lhs = rhs; } + bool + IAlloc::check_owned(Object * /*obj*/) const + { + return false; + } + + bool + IAlloc::check_move(Object * /*obj*/) const + { + return false; + } + // LCOV_EXCL_START std::byte * IAlloc::alloc_gc_copy(std::size_t /*z*/, const void * /*src*/) diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index 309e0886..560f941b 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -14,9 +14,9 @@ operator new (std::size_t z, const xo::Cpof & cpof) { using xo::gc::GC; - GC * gc = reinterpret_cast(cpof.mm_); + //GC * gc = reinterpret_cast(cpof.mm_); - return gc->alloc_gc_copy(z, cpof.src_); + return cpof.mm_->alloc_gc_copy(z, cpof.src_); } namespace xo { @@ -32,18 +32,15 @@ namespace xo { if (src->_is_forwarded()) return src->_offset_destination(src); - bool full_move = gc->runstate().full_move(); + if (gc->check_move(src)) { + Object::_shallow_move(src, gc); - if (!full_move && (gc->tospace_generation_of(src) == gc::generation_result::tenured)) { + /* *src is now a forwarding pointer to a copy in to-space */ + return src->_offset_destination(src); + } else { /* don't move tenured objects during incremental collection */ return src; } - - Object::_shallow_move(src, gc); - - /* *src is now a forwarding pointer to copy in to-space */ - - return src->_offset_destination(src); } Object * @@ -59,9 +56,7 @@ namespace xo { if (retval) return retval; - bool full_move = gc->runstate().full_move(); - - if (!full_move && gc->tospace_generation_of(from_src) == gc::generation_result::tenured) { + if (!gc->check_move(from_src)) { /** incremental collection does not move already-tenured objects **/ return from_src; } @@ -70,7 +65,8 @@ namespace xo { * To-space: * * to_lo = start of to-space - * w,W = white objects. An object x is white if x + all immediate children of x are in to-space + * w,W = white objects. An object x is white if x + * + all immediate children of x are in to-space * (also implies this GC cycle put it there) * g,G = grey objects. An object x is gray if it's in to-space, * but possibly has >0 black children @@ -141,7 +137,7 @@ namespace xo { // update per-class stats here - std::size_t xz = x->_forward_children(); + std::size_t xz = x->_forward_children(gc); // must pad xz to multiple of word size, // to match behavior of LinearAlloc::alloc() @@ -163,12 +159,12 @@ namespace xo { } /*deep_move*/ Object * - Object::_shallow_move(Object * src, gc::GC * gc) + Object::_shallow_move(Object * src, gc::IAlloc * gc) { /* filter for source objects that are owned by GC. * Care required though -- during GC from/to spaces have been swapped already */ - if (gc->fromspace_contains(src)) + if (gc->check_owned(src)) { Object * dest = src->_shallow_copy(gc); From e10380a792470dcddcd87dcfa370f6688deaa43f Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 24 Nov 2025 12:47:44 -0500 Subject: [PATCH 37/69] xo-alloc: utest: fix forwading unit test after upstream refactor --- utest/Forwarding1.test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utest/Forwarding1.test.cpp b/utest/Forwarding1.test.cpp index b39c35a1..2dcb5efe 100644 --- a/utest/Forwarding1.test.cpp +++ b/utest/Forwarding1.test.cpp @@ -33,8 +33,8 @@ namespace xo { void display(std::ostream & os) const final override { os << data_; } virtual std::size_t _shallow_size() const final override { return sizeof(*this); } - virtual Object * _shallow_copy() const final override { return new (Cpof(Object::mm, this)) DummyObject(*this); } - virtual std::size_t _forward_children() final override { return _shallow_size(); } + virtual Object * _shallow_copy(gc::IAlloc * mm) const final override { return new (Cpof(mm, this)) DummyObject(*this); } + virtual std::size_t _forward_children(gc::GC * gc) final override { return _shallow_size(); } private: std::array data_; From 66235079a86c7ddf0964ae9ee7758f64a8dfd48f Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 24 Nov 2025 12:58:54 -0500 Subject: [PATCH 38/69] xo-alloc: IAlloc* i/face sufficient for Object._forward_children --- include/xo/alloc/Forwarding1.hpp | 2 +- include/xo/alloc/Object.hpp | 8 ++++---- src/alloc/Forwarding1.cpp | 2 +- src/alloc/Object.cpp | 2 +- utest/Forwarding1.test.cpp | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/xo/alloc/Forwarding1.hpp b/include/xo/alloc/Forwarding1.hpp index 88ca849a..8999849a 100644 --- a/include/xo/alloc/Forwarding1.hpp +++ b/include/xo/alloc/Forwarding1.hpp @@ -32,7 +32,7 @@ namespace xo { /** required by Object i/face, but never called on Forwarding1 **/ virtual Object * _shallow_copy(gc::IAlloc * mm) const final override; /** required by Object i/face, but never called on Forwarding1 **/ - virtual std::size_t _forward_children(gc::GC * mm) final override; + virtual std::size_t _forward_children(gc::IAlloc * mm) final override; private: /** the object that used to be located at this address (i.e. @c this) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index f1d03225..f16dd6ac 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -119,17 +119,17 @@ namespace xo { * @p src. source object to be forwarded * @p gc. garbage collector */ - static Object * _forward(Object * src, gc::GC * gc); + static Object * _forward(Object * src, gc::IAlloc * gc); template - static void _forward_inplace(T ** src_addr, gc::GC * gc) { + static void _forward_inplace(T ** src_addr, gc::IAlloc * gc) { Object * fwd = _forward(*src_addr, gc); *src_addr = reinterpret_cast(fwd); } template - static void _forward_inplace(gp & src, gc::GC * gc) { + static void _forward_inplace(gp & src, gc::IAlloc * gc) { _forward_inplace(src.ptr_address(), gc); } @@ -243,7 +243,7 @@ namespace xo { * allocated by @ref _shallow_move * **/ - virtual std::size_t _forward_children(gc::GC * gc) = 0; + virtual std::size_t _forward_children(gc::IAlloc * gc) = 0; }; template diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp index 2cc183d5..f94bf4f1 100644 --- a/src/alloc/Forwarding1.cpp +++ b/src/alloc/Forwarding1.cpp @@ -60,7 +60,7 @@ namespace xo { // LCOV_EXCL_START std::size_t - Forwarding1::_forward_children(gc::GC *) { + Forwarding1::_forward_children(gc::IAlloc *) { assert(false); return 0; } diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index 560f941b..dbb86ae2 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -24,7 +24,7 @@ namespace xo { Object::mm = nullptr; Object * - Object::_forward(Object * src, gc::GC * gc) + Object::_forward(Object * src, gc::IAlloc * gc) { if (!src) return src; diff --git a/utest/Forwarding1.test.cpp b/utest/Forwarding1.test.cpp index 2dcb5efe..19d9bf1e 100644 --- a/utest/Forwarding1.test.cpp +++ b/utest/Forwarding1.test.cpp @@ -34,7 +34,7 @@ namespace xo { virtual std::size_t _shallow_size() const final override { return sizeof(*this); } virtual Object * _shallow_copy(gc::IAlloc * mm) const final override { return new (Cpof(mm, this)) DummyObject(*this); } - virtual std::size_t _forward_children(gc::GC * gc) final override { return _shallow_size(); } + virtual std::size_t _forward_children(gc::IAlloc * gc) final override { return _shallow_size(); } private: std::array data_; From 5c032834c6afc0db3a89f0962af373293eb00c21 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 24 Nov 2025 18:01:24 -0500 Subject: [PATCH 39/69] xo-interpreter: handle define-expressions. --- include/xo/alloc/Object.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index f16dd6ac..73a60edd 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -97,6 +97,8 @@ namespace xo { /** memory allocator for objects. Likely this will be a GC instance, * but simple arena also supported. + * + * Load-bearing for .assign_member() **/ static gc::IAlloc * mm; From e5a72bce36601aa9c088f3d4b4ed433c42aa03c8 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 25 Nov 2025 12:43:57 -0500 Subject: [PATCH 40/69] xo-interpreter: implement variable lookup --- include/xo/alloc/Object.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 73a60edd..c8772db9 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -39,6 +39,10 @@ namespace xo { template gc_ptr(const gc_ptr & x) : ptr_{x.ptr()} {} + /** runtime downcast. shorthand for dynamic_cast **/ + template + static gc_ptr from(const gc_ptr & x) { return gc_ptr{dynamic_cast(x.ptr())}; } + /** convenience for static asserts **/ static constexpr bool is_gc_ptr = true; /** see also: xo/refcnt/Refcounted.hpp **/ From daf729292e6ba9ac23d4f8d78f40b210bc498a38 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 26 Nov 2025 20:15:03 -0500 Subject: [PATCH 41/69] xo-interpreter: Object->TaggedPtr conversion (prep for primitives) --- include/xo/alloc/Blob.hpp | 40 +++++++++++++++++++++++++++ src/alloc/Blob.cpp | 57 +++++++++++++++++++++++++++++++++++++++ src/alloc/CMakeLists.txt | 1 + 3 files changed, 98 insertions(+) create mode 100644 include/xo/alloc/Blob.hpp create mode 100644 src/alloc/Blob.cpp diff --git a/include/xo/alloc/Blob.hpp b/include/xo/alloc/Blob.hpp new file mode 100644 index 00000000..9e3ae44a --- /dev/null +++ b/include/xo/alloc/Blob.hpp @@ -0,0 +1,40 @@ +/** @file Blob.hpp + * + * @author Roland Conybeare, Nov 2025 + **/ + +#pragma once + +#include "Object.hpp" +#include "IAlloc.hpp" + +namespace xo { + /** Use to allocate opaque binary data, + * with object header. + * + * Not sure if we want to bother implementing reflection for this... + **/ + class Blob : public Object { + public: + Blob(std::size_t z) : z_{z} {}; + + static gp make(gc::IAlloc * mm, std::size_t z); + + std::size_t size() const { return z_; } + std::byte * data() { return data_; } + + virtual TaggedPtr self_tp() const final override; + virtual void display(std::ostream & os) const final override; + + virtual std::size_t _shallow_size() const final override; + virtual Object * _shallow_copy(gc::IAlloc * gc) const final override; + virtual std::size_t _forward_children(gc::IAlloc * gc) final override; + + private: + std::size_t z_ = 0; + /** flexible array, with @ref z_ bytes **/ + std::byte data_[]; + }; +} + +/* end Blob.hpp */ diff --git a/src/alloc/Blob.cpp b/src/alloc/Blob.cpp new file mode 100644 index 00000000..97932844 --- /dev/null +++ b/src/alloc/Blob.cpp @@ -0,0 +1,57 @@ +/** @file Blob.cpp + * + * @author Roland Conybeare, Nov 2025 + **/ + +#include "Blob.hpp" +#include "xo/reflect/Reflect.hpp" +#include "xo/alloc/IAlloc.hpp" + +namespace xo { + using xo::reflect::Reflect; + using xo::reflect::TaggedPtr; + + gp + Blob::make(gc::IAlloc * mm, std::size_t z) { + std::byte * mem = mm->alloc(sizeof(Blob) + z); + + return new (mem) Blob(z); + } + + TaggedPtr + Blob::self_tp() const + { + return Reflect::make_tp(const_cast(this)); + } + + void + Blob::display(std::ostream & os) const + { + os << ""; + } + + std::size_t + Blob::_shallow_size() const { + return sizeof(Blob) + z_; + } + + Object * + Blob::_shallow_copy(gc::IAlloc * mm) const { + Cpof cpof(mm, this); + std::byte * cp_mem = mm->alloc_gc_copy(sizeof(Blob) + z_, this); + + gp copy = new (cp_mem) Blob(z_); + + ::memcpy(copy->data(), data_, z_); + + return copy.get(); + } + + std::size_t + Blob::_forward_children(gc::IAlloc *) + { + return this->_shallow_size(); + } +} + +/* end Blob.cpp */ diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index 6bbc2f32..4a9d1db4 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -9,6 +9,7 @@ set(SELF_SRCS GcStatistics.cpp ObjectStatistics.cpp Object.cpp + Blob.cpp Forwarding1.cpp generation.cpp ) From 2febec3c8ca16896078fb5d42c81b7451edef03a Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 29 Nov 2025 16:58:44 -0500 Subject: [PATCH 42/69] xo-alloc, xo-object: fix alloc,gc unit tests after gc improvements --- include/xo/alloc/ArenaAlloc.hpp | 1 + include/xo/alloc/GC.hpp | 2 ++ include/xo/alloc/IAlloc.hpp | 5 ++++- include/xo/alloc/ListAlloc.hpp | 3 +++ src/alloc/GC.cpp | 6 ++++++ src/alloc/ListAlloc.cpp | 6 ++++++ utest/ArenaAlloc.test.cpp | 24 ++++++++++++------------ utest/GC.test.cpp | 9 ++++++--- utest/ListAlloc.test.cpp | 5 ++++- 9 files changed, 44 insertions(+), 17 deletions(-) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index ed3fd59c..d9731761 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -130,6 +130,7 @@ namespace xo { std::size_t reserved() const { return hi_ - lo_; }; std::size_t page_size() const { return page_z_; } + std::size_t hugepage_z() const { return hugepage_z_; } std::byte * free_ptr() const { return free_ptr_; } void set_free_ptr(std::byte * x); diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 423dd2f8..6146cba9 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -177,6 +177,8 @@ namespace xo { /** @return pagesize (will be the same for {nursery, tenured} spaces) **/ std::size_t pagesize() const; + /** @return hugepage size (will be the same for {nursery, tenured} spaces) **/ + std::size_t hugepage_z() const; /** @return allocation portion of Nursery to-space **/ std::size_t nursery_to_allocated() const; diff --git a/include/xo/alloc/IAlloc.hpp b/include/xo/alloc/IAlloc.hpp index fceaca94..3b36dc98 100644 --- a/include/xo/alloc/IAlloc.hpp +++ b/include/xo/alloc/IAlloc.hpp @@ -16,12 +16,15 @@ namespace xo { namespace gc { /** @class IAllocator - * @brief memory allocation interface with limited garbage collector support + * @brief arena allocation interface with limited garbage collector support * * Garbage collector support methods: * - checkpoint() * - assign_member() * - alloc_gc_copy() + * + * See class GC for copying incremental collector. + * See class ArenaAlloc for arena allocator **/ class IAlloc { public: diff --git a/include/xo/alloc/ListAlloc.hpp b/include/xo/alloc/ListAlloc.hpp index 75b148ac..30d91cfb 100644 --- a/include/xo/alloc/ListAlloc.hpp +++ b/include/xo/alloc/ListAlloc.hpp @@ -37,6 +37,9 @@ namespace xo { /** page size used by underlying ArenaAlloc **/ std::size_t page_size() const; + /** hugepage size used by underlying ArenaAlloc **/ + std::size_t hugepage_z() const; + /** reset to have at least @p z bytes of storage **/ bool reset(std::size_t z); diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index bb556620..3e636cad 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -233,6 +233,12 @@ namespace xo { return nursery_to()->page_size(); } + std::size_t + GC::hugepage_z() const + { + return nursery_to()->hugepage_z(); + } + std::size_t GC::nursery_from_allocated() const { diff --git a/src/alloc/ListAlloc.cpp b/src/alloc/ListAlloc.cpp index 8b0a2dbb..9961d59b 100644 --- a/src/alloc/ListAlloc.cpp +++ b/src/alloc/ListAlloc.cpp @@ -75,6 +75,11 @@ namespace xo { return hd_->page_size(); } + std::size_t + ListAlloc::hugepage_z() const { + return hd_->hugepage_z(); + } + std::size_t ListAlloc::size() const { return total_z_; @@ -336,6 +341,7 @@ namespace xo { std::unique_ptr new_alloc = ArenaAlloc::make(name, cz, debug_flag_); + cz = new_alloc->size(); if (!new_alloc) return false; diff --git a/utest/ArenaAlloc.test.cpp b/utest/ArenaAlloc.test.cpp index 52f135f3..78055eed 100644 --- a/utest/ArenaAlloc.test.cpp +++ b/utest/ArenaAlloc.test.cpp @@ -13,17 +13,16 @@ namespace xo { namespace { struct testcase_alloc { - testcase_alloc(std::size_t rz, std::size_t z) + explicit testcase_alloc(std::size_t z) : arena_z_{z} {} std::size_t arena_z_; - }; std::vector s_testcase_v = { - testcase_alloc(0, 4096) + testcase_alloc(4096) }; } @@ -37,11 +36,12 @@ namespace xo { auto alloc = ArenaAlloc::make("linearalloc", tc.arena_z_, c_debug_flag); + alloc->expand(tc.arena_z_); REQUIRE(alloc.get()); REQUIRE(alloc->name() == "linearalloc"); - REQUIRE(alloc->size() == tc.arena_z_); - REQUIRE(alloc->available() == tc.arena_z_); + REQUIRE(alloc->size() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->available() == std::max(tc.arena_z_, alloc->hugepage_z())); REQUIRE(alloc->allocated() == 0); REQUIRE(alloc->is_before_checkpoint(alloc->free_ptr()) == false); REQUIRE(alloc->before_checkpoint() == 0); @@ -49,23 +49,23 @@ namespace xo { auto free0 = alloc->free_ptr(); - auto mem = alloc->alloc(tc.arena_z_); + auto mem = alloc->alloc(std::max(tc.arena_z_, alloc->hugepage_z())); REQUIRE(mem != nullptr); REQUIRE(mem == free0); - REQUIRE(alloc->size() == tc.arena_z_); + REQUIRE(alloc->size() == std::max(tc.arena_z_, alloc->hugepage_z())); REQUIRE(alloc->available() == 0); - REQUIRE(alloc->allocated() == tc.arena_z_); + REQUIRE(alloc->allocated() == std::max(tc.arena_z_, alloc->hugepage_z())); REQUIRE(alloc->is_before_checkpoint(mem) == false); REQUIRE(alloc->before_checkpoint() == 0); - REQUIRE(alloc->after_checkpoint() == tc.arena_z_); + REQUIRE(alloc->after_checkpoint() == std::max(tc.arena_z_, alloc->hugepage_z())); alloc->clear(); REQUIRE(alloc->free_ptr() == free0); - REQUIRE(alloc->available() == tc.arena_z_); + REQUIRE(alloc->available() == std::max(tc.arena_z_, alloc->hugepage_z())); REQUIRE(alloc->allocated() == 0); REQUIRE(alloc->is_before_checkpoint(free0) == false); REQUIRE(alloc->before_checkpoint() == 0); @@ -74,8 +74,8 @@ namespace xo { mem = alloc->alloc(1); auto used = sizeof(void*); - REQUIRE(alloc->size() == tc.arena_z_); - REQUIRE(alloc->available() == tc.arena_z_ - used); + REQUIRE(alloc->size() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->available() == std::max(tc.arena_z_, alloc->hugepage_z()) - used); REQUIRE(alloc->allocated() == used); REQUIRE(alloc->is_before_checkpoint(free0) == false); REQUIRE(alloc->before_checkpoint() == 0); diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index 67a0fdc0..9e6c17e3 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -26,8 +26,11 @@ namespace xo { std::vector s_testcase_v = { + // n_gct: nursery gc threshold + // t_gct: tenured gc threshold + // // nz tz n_gct t_gct - testcase_gc(1024, 4096, 1024, 4096) + testcase_gc(1024, 4096, 1024, 1024) }; } @@ -49,9 +52,9 @@ namespace xo { REQUIRE(gc->nursery_to_allocated() == 0); REQUIRE(gc->nursery_to_committed() >= tc.nursery_z_); REQUIRE(gc->nursery_to_reserved() >= tc.nursery_z_); - REQUIRE(gc->nursery_to_reserved() < tc.nursery_z_ + gc->pagesize()); + REQUIRE(gc->nursery_to_reserved() < tc.nursery_z_ + gc->hugepage_z()); REQUIRE(gc->size() >= tc.nursery_z_ + tc.tenured_z_); - REQUIRE(gc->size() < tc.nursery_z_ + gc->pagesize() + tc.tenured_z_ + gc->pagesize()); + REQUIRE(gc->size() < tc.nursery_z_ + gc->hugepage_z() + tc.tenured_z_ + gc->hugepage_z()); REQUIRE(gc->allocated() == 0); REQUIRE(gc->available() == gc->nursery_to_reserved()); REQUIRE(gc->before_checkpoint() == 0); diff --git a/utest/ListAlloc.test.cpp b/utest/ListAlloc.test.cpp index 0a62fd0b..5f425568 100644 --- a/utest/ListAlloc.test.cpp +++ b/utest/ListAlloc.test.cpp @@ -10,6 +10,8 @@ namespace xo { using xo::gc::ListAlloc; namespace ut { +#ifdef NOT_USING // ListAlloc probably permanently retired. Not maintaining + TEST_CASE("ListAlloc", "[alloc][gc]") { /** teeny weeny allocator. @@ -27,7 +29,7 @@ namespace xo { std::byte * mem1 = alloc->alloc(20); REQUIRE(mem1); - REQUIRE(alloc->size() == alloc->page_size()); + REQUIRE(alloc->size() == std::max(alloc->page_size(), alloc->hugepage_z())); /* round up to multiple of 8 */ REQUIRE(alloc->before_checkpoint() == 24); REQUIRE(alloc->after_checkpoint() == 0); @@ -54,6 +56,7 @@ namespace xo { REQUIRE(alloc->is_before_checkpoint(mem2) == false); REQUIRE(alloc->is_before_checkpoint(mem3) == false); } +#endif } /*namespace ut*/ } /*namespace xo*/ From 50b0f7698c7ac6357b49e2a5efcae19f79c492e4 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 29 Nov 2025 16:59:36 -0500 Subject: [PATCH 43/69] xo-alloc: + ArenaAllocT for use with std::map() etc. --- include/xo/alloc/ArenaAllocT.hpp | 74 ++++++++++++++++++++++++++++++++ utest/ArenaAllocT.test.cpp | 65 ++++++++++++++++++++++++++++ utest/CMakeLists.txt | 1 + 3 files changed, 140 insertions(+) create mode 100644 include/xo/alloc/ArenaAllocT.hpp create mode 100644 utest/ArenaAllocT.test.cpp diff --git a/include/xo/alloc/ArenaAllocT.hpp b/include/xo/alloc/ArenaAllocT.hpp new file mode 100644 index 00000000..6c8a9b41 --- /dev/null +++ b/include/xo/alloc/ArenaAllocT.hpp @@ -0,0 +1,74 @@ +/** @file Allocator.hpp + * + * @author Roland Conybeare, Nov 2025 + **/ + +#pragma once + +#include "xo/alloc/ArenaAlloc.hpp" + +namespace xo { + namespace gc { + /** @class allocator + * @brief c++ allocator with allocator traits + * + * Can use ArenaAllocT with std::map etc. + **/ + template + class ArenaAllocT { + public: + using value_type = T; + /** copy assignment: leave lhs allocator in place **/ + using propagate_on_container_copy_assignment = std::false_type; + /** move assignment: adopt rhs allocator + * (Forced: cannot mix allocations from different allocators + * within a container) + **/ + using propagate_on_container_move_assignment = std::true_type; + /** swap: also swap allocators + * (Forced: cannot mix allocations from different allocators + * within a containers) + **/ + using propagate_on_container_swap = std::true_type; + /** An ArenaAlloc instance is unique owner of its own memory: + * no other instance can dealloc + **/ + using is_always_equal = std::false_type; + + public: + explicit ArenaAllocT(ArenaAlloc * mm) : mm_{mm} {} + ArenaAllocT(const ArenaAllocT & other) = default; + + /** rebind ctor. Allows container to use supplied allocator + * for multiple types + **/ + template + ArenaAllocT(const ArenaAllocT & other) noexcept : mm_{other.mm_} {} + + T * allocate(size_t n) { + void * mem = mm_->alloc(n * sizeof(T)); + + return reinterpret_cast(mem); + } + + void deallocate(T * p, size_t n) noexcept { + assert(mm_->contains(p)); + assert(n == 0 || mm_->contains(p + n - 1)); + + //arena_->deallocate(p, n * sizeof(T)); + } + + bool operator==(const ArenaAllocT & other) const { + return mm_ == other.mm_; + } + + bool operator!=(const ArenaAllocT & other) const { + return mm_ != other.mm_; + } + + ArenaAlloc * mm_ = nullptr; + }; + } /*namespace gc*/ +} /*namespace xo*/ + +/* end Allocator.hpp */ diff --git a/utest/ArenaAllocT.test.cpp b/utest/ArenaAllocT.test.cpp new file mode 100644 index 00000000..44b350be --- /dev/null +++ b/utest/ArenaAllocT.test.cpp @@ -0,0 +1,65 @@ +/** @file ArenaAllocT.test.cpp + * + * @author Roland Conybeare, Nov 2025 + **/ + +#include "xo/alloc/ArenaAllocT.hpp" +#include +#include + +namespace xo { + using xo::gc::ArenaAllocT; + using xo::gc::ArenaAlloc; + + namespace ut { + + namespace { + struct testcase_ArenaAllocT { + testcase_ArenaAllocT(std::size_t z) : arena_z_{z} {} + + std::size_t arena_z_; + std::vector> kv_pairs_; + }; + + std::vector + s_testcase_v = { + testcase_ArenaAllocT(4096) + }; + } + + TEST_CASE("ArenaAllocT", "[alloc][traits]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + const testcase_ArenaAllocT & tc = s_testcase_v[i_tc]; + + constexpr bool c_debug_flag = true; + + auto arena = ArenaAlloc::make("arena", tc.arena_z_, c_debug_flag); + auto alloc = ArenaAllocT>(arena.get()); + + using TestMapType = std::map, + ArenaAllocT>>; + + TestMapType test_map(alloc); + + size_t n = 0; + for (const auto & kv_ix : tc.kv_pairs_) { + test_map[kv_ix.first] = kv_ix.second; + ++n; + + REQUIRE(test_map.size() == n); + + for (const auto & map_ix : test_map) { + map_ix.first; + map_ix.second; + } + } + + } + } + } +} /*namespace xo*/ + +/* end ArenaAllocT.test.cpp */ diff --git a/utest/CMakeLists.txt b/utest/CMakeLists.txt index 366cf664..6c644f51 100644 --- a/utest/CMakeLists.txt +++ b/utest/CMakeLists.txt @@ -7,6 +7,7 @@ set(UTEST_SRCS alloc_utest_main.cpp IAlloc.test.cpp ArenaAlloc.test.cpp + ArenaAllocT.test.cpp ListAlloc.test.cpp GC.test.cpp GcStatistics.test.cpp From 5e3df1c7837746c02acbd845040cde42a1d728a1 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 29 Nov 2025 17:11:53 -0500 Subject: [PATCH 44/69] xo-alloc: + ArenaAllocT unit test --- utest/ArenaAllocT.test.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/utest/ArenaAllocT.test.cpp b/utest/ArenaAllocT.test.cpp index 44b350be..a9257173 100644 --- a/utest/ArenaAllocT.test.cpp +++ b/utest/ArenaAllocT.test.cpp @@ -15,15 +15,16 @@ namespace xo { namespace { struct testcase_ArenaAllocT { - testcase_ArenaAllocT(std::size_t z) : arena_z_{z} {} - std::size_t arena_z_; std::vector> kv_pairs_; }; std::vector s_testcase_v = { - testcase_ArenaAllocT(4096) + { 4096, {} }, + { 4096, {{"a", "apple"}} }, + { 4096, {{"a", "apple"}, {"b", "banana"}, {"c", "carrot"}} }, + { 4096, {{"a", "apple"}, {"b", "banana"}, {"c", "carrot"}, {"e", "eggplant"}} }, }; } @@ -32,7 +33,7 @@ namespace xo { for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { const testcase_ArenaAllocT & tc = s_testcase_v[i_tc]; - constexpr bool c_debug_flag = true; + constexpr bool c_debug_flag = false; auto arena = ArenaAlloc::make("arena", tc.arena_z_, c_debug_flag); auto alloc = ArenaAllocT>(arena.get()); @@ -52,8 +53,9 @@ namespace xo { REQUIRE(test_map.size() == n); for (const auto & map_ix : test_map) { - map_ix.first; - map_ix.second; + // verify alloc was used for both Key + Value. + REQUIRE(arena->contains(&map_ix.first)); + REQUIRE(arena->contains(&map_ix.second)); } } From 30a00be2620dc0ad40a71b1258454db269a6c428 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 1 Dec 2025 01:20:49 -0500 Subject: [PATCH 45/69] xo-alloc + xo-allocutil: refactor to shrink dep surface area --- include/xo/alloc/ArenaAlloc.hpp | 4 +- include/xo/alloc/Forwarding1.hpp | 10 +- include/xo/alloc/GC.hpp | 37 +++---- include/xo/alloc/IAlloc.hpp | 129 ----------------------- include/xo/alloc/Object.hpp | 171 +++++-------------------------- src/alloc/ArenaAlloc.cpp | 2 +- src/alloc/Blob.cpp | 2 +- src/alloc/CMakeLists.txt | 2 +- src/alloc/Forwarding1.cpp | 21 ++-- src/alloc/GC.cpp | 41 ++++---- src/alloc/IAlloc.cpp | 74 ------------- src/alloc/Object.cpp | 20 ++-- 12 files changed, 100 insertions(+), 413 deletions(-) delete mode 100644 include/xo/alloc/IAlloc.hpp delete mode 100644 src/alloc/IAlloc.cpp diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index d9731761..1f48121b 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -5,7 +5,7 @@ #pragma once -#include "IAlloc.hpp" +#include "xo/allocutil/IAlloc.hpp" #include "ObjectStatistics.hpp" namespace xo { @@ -175,8 +175,8 @@ namespace xo { virtual void clear() final override; virtual void checkpoint() final override; virtual std::byte * alloc(std::size_t z) final override; + virtual bool check_owned(IObject * src) const final override; - virtual bool check_owned(Object * src) const final override; ArenaAlloc & operator=(const ArenaAlloc &) = delete; ArenaAlloc & operator=(ArenaAlloc &&) = delete; diff --git a/include/xo/alloc/Forwarding1.hpp b/include/xo/alloc/Forwarding1.hpp index 8999849a..90ff0198 100644 --- a/include/xo/alloc/Forwarding1.hpp +++ b/include/xo/alloc/Forwarding1.hpp @@ -19,18 +19,18 @@ namespace xo { **/ class Forwarding1 : public Object { public: - explicit Forwarding1(gp dest); + explicit Forwarding1(gp dest); // inherited from Object.. virtual TaggedPtr self_tp() const final override; virtual void display(std::ostream & os) const final override; virtual bool _is_forwarded() const final override { return true; } - virtual Object * _offset_destination(Object * src) const final override; - virtual Object * _destination() final override; + virtual IObject * _offset_destination(IObject * src) const final override; + virtual IObject * _destination() final override; /** required by Object i/face, but never called on Forwarding1 **/ virtual std::size_t _shallow_size() const final override; /** required by Object i/face, but never called on Forwarding1 **/ - virtual Object * _shallow_copy(gc::IAlloc * mm) const final override; + virtual IObject * _shallow_copy(gc::IAlloc * mm) const final override; /** required by Object i/face, but never called on Forwarding1 **/ virtual std::size_t _forward_children(gc::IAlloc * mm) final override; @@ -47,7 +47,7 @@ namespace xo { * UB revealed when GC traverses a pointer that relies on the 2nd * vtable to index virtual methods. **/ - gp dest_; + gp dest_; }; } /*namespace obj*/ diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 6146cba9..aeb918c9 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -88,29 +88,30 @@ namespace xo { class MutationLogEntry { public: - MutationLogEntry(Object * parent, Object ** lhs) : parent_{parent}, lhs_{lhs} {} + MutationLogEntry(IObject * parent, IObject ** lhs) + : parent_{parent}, lhs_{lhs} {} - Object * parent() const { return parent_; } - Object ** lhs() const { return lhs_; } + IObject * parent() const { return parent_; } + IObject ** lhs() const { return lhs_; } - Object * child() const { return *lhs_; } + IObject * child() const { return *lhs_; } bool is_child_forwarded() const; bool is_parent_forwarded() const; - Object * parent_destination() const; + IObject * parent_destination() const; /** Flag obsolete mutation. * Future proofing, never happens for regular objects **/ bool is_dead() const { return false; } - MutationLogEntry update_parent_moved(Object * parent_to) const; - void fixup_parent_child_moved(Object * child_to) { *lhs_ = child_to; } + MutationLogEntry update_parent_moved(IObject * parent_to) const; + void fixup_parent_child_moved(IObject * child_to) { *lhs_ = child_to; } private: - Object * parent_; - Object ** lhs_; + IObject * parent_ = nullptr; + IObject ** lhs_ = nullptr; }; using MutationLog = std::vector; @@ -235,15 +236,15 @@ namespace xo { /** add gc root at address @p addr . Gc will keep alive anything reachable * from @c *addr **/ - void add_gc_root(Object ** addr); + void add_gc_root(IObject ** addr); /** reverse the effect of previous call to @ref add_gc_root **/ - void remove_gc_root(Object ** addr); + void remove_gc_root(IObject ** addr); /** convenience wrapper **/ template - void add_gc_root_dwim(gp * p) { this->add_gc_root(reinterpret_cast(p->ptr_address())); } + void add_gc_root_dwim(gp * p) { this->add_gc_root(reinterpret_cast(p->ptr_address())); } template - void remove_gc_root_dwim(gp * p) { this->remove_gc_root(reinterpret_cast(p->ptr_address())); } + void remove_gc_root_dwim(gp * p) { this->remove_gc_root(reinterpret_cast(p->ptr_address())); } /** may optionally use this to observe GC copy phase. * Will be invoked once _per surviving object_, so not cheap. @@ -308,17 +309,17 @@ namespace xo { * @param lhs. address of a member variable within the allocation of @p parent. * @param rhs. new target for @p *lhs **/ - virtual void assign_member(Object * parent, Object ** lhs, Object* rhs) final override; + virtual void assign_member(IObject * parent, IObject ** lhs, IObject* rhs) final override; /** during GC check for source objects owned by GC. * See Object::_shallow_move. **/ - virtual bool check_owned(Object * src) const final override; + virtual bool check_owned(IObject * src) const final override; /** queries during GC to determine if object at address @p src should move: * - full GC -> always * - incr GC -> if not tenured **/ - virtual bool check_move(Object * src) const final override; + virtual bool check_move(IObject * src) const final override; virtual std::byte * alloc(std::size_t z) final override; virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; @@ -349,7 +350,7 @@ namespace xo { /** scan to-space for object statistics before GC */ void capture_object_statistics(generation upto, capture_phase phase); /** copy object **/ - void copy_object(Object ** addr, generation upto, ObjectStatistics * object_stats); + void copy_object(IObject ** addr, generation upto, ObjectStatistics * object_stats); /** copy everything reachable from global gc roots **/ void copy_globals(generation g); /** review mutation log; may discover+rescue reachable objects. @@ -426,7 +427,7 @@ namespace xo { * Application can introduce new root object pointers at any time provided GC not running, * but cannot withdraw them. **/ - std::vector gc_root_v_; + std::vector gc_root_v_; /** log cross-generational and cross-checkpoint mutations. * These need to be adjusted on next incremental collection diff --git a/include/xo/alloc/IAlloc.hpp b/include/xo/alloc/IAlloc.hpp deleted file mode 100644 index 3b36dc98..00000000 --- a/include/xo/alloc/IAlloc.hpp +++ /dev/null @@ -1,129 +0,0 @@ -/* file IAlloc.hpp - * - * author: Roland Conybeare, Jul 2025 - */ - -#pragma once - -#include -#include - -namespace xo { - template - using up = std::unique_ptr; - - class Object; - - namespace gc { - /** @class IAllocator - * @brief arena allocation interface with limited garbage collector support - * - * Garbage collector support methods: - * - checkpoint() - * - assign_member() - * - alloc_gc_copy() - * - * See class GC for copying incremental collector. - * See class ArenaAlloc for arena allocator - **/ - class IAlloc { - public: - virtual ~IAlloc() {} - - /** compute padding to add to an allocation of size z to bring it up to - * a multiple of word size (8 bytes on x86_64) - **/ - static std::uint32_t alloc_padding(std::size_t z); - /** z + alloc_padding(z) **/ - static std::size_t with_padding(std::size_t z); - - /** optional name for this allocator; labelling for diagnostics **/ - virtual const std::string & name() const = 0; - /** allocator size in bytes (up to reserved limit) - * Includes unallocated mmeory - **/ - virtual std::size_t size() const = 0; - /** committed size in bytes **/ - virtual std::size_t committed() const = 0; - /** number of unallocated bytes available (up to soft limit) - * from this allocator - **/ - virtual std::size_t available() const = 0; - /** number of bytes allocated from this allocator **/ - virtual std::size_t allocated() const = 0; - /** true iff pointer x comes from this allocator **/ - virtual bool contains(const void * x) const = 0; - /** true iff object at address @p x was allocated by this allocator, - * and before checkpoint - **/ - virtual bool is_before_checkpoint(const void * x) const = 0; - /** number of bytes allocated before @ref checkpoint **/ - virtual std::size_t before_checkpoint() const = 0; - /** number of bytes allocated since @ref checkpoint **/ - virtual std::size_t after_checkpoint() const = 0; - /** @return true iff debug logging enabled **/ - virtual bool debug_flag() const = 0; - - /** remember allocator state. All currently-allocated addresses xo - * will satisfy is_before_checkpoint(x). Subsequent allocations x - * will fail is_before_checkpoint(x), until checkpoint superseded - * by @ref clear or another call to @ref checkpoint - **/ - virtual void checkpoint() = 0; - - /** allocate @p z bytes of memory. returns pointer to first address **/ - virtual std::byte * alloc(std::size_t z) = 0; - /** reset allocator to empty state. **/ - virtual void clear() = 0; - - // ----- GC-specific methods ----- - - /** true iff this allocator owns object at address @p src. - * Use to assist Object::_shallow_move - **/ - virtual bool check_owned(Object * src) const; - /** true iff object at address @p src must move as part of - * in-progress collection phase - **/ - virtual bool check_move(Object * src) const; - /** perform assignment - * @code - * *lhs = rhs - * @endcode - * plus additional book keeping if needed (e.g. in @ref GC) - * Default implementation just does the assignment. - **/ - virtual void assign_member(Object * parent, Object ** lhs, Object * rhs); - /** allocate @p z bytes for copy of object at @p src. - * Only used in @ref GC. Default implementation asserts and returns nullptr - **/ - virtual std::byte * alloc_gc_copy(std::size_t z, const void * src); - }; - } /*namespace gc*/ - - class MMPtr { - public: - explicit MMPtr(gc::IAlloc * mm) : mm_{mm} {} - - gc::IAlloc * mm_ = nullptr; - }; -} /*namespace xo*/ - -inline void * operator new (std::size_t z, const xo::MMPtr & mmp) { - return mmp.mm_->alloc(z); -} - -//inline void operator delete (void * p, const MMPtr & mmp) { -// mmp.mm_->free(reinterpret_cast(p)); -//} - -inline void * operator new[] (std::size_t z, const xo::MMPtr & mmp) { - return mmp.mm_->alloc(z); -} - -//inline void operator delete[] (void * p, const MMPtr & mmp) { -// mmp.mm_->free(reinterpret_cast(p)); -//} - - -/* end IAlloc.hpp */ diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index c8772db9..2468c8bc 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -7,6 +7,8 @@ #include "xo/reflect/TaggedPtr.hpp" #include "IAlloc.hpp" +#include "xo/allocutil/IObject.hpp" +#include "xo/allocutil/gc_ptr.hpp" #include #include @@ -16,71 +18,6 @@ namespace xo { class ObjectStatistics; }; - template - class gc_ptr; - - template - using gp = gc_ptr; - - /** wrapper for a pointer to garbage-collector-eligible T. - * Application code will usually use the alias template gp - **/ - template - class gc_ptr { - public: - using element_type = T; - - public: - gc_ptr() = default; - gc_ptr(T * p) : ptr_{p} {} - gc_ptr(const gc_ptr & x) : ptr_{x.ptr_} {} - - /** create from gc_ptr to some related type @tparam S **/ - template - gc_ptr(const gc_ptr & x) : ptr_{x.ptr()} {} - - /** runtime downcast. shorthand for dynamic_cast **/ - template - static gc_ptr from(const gc_ptr & x) { return gc_ptr{dynamic_cast(x.ptr())}; } - - /** convenience for static asserts **/ - static constexpr bool is_gc_ptr = true; - /** see also: xo/refcnt/Refcounted.hpp **/ - static constexpr bool is_rc_ptr = false; - - static bool is_eq(gc_ptr x1, gc_ptr x2) { - std::uintptr_t u1 = reinterpret_cast(x1.ptr()); - std::uintptr_t u2 = reinterpret_cast(x2.ptr()); - - // multiple inheritance shenanigans. - // (allow interface pointers separated by one pointer) - - if (u1 >= u2) - return (u1 <= u2 + sizeof(std::uintptr_t)); - else - return (u2 <= u1 + sizeof(std::uintptr_t)); - } - - /** (for consistency's sake) **/ - T * get() const { return ptr_; } - - T * ptr() const { return ptr_; } - T ** ptr_address() { return &ptr_; } - - bool is_null() const { return ptr_ == nullptr; } - void make_null() { ptr_ = nullptr; } - - void assign_ptr(T * x) { ptr_ = x; } - - gc_ptr & operator=(const gc_ptr & x) { ptr_ = x.ptr(); return *this; } - - T * operator->() const { return ptr_; } - T & operator*() const { return *ptr_; } - - private: - T * ptr_ = nullptr; - }; - /** Root class for all xo GC-collectable objects. * * Design note: @@ -92,11 +29,15 @@ namespace xo { * Would be feasible to relax the must-inherit-from-Object constraint * by having GC use its own wrapper, at cost of an extra layer of indirection **/ - class Object { + class Object : public IObject { public: using TaggedPtr = xo::reflect::TaggedPtr; public: + static gp from(gp x) { + return dynamic_cast(x.ptr()); + } + virtual ~Object() = default; /** memory allocator for objects. Likely this will be a GC instance, @@ -111,7 +52,7 @@ namespace xo { * add mutation log entry **/ template - static void assign_member(gp parent, gp * lhs, gp rhs); + static void assign_member(gp parent, gp * lhs, gp rhs); /** use from GC aux functions **/ static gc::GC * _gc() { return reinterpret_cast(mm); } @@ -125,11 +66,11 @@ namespace xo { * @p src. source object to be forwarded * @p gc. garbage collector */ - static Object * _forward(Object * src, gc::IAlloc * gc); + static IObject * _forward(IObject * src, gc::IAlloc * gc); template static void _forward_inplace(T ** src_addr, gc::IAlloc * gc) { - Object * fwd = _forward(*src_addr, gc); + IObject * fwd = _forward(*src_addr, gc); *src_addr = reinterpret_cast(fwd); } @@ -160,12 +101,12 @@ namespace xo { * @param gc garbage collector * @param stats per-object-type GC statistics **/ - static Object * _deep_move(Object * src, gc::GC * gc, gc::ObjectStatistics * stats); + static IObject * _deep_move(IObject * src, gc::GC * gc, gc::ObjectStatistics * stats); /** copy @p src to to-space. Overwrite original with forwarding pointer to new location. * return the new location **/ - static Object * _shallow_move(Object * src, gc::IAlloc * gc); + static IObject * _shallow_move(IObject * src, gc::IAlloc * gc); // Reflection support @@ -176,89 +117,25 @@ namespace xo { /** print on stream @p os **/ virtual void display(std::ostream & os) const = 0; - // GC support + // Inherited from IObject.. - /** true iff this object represents a forwarding pointer. - * Forwarding pointers are exclusively created by the garbage collector; - * forwarding pointers (and only forwarding pointers) return true here. - **/ - virtual bool _is_forwarded() const { return false; } + //virtual bool _is_forwarded() const override { return false; } + //virtual IObject * _offset_destination(IObject * src) const override { return src; }; + virtual void _forward_to(IObject * dest) override; + //virtual IObject * _destination() override { return nullptr; } - /** offset for uncommon situation where pointer address is offset from object - * base address - **/ - virtual Object * _offset_destination(Object * src) const { return src; }; - - /** replace this object with a forwarding pointer referring to @p dest. - **/ - virtual void _forward_to(Object * dest); - - /** if this object represents a forwarding pointer, return its new location. - * forwarding pointers belong to the garbage collector implementation. - * (if you have to ask -- no, your class is not a forwarding pointer) - * all other objects return nullptr here. - **/ - virtual Object * _destination() { return nullptr; } - - /** return amount of storage (including padding) consumed by this object, - * excluding immediate Object-pointer children - **/ - virtual std::size_t _shallow_size() const = 0; - - /** if subject is allocated by GC: - * - create copy C in to-space - * - destination C will be nursery|tenured depending on location of this. - * else - * - return this to disengage from GC - * - * Require: @ref mm is an instance of @ref gc::GC - **/ - virtual Object * _shallow_copy(gc::IAlloc * gc) const = 0; - - /** update child pointers that refer to forwarding pointers, - * replacing them with the correct destination. - * See @ref Object::deep_move - * - * this gray object, located in to-space. - * fwd1 forwarding objects. - * Located in from-space. Invalid at end of GC cycle. - * p1,p2 source pointers. - * D1,D2 already-forwarded objects. located in to-space. - * - * before: - * this fwd1 - * +----+ +-+ - * | p1 ----->|x|-------> D1 - * | | +-+ - * | | - * | p2 ----------------> D2 - * +----+ - * - * after: - * this - * +----+ - * | p1 ----------------> D1 - * | | - * | | - * | p2 ----------------> D2 - * +----+ - * - * this is now white - * - * @return shallow size of *this. Must exactly match the amount of memory in to-space - * allocated by @ref _shallow_move - * - **/ - virtual std::size_t _forward_children(gc::IAlloc * gc) = 0; + virtual std::size_t _shallow_size() const override = 0; + virtual IObject * _shallow_copy(gc::IAlloc * gc) const override = 0; + virtual std::size_t _forward_children(gc::IAlloc * gc) override = 0; }; template void - Object::assign_member(gp parent, gp * lhs, gp rhs) + Object::assign_member(gp parent, gp * lhs, gp rhs) { - Object::mm->assign_member(parent.ptr(), - reinterpret_cast(lhs->ptr_address()), - rhs.ptr()); + Object::mm->assign_member(reinterpret_cast(parent.ptr()), + reinterpret_cast(lhs->ptr_address()), + reinterpret_cast(rhs.ptr())); } std::ostream & diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index 15078a0e..886b6a70 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -357,7 +357,7 @@ namespace xo { } bool - ArenaAlloc::check_owned(Object * src) const + ArenaAlloc::check_owned(IObject * src) const { byte * addr = reinterpret_cast(src); diff --git a/src/alloc/Blob.cpp b/src/alloc/Blob.cpp index 97932844..83e4121b 100644 --- a/src/alloc/Blob.cpp +++ b/src/alloc/Blob.cpp @@ -5,7 +5,7 @@ #include "Blob.hpp" #include "xo/reflect/Reflect.hpp" -#include "xo/alloc/IAlloc.hpp" +#include "xo/allocutil/IAlloc.hpp" namespace xo { using xo::reflect::Reflect; diff --git a/src/alloc/CMakeLists.txt b/src/alloc/CMakeLists.txt index 4a9d1db4..67e9759b 100644 --- a/src/alloc/CMakeLists.txt +++ b/src/alloc/CMakeLists.txt @@ -2,7 +2,6 @@ set(SELF_LIB xo_alloc) set(SELF_SRCS - IAlloc.cpp ArenaAlloc.cpp ListAlloc.cpp GC.cpp @@ -15,6 +14,7 @@ set(SELF_SRCS ) xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) +xo_headeronly_dependency(${SELF_LIB} xo_allocutil) # xo-unit used for time measurement xo_headeronly_dependency(${SELF_LIB} xo_unit) xo_dependency(${SELF_LIB} indentlog) diff --git a/src/alloc/Forwarding1.cpp b/src/alloc/Forwarding1.cpp index f94bf4f1..a42a3f57 100644 --- a/src/alloc/Forwarding1.cpp +++ b/src/alloc/Forwarding1.cpp @@ -13,7 +13,7 @@ namespace xo { using xo::reflect::TaggedPtr; namespace obj { - Forwarding1::Forwarding1(gp dest) + Forwarding1::Forwarding1(gp dest) : dest_{dest} {} @@ -26,18 +26,21 @@ namespace xo { void Forwarding1::display(std::ostream & os) const { - os << "self_tp().td()->short_name()) << ">"; + os << "self_tp().td()->short_name()) + << ">"; } - Object * - Forwarding1::_offset_destination(Object * src) const + IObject * + Forwarding1::_offset_destination(IObject * src) const { - intptr_t offset = src - static_cast(this); + intptr_t offset = src - static_cast(this); return dest_.ptr() + offset; } - Object * + IObject * Forwarding1::_destination() { return dest_.ptr(); } @@ -51,8 +54,10 @@ namespace xo { // LCOV_EXCL_STOP // LCOV_EXCL_START - Object * + IObject * Forwarding1::_shallow_copy(gc::IAlloc *) const { + /* forwarding objects are never copied */ + assert(false); return nullptr; } @@ -61,6 +66,8 @@ namespace xo { // LCOV_EXCL_START std::size_t Forwarding1::_forward_children(gc::IAlloc *) { + /* forwarding objects are never traced */ + assert(false); return 0; } diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 3e636cad..ca2506d9 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -27,7 +27,7 @@ namespace xo { return parent_->_is_forwarded(); } - Object * + IObject * MutationLogEntry::parent_destination() const { //const bool c_debug_flag = true; @@ -45,7 +45,7 @@ namespace xo { } MutationLogEntry - MutationLogEntry::update_parent_moved(Object * parent_to) const + MutationLogEntry::update_parent_moved(IObject * parent_to) const { std::byte * parent_from = reinterpret_cast(parent_); std::byte * lhs_from = reinterpret_cast(lhs_); @@ -55,7 +55,7 @@ namespace xo { std::byte * lhs_to = reinterpret_cast(parent_to) + offset; return MutationLogEntry(parent_to, - reinterpret_cast(lhs_to)); + reinterpret_cast(lhs_to)); } GC::GC(const Config & config) @@ -395,13 +395,13 @@ namespace xo { } void - GC::add_gc_root(Object ** addr) + GC::add_gc_root(IObject ** addr) { gc_root_v_.push_back(addr); } void - GC::remove_gc_root(Object ** addr) + GC::remove_gc_root(IObject ** addr) { /* Multithreaded GC not supported */ @@ -450,7 +450,9 @@ namespace xo { std::byte * GC::alloc_gc_copy(std::size_t z, const void * src) { - scope log(XO_DEBUG(config_.debug_flag_), xtag("z", z), xtag("+pad", IAlloc::alloc_padding(z))); + scope log(XO_DEBUG(config_.debug_flag_), + xtag("z", z), + xtag("+pad", IAlloc::alloc_padding(z))); generation_result src_gr = this->fromspace_generation_of(src); @@ -483,7 +485,8 @@ namespace xo { gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, z, src, retval, generation::nursery, generation::tenured); - this->gc_statistics_.total_promoted_ += IAlloc::with_padding(z); + this->gc_statistics_.total_promoted_ + += IAlloc::with_padding(z); } else { log && log("nursery"); @@ -509,7 +512,7 @@ namespace xo { } void - GC::assign_member(Object * parent, Object ** lhs, Object * rhs) + GC::assign_member(IObject * parent, IObject ** lhs, IObject * rhs) { ++gc_statistics_.n_mutation_; @@ -566,13 +569,13 @@ namespace xo { } bool - GC::check_owned(Object * src) const + GC::check_owned(IObject * src) const { return this->fromspace_contains(src); } bool - GC::check_move(Object * src) const + GC::check_move(IObject * src) const { return (this->runstate().full_move() || (this->tospace_generation_of(src) != gc::generation_result::tenured)); @@ -682,7 +685,9 @@ namespace xo { } void - GC::copy_object(Object ** pp_object, generation upto, ObjectStatistics * object_stats) + GC::copy_object(IObject ** pp_object, + generation upto, + ObjectStatistics * object_stats) { void * object_address = *pp_object; @@ -707,7 +712,7 @@ namespace xo { scope log(XO_DEBUG(config_.debug_flag_), xtag("roots", gc_root_v_.size())); - for (Object ** pp_root : gc_root_v_) { + for (IObject ** pp_root : gc_root_v_) { this->copy_object(pp_root, upto, &object_statistics_sae_[gen2int(upto)]); } @@ -778,7 +783,7 @@ namespace xo { // obsolete mutation -- no longer belongs to parent, discard } else { // note: child obtained (as it must be) by reading from parent's memory _now_. - Object * child_from = from_entry.child(); + IObject * child_from = from_entry.child(); if (child_from) { if (!child_from->_is_forwarded()) { @@ -813,7 +818,7 @@ namespace xo { // P->C, C moved to C' // Includes cases (a),(c) from above - Object * child_to = child_from->_destination(); + IObject * child_to = child_from->_destination(); from_entry.fixup_parent_child_moved(child_to); @@ -843,7 +848,7 @@ namespace xo { // follows that loc(P') = T // already have P'->C' when parent moved separately - Object * parent_to = from_entry.parent_destination(); + IObject * parent_to = from_entry.parent_destination(); log && log(xtag("parent_to", (void*)parent_to)); @@ -851,7 +856,7 @@ namespace xo { MutationLogEntry to_entry = from_entry.update_parent_moved(parent_to); - Object * child_to = to_entry.child(); // after moving + IObject * child_to = to_entry.child(); // after moving if (tospace_generation_of(child_to) == generation_result::nursery) { if (to_entry.is_dead()) { @@ -954,7 +959,7 @@ namespace xo { log && (i_from % 10000 == 0) && log(xtag("i_from", i_from)); if (from_entry.is_parent_forwarded()) { - Object * parent_to = from_entry.parent_destination(); + IObject * parent_to = from_entry.parent_destination(); log && log(xtag("parent_to", (void*)parent_to)); @@ -964,7 +969,7 @@ namespace xo { // note: child obtained (as it must be) by reading from prarent's memory _now_. // Since parent has moved, child has too - Object * child_to = to_entry.child(); // after moveing + IObject * child_to = to_entry.child(); // after moveing if (tospace_generation_of(parent_to) == generation_result::tenured) { diff --git a/src/alloc/IAlloc.cpp b/src/alloc/IAlloc.cpp deleted file mode 100644 index 9754fb47..00000000 --- a/src/alloc/IAlloc.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/* @file IAlloc.cpp - * - * author: Roland Conybeare, Aug 2025 - */ - -#include "IAlloc.hpp" -#include -#include - -namespace xo { - namespace gc { - - std::uint32_t - IAlloc::alloc_padding(std::size_t z) - { - /* word size for alignment */ - constexpr uint32_t c_bpw = sizeof(std::uintptr_t); - - /* round up to multiple of c_bpw, but map 0 -> 0 - * (table assuming c_bpw==8) - * - * z%c_bpw dz - * ------------ - * 0 0 - * 1 7 - * 2 6 - * .. .. - * 7 1 - */ - std::uint32_t dz = (c_bpw - (z % c_bpw)) % c_bpw; - z += dz; - - assert(z % c_bpw == 0ul); - - return dz; - } - - std::size_t - IAlloc::with_padding(std::size_t z) - { - return z + alloc_padding(z); - } - - void - IAlloc::assign_member(Object * /*parent*/, Object ** lhs, Object * rhs) - { - *lhs = rhs; - } - - bool - IAlloc::check_owned(Object * /*obj*/) const - { - return false; - } - - bool - IAlloc::check_move(Object * /*obj*/) const - { - return false; - } - - // LCOV_EXCL_START - std::byte * - IAlloc::alloc_gc_copy(std::size_t /*z*/, const void * /*src*/) - { - assert(false); - return nullptr; - } - // LCOV_EXCL_STOP - - } /*namespace gc*/ -} /*namespace xo*/ - -/* end IAlloc.cpp */ diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index dbb86ae2..82583a0a 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -23,8 +23,8 @@ namespace xo { gc::IAlloc * Object::mm = nullptr; - Object * - Object::_forward(Object * src, gc::IAlloc * gc) + IObject * + Object::_forward(IObject * src, gc::IAlloc * gc) { if (!src) return src; @@ -43,15 +43,15 @@ namespace xo { } } - Object * - Object::_deep_move(Object * from_src, gc::GC * gc, gc::ObjectStatistics * /*stats*/) + IObject * + Object::_deep_move(IObject * from_src, gc::GC * gc, gc::ObjectStatistics * /*stats*/) { using gc::generation; if (!from_src) return nullptr; - Object * retval = from_src->_destination(); + IObject * retval = from_src->_destination(); if (retval) return retval; @@ -124,7 +124,7 @@ namespace xo { std::array gray_lo_v = { gc->free_ptr(generation::nursery), gc->free_ptr(generation::tenured) }; - Object * to_src = Object::_shallow_move(from_src, gc); + IObject * to_src = Object::_shallow_move(from_src, gc); std::size_t fixup_work = 0; do { @@ -158,15 +158,15 @@ namespace xo { return to_src; } /*deep_move*/ - Object * - Object::_shallow_move(Object * src, gc::IAlloc * gc) + IObject * + Object::_shallow_move(IObject * src, gc::IAlloc * gc) { /* filter for source objects that are owned by GC. * Care required though -- during GC from/to spaces have been swapped already */ if (gc->check_owned(src)) { - Object * dest = src->_shallow_copy(gc); + IObject * dest = src->_shallow_copy(gc); if (dest != src) src->_forward_to(dest); @@ -178,7 +178,7 @@ namespace xo { } void - Object::_forward_to(Object * dest) + Object::_forward_to(IObject * dest) { char * mem = reinterpret_cast(this); From b32b9151dad7dda47b6fe13ae0d4309ce51ee016 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 1 Dec 2025 14:22:41 -0500 Subject: [PATCH 46/69] xo-alloc xo-ordinaltree: GC option work in progress --- include/xo/alloc/Object.hpp | 2 +- utest/Forwarding1.test.cpp | 4 ++-- utest/IAlloc.test.cpp | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 2468c8bc..2c11ea9e 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -64,7 +64,7 @@ namespace xo { * 3. return the location of the copy make in step 1. * * @p src. source object to be forwarded - * @p gc. garbage collector + * @p gc. allocator (poassibly garbage collector) */ static IObject * _forward(IObject * src, gc::IAlloc * gc); diff --git a/utest/Forwarding1.test.cpp b/utest/Forwarding1.test.cpp index 19d9bf1e..f0a81c14 100644 --- a/utest/Forwarding1.test.cpp +++ b/utest/Forwarding1.test.cpp @@ -23,7 +23,7 @@ namespace xo { gp member() const { return member_; } void assign_member(Object * x) { - Object::mm->assign_member(this, member_.ptr_address(), x); + Object::mm->assign_member(this, reinterpret_cast(member_.ptr_address()), x); } TaggedPtr self_tp() const final override { @@ -33,7 +33,7 @@ namespace xo { void display(std::ostream & os) const final override { os << data_; } virtual std::size_t _shallow_size() const final override { return sizeof(*this); } - virtual Object * _shallow_copy(gc::IAlloc * mm) const final override { return new (Cpof(mm, this)) DummyObject(*this); } + virtual IObject * _shallow_copy(gc::IAlloc * mm) const final override { return new (Cpof(mm, this)) DummyObject(*this); } virtual std::size_t _forward_children(gc::IAlloc * gc) final override { return _shallow_size(); } private: diff --git a/utest/IAlloc.test.cpp b/utest/IAlloc.test.cpp index b0214749..823791ab 100644 --- a/utest/IAlloc.test.cpp +++ b/utest/IAlloc.test.cpp @@ -3,7 +3,7 @@ * author: Roland Conybeare, Aug 2025 */ -#include "xo/alloc/IAlloc.hpp" +#include "xo/allocutil/IAlloc.hpp" #include namespace xo { From fd6bdd93c37e1dec26f139308baa336a68140bb6 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 1 Dec 2025 22:25:41 -0500 Subject: [PATCH 47/69] xo-ordinaltree: GC test [wip] --- include/xo/alloc/GC.hpp | 8 ++++++++ include/xo/alloc/Object.hpp | 13 ------------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index aeb918c9..092f2f36 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -144,9 +144,17 @@ namespace xo { using GcCopyCallbackSet = xo::fn::UpCallbackSet; using nanos = decltype(xo::qty::qty::nanosecond); + /** rebind is for typed allocators. since IAlloc is untyped, + * we want degenerate version + **/ + template + struct rebind { using other = GC; }; + public: /** create new GC instance with configuration @p config **/ explicit GC(const Config & config); + /** noncopyable **/ + GC(const GC & other) = delete; virtual ~GC(); /** create GC allocator. diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 2c11ea9e..9681afab 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -141,19 +141,6 @@ namespace xo { std::ostream & operator<< (std::ostream & os, gp x); - /** @class Cpof - * @brief argument to operator new used for garbage collector evacuation phase - * - * Tag overloaded operator new to activate allocation policy based on location - * in memory of source object. - **/ - class Cpof { - public: - explicit Cpof(gc::IAlloc * mm, const Object * src) : mm_{mm}, src_{src} {} - - gc::IAlloc * mm_ = nullptr; - const void * src_ = nullptr; - }; } /*namespace xo*/ void * operator new (std::size_t z, const xo::Cpof & copy); From 77f84cabbbc94cb362c68e65f22446bec8792148 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 2 Dec 2025 10:37:07 -0500 Subject: [PATCH 48/69] xo-alloc / xo-ordinaltree: work on dual-alloc-policy trees --- include/xo/alloc/GC.hpp | 3 ++- src/alloc/GC.cpp | 6 ++++++ src/alloc/Object.cpp | 3 ++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 092f2f36..be2ff071 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -318,7 +318,8 @@ namespace xo { * @param rhs. new target for @p *lhs **/ virtual void assign_member(IObject * parent, IObject ** lhs, IObject* rhs) final override; - + /** evacuate @p *lhs and replace with forwarding pointer **/ + virtual void forward_inplace(IObject ** lhs) final override; /** during GC check for source objects owned by GC. * See Object::_shallow_move. **/ diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index ca2506d9..40234bf3 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -568,6 +568,12 @@ namespace xo { } } + void + GC::forward_inplace(IObject ** lhs) + { + Object::_forward_inplace(lhs, this); + } + bool GC::check_owned(IObject * src) const { diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index 82583a0a..75abec14 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -24,7 +24,8 @@ namespace xo { Object::mm = nullptr; IObject * - Object::_forward(IObject * src, gc::IAlloc * gc) + Object::_forward(IObject * src, + gc::IAlloc * gc) { if (!src) return src; From 764e98e12e10c644b7347dd82d4d4e7f35eb1a0b Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 2 Dec 2025 17:07:19 -0500 Subject: [PATCH 49/69] xo-alloc: UT for allocator interation + misc improvements --- include/xo/alloc/ArenaAllocT.hpp | 74 ------------- utest/ArenaAllocT.test.cpp | 67 ------------ utest/CMakeLists.txt | 1 - utest/Forwarding1.test.cpp | 10 +- utest/GC.test.cpp | 179 ++++++++++++++++++++++++++++++- utest/IAlloc.test.cpp | 118 ++++++++++++++++++-- 6 files changed, 295 insertions(+), 154 deletions(-) delete mode 100644 include/xo/alloc/ArenaAllocT.hpp delete mode 100644 utest/ArenaAllocT.test.cpp diff --git a/include/xo/alloc/ArenaAllocT.hpp b/include/xo/alloc/ArenaAllocT.hpp deleted file mode 100644 index 6c8a9b41..00000000 --- a/include/xo/alloc/ArenaAllocT.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/** @file Allocator.hpp - * - * @author Roland Conybeare, Nov 2025 - **/ - -#pragma once - -#include "xo/alloc/ArenaAlloc.hpp" - -namespace xo { - namespace gc { - /** @class allocator - * @brief c++ allocator with allocator traits - * - * Can use ArenaAllocT with std::map etc. - **/ - template - class ArenaAllocT { - public: - using value_type = T; - /** copy assignment: leave lhs allocator in place **/ - using propagate_on_container_copy_assignment = std::false_type; - /** move assignment: adopt rhs allocator - * (Forced: cannot mix allocations from different allocators - * within a container) - **/ - using propagate_on_container_move_assignment = std::true_type; - /** swap: also swap allocators - * (Forced: cannot mix allocations from different allocators - * within a containers) - **/ - using propagate_on_container_swap = std::true_type; - /** An ArenaAlloc instance is unique owner of its own memory: - * no other instance can dealloc - **/ - using is_always_equal = std::false_type; - - public: - explicit ArenaAllocT(ArenaAlloc * mm) : mm_{mm} {} - ArenaAllocT(const ArenaAllocT & other) = default; - - /** rebind ctor. Allows container to use supplied allocator - * for multiple types - **/ - template - ArenaAllocT(const ArenaAllocT & other) noexcept : mm_{other.mm_} {} - - T * allocate(size_t n) { - void * mem = mm_->alloc(n * sizeof(T)); - - return reinterpret_cast(mem); - } - - void deallocate(T * p, size_t n) noexcept { - assert(mm_->contains(p)); - assert(n == 0 || mm_->contains(p + n - 1)); - - //arena_->deallocate(p, n * sizeof(T)); - } - - bool operator==(const ArenaAllocT & other) const { - return mm_ == other.mm_; - } - - bool operator!=(const ArenaAllocT & other) const { - return mm_ != other.mm_; - } - - ArenaAlloc * mm_ = nullptr; - }; - } /*namespace gc*/ -} /*namespace xo*/ - -/* end Allocator.hpp */ diff --git a/utest/ArenaAllocT.test.cpp b/utest/ArenaAllocT.test.cpp deleted file mode 100644 index a9257173..00000000 --- a/utest/ArenaAllocT.test.cpp +++ /dev/null @@ -1,67 +0,0 @@ -/** @file ArenaAllocT.test.cpp - * - * @author Roland Conybeare, Nov 2025 - **/ - -#include "xo/alloc/ArenaAllocT.hpp" -#include -#include - -namespace xo { - using xo::gc::ArenaAllocT; - using xo::gc::ArenaAlloc; - - namespace ut { - - namespace { - struct testcase_ArenaAllocT { - std::size_t arena_z_; - std::vector> kv_pairs_; - }; - - std::vector - s_testcase_v = { - { 4096, {} }, - { 4096, {{"a", "apple"}} }, - { 4096, {{"a", "apple"}, {"b", "banana"}, {"c", "carrot"}} }, - { 4096, {{"a", "apple"}, {"b", "banana"}, {"c", "carrot"}, {"e", "eggplant"}} }, - }; - } - - TEST_CASE("ArenaAllocT", "[alloc][traits]") - { - for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { - const testcase_ArenaAllocT & tc = s_testcase_v[i_tc]; - - constexpr bool c_debug_flag = false; - - auto arena = ArenaAlloc::make("arena", tc.arena_z_, c_debug_flag); - auto alloc = ArenaAllocT>(arena.get()); - - using TestMapType = std::map, - ArenaAllocT>>; - - TestMapType test_map(alloc); - - size_t n = 0; - for (const auto & kv_ix : tc.kv_pairs_) { - test_map[kv_ix.first] = kv_ix.second; - ++n; - - REQUIRE(test_map.size() == n); - - for (const auto & map_ix : test_map) { - // verify alloc was used for both Key + Value. - REQUIRE(arena->contains(&map_ix.first)); - REQUIRE(arena->contains(&map_ix.second)); - } - } - - } - } - } -} /*namespace xo*/ - -/* end ArenaAllocT.test.cpp */ diff --git a/utest/CMakeLists.txt b/utest/CMakeLists.txt index 6c644f51..366cf664 100644 --- a/utest/CMakeLists.txt +++ b/utest/CMakeLists.txt @@ -7,7 +7,6 @@ set(UTEST_SRCS alloc_utest_main.cpp IAlloc.test.cpp ArenaAlloc.test.cpp - ArenaAllocT.test.cpp ListAlloc.test.cpp GC.test.cpp GcStatistics.test.cpp diff --git a/utest/Forwarding1.test.cpp b/utest/Forwarding1.test.cpp index f0a81c14..64c9f3f8 100644 --- a/utest/Forwarding1.test.cpp +++ b/utest/Forwarding1.test.cpp @@ -7,6 +7,7 @@ #include "ArenaAlloc.hpp" #include "xo/reflect/Reflect.hpp" #include +#include #include namespace xo { @@ -58,7 +59,14 @@ namespace xo { std::stringstream ss; ss << fwd; - REQUIRE(ss.str() == ""); + // forwarding printer looks like + // "" + // + + std::regex pattern(R"()"); + REQUIRE(std::regex_match(ss.str(), pattern)); + + //REQUIRE(ss.str() == ""); tag_config::tag_color_enabled = saved; } diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index 9e6c17e3..c5245243 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -4,12 +4,16 @@ */ #include "xo/alloc/GC.hpp" +#include "xo/allocutil/gc_allocator_traits.hpp" #include namespace xo { + using xo::gc::IAlloc; using xo::gc::GC; + using xo::gc::gc_allocator_traits; using xo::gc::generation; using xo::gc::Config; + using xo::reflect::TaggedPtr; namespace ut { @@ -80,13 +84,186 @@ namespace xo { REQUIRE(gc->gc_in_progress() == false); REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 1); - } catch (std::exception &ex) { + } catch (std::exception & ex) { std::cerr << "caught exception: " << ex.what() << std::endl; REQUIRE(false); } } } + /** gc-enabled allocator **/ + namespace { + /** Setup test with custom allocator + * + **/ + template + struct TestClass : public GcObjectInterface { + TestClass() = default; + explicit TestClass(const Nested & member1) : member1_{member1} {} + + // using allocator_type = Allocator; + // using allocator_traits = xo::gc::gc_allocator_traits; + + /** stage1 - just allocates some memory using allocator **/ + template + static TestClass * make_0(Allocator & alloc) { + TestClass * mem = alloc.allocate(sizeof(TestClass)); + + /* but ctor will not have run, so ub to visit object */ + + return mem; + } + + /** stage2 - use allocator_traits construct **/ + template + static TestClass * make_1(Allocator & alloc) { + using traits = gc_allocator_traits; + + TestClass * mem = traits::allocate(alloc, 1); + + /* ctor will not have run here either */ + return mem; + } + + /** stage3 - invoke construct **/ + template + static TestClass * make_2(Allocator & alloc) { + using traits = gc_allocator_traits; + + TestClass * obj = traits::allocate(alloc, 1); + try { + // placement new + traits::construct(alloc, obj); + + return obj; + } catch(...) { + traits::deallocate(alloc, obj, 1); + throw; + } + } + + /** stage4 - init nested type **/ + template + static TestClass * make_3(Allocator & alloc) { + using traits = gc_allocator_traits; + + TestClass * obj = traits::allocate(alloc, 1); + try { + Nested nested; + + // placemenet new + traits::construct(alloc, obj); + + return obj; + } catch(...) { + traits::deallocate(alloc, obj, 1); + throw; + } + } + + // ----- inherited from Object ----- + + virtual TaggedPtr self_tp() const final override { + assert(false); return TaggedPtr::universal_null(); + } + virtual void display(std::ostream & os) const final override { + os << ""; + } + virtual std::size_t _shallow_size() const final override { + assert(false); return sizeof(*this); + } + virtual IObject * _shallow_copy(IAlloc * gc) const final override { + assert(false); return nullptr; + } + virtual std::size_t _forward_children(IAlloc * gc) final override { + assert(false); return _shallow_size(); + } + + Nested member1_; + }; + + struct MemberType { + MemberType() : ctor_ran_{true} {} + explicit MemberType(const std::vector> & mem2) : member2_{mem2} {} + + std::vector> member2_; + std::size_t ctor_ran_ = false; + }; + } + + TEST_CASE("gc_allocator_traits", "[alloc][gc]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + try { + const testcase_gc & tc = s_testcase_v[i_tc]; + + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_, + .incr_gc_threshold_ = tc.incr_gc_threshold_, + .full_gc_threshold_ = tc.full_gc_threshold_, + }); + + REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); + + using ex_allocator = xo::gc::allocator; + using MyObjectInterface = gc_allocator_traits::template object_interface; + using NestedElementAllocator = xo::gc::allocator>; + using NestedType = MemberType; + using MyType = TestClass; + using MyAllocator = xo::gc::allocator; + + MyAllocator alloc(gc.get()); + + { + /* verify that MyType is constructible */ + MyType obj0; + + REQUIRE(obj0.member1_.ctor_ran_ == true); + } + + { + MyType * mem0 = MyType::make_0(alloc); + + REQUIRE(mem0 != nullptr); + REQUIRE(mem0->member1_.ctor_ran_ == false); + } + + { + MyType * mem1 = MyType::make_1(alloc); + + REQUIRE(mem1 != nullptr); + REQUIRE(mem1->member1_.ctor_ran_ == false); + } + + { + MyType * mem2 = MyType::make_2(alloc); + + REQUIRE(mem2 != nullptr); + REQUIRE(mem2->member1_.ctor_ran_ == true); + } + + { + MyType * mem3 = MyType::make_3(alloc); + + REQUIRE(mem3 != nullptr); + REQUIRE(mem3->member1_.ctor_ran_ == true); + } + + gp ptr; + { + REQUIRE(ptr.is_null()); + //ptr = MyType::make_0(); + } + } catch (std::exception & ex) { + std::cerr << "caught exception: " << ex.what() << std::endl; + REQUIRE(false); + } + } + + } + } /*namespace ut*/ } /*namespace xo*/ diff --git a/utest/IAlloc.test.cpp b/utest/IAlloc.test.cpp index 823791ab..7c1fdc56 100644 --- a/utest/IAlloc.test.cpp +++ b/utest/IAlloc.test.cpp @@ -3,26 +3,124 @@ * author: Roland Conybeare, Aug 2025 */ -#include "xo/allocutil/IAlloc.hpp" +//#include "xo/allocutil/IAlloc.hpp" +#include "xo/alloc/ArenaAlloc.hpp" +#include "xo/indentlog/print/tag.hpp" #include namespace xo { using xo::gc::IAlloc; + using xo::gc::ArenaAlloc; namespace ut { TEST_CASE("ialloc", "[alloc]") { + static_assert((sizeof(std::uintptr_t) == 8) && "possibly fine if this fails, but would want to know"); + REQUIRE(IAlloc::alloc_padding(0) == 0); - REQUIRE(IAlloc::alloc_padding(1) == 7); - REQUIRE(IAlloc::alloc_padding(2) == 6); - REQUIRE(IAlloc::alloc_padding(3) == 5); - REQUIRE(IAlloc::alloc_padding(4) == 4); - REQUIRE(IAlloc::alloc_padding(5) == 3); - REQUIRE(IAlloc::alloc_padding(6) == 2); - REQUIRE(IAlloc::alloc_padding(7) == 1); - REQUIRE(IAlloc::alloc_padding(8) == 0); - REQUIRE(IAlloc::alloc_padding(9) == 7); + + for (std::size_t i = 1; i < sizeof(std::uintptr_t); ++i) { + REQUIRE(IAlloc::alloc_padding(i) + i == IAlloc::c_alloc_alignment); + } + + REQUIRE(IAlloc::alloc_padding(IAlloc::c_alloc_alignment) == 0); + + for (std::size_t i = 1; i < sizeof(std::uintptr_t); ++i) { + REQUIRE(IAlloc::alloc_padding(IAlloc::c_alloc_alignment + i) + i == IAlloc::c_alloc_alignment); + } } + + /* although xo::gc::allocator is intended for + * IAlloc derivatives (so T is ArenaAlloc | GC), + * + * it only relies on allocate() and deallocate() methods + */ + + namespace { + struct TestCase { + explicit TestCase(size_t arena_z, size_t n, size_t n2) : arena_z_{arena_z}, n_{n}, n2_{n2} {} + + size_t arena_z_ = 0; + size_t n_ = 0; + size_t n2_ = 0; + }; + + std::vector s_testcase_v = { TestCase{1024*1024, 9, 13} }; + } + + TEST_CASE("gc.allocator", "[alloc]") + { + using xo::gc::allocator; + + constexpr bool c_debug_flag = false; + + for (size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + INFO(xtag("i_tc", i_tc)); + + const TestCase & tc = s_testcase_v[i_tc]; + + up mm1 = ArenaAlloc::make("arena1", + tc.arena_z_, + c_debug_flag); + up mm2 = ArenaAlloc::make("arena2", + tc.arena_z_, + c_debug_flag); + + REQUIRE(mm1.get()); + REQUIRE(mm1->allocated() == 0); + + allocator alloc1(mm1.get()); + allocator alloc1a(mm1.get()); + + REQUIRE(mm2.get()); + REQUIRE(mm2->allocated() == 0); + + allocator alloc2(mm2.get()); + + SECTION("IAlloc identity determines allocator equality") { + REQUIRE(alloc1 == alloc1a); + REQUIRE(alloc1 != alloc2); + } + + int * p1 = nullptr; + size_t z1 = 0; + + SECTION("alloc space for ints") { + p1 = alloc1.allocate(tc.n_); + + REQUIRE(p1 != nullptr); + + // note: allowing for alignment + REQUIRE(mm1->allocated() >= sizeof(int32_t) * tc.n_); + REQUIRE(mm1->allocated() < sizeof(int32_t) * tc.n_ + IAlloc::c_alloc_alignment); + z1 = mm1->allocated(); + + // deallocate exists.. + alloc1.deallocate(p1, tc.n_); + + // ..but is a no-op + REQUIRE(mm1->allocated() == z1); + } + + int * p2 = nullptr; + + SECTION("allocator independence") { + REQUIRE(mm2->allocated() == 0); + + p2 = alloc2.allocate(tc.n2_); + + REQUIRE(p2 != nullptr); + REQUIRE(p1 != p2); + + REQUIRE(mm2->allocated() >= sizeof(int32_t) * tc.n2_); + REQUIRE(mm2->allocated() < sizeof(int32_t) * tc.n2_ + IAlloc::c_alloc_alignment); + + // mm1 unaffected by mm2 allocation + REQUIRE(mm1->allocated() == z1); + } + } + } + } /*namespace ut*/ } /*namespace xo*/ From 43d79d78096284051edd076526d46e7916b28c91 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 2 Dec 2025 21:47:34 -0500 Subject: [PATCH 50/69] Gc.test.cpp expansion. Not working yet --- utest/GC.test.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index c5245243..3912e8e4 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -182,11 +182,23 @@ namespace xo { Nested member1_; }; + template struct MemberType { - MemberType() : ctor_ran_{true} {} - explicit MemberType(const std::vector> & mem2) : member2_{mem2} {} + public: + using allocator_type = Allocator; + using vector_allocator_type = typename std::allocator_traits::template rebind_alloc>; + using vector_type = std::vector, vector_allocator_type>; - std::vector> member2_; + public: + MemberType() : ctor_ran_{true} {} + explicit MemberType(const Allocator & alloc) + : member2_{vector_allocator_type(alloc)}, ctor_ran_{true} {} + + explicit MemberType(const vector_type & mem2) : member2_{mem2}, ctor_ran_{true} {} + MemberType(const vector_type & mem2, const Allocator & alloc) + : member2_{mem2, vector_allocator_type(alloc)}, ctor_ran_{true} {} + + vector_type member2_; std::size_t ctor_ran_ = false; }; } From 146b730447e9b9fadbb86a699b20be0989774505 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 3 Dec 2025 00:14:09 -0500 Subject: [PATCH 51/69] fix unit tests to build on osx / clang16 --- utest/generation.test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/utest/generation.test.cpp b/utest/generation.test.cpp index 415d6258..edbebffa 100644 --- a/utest/generation.test.cpp +++ b/utest/generation.test.cpp @@ -4,6 +4,7 @@ */ #include "xo/alloc/generation.hpp" +#include #include #include From 67cf4cc625e573cc7e85cb4d020a510babb54dc8 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 3 Dec 2025 00:14:38 -0500 Subject: [PATCH 52/69] xo-alloc: utest: revert allocator changes in nested type --- utest/GC.test.cpp | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index 3912e8e4..398336a8 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -94,13 +94,13 @@ namespace xo { /** gc-enabled allocator **/ namespace { /** Setup test with custom allocator - * + * **/ template struct TestClass : public GcObjectInterface { TestClass() = default; explicit TestClass(const Nested & member1) : member1_{member1} {} - + // using allocator_type = Allocator; // using allocator_traits = xo::gc::gc_allocator_traits; @@ -182,21 +182,22 @@ namespace xo { Nested member1_; }; - template + //template struct MemberType { public: - using allocator_type = Allocator; - using vector_allocator_type = typename std::allocator_traits::template rebind_alloc>; - using vector_type = std::vector, vector_allocator_type>; + //using allocator_type = Allocator; + //using vector_allocator_type = typename std::allocator_traits::template rebind_alloc>; + using vector_type = std::vector>; + //using vector_type = std::vector, vector_allocator_type>; public: MemberType() : ctor_ran_{true} {} - explicit MemberType(const Allocator & alloc) - : member2_{vector_allocator_type(alloc)}, ctor_ran_{true} {} + //explicit MemberType(const Allocator & alloc) + //: member2_{vector_allocator_type(alloc)}, ctor_ran_{true} {} explicit MemberType(const vector_type & mem2) : member2_{mem2}, ctor_ran_{true} {} - MemberType(const vector_type & mem2, const Allocator & alloc) - : member2_{mem2, vector_allocator_type(alloc)}, ctor_ran_{true} {} + //MemberType(const vector_type & mem2, const Allocator & alloc) + //: member2_{mem2, vector_allocator_type(alloc)}, ctor_ran_{true} {} vector_type member2_; std::size_t ctor_ran_ = false; @@ -223,6 +224,7 @@ namespace xo { using MyObjectInterface = gc_allocator_traits::template object_interface; using NestedElementAllocator = xo::gc::allocator>; using NestedType = MemberType; + //using NestedType = MemberType; using MyType = TestClass; using MyAllocator = xo::gc::allocator; @@ -273,7 +275,7 @@ namespace xo { REQUIRE(false); } } - + } } /*namespace ut*/ From 1a8771dc5d00f0453172d5a342b268e1500deb4a Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 3 Dec 2025 10:22:52 -0500 Subject: [PATCH 53/69] xo-alloc: + utest vector w/ custom allocator --- utest/GC.test.cpp | 88 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index 398336a8..03b5c50a 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -204,6 +204,94 @@ namespace xo { }; } + TEST_CASE("vector_custom_allocator", "[alloc][vector]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + try { + const testcase_gc & tc = s_testcase_v[i_tc]; + + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_, + .incr_gc_threshold_ = tc.incr_gc_threshold_, + .full_gc_threshold_ = tc.full_gc_threshold_, + }); + + REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); + + using NestedElementAllocator = xo::gc::allocator>; + + NestedElementAllocator alloc(gc.get()); + + /** testv will use GC to allocaate element storage + * Attempt to gc will fail, because memory iteration + * won't work. + **/ + std::vector, + NestedElementAllocator> testv(alloc); + + testv.push_back(gp()); + +#ifdef NOPE + using ex_allocator = xo::gc::allocator; + using MyObjectInterface = gc_allocator_traits::template object_interface; + using NestedType = MemberType; + //using NestedType = MemberType; + using MyType = TestClass; + using MyAllocator = xo::gc::allocator; + + MyAllocator alloc(gc.get()); + + { + /* verify that MyType is constructible */ + MyType obj0; + + REQUIRE(obj0.member1_.ctor_ran_ == true); + } + + { + MyType * mem0 = MyType::make_0(alloc); + + REQUIRE(mem0 != nullptr); + REQUIRE(mem0->member1_.ctor_ran_ == false); + } + + { + MyType * mem1 = MyType::make_1(alloc); + + REQUIRE(mem1 != nullptr); + REQUIRE(mem1->member1_.ctor_ran_ == false); + } + + { + MyType * mem2 = MyType::make_2(alloc); + + REQUIRE(mem2 != nullptr); + REQUIRE(mem2->member1_.ctor_ran_ == true); + } + + { + MyType * mem3 = MyType::make_3(alloc); + + REQUIRE(mem3 != nullptr); + REQUIRE(mem3->member1_.ctor_ran_ == true); + } + + gp ptr; + { + REQUIRE(ptr.is_null()); + //ptr = MyType::make_0(); + } +#endif + } catch (std::exception & ex) { + std::cerr << "caught exception: " << ex.what() << std::endl; + REQUIRE(false); + } + } + + } + TEST_CASE("gc_allocator_traits", "[alloc][gc]") { for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { From 3c3709ba15a98e34a81d3d797436465bbb0a798b Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 3 Dec 2025 15:36:59 -0500 Subject: [PATCH 54/69] xo-alloc / xo-ordinaltree: + concepts + allocator-aware --- include/xo/alloc/Object.hpp | 5 ++++- utest/GC.test.cpp | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 9681afab..d2e742a0 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -38,7 +38,7 @@ namespace xo { return dynamic_cast(x.ptr()); } - virtual ~Object() = default; + virtual ~Object() noexcept = default; /** memory allocator for objects. Likely this will be a GC instance, * but simple arena also supported. @@ -129,6 +129,9 @@ namespace xo { virtual std::size_t _forward_children(gc::IAlloc * gc) override = 0; }; + static_assert(std::is_destructible_v, "Object must be destructible"); + static_assert(std::is_nothrow_destructible_v, "Object must be noexcept destructible"); + template void Object::assign_member(gp parent, gp * lhs, gp rhs) diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index 03b5c50a..90e22dfa 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -200,7 +200,7 @@ namespace xo { //: member2_{mem2, vector_allocator_type(alloc)}, ctor_ran_{true} {} vector_type member2_; - std::size_t ctor_ran_ = false; + bool ctor_ran_ = false; }; } From e4a4e0dc872bee27d6833ad299533531dc2b3fb3 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 4 Dec 2025 17:28:59 -0500 Subject: [PATCH 55/69] xo-alloc: + static_asserts in GC root helpers --- include/xo/alloc/GC.hpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index be2ff071..214027b7 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -249,10 +249,18 @@ namespace xo { void remove_gc_root(IObject ** addr); /** convenience wrapper **/ + template - void add_gc_root_dwim(gp * p) { this->add_gc_root(reinterpret_cast(p->ptr_address())); } + void add_gc_root_dwim(gp * p) { + static_assert(std::is_convertible_v); + this->add_gc_root(reinterpret_cast(p->ptr_address())); + } + template - void remove_gc_root_dwim(gp * p) { this->remove_gc_root(reinterpret_cast(p->ptr_address())); } + void remove_gc_root_dwim(gp * p) { + static_assert(std::is_convertible_v); + this->remove_gc_root(reinterpret_cast(p->ptr_address())); + } /** may optionally use this to observe GC copy phase. * Will be invoked once _per surviving object_, so not cheap. From 676a9d0d62e7fde60aa0af22ce9e054be229eee9 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 4 Dec 2025 17:29:27 -0500 Subject: [PATCH 56/69] xo-alloc: + comments on design --- include/xo/alloc/Object.hpp | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index d2e742a0..c1217418 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -20,14 +20,18 @@ namespace xo { /** Root class for all xo GC-collectable objects. * - * Design note: - * - * relying on inheritance means we insist that GC traits - * for a type appear directly in that type's vtable, and at specific locations. - * This implies one level of indirection when GC traverses an instance. - * - * Would be feasible to relax the must-inherit-from-Object constraint - * by having GC use its own wrapper, at cost of an extra layer of indirection + * Design notes: + * 1. xo::IObject -> xo-allocutil header-only library + * xo::Object -> xo-alloc ordinary library + * 2. relying on inheritance means we insist that GC traits + * for a type appear directly in that type's vtable, and at specific locations. + * This implies one level of indirection when GC traverses an instance. + * 3. Could adapt a gc-aware XO library (such as xo-ordinaltree) + * to a non-xo garbage collector. + * Would still need to use xo::IObject and xo::gc::gc_allocator_traits, + * but not necessarily xo::Object + * 4. Would be feasible to relax the must-inherit-from-Object constraint + * by having GC use its own wrapper, at cost of an extra layer of indirection **/ class Object : public IObject { public: @@ -49,7 +53,9 @@ namespace xo { /** assign value @p rhs to member @p *lhs of @p parent. * if assignment creates a cross-generational or cross-checkpoint pointer, - * add mutation log entry + * add mutation log entry. + * + * DEPRECATED. prefer IObject::_gc_assign_member, for explicit alloc **/ template static void assign_member(gp parent, gp * lhs, gp rhs); From bd0b1b1f719706974abce1a9706b11625df9a4d0 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 4 Dec 2025 21:31:55 -0500 Subject: [PATCH 57/69] xo-alloc/xo-ordinaltree: refactor rbtree Node alloc progress toward careful gc-aware assignment --- include/xo/alloc/Object.hpp | 2 +- src/alloc/Object.cpp | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index c1217418..742fba4c 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -118,7 +118,7 @@ namespace xo { /** tagged pointer with runtime type information **/ - virtual TaggedPtr self_tp() const = 0; + virtual TaggedPtr self_tp() const; /** print on stream @p os **/ virtual void display(std::ostream & os) const = 0; diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index 75abec14..ab23b76a 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -20,9 +20,18 @@ operator new (std::size_t z, const xo::Cpof & cpof) } namespace xo { + using xo::reflect::TaggedPtr; + gc::IAlloc * Object::mm = nullptr; + TaggedPtr + Object::self_tp() const + { + assert(false); + return TaggedPtr::universal_null(); + } + IObject * Object::_forward(IObject * src, gc::IAlloc * gc) From e8d755252a42b291ceb27509719901e09f9c503a Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 4 Dec 2025 21:44:22 -0500 Subject: [PATCH 58/69] xo-alloc: provide default Object::display() method So we can remove from FallbackObjectInterface and IObject --- src/alloc/Object.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index ab23b76a..b0be501b 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -32,6 +32,12 @@ namespace xo { return TaggedPtr::universal_null(); } + void + Object::display(std::ostream & os) const + { + os << ""; + } + IObject * Object::_forward(IObject * src, gc::IAlloc * gc) From 40281c4e0ab5e95a8180cecaeb1a2880ab7b7060 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 4 Dec 2025 23:38:56 -0500 Subject: [PATCH 59/69] xo-ordinaltree: rbtree ops satisfy gc write barriers --- include/xo/alloc/Object.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 742fba4c..5f31356f 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -121,7 +121,7 @@ namespace xo { virtual TaggedPtr self_tp() const; /** print on stream @p os **/ - virtual void display(std::ostream & os) const = 0; + virtual void display(std::ostream & os) const; // Inherited from IObject.. From b6ccde3ddcf9858a1981731fec14c04de7a5acd6 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 5 Dec 2025 10:00:36 -0500 Subject: [PATCH 60/69] xo-alloc: check_write_barrier to verify mutation log --- include/xo/alloc/ArenaAlloc.hpp | 1 - include/xo/alloc/GC.hpp | 16 ++++- include/xo/alloc/generation.hpp | 7 +++ src/alloc/GC.cpp | 104 +++++++++++++++++++++++++++++++- src/alloc/generation.cpp | 9 +++ 5 files changed, 133 insertions(+), 4 deletions(-) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 1f48121b..df4d2a01 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -177,7 +177,6 @@ namespace xo { virtual std::byte * alloc(std::size_t z) final override; virtual bool check_owned(IObject * src) const final override; - ArenaAlloc & operator=(const ArenaAlloc &) = delete; ArenaAlloc & operator=(ArenaAlloc &&) = delete; diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 214027b7..e5dd6781 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -337,6 +337,12 @@ namespace xo { * - incr GC -> if not tenured **/ virtual bool check_move(IObject * src) const final override; + /** if src is cross-generational (or cross-checkpoint), verify that it + * is recorded in mutation log, + * given an object @p parent that contains object pointer @p lhs + **/ + virtual bool check_write_barrier(IObject * parent, IObject ** lhs, bool may_throw) const final; + virtual std::byte * alloc(std::size_t z) final override; virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; @@ -447,7 +453,15 @@ namespace xo { std::vector gc_root_v_; /** log cross-generational and cross-checkpoint mutations. - * These need to be adjusted on next incremental collection + * These need to be adjusted on next incremental collection. + * + * mutation_log_[tospace] accumulates {xgen,xckp} pointers until + * the next GC. + * + * See GC aux functions + * @ref incremental_gc_forward_mlog + * @ref full_gc_forward_mlog + * **/ std::array, role2int(role::N)> mutation_log_; /** temporary mutation log (for deferred entries) **/ diff --git a/include/xo/alloc/generation.hpp b/include/xo/alloc/generation.hpp index 0acd943a..9f122044 100644 --- a/include/xo/alloc/generation.hpp +++ b/include/xo/alloc/generation.hpp @@ -41,6 +41,13 @@ namespace xo { return generation::tenured; } + const char * genresult2str(generation_result x); + + inline std::ostream & operator<<(std::ostream & os, generation_result x) { + os << genresult2str(x); + return os; + } + } /*namespace gc*/ } /*namespace xo*/ diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 40234bf3..5cfdd9fc 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -587,6 +587,104 @@ namespace xo { || (this->tospace_generation_of(src) != gc::generation_result::tenured)); } + bool + GC::check_write_barrier(IObject * parent, + IObject ** lhs, + bool may_throw_flag) const + { + if (!this->contains(parent)) { + if (may_throw_flag) { + throw std::runtime_error(tostr("GC::check_write_barrier", + ": expected parent object P in GC to-space", + xtag("P", parent))); + } + return false; + } + + std::size_t parent_z = parent->_shallow_size(); + + std::byte * parent_addr = reinterpret_cast(parent); + std::byte * lhs_addr = reinterpret_cast(lhs); + + if ((lhs_addr < parent_addr) || (parent_addr + parent_z < lhs_addr)) { + if (may_throw_flag) { + throw std::runtime_error + (tostr("GC::check_write_barrier", + ": expected lhs address L within address extent z of parent P", + xtag("P", parent), xtag("z", parent_z), + xtag("P+z", parent_addr + parent_z), + xtag("L", lhs))); + } + return false; + } + + IObject * rhs = *lhs; + + auto parent_gen = tospace_generation_of(parent); + auto rhs_gen = tospace_generation_of(rhs); + + switch(parent_gen) { + case generation_result::nursery: + if (is_before_checkpoint(parent)) { + switch(rhs_gen) { + case generation_result::nursery: + if (is_before_checkpoint(rhs)) { + /* no mlog entry needed */ + return true; + } else { + /* need to check mlog */ + ; + } + break; + case generation_result::tenured: + /* no mlog entry needed */ + return true; + case generation_result::not_found: + /* possible non-gc rhs */ + return true; + } + } else { + /* no mlog entry needed */ + return true; + } + break; + case generation_result::tenured: + switch(rhs_gen) { + case generation_result::nursery: + /* need to check mlog */ + break; + case generation_result::tenured: + /* no mlog entry needed */ + return true; + case generation_result::not_found: + /* possible non-gc rhs */ + return true; + } + case generation_result::not_found: + /* already excluded -> impossible */ + assert(false && "already verified parent owned by GC"); + } + + /* control here -> expect mutation log entry. + * search mutation log + verify such entry exists + */ + for (MutationLogEntry & mlog : *(mutation_log_[role2int(role::to_space)])) { + if ((mlog.parent() == parent) && (mlog.lhs() == lhs)) { + return true; + } + mlog.lhs(); + } + + if (may_throw_flag) { + throw std::runtime_error + (tostr("GC::check_write_barrier", + ": expected mlog entry for xgen pointer L->C within parent P", + xtag("P", parent), xtag("L", lhs), xtag("C", rhs), + xtag("gen(P)", parent_gen), xtag("gen(C)", rhs_gen))); + } + return false; + } + void GC::swap_nursery() { @@ -1126,9 +1224,11 @@ namespace xo { scope log(XO_DEBUG(config_.debug_flag_)); if (upto == generation::tenured) { - this->full_gc_forward_mlog(&object_statistics_sae_[gen2int(generation::tenured)]); + this->full_gc_forward_mlog + (&object_statistics_sae_[gen2int(generation::tenured)]); } else { - this->incremental_gc_forward_mlog(&object_statistics_sae_[gen2int(generation::nursery)]); + this->incremental_gc_forward_mlog + (&object_statistics_sae_[gen2int(generation::nursery)]); } } diff --git a/src/alloc/generation.cpp b/src/alloc/generation.cpp index 54c151ac..a0ae9e65 100644 --- a/src/alloc/generation.cpp +++ b/src/alloc/generation.cpp @@ -16,6 +16,15 @@ namespace xo { return "?generation"; } + const char * genresult2str(generation_result x) { + switch (x) { + case generation_result::nursery: return "nursery"; + case generation_result::tenured: return "tenured"; + case generation_result::not_found: return "not-found"; + } + return "?generation_result"; + } + } /*namespace gc*/ } /*namespace xo*/ From bd8ca68e7c4eb1410994c6dceecff9d47077760b Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 5 Dec 2025 18:38:29 -0500 Subject: [PATCH 61/69] xo-ordinaltree: expand unittest + debug logging --- include/xo/alloc/GC.hpp | 4 +-- src/alloc/GC.cpp | 79 +++++++++++++++++++++++++++++++++++------ src/alloc/Object.cpp | 14 ++++++-- utest/GC.test.cpp | 23 ++++++++++++ 4 files changed, 106 insertions(+), 14 deletions(-) diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index e5dd6781..d8acc08c 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -107,7 +107,7 @@ namespace xo { bool is_dead() const { return false; } MutationLogEntry update_parent_moved(IObject * parent_to) const; - void fixup_parent_child_moved(IObject * child_to) { *lhs_ = child_to; } + void fixup_parent_child_moved(IObject * child_to); private: IObject * parent_ = nullptr; @@ -341,7 +341,7 @@ namespace xo { * is recorded in mutation log, * given an object @p parent that contains object pointer @p lhs **/ - virtual bool check_write_barrier(IObject * parent, IObject ** lhs, bool may_throw) const final; + virtual bool check_write_barrier(const void * parent, const void * const * lhs, bool may_throw) const final; virtual std::byte * alloc(std::size_t z) final override; virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 5cfdd9fc..dd17fd3b 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -58,6 +58,12 @@ namespace xo { reinterpret_cast(lhs_to)); } + void + MutationLogEntry::fixup_parent_child_moved(IObject * child_to) + { + *(this->lhs_) = child_to; + } + GC::GC(const Config & config) : config_{config} { @@ -508,12 +514,17 @@ namespace xo { assert(retval); + log && log(xtag("retval", retval)); + return retval; } void GC::assign_member(IObject * parent, IObject ** lhs, IObject * rhs) { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("parent", parent), xtag("lhs", lhs), xtag("rhs", rhs)); + ++gc_statistics_.n_mutation_; *lhs = rhs; @@ -532,23 +543,28 @@ namespace xo { { case generation_result::tenured: /* only need to log mutations that create tenured->nursery pointers */ + log && log(xtag("act", "any->T no log")); return; case generation_result::nursery: switch (tospace_generation_of(parent)) { case generation_result::nursery: if (is_before_checkpoint(parent)) { + log && log(xtag("act", "N1->N0 must mlog")); + // N1->N0, so must log this->mutation_log_[role2int(role::to_space)]->push_back(MutationLogEntry(parent, lhs)); ++(this->gc_statistics_.n_logged_mutation_); ++(this->gc_statistics_.n_xckp_mutation_); } else { // parent in N0, not an xckp mutation - return; + log && log(xtag("act", "N0->any no long")); } break; case generation_result::tenured: // T->N, so must log + log && log(xtag("act", "T->N must mlog")); + this->mutation_log_[role2int(role::to_space)]->push_back(MutationLogEntry(parent, lhs)); ++(this->gc_statistics_.n_logged_mutation_); ++(this->gc_statistics_.n_xgen_mutation_); @@ -556,11 +572,13 @@ namespace xo { case generation_result::not_found: // parent is global // This may be ok (provided lhs is a gc root) + log && log(xtag("act", "root->any no log")); break; } break; case generation_result::not_found: + log && log(xtag("act", "any->root no log")); // child is global; // logging not required @@ -571,6 +589,8 @@ namespace xo { void GC::forward_inplace(IObject ** lhs) { + scope log(XO_DEBUG(config_.debug_flag_), xtag("lhs", lhs)); + Object::_forward_inplace(lhs, this); } @@ -588,10 +608,13 @@ namespace xo { } bool - GC::check_write_barrier(IObject * parent, - IObject ** lhs, + GC::check_write_barrier(const void * parent, + const void * const * lhs, bool may_throw_flag) const { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("P", parent), xtag("L", lhs)); + if (!this->contains(parent)) { if (may_throw_flag) { throw std::runtime_error(tostr("GC::check_write_barrier", @@ -601,10 +624,11 @@ namespace xo { return false; } +#ifdef NOPE // don't want to assume IObject* std::size_t parent_z = parent->_shallow_size(); - std::byte * parent_addr = reinterpret_cast(parent); - std::byte * lhs_addr = reinterpret_cast(lhs); + const std::byte * parent_addr = reinterpret_cast(parent); + const std::byte * lhs_addr = reinterpret_cast(lhs); if ((lhs_addr < parent_addr) || (parent_addr + parent_z < lhs_addr)) { if (may_throw_flag) { @@ -617,12 +641,21 @@ namespace xo { } return false; } +#endif - IObject * rhs = *lhs; + const void * rhs = *lhs; + + if (!rhs) + return true; auto parent_gen = tospace_generation_of(parent); auto rhs_gen = tospace_generation_of(rhs); + log && log(xtag("C", rhs), + xtag("gen(P)", parent_gen), xtag("gen(C)", rhs_gen), + xtag("P.before-ckp", is_before_checkpoint(parent)), + xtag("C.before-ckp", is_before_checkpoint(rhs))); + switch(parent_gen) { case generation_result::nursery: if (is_before_checkpoint(parent)) { @@ -630,21 +663,25 @@ namespace xo { case generation_result::nursery: if (is_before_checkpoint(rhs)) { /* no mlog entry needed */ + log && log(xtag("msg", "N1->N1 - trivial")); return true; } else { /* need to check mlog */ - ; + log && log(xtag("msg", "N1->N0 - xgen")); } break; case generation_result::tenured: /* no mlog entry needed */ + log && log(xtag("msg", "N1->T - trivial")); return true; case generation_result::not_found: /* possible non-gc rhs */ + log && log(xtag("msg", "non-gc rhs - trivial")); return true; } } else { /* no mlog entry needed */ + log && log(xtag("msg", "N0->any - trivial")); return true; } break; @@ -652,16 +689,21 @@ namespace xo { switch(rhs_gen) { case generation_result::nursery: /* need to check mlog */ + log && log(xtag("msg", "T->N - xgen")); break; case generation_result::tenured: /* no mlog entry needed */ + log && log(xtag("msg", "T->T - trivial")); return true; case generation_result::not_found: /* possible non-gc rhs */ + log && log(xtag("msg", "non-gc rhs - trivial")); return true; } + break; case generation_result::not_found: /* already excluded -> impossible */ + log && log(xtag("msg", "assert")); assert(false && "already verified parent owned by GC"); } @@ -669,7 +711,7 @@ namespace xo { * search mutation log + verify such entry exists */ for (MutationLogEntry & mlog : *(mutation_log_[role2int(role::to_space)])) { - if ((mlog.parent() == parent) && (mlog.lhs() == lhs)) { + if ((mlog.parent() == parent) && ((const void * const *)mlog.lhs() == lhs)) { return true; } mlog.lhs(); @@ -872,7 +914,7 @@ namespace xo { for (MutationLogEntry & from_entry : *from_mlog) { if (log) { - if (i_from % 10000 == 0) + if (i_from % 10000 == 0 || true) log(xtag("i_from", i_from)); } @@ -905,8 +947,12 @@ namespace xo { ++n_rescue; + log && log(xtag("parent", parent), xtag("act", "move child"), xtag("child.from", child_from)); + Object::_deep_move(child_from, this, per_type_stats); + log && log(xtag("child.to", child_from->_destination())); + // C forwards to C', fall thru to parent fixup below // (a) T->N1' // (c) T->T @@ -924,13 +970,24 @@ namespace xo { IObject * child_to = child_from->_destination(); + log && log(xtag("act", "fixup parent"), xtag("parent", parent), xtag("lhs", from_entry.lhs()), xtag("child.from", child_from), xtag("child.to", child_to)); + from_entry.fixup_parent_child_moved(child_to); + { + // verify fixup was effective + IObject * child_from2 = from_entry.child(); + assert(child_from2 == child_to); + } + + // P->C', loc(C') in {N1', T'} if (tospace_generation_of(child_to) == generation_result::nursery) { // (b) loc(P)=T, loc(C')=N1'; also case (a) + log && log(xtag("act", "still xgen -> keep mlog entry")); + // still have xgen pointer, so need mlog for it to_mlog->push_back(from_entry); } else { @@ -972,6 +1029,8 @@ namespace xo { } } else { + log && log("defer"); + // loc(P) = N1, loc(C) = N0, P may be garbage // Includes cases: // (e) P->C, C not moved @@ -1347,7 +1406,7 @@ namespace xo { void GC::execute_gc(generation upto) { - scope log(XO_DEBUG(config_.stats_flag_)); + scope log(XO_DEBUG(config_.stats_flag_ || config_.debug_flag_)); auto t0 = std::chrono::steady_clock::now(); diff --git a/src/alloc/Object.cpp b/src/alloc/Object.cpp index b0be501b..b0a52f59 100644 --- a/src/alloc/Object.cpp +++ b/src/alloc/Object.cpp @@ -42,18 +42,24 @@ namespace xo { Object::_forward(IObject * src, gc::IAlloc * gc) { + scope log(XO_DEBUG(gc->debug_flag()), xtag("src", src)); + if (!src) return src; - if (src->_is_forwarded()) + if (src->_is_forwarded()) { + log && log("already forwarded", xtag("dest", src->_offset_destination(src))); return src->_offset_destination(src); + } if (gc->check_move(src)) { + log && log("needs forwarding"); Object::_shallow_move(src, gc); /* *src is now a forwarding pointer to a copy in to-space */ return src->_offset_destination(src); } else { + log && log("already tenured + incr collection"); /* don't move tenured objects during incremental collection */ return src; } @@ -62,6 +68,8 @@ namespace xo { IObject * Object::_deep_move(IObject * from_src, gc::GC * gc, gc::ObjectStatistics * /*stats*/) { + scope log(XO_DEBUG(gc->config().debug_flag_)); + using gc::generation; if (!from_src) @@ -146,13 +154,15 @@ namespace xo { do { fixup_work = 0; - auto fixup_generation = [gc, &gray_lo_v](generation gen) { + auto fixup_generation = [gc, &log, &gray_lo_v](generation gen) { std::size_t work = 0; while(gray_lo_v[gen2int(gen)] < gc->free_ptr(gen)) { Object * x = reinterpret_cast(gray_lo_v[gen2int(gen)]); // update per-class stats here + log && log("fwd children", xtag("x", x)); + std::size_t xz = x->_forward_children(gc); // must pad xz to multiple of word size, diff --git a/utest/GC.test.cpp b/utest/GC.test.cpp index 90e22dfa..f178f4a8 100644 --- a/utest/GC.test.cpp +++ b/utest/GC.test.cpp @@ -202,6 +202,29 @@ namespace xo { vector_type member2_; bool ctor_ran_ = false; }; + +#ifdef NOT_YET + struct MemberType2 { + public: + MemberType2() = default; + /** GC hooks rely on copy constructor. But can't write it without allocator state. + * Therefore: need copy-like constructor that takes allocator argument + **/ + + template + explicit MemberType2(Allocator & alloc, uint64 payload) { + using traits = gc_allocator_traits; + + uint64_t * ptr = traits::allocate(alloc, 1); + + this->payload_ = payload; + this->ctor_ran_ = true; + } + + uint64_t * payload_ = nullptr; + bool ctor_ran_ = false; + } +#endif } TEST_CASE("vector_custom_allocator", "[alloc][vector]") From 1b93c9a427e6d8673a95fc2d4d738f45438d3f23 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 5 Dec 2025 19:54:00 -0500 Subject: [PATCH 62/69] xo-ordinaltree: start work on gc-aware Key,Value in rbtree --- include/xo/alloc/Object.hpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index 5f31356f..dce67c11 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -5,15 +5,16 @@ #pragma once -#include "xo/reflect/TaggedPtr.hpp" -#include "IAlloc.hpp" #include "xo/allocutil/IObject.hpp" +#include "xo/reflect/TaggedPtr.hpp" +#include "xo/allocutil/ObjectVisitor.hpp" #include "xo/allocutil/gc_ptr.hpp" #include #include namespace xo { namespace gc { + class IAlloc; class GC; class ObjectStatistics; }; @@ -147,6 +148,18 @@ namespace xo { reinterpret_cast(rhs.ptr())); } + namespace gc { + template + class ObjectVisitor> { + public: + void forward_children(gp & target, + IAlloc * gc) + { + Object::_forward_inplace(target, gc); + } + }; + } + std::ostream & operator<< (std::ostream & os, gp x); From 7b5198be081699ade480dcb4d0a0b8b01f4d5473 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 6 Dec 2025 19:42:36 -0500 Subject: [PATCH 63/69] xo-ordinaltree: work on gp-key unit test --- include/xo/alloc/Object.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/xo/alloc/Object.hpp b/include/xo/alloc/Object.hpp index dce67c11..9665e66a 100644 --- a/include/xo/alloc/Object.hpp +++ b/include/xo/alloc/Object.hpp @@ -152,8 +152,8 @@ namespace xo { template class ObjectVisitor> { public: - void forward_children(gp & target, - IAlloc * gc) + static void forward_children(gp & target, + IAlloc * gc) { Object::_forward_inplace(target, gc); } From 05ab69384af65dc9da25885d6a9e4641e8b52b96 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 11 Dec 2025 11:14:46 -0500 Subject: [PATCH 64/69] xo-alloc2: work on fomo Arena --- src/alloc/ArenaAlloc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index 886b6a70..d7f794f3 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -374,7 +374,7 @@ namespace xo { ArenaAlloc::clear() { this->set_free_ptr(lo_); - this->limit_ = hi_; + //this->limit_ = hi_; } void From d720d89fd26a76101fb1238c917a0b1cd5b38a78 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 26 Jan 2026 19:26:54 -0500 Subject: [PATCH 65/69] xo-reflect: + pretty printing for xo::reflect::TypeDescr --- include/xo/alloc/GcStatistics.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/xo/alloc/GcStatistics.hpp b/include/xo/alloc/GcStatistics.hpp index b591bf22..69053457 100644 --- a/include/xo/alloc/GcStatistics.hpp +++ b/include/xo/alloc/GcStatistics.hpp @@ -1,16 +1,16 @@ -/* GcStatistics.hpp +/** @file GcStatistics.hpp * - * author: Roland Conybeare, Aug 2025 - */ + * @author Roland Conybeare, Aug 2025 + **/ #pragma once #include "generation.hpp" #include "CircularBuffer.hpp" -#include "xo/reflect/TypeDescr.hpp" -#include "xo/unit/quantity.hpp" -#include "xo/unit/quantity_iostream.hpp" -#include "xo/indentlog/print/pretty.hpp" +#include +#include +#include +#include #include #include From caa03566da2858b368de16dc1c0dd4d2eb5bca25 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 26 Feb 2026 12:20:34 +1100 Subject: [PATCH 66/69] xo-alloc: nix-build + compiler nits --- src/alloc/ArenaAlloc.cpp | 2 ++ src/alloc/GC.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index d7f794f3..dc80eb4e 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -390,8 +390,10 @@ namespace xo { /* word size for alignment */ constexpr uint32_t c_bpw = sizeof(std::uintptr_t); + (void)c_bpw; std::uintptr_t free_u64 = reinterpret_cast(free_ptr_); + (void)free_u64; assert(free_u64 % c_bpw == 0ul); diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index dd17fd3b..f1c52d1b 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -551,7 +551,7 @@ namespace xo { case generation_result::nursery: if (is_before_checkpoint(parent)) { log && log(xtag("act", "N1->N0 must mlog")); - + // N1->N0, so must log this->mutation_log_[role2int(role::to_space)]->push_back(MutationLogEntry(parent, lhs)); ++(this->gc_statistics_.n_logged_mutation_); @@ -974,12 +974,14 @@ namespace xo { from_entry.fixup_parent_child_moved(child_to); +#ifndef NDEBUG { // verify fixup was effective IObject * child_from2 = from_entry.child(); assert(child_from2 == child_to); } - +#endif + // P->C', loc(C') in {N1', T'} From a1e5449d5a0910ba71c43dc18bcc3efe451c81c9 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 27 Feb 2026 11:33:40 +1100 Subject: [PATCH 67/69] xo-alloc: nix requires specific include path --- include/xo/alloc/Blob.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/xo/alloc/Blob.hpp b/include/xo/alloc/Blob.hpp index 9e3ae44a..205e984b 100644 --- a/include/xo/alloc/Blob.hpp +++ b/include/xo/alloc/Blob.hpp @@ -6,7 +6,7 @@ #pragma once #include "Object.hpp" -#include "IAlloc.hpp" +#include namespace xo { /** Use to allocate opaque binary data, From fe9ae6389240138d57db72a1adc807f07f2d3715 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 27 Feb 2026 11:34:18 +1100 Subject: [PATCH 68/69] xo-alloc: fix broken cmake export deps --- cmake/xo_allocConfig.cmake.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/xo_allocConfig.cmake.in b/cmake/xo_allocConfig.cmake.in index f5afd837..97574de5 100644 --- a/cmake/xo_allocConfig.cmake.in +++ b/cmake/xo_allocConfig.cmake.in @@ -1,7 +1,10 @@ @PACKAGE_INIT@ include(CMakeFindDependencyMacro) +find_dependency(xo_allocutil) +find_dependency(xo_unit) find_dependency(indentlog) -#find_dependency(xo_flatstring) +find_dependency(reflect) +find_dependency(callback) include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") check_required_components("@PROJECT_NAME@") From 1d6f1c22eaaee04a263117dd89be9d3ef5457d22 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 27 Feb 2026 19:38:53 +1100 Subject: [PATCH 69/69] xo-cmake: setup to make share target available via cmake install --- cmake/xo_allocConfig.cmake.in | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/xo_allocConfig.cmake.in b/cmake/xo_allocConfig.cmake.in index 97574de5..e627df64 100644 --- a/cmake/xo_allocConfig.cmake.in +++ b/cmake/xo_allocConfig.cmake.in @@ -7,4 +7,5 @@ find_dependency(indentlog) find_dependency(reflect) find_dependency(callback) include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Share.cmake") check_required_components("@PROJECT_NAME@")