From 2c8faf6e43324ef3a8bb0679956451f8f01df993 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 6 Jun 2026 22:03:21 -0400 Subject: [PATCH] git subrepo clone git@github.com:Rconybea/xo-alloc.git xo-alloc subrepo: subdir: "xo-alloc" merged: "fc656313" upstream: origin: "git@github.com:Rconybea/xo-alloc.git" branch: "main" commit: "fc656313" git-subrepo: version: "0.4.9" origin: "???" commit: "???" --- xo-alloc/.gitrepo | 12 + xo-alloc/CMakeLists.txt | 32 + xo-alloc/README.md | 16 + xo-alloc/cmake/xo-bootstrap-macros.cmake | 41 + xo-alloc/cmake/xo_allocConfig.cmake.in | 11 + xo-alloc/docs/CMakeLists.txt | 9 + xo-alloc/docs/README | 41 + xo-alloc/docs/_static/README | 1 + xo-alloc/docs/_static/img/favicon.ico | Bin 0 -> 309936 bytes xo-alloc/docs/conf.py | 39 + xo-alloc/docs/glossary.rst | 28 + xo-alloc/docs/implementation.rst | 202 +++ xo-alloc/docs/index.rst | 17 + xo-alloc/docs/install.rst | 120 ++ xo-alloc/docs/introduction.rst | 268 +++ xo-alloc/include/xo/alloc/AllocPolicy.hpp | 58 + xo-alloc/include/xo/alloc/ArenaAlloc.hpp | 231 +++ xo-alloc/include/xo/alloc/Blob.hpp | 40 + xo-alloc/include/xo/alloc/CircularBuffer.hpp | 245 +++ xo-alloc/include/xo/alloc/Forwarding1.hpp | 56 + xo-alloc/include/xo/alloc/GC.hpp | 496 ++++++ xo-alloc/include/xo/alloc/GcStatistics.hpp | 287 ++++ xo-alloc/include/xo/alloc/ListAlloc.hpp | 100 ++ xo-alloc/include/xo/alloc/Object.hpp | 170 ++ .../include/xo/alloc/ObjectStatistics.hpp | 87 + xo-alloc/include/xo/alloc/Stack.hpp | 49 + xo-alloc/include/xo/alloc/generation.hpp | 54 + xo-alloc/src/alloc/AllocPolicy.cpp | 13 + xo-alloc/src/alloc/ArenaAlloc.cpp | 426 +++++ xo-alloc/src/alloc/Blob.cpp | 57 + xo-alloc/src/alloc/CMakeLists.txt | 24 + xo-alloc/src/alloc/Forwarding1.cpp | 79 + xo-alloc/src/alloc/GC.cpp | 1526 +++++++++++++++++ xo-alloc/src/alloc/GcStatistics.cpp | 214 +++ xo-alloc/src/alloc/ListAlloc.cpp | 400 +++++ xo-alloc/src/alloc/Object.cpp | 230 +++ xo-alloc/src/alloc/ObjectStatistics.cpp | 73 + xo-alloc/src/alloc/generation.cpp | 31 + xo-alloc/utest/ArenaAlloc.test.cpp | 88 + xo-alloc/utest/CMakeLists.txt | 26 + xo-alloc/utest/CircularBuffer.test.cpp | 174 ++ xo-alloc/utest/Forwarding1.test.cpp | 93 + xo-alloc/utest/GC.test.cpp | 395 +++++ xo-alloc/utest/GcStatistics.test.cpp | 216 +++ xo-alloc/utest/IAlloc.test.cpp | 127 ++ xo-alloc/utest/ListAlloc.test.cpp | 64 + xo-alloc/utest/ObjectStatistics.test.cpp | 185 ++ xo-alloc/utest/alloc_utest_main.cpp | 6 + xo-alloc/utest/generation.test.cpp | 39 + 49 files changed, 7196 insertions(+) create mode 100644 xo-alloc/.gitrepo create mode 100644 xo-alloc/CMakeLists.txt create mode 100644 xo-alloc/README.md create mode 100755 xo-alloc/cmake/xo-bootstrap-macros.cmake create mode 100644 xo-alloc/cmake/xo_allocConfig.cmake.in create mode 100644 xo-alloc/docs/CMakeLists.txt create mode 100644 xo-alloc/docs/README create mode 100644 xo-alloc/docs/_static/README create mode 100644 xo-alloc/docs/_static/img/favicon.ico create mode 100644 xo-alloc/docs/conf.py create mode 100644 xo-alloc/docs/glossary.rst create mode 100644 xo-alloc/docs/implementation.rst create mode 100644 xo-alloc/docs/index.rst create mode 100644 xo-alloc/docs/install.rst create mode 100644 xo-alloc/docs/introduction.rst create mode 100644 xo-alloc/include/xo/alloc/AllocPolicy.hpp create mode 100644 xo-alloc/include/xo/alloc/ArenaAlloc.hpp create mode 100644 xo-alloc/include/xo/alloc/Blob.hpp create mode 100644 xo-alloc/include/xo/alloc/CircularBuffer.hpp create mode 100644 xo-alloc/include/xo/alloc/Forwarding1.hpp create mode 100644 xo-alloc/include/xo/alloc/GC.hpp create mode 100644 xo-alloc/include/xo/alloc/GcStatistics.hpp create mode 100644 xo-alloc/include/xo/alloc/ListAlloc.hpp create mode 100644 xo-alloc/include/xo/alloc/Object.hpp create mode 100644 xo-alloc/include/xo/alloc/ObjectStatistics.hpp create mode 100644 xo-alloc/include/xo/alloc/Stack.hpp create mode 100644 xo-alloc/include/xo/alloc/generation.hpp create mode 100644 xo-alloc/src/alloc/AllocPolicy.cpp create mode 100644 xo-alloc/src/alloc/ArenaAlloc.cpp create mode 100644 xo-alloc/src/alloc/Blob.cpp create mode 100644 xo-alloc/src/alloc/CMakeLists.txt create mode 100644 xo-alloc/src/alloc/Forwarding1.cpp create mode 100644 xo-alloc/src/alloc/GC.cpp create mode 100644 xo-alloc/src/alloc/GcStatistics.cpp create mode 100644 xo-alloc/src/alloc/ListAlloc.cpp create mode 100644 xo-alloc/src/alloc/Object.cpp create mode 100644 xo-alloc/src/alloc/ObjectStatistics.cpp create mode 100644 xo-alloc/src/alloc/generation.cpp create mode 100644 xo-alloc/utest/ArenaAlloc.test.cpp create mode 100644 xo-alloc/utest/CMakeLists.txt create mode 100644 xo-alloc/utest/CircularBuffer.test.cpp create mode 100644 xo-alloc/utest/Forwarding1.test.cpp create mode 100644 xo-alloc/utest/GC.test.cpp create mode 100644 xo-alloc/utest/GcStatistics.test.cpp create mode 100644 xo-alloc/utest/IAlloc.test.cpp create mode 100644 xo-alloc/utest/ListAlloc.test.cpp create mode 100644 xo-alloc/utest/ObjectStatistics.test.cpp create mode 100644 xo-alloc/utest/alloc_utest_main.cpp create mode 100644 xo-alloc/utest/generation.test.cpp diff --git a/xo-alloc/.gitrepo b/xo-alloc/.gitrepo new file mode 100644 index 00000000..e9c6a776 --- /dev/null +++ b/xo-alloc/.gitrepo @@ -0,0 +1,12 @@ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/ingydotnet/git-subrepo#readme +; +[subrepo] + remote = git@github.com:Rconybea/xo-alloc.git + branch = main + commit = fc656313e9582957f13446364299a8e79cbd51f0 + parent = d16545d815d055837e0973cca8483277a925d7fb + method = merge + cmdver = 0.4.9 diff --git a/xo-alloc/CMakeLists.txt b/xo-alloc/CMakeLists.txt new file mode 100644 index 00000000..0e9de5c4 --- /dev/null +++ b/xo-alloc/CMakeLists.txt @@ -0,0 +1,32 @@ +# xo-alloc/CMakeLists.txt + +cmake_minimum_required(VERSION 3.10) + +project(xo_alloc VERSION 0.1) + +include(GNUInstallDirs) +include(cmake/xo-bootstrap-macros.cmake) + +xo_cxx_toplevel_options3() + +# ---------------------------------------------------------------- +# c++ settings + +set(PROJECT_CXX_FLAGS "") +#set(PROJECT_CXX_FLAGS "-fconcepts-diagnostics-depth=2") # gcc-only! +add_definitions(${PROJECT_CXX_FLAGS}) + +# ---------------------------------------------------------------- + +# must complete definition of expression lib before configuring examples +add_subdirectory(src/alloc) +add_subdirectory(utest) +xo_export_cmake_config(${PROJECT_NAME} ${PROJECT_VERSION} ${PROJECT_NAME}Targets) + +# ---------------------------------------------------------------- +# docs targets depend on other library/utest/exec targets above, +# --> must come after them. +# +add_subdirectory(docs) + +# end CmakeLists.txt diff --git a/xo-alloc/README.md b/xo-alloc/README.md new file mode 100644 index 00000000..5a42f922 --- /dev/null +++ b/xo-alloc/README.md @@ -0,0 +1,16 @@ +# xo-alloc -- arena allocator and incremental garbage collector + +# Rules for writing garbage-collected classes. + +Topics +* allocation - allocate Objects (inheriting xo::Object) before owned scratch space. + Can relax this if/when abandon the bad-for-locality use of two pointers + into to-space to keep track of grey objects. Want to use stack anyway + so we can do depth-first search. +* destructors - can omit except for finalization +* assignment - MUST USE Object::assign_member() to assign pointers to gc-owned memory. + Only necessary for old->new pointers, so don't need to worry about this + for initialization. +* finalization - not supported (yet) + +- padding - use IAlloc::with_padding(z) for hand-allocated objects. diff --git a/xo-alloc/cmake/xo-bootstrap-macros.cmake b/xo-alloc/cmake/xo-bootstrap-macros.cmake new file mode 100755 index 00000000..592272c0 --- /dev/null +++ b/xo-alloc/cmake/xo-bootstrap-macros.cmake @@ -0,0 +1,41 @@ +# ---------------------------------------------------------------- +# for example: +# $ PREFIX=/usr/local # for example +# $ cmake -DCMAKE_MODULE_PATH=prefix -DCMAKE_INSTALL_PREFIX=$PREFIX -B .build +# +# will get +# CMAKE_MODULE_PATH +# from xo-cmake-config --cmake-module-path +# +# and expect .cmake macros in +# CMAKE_MODULE_PATH/xo_macros/xo_cxx.cmake +# ---------------------------------------------------------------- + +find_program(XO_CMAKE_CONFIG_EXECUTABLE NAMES xo-cmake-config REQUIRED) + +if ("${XO_CMAKE_CONFIG_EXECUTABLE}" STREQUAL "XO_CMAKE_CONFIG_EXECUTABLE-NOT_FOUND") + message(FATAL "could not find xo-cmake-config executable") +endif() + +message(STATUS "XO_CMAKE_CONFIG_EXECUTABLE=${XO_CMAKE_CONFIG_EXECUTABLE}") + +if (XO_SUBMODULE_BUILD) + if (("${CMAKE_MODULE_PATH}" STREQUAL "") OR ("${CMAKE_MODULE_PATH}" STREQUAL prefix)) + # local version of xo-cmake macros + set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/xo-cmake/cmake") + message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + endif() +else() + if (("${CMAKE_MODULE_PATH}" STREQUAL "") OR ("${CMAKE_MODULE_PATH}" STREQUAL prefix)) + # default to typical install location for xo-project-macros + execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --cmake-module-path OUTPUT_VARIABLE CMAKE_MODULE_PATH) + message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + endif() +endif() + +# needs to have been installed somewhere on CMAKE_MODULE_PATH, +# (e.g. from xo-cmake with the same value for CMAKE_INSTALL_PREFIX) +# +include(xo_macros/xo_cxx) + +xo_cxx_bootstrap_message() diff --git a/xo-alloc/cmake/xo_allocConfig.cmake.in b/xo-alloc/cmake/xo_allocConfig.cmake.in new file mode 100644 index 00000000..e627df64 --- /dev/null +++ b/xo-alloc/cmake/xo_allocConfig.cmake.in @@ -0,0 +1,11 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(xo_allocutil) +find_dependency(xo_unit) +find_dependency(indentlog) +find_dependency(reflect) +find_dependency(callback) +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Share.cmake") +check_required_components("@PROJECT_NAME@") diff --git a/xo-alloc/docs/CMakeLists.txt b/xo-alloc/docs/CMakeLists.txt new file mode 100644 index 00000000..e13b26a0 --- /dev/null +++ b/xo-alloc/docs/CMakeLists.txt @@ -0,0 +1,9 @@ +# xo-alloc/docs/CMakeLists.txt + +xo_doxygen_collect_deps() +xo_docdir_doxygen_config() +xo_docdir_sphinx_config( + index.rst install.rst introduction.rst implementation.rst) + +# see xo-reader/doc or xo-unit/doc for working examples +# example.rst install.rst implementation.rst diff --git a/xo-alloc/docs/README b/xo-alloc/docs/README new file mode 100644 index 00000000..6aff5d41 --- /dev/null +++ b/xo-alloc/docs/README @@ -0,0 +1,41 @@ +standalone build + + +-----------------------------------------------+ + | cmake | + | CMakeLists.txt | + | $PREFIX/share/cmake/xo_macros/xo_cxx.cmake | + +-----------------------------------------------+ + | + | +----------------------+ + +------------------------------------------------->| .build/docs/Doxyfile | + | +----------------------+ + | ^ + | (cmake) | + | /------------/ + | | + | +---------------------------------------+ +-----------------+ + +---->| doxygen |--------->| .build/docs/dox | + | | $PREFIX/share/xo-macros/Doxyfile.in | (doxygen)| +- html/ | + | +---------------------------------------+ | +- xml/ | + | +-----------------+ + | | + | |(sphinx) + | | + | v + | +---------------------------------------+ +--------------------+ + \---->| sphinx |------->| .build/docs/sphinx | + | +- conf.py | | +- html/ | + | +- _static/ | +--------------------+ + | +- *.rst | + +---------------------------------------+ + +umbrella build relies on top-level cmake macros + +files + + README this file + CMakeLists.txt build entry point + conf.py sphinx config + _static static files for sphinx + + index.rst toplevel sphinx document; entry point diff --git a/xo-alloc/docs/_static/README b/xo-alloc/docs/_static/README new file mode 100644 index 00000000..7297d046 --- /dev/null +++ b/xo-alloc/docs/_static/README @@ -0,0 +1 @@ +add any static {.html, .js, ..} files for sphinx to pickup here diff --git a/xo-alloc/docs/_static/img/favicon.ico b/xo-alloc/docs/_static/img/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..4163dd69c734f8186cf1ce5e726213cebd231c31 GIT binary patch literal 309936 zcmZQzU}WH800Bk@1%|zv3=GQ{7#I#50EsIwXaq4aBx^A+G&Df@9E=RzHB1Z%2@w8@ zDGUsoTbLOf93XrRCkBRSNfrhJ0|#lWE5$ikqY0O79?U|^U$ zn}tC_0>bZ5U|_Ib!@?jS0O4n_Ffbh6%EHhY;OEXI1#%~^r-w@rND_oO*cccXVv1Iz zF)(Phc)B=-RNQ)dx4b6i>dX%x@9!*6Gnu4wa+A(!HtFQYZ7*je9SRb%TBzP4cZ$tP zVYbt&Pn&i>;q_KHJ!zBSLY3J9UJ4VK77ABwWnMNbtss9=YN_w;@ALjX?m2M7c-yHn zb2QI?|2$Jb#qjf;nbq?w|27CXaVWNsOmI2hTLyV&ero8F#WKIIuVn&NhzE8B9^U#LMNYX4rLd$CPc&nsSS&rGvkI^&_$nF1L# z_d{D)7<)DQJFe|mC~$Iu_x|S_4_;rx#UxfO_dfqc+Q(0(Ypg%a_slVu3-bv2A>zrf zXvLrJsUOz94$zigSokL68AJOnzTeh-m!~H22n0p%%r5;^zh#*{N5pIPJqsUpDm+*z z`nsb2nVibGJsf%U7rtG+rLLghAksRuzu}sNm*LiZ(rVxN`0MS2Ch9vir>{Bw79 zl(7G@ik0@~dUIuM7MR)^Z(o1LMF9jHA_`AEEcku(!@Zza+ojJ-Uex+`v~=dw@6th^ zKgP+jG@c{8zvI-Y3y=8?(UF<-aX6vKr6HmCfFzyMOK4XUi8l ziPZhL5^k@0v z8PmGW{&{5fPXF{H`!$Z~G4Fjg`)c62LfLJaZHM~V zmKMmg-&~wEBRcwQ^uxUoH!raF?@6%U@kgs|dwfX4xpc*ui?(f?C%=Plx~A)uGM}3j zcJf(sY1wz)C(IVYD+*;{?Pa%ZYo z_Pq#>_j+4pBkR_McI>J6<7V-M=|%O#;MJ>tOnrPI!Ft)kF7=A4TFNz{JBqxYF?Q^k zVZ%6W&7y;aKgzEsN(B_P$~$Osv|Ci)k?_l%#jOA5T$#&GRRslyCwKPRH%@b2uydmE zAw|1<>vgZB0!ELhsdKP?}rQHfY z{ytq8Zl7j7OD1mmTLzcP$e$c0G7VCpXD!#A*17%hNmr0Sk!9{Jm2VQU>{a{Tb`&`a zHJn&kn7rV3<+9y+%cmwxuJ4|bR8uPY{LkauJF089Ek4QX$)J>8_g|UgJR7SzV{q>A z$=NS6Pai)&vt-lxHOQL8WZpT^hY_1DTwl-0qB_{ov@f6{ks7h0WK z=oOf3J+G!nwk$6v;o6Oh+mt0OJsFhL)7~@4-*I<7@znTf)L;1pBE9R9^1hnEWTTEb z1T#;TZ8w^&wb^p%fzq4bf6aevSGoCZhTF0xGkc+*A@TgRwLaw{z1xyCS)Oiqypi27 zD*IOMaU-Eg3@&*t3X@XQM2q}eV~_dIb0|Kc`<6w)m%%WGYx8^;4vv+w*>@_MPCmW& z@4iR#d*gd$0?MI+H=}rFa$jNi>L6^O_cw#>7{lVmKnIaxe#@_>D^oT5Wpmi$G@^p9 z-aWqK*`j;5GhVAOc4+x5oX2s7VcBx|$%d`_1cPrV=PW!XQMBiJ!+~!{MY@7^EPgE! z`^r?#lVPfB;=ZlwOxBFMnX41H{#BeY|F>aQtJg%G=~L^QcPDVPuF(AI`&ET;ioJMn z>3!B8>Iw2b59RrHYHxVBL7KaLkB~lJ^Ynj@%0J|~8yQVvI5oNZ=d}C$7A!NmBj-%s zE?K1+U+VlZ|Hn*iDGn2b@Lvor6AKJ-ztuDKvGB1hn=t=;MDnfQb-(8486DmAD&wgN zgUK#?xBA%5{>7e58nvb~w*4tQaPfeT#K(QV?JjVaFuZQOdLSwx_Sb#uM!5y{f45sb zVOuSGRCL?@9;Q!!8ou5t6H!)CP*7N4qPc+8LdJsi2Txi?o<)sMV;oxzgAUU@=4nY> zwdNPtml)jtGyCNZX*c;7i^DfYeg3%XXVLX=+0yxfBb% zAHfn&)NXTKSYh!(^}y)^!3VTwHM=ZGxZFrRl$^Uq`QJq!_yHjIB*XS{#8KvaL`cgd=5yQXz<*WT0E|}5Tm2s4_gZ^{M_a&De1{@bh)5~Q62Vl_%2PFc zvZKC|==*1xYkxXfmb3h_(>5#l>{UMht%sYRki$m~W_HH=jCQ9Vq`lml$n_`pK=%wk zo9v+41*ZN%w_Rl$EGe13l@BF95X}9}Qo}9AqpFTzN!1YG`!wbx=M6llJ*8H)a z+4HtI!-seA3eyxeedhaP_M~Xm+lo()*^fT5ED!B}q`!)1o845YhS^V{vc<#ADqElFt+jO2Je3|`P`hd4^Uq#3M~zSV-DZ5+rovEN z#;~vb<=y+Y429nwxO>3&z-*DogYpHBwkKR&{NT5Z>AKG6s}IR!SudLtD3I~zpTz$B zI}>@A&&jl2a7W`~CXcFS&&+Mr`~`N=dVe;4xj(ldQ)!ReM6rL2=l5|jkz2bfPViR*O#hYg(`0SuQky!%+VZIT@mbch?o9MMaOkFj!VmukpEn%MRf%ep zZj4X7^+$E(f7chi8}pNGzqiF~lg)6RuC!M1W#RKlQ|H8sF3f*rxJ~(#>`KNb^O5S>#uF3_L1%@fIe3xe^C@9#3Gx!IyvHlMh-!8DL%29LK zwXovZuQ$wpJA+N!BJG6!f&W+U>ioS{+Vj;my?3>R{;E6T)lHLL^ht_%GW;|Y+`+cF zNI!b2ecMZ(JzNo))(>hT{~4I{G)*~v@9Wk-dNN7ww+(YIIy}7oV)nlkzxGw{<@Gnw%$L*4r$(;;uI#bN$^kh1TysuKYZ9ptphPVc7?Lg>dP@DIMX^Fo|=Pf$0TCdpGjj z?re?u`+@t*mm@!#+v<|IbC%$us|xzI^Y!76=Hn?-Eu@+$Udjb8W)b#$Gj5)>#n(1^MCn;pVbO7S-S3tFAXXZ_g9M_J@7u zd7F&GXRg#-coc1^G>O42&Hq9D$$gX7OaD;(VDE6KY{SvCpUwuE@e!YxCLiy8*z`&x zUGwM?r#n*`kA1d@^IP(x@;2K)v5a5KeVMi%Ppgk&j^T~*diLoPTiTZ8D%oX|7~1mH z&#=tT$^FOYBl=0qYtP&n`K#qx`||Fr6ux<})Z%!d#l_Maa;=k3ONY$<-}#e~yXyDO zt!tZqFOzq?oxi@d_c2F1=LHH&P4_w; z_~o!!<-(qMKA)1}=Ul%RcV-py^9P}Cjo)nGJFN2cY@5O1Q;|=F=PY_qxxeJpew7AA zDb|?wmyG9@2!B5NUwX-h=RUuKKU|t^VZ43$I;}G|p1DN{dCpp;WYQM1sIPgk>B@C+ zRfiR={xn?c%Zr-6Hu13If1jq-jIGZ*4l@MD?c9qg0 zb01!8eyK4jcG{`i8fW5qX1-(O5B@Ej`r`DAxW7K~HB3M1d;SFY1xlqQeoHx1qOYU; zyG9^z5#N!T4KwWu2JwgEa$we`jMH&V%|XQJ0cHMjp{E*?Pxxj`&9Ro zx5(w|lV8qz!MZQO+QG1>bjH0uIj#Ra+sfkRYv@}uzMuT_UWHi3HrWLdcC9nnX05&a zJ~pB6twL+*^rUN5DX#?n%zFFa$cI|K&3X305A)T3uQ3RWoYk;@@=v=;*_DdZmgE{n z7pDIT2tL3+Vd<4e!p9Ta8qN2HGdAT|Z zv8mnW3*fouIU~5txOT?+#Ahws8&@(L`X2V#+K_*f&yVL^$^-cny+5)CK5ku1{SFnb_yo3U$BAX)$;h#Ean zXIsy=S;yk?-V)^^ebKpd8~acEY23?@lWgq}Saj~-mfz-k=TBp}oXf6mCwOemqsh6o z|2?-ci>PKQ&7987KBHLCWuk@1=GO3KEFZGZbm=R$J&rQEAI`7feZ2dS#smG7e-oxn zs$HG0$oAObj9B69d(mqT$Q`bany9Uwe6H+ti0|$@Wd)0#T%A~(^3v$`!O&fXGq(Pj zX*Y*;#!&~So-5x^)b!i7m@u!gntkT=pUEeBbSGL@Grqq0$1a-rO_KEvud6=amhS6U zzFpmU=j1gWj)u>hNmnB2s*2Gp|O z93H(a|Ns6(qiq5^zgV6zdEoP#M%goxXjGxaRVY%@7%=UjT zQ{<{^`7Zy~ciE?NB**yrhLh*Z51sMM+$tJzQLUk5;rdrx8|OaXTl-q3_v~W2H;37O%S^=sJ7>HSZ&?1ojP-8Ol=UYTs~+Go z^Jfq?{JwcBOF+n_y~=%UZ)+^BH&@q2$T@h;R({<6`Ge`2YX$dAMu8 ze4!%!Wo-eGK@L2N80QEc$d+ni68+OJ&zQk)=W)QbJNDJIkV`W`ua^^Cp) zrxqBt?iW@2`C;{eFL%VZJV^Io=@j{Muvt>7SZ0HOr5R&I`&r=wT3SyxNcO&5^gS`3 zf1X}KY5j)IgoW(wHhfvlZ_T}@%+5Mf@EMdd6&Ca>%@CYX-dL%@>(2HeZlmbG-bn|# z82qk2IB{XY2}RaBPS2(`rr+XPrLZ<`?!GGC;~6^w5AvkG?%M8j(1rE?;)iSa55C+c zJ3&KSDq`Bh=Uqj1PBsr7o0LZ|cz1uDZSU;6;HApRDgj(*t3 zj|TVWG3soJUC};~Wg7d3NcqVpS|k=KH(JbAZ?F$Mvbm`J`SN8KR1%yR`fuxpl{yEt{dD*t<{Jcj*CaT25YhC@9!!KXApUZ9S*1yZTg3M(jPCipm zc))wcYU(G03-7)^Ji@RmNBn`z6(*s=9SV1+D;_E6Wf$96$rRDFwC}*49;PpC3r!EP zJg{D6)92FX@?iF^Ahn#1noi4^-{-D)Hd%BP>(tLI9~1(XFH}4^^OVkB-O5uoX^i3u zpP7$*Sjx0n^v1GP8q--e*k5>SsLJwOE4b{`$(1K2CMNx`F;Z{fSSA)wVCl=im==B4 z%Hm|BY;H_|on~>;g_oz!WQZiKusM_S!?#tVokhU7f8&9ZPrqH=Kk>0?j=q8ek8^{j z+Uz2Yc9sBUmqhynb7}El4z6X1->SruEFMYLJxln-$fcLAps+xEM*gio?u>!b;=u}i zO^ZcbmWU~|DNcQu&gDJ3>4^@bOQYq5H3|QX*8O8%keDyOZSqA1L!P*4^P?@*bkY?R z6b_j7&*WNmW5)!h*$)n^dphrhOI*q-3oV_w7bjj8zq-L%v|!&nqpMpjU#gyS>bY{D z=RnhGPo^6hpXTPow5V=2ZBb?HP&^fO^?NSMHB8#H{%yyc(r#nZPh z^xT%M$YQX(*{U+vbK{b>ZaV$cAmPk4*N90MW|o%=D$SNM;+H*cpw1#1b8ef` zwF}G7^evydNPXRlvdiN7M-;A1+Nq#$p?9hN`8lc#sYW-XK>oh8@NKVQ2|HKDX4zg1 z!O|2zhS-!iwK?UMo(!G}OrE^*4nN;APxDlHaCl|xm$1{591JUxPwWZVC+z*#)AA*= zgv6izgR?{YT7UFT6ZyBx>(w_A<*QE5ST;<1bGqfTR{GoqI{`y>nJIq*t8SIt+Ouk6 z?V+O+denCM?cZiiw4b{|=W*jQ9{c>GJj_uOU)wPVJ^R?vCY~D`vM-}z@#Tm2 z;&e|l9G#GQ+cC>hZ26SuHR_l27D$;feBQED)bHnlgZ>9}g>AyNgq`wXpXhBpq4Lq) z_J~=`43!=Xj}7NGa>+5g)1SU+(ZiQ`P+k8YC!c|JM1G$-zd zX@T(5eHCG!KW=+?nYrRt*~5Fc7Bhdj?$GXFy}|9Ef`UTof5pz4paW-9{1~R+;#;TC z>9w2msyDYYuf1BJXL-lXH>BmVp>VC)-sN%%pPwl_)&I-qVmfDe;>NR0@-vQW z@!r29`|Yq#;58G`5a6`wlHTuy=h z+p>5%SLU+lnlHW{J7eb3#RmCt&Leah3P^R^DXwLju#f2<}R4d=J+Y}e*gX_ zzwaIT_wQ=<|6j|Q{|cN;PG0#$zP9A5n*Lv_0*ht&2X;;JWN=*|wL@Yuht%djz3hJk zgOC2cP<9}x`2Nph?N?f+?~qPn_hIh)#{4}Z!F=f)w}AOt{4Zxc4O=Sg?3BG%WRE)Y z{_pdaJjgcq%Y1WBlcz|3aHRV( z@wMIiK5?+TRaA0+^KRF~V$-e1877Z#A zgy5s+Yag-)Xvt(|-LjPF?NgljMs4B?m)8tF@fWW~++isRp2X0WZn;c2ohfbga-Z66 z(*uqN{AOI9bnPbJu1hgRoyR`^TD8~ZkIt19@lIC_y)3tSt+c6mFCEqVc*m9N{%Mz% z9G+qI)n(S~=K@@$iUWVm38> z{Kxv;4AV{GCmB5%l*QTnj+}WrZ(r+v?}Ro`>R5lwgu82z@QOPzMV)F-oRntXxLxXG z&z!!0Pwid)nQqy86*(q_v)4V>|EnPM`|JAIZvVC@EBH1{ZK|^NkTCQsS z_nW`-3w;*<5lzhx5gK#{4=kKf`AFOF`61TkO2&zIf5u1p<{ox& zTmH!1`H5RWnZ~+9{}mLLhH%WOHF?P9t~&32_Ihn0Zm~bhr$(qSJ_+O5{OfgT0^3b@ z1qFpWU*7j0RB1UX5_rB~rRCEp*ClS-^zOU&Bi(f3>?uWR;+Z%9&$xCf(Ch8WQjJUU zudDLb@74PrSNnBV({V*m=qP+QS>|Bh^fGxSZ%FFT+ffPiKD`Hymx!I3@K8Xu@<(aK z%6YL1eyJ6ti&Z~tZ{%}&nlZ<6VZf5rYMWC2+5P?#u;1ptI%}`9ki)+cX8)vL?8_3{ zc#QAP;*w~eX>K4MEF96dMpI%+e#|bm^RMNlUl{&x=8|XlIpfx3`pv-17UsgAc~V3pv#NoMFkppUCw^XVZ7y9W$Jj`i_24+>-c6>C zLbZB%Q7iw~HNJ08>Msr9U2(H+)#HzC$7-#c)@esxj+=3P8cTr0-2=e~YR-J@oP6@% zJ$Hk%ZI2DLPu?<+?U}@3R`j!cy0HJ&2RnG%9ezlyopUQl=a0dLXAZsLU8_2xe!Td0 zx$eP!lmE|ID|0;=T7?4A7lt>xUVnEeH`iMJ zuc~*q_e!xkgItGRC(ZI@;wJmVEOg(??>s--#`yo-J^Qz*G#u(?{?8zHOypVSjB^Ka zvKsChvQK%_G}Cj3*yjI^uD|{*wO2g$FhkZZc;$(uUMu&=6h(V@HMVKF?@Sgwct5E| zb6>(i{?Gq*O=5U+li`o7apMZH8}oGEo5l1-Hk2@J$p4l8+Ti!HkSVs`c8a&}D?GLG zuY!Vt#qFt|w*NmiC1}AFmy!=<3!L(0ZQMT|vY&VAb@sHHXF7#)Y0{rQH(WN@+O(FJ zxsR!j>7RIozz?|%%A!Sajd6UQ>-p?gSbAlr`|T9@*T3F>wbqo+sp0F_c3V$>^7rc! zwg4HnWo-BRBh@doLhyNm|7QX|YZIP-`Mf3gSh0eFg4G_DMDx(aoiPf2o;^SL zTZ0yu>K@oUYnLjw*q{G<_RYJ((B*AjV4ik);%5WjmbId3tvBo6$(+e^;ZHJCzS;f8 zxli>pze&{cvZFGOf|iBMy>{{Ov$Nc*z4dCW_peX+9duO8G$Bo4UiZ!D1daBITaw#k zE@kz9R%D-FpKfjqS zv)%6J{{6ShpZL#cyu#oW+z@F{B3xi?z;2xUf#b}Yg9pX>q|R)4eDlC`L*KsshSnLq z{`dX&Tivd`{J%A)5D zuN#;Re)Gt5-0V--;JlYhqW7ip4dEZ@XVx4xJy1O(Zz z%Z8q+2UJ?a&g8kBZs0fA+H!S<<|$c+^+k7{Jud3iYn(JNi5NZ-bf7^JMW0&7MjMXAEC!o?%hrzkFFV;hi_@j)fDX%ck(D20Nyl znK5@(zxBI~v$L|*-(>yv)n!?`m32R~*f$ankzW@vT0~UwFPhSB=nQ1~We#Bgh$;#HeV>aS}4tl7I)TX%11>r%tRjfvNblRZ5dl)jrRWB&N|%VR&E z+4a_E9xaMB>`YX97F{diYEY;nr{CpV`Z;gGA^D7^nzBj{CIM!XWwmw_1l}Fg@t5oN zP%ifGkiPk|85O8rOuak<9YW? z`4`C&ES+(zN5|)wO2^5U_4#ba7)ocns%w!^>$ta%@9A$(r-fZRN+&PaZFiUF)Y`5J zaaTW8L)%}{2E|Nl7Bk+S&o)8%KaI$RbRp3$*BVY2|UIkKLV%iFE*=c`LX1|?wq*b z4~+L2^_xEZ+A{rC`F;7=)l;fxzU|g!>AmlGVWZvr;~rj20|3{Q)kOP5u}KUrCj+XO`Sx}zMx%22=Z{qc7@ejKvP z+xz&;p8c~_7y^rwt`@J9t?hfi>z{Ab>(r;494B0p^7#Ah+3e$cCcoIZUtztbf`Wp> zlsWPLf-c_tDQ5rQ^=)O{c+n;JZb3y z`{U=j-Ll2DY}0#m&bnNyf@czg%T0^-d%~FP?*+u({o#Ck?Ou+quYVkT1z6T+zm0uj z@lc%c+1bsL*DnDz&rTi7uQTr3^~*Ch(Y$Quq1{p0ceAaR`DU)Y7-j$X_s8OGH3 z_jujzZyTg0e%Eu3agA*G_HfD%)twiQSIA#?pCD|3AAPwRi#l)4T6~ zcg|;-tG-8U)n0`e-(Lsk>^GI4%%X7W-p=Ri7A@cIn`>Qn>dO-GUtK{HgQpg$KU=J~ zz;uC0#;=3*ZtN$evlhqp|Gd>3`+u|8?fj!@DvTXnh370+cc|~WyX+#baes6k9TNt9Nlv5_116U8cE%<>R*f#CoH149ENXsxFLNQ!~Mfr)`3 zg#m&gBBSJJ2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kin zXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD zjE2By2#kinXb6mkz-S1JhQMeDjE2C73;{+41}1(6A#QmFE*5qM4rb;N849DG9^N7F zKRcV@za0a^^NcL6Pg&V&U$e5ker09l{LRYh_>-M8{aaSn+#lIFv;Jjg_x;JrD*Kj| z75X(h$M8i?zTowua>lDg6~jA#MtwIdL*Qdd3d7ypLbi{YSw_FJva0@OXYc--o&Drr zcJ`0|+1dYb;eXlLfBt1>zx|t?efn2+&ZKYIIi6p#vIPJC|IctFICxlw)u^Y2PY8U> z&SiL+SIF@xGt=W&cJ_+D+1amgrFCM&|7B-?`@Fgc#;&*n=?EeJX_PEkCIpY7avycDE%(VR9(ZO&pHg-sc->9R9 zZU{U}Ok{YMk;(8TEnWV1cJ`+K+1dZdNyn7v`InV-`&VYB`~UiShL5SKLpLx-T|T5j zz*t0t;cI3V!{@9l>EBsdTPaDyl?p@NTj2o=t!TyXommUSq$H^vfTbMYuaEHLh zyaI*~c?BY%IoN?tqh9mz6cc6~CZb*bcn}r1fs7>)RJ16{K zcJ@zv>2EN|{m#yweXppT=~I5ukO<#VM-J`~c#)dMa4#u^E(2)3Vlap8sM`j22wabfV)$QJ$nY~WvjLX2h64RJJNxj< ztZbeaSvi9{fJa?NuMl8jU|?WpU|`^5U|*VU|`T?U|=v|U|_IhU|_Ie zU|_I?VoL@FMq@?>HeCUB1qHRI2?-m9Lc)i+@o#qagAZBR3Li!%gz1&Q27^7U3=9lH z3=9ma(8L_hz`y_+1?gd6U|7Pyz_6Wxf#Em<1H%;t28MeK3=Gd07#Q9#FfhDlU|{&f zz`*dCfq~%*lm_wNGcqu}W?*1^T4`+j4VJcs0{t&L`~9!%Y?I&FqZ46+Az{+M4II!y z98_R~W_x-W7#KD%Ffg2BU|@I(O~Zc}7#KjK;1uIZQ`3L{b8@IQSxbjJ^!;CUwhL(B zmIh%zY9xI_fDzho(P3a0l5K0|SFGw9USS zfq~%>0|Ub^TxBvI@xRb^{Bs5dhAWJW%m>+6*;g?#F--y8ch11T0Gb(!XJB9mVqjqK zU|?WyU|?XdVqjn}XJBA3Wnf@11M!)eSwP|*^1QqW_v7O354HBs->fX>(ezJ;R7;r^ zT+lRE&A`C0pMinlEdv7ssB9*-&HkN%f#C_XU%MEZzGD~|80;7r7}P<7Jxq+uT&iMH z;1kwnmXwg>+PlfA4EIw~x&CBj4a!BIxcU+Qva{cP%gi+TIeJ(MN%2i5#h|nZ8hiu| zPJr6+pfpcRdIy#3?-&>u&N47CfYN#~0|SFAbi4;Nun8LEq^cJ#M@BNN@$g{!ote1= zS6Um4;{UR=QRTZ!(Ijk22h*Go`HcujDdlHF6kXxuzg7&;gj81573 z%Yy2HV+;%oRSXOaMhpxL3ePS-@uKm5(kzWF&bOYU3d z=s}^>NdZKe0&3@g%KLU`+a6aN2UONShPLNHeE|WYOdnuHf3mX~erM-^7J+^r?0MjK zcJ`FSumr}7siSRwIwc!W`ySL*zk{c|2aV&OXJBBcWME*>U|?VXjRDcgd6Zl8F(ZTF zLq>++-|Xx|gE{?!4p0A)lWX&5blivXNGEIpXgmkh!2+dgTw^()`PG9A3=AN3;)IPH zoGLC=)HD3gNN4z&l@&LbHv;_2&R*~@JBJO_XCIs)IV@ZSD!W1BGFurK7(nS8TU!St zzMFx80kj5lNVIVX#mu{`9ER7K+5CU9v$qV!IskOR#@DPYQ&3xNZC*Jc?~KZm7Xq>j z3=CZi3=E(#N^I#K)Xo8w_Yu%FYvkE72z9yay~F@oPw*o<+xA~}_VYoO2mXA|%Bp1W zPi8!yKHBf4LGoc^U|;~|#EbAf?@0W?PinxCS969(SMS6Mj>GqTGV ze`ea*5B*JFapcw`J}|6F8XV2A|GSt%$C%4sX8H)uv3Y3gP8osq%tH78f% zZ&ns)c^uu(gZ`JDeep+5uJ!+%Y=&34pqX=;d3U543>uUj?->~kKQl4}e`ja+{L9Y%Li6Guv<)*S=kK4atX}2931}{85u!G zSV#`HFdRVt2tv1&f5Xz=dBMQIP{6>z0NS27fKD75R^QIZW;m6c&iWxM+wxa-&h&rT z*$@6_Xa6O;PRPl5+iz=om5GrNv|Sli9{pfsVDK9nAwn%zg1yej!0?}$f#E+Z1H*q7 z28RDkh!Z}@wg)t9xeUh=WzcaVpeHL1_}sbNTyrA{SyLuPR?x97;6wl_qZk<&`iF10 zf=*<5#>l|%ho6Drzaay||0o8A|Md(E|7SBW{9nz$@P9J{!~cy84F8ugF#Maq!1%L} zf$^s&1H&&R28KUu3=FVhoL~rnw#e=Rg(n&Yl{M?3XQUA{WK;p}5cpqO%J3mQgW*YD zA?wTBe5r3)*;c=^vZB6c=M?|V&Tjsno!#{>JG<>mW@hD&%&g=u>FKVY)6!IL$Hj34 z7+W&%vU4!7FtczlFff9ao50gQ0|UcJ1_lODy^Y)QAteLy8ff*vUoHlQzt#*4f7=-t z{vT&x`2U81;r|~7hX4Nw;2#VO{~s_g{NKR9@IRe_;fE3f!*6K)fHfdMOYTpgrF)Pg zKwHZDphu};)i^3RARz!+(*ueh7zP!R76alEs#*&3&VMcjhW~yH4F5MVF#P{OAgvS1 z3x63H{@rC@_}k6E@JE}0;TI$1BoVm7LFZk9hVbE*L-PS>dwC%P0|V%IHMrtY#-IoR z(D|gGv1jBs%N-OhBij*RuYlV0X3%!cR|?ZTwtVq`f$?`01Jfrl2BtKSOF>72T!W@} zumhm%rwj~?p!svqv@b~AC>~rP06Mo9w8RQI&Op=eLo~j?-r#0n_@B?f@c#)7(mirs z0JWd?Ffb%$FfcG+N&8H985kJ6m>5ArprD;xgDc`kT>}mQ&=|x5XjuhLJPZsBpm`e5 z@c>{2LxTW09n{4BA{K;$d~c1m z3>UN%D`n_7AC&a|Npugd&?5#$VM_)^&?&P+Cj>{hOF?tK z*U{QQpyPpuK(K(a>K`5khJSPETjpcSSC9WQFkF7i!1yl!Jbxwz+QB;nf^gJ{pmkRp z(b7NYs4&nms)ILln4d8)@T_HIVE9+U!0`Xqz^8v$p7?v6f$@hf10$$EGI)b;)MY3k z(1w=&L1P!7ZK)`V1`p>y0|O&yUhB6F1H->(gD367^1%O93=H3S7#KiH+y+mmjk*M9 z2!M{a25mP5HKsurG$#N$z8$BAft3KYd;ao5*DDNAn;)LmvDyEFf#F9yxE%!E)jqJH zHEKJ-5CDz$fUfBPrF{?vB`{E)AgGc43P884ft>g!oPpv0_d!_pL-WJmlMKvn#Tl4D zw+YcdltyhNDFi^baxW;) z(k*~SEh0SxK7OeD!~f5NJN<+6z#ayMAH0wgt!Nr3 zqlQx!0-$sbTCWZ(`$075>YPC~1_W{zDE3Z!IMKgN77n9*3hAkRJl*>mWc^{ejLb zAm2o)>-@{W!1xnXo_-oS>Hq&$28KWEkXuHn9w4IzQZEDo(az=o4P1cMMo`ZNicJDl zO@A_mP}&EL1>I&~{4K)(-fBv50E`+yyAS}Sbx@iIB{C2Ot!oURopq#|3~~|}_YJA^ z&-nKP1M3HK1{ToCZlwBoR6QL-0CeaH`Z*oRbhMvGF#Ov;^wR(Tw9)iW$FxG8RghDM_drX2ST0z|z`!s_wy%OZ z{#prwD%&KOX>AkRdq>H?>Kkn>uGSo#N@wxvIs{;8T6Xki#=70_}} z0z<-}6XHNS#Asn38D@g>0H`hTcL?QyOAHME#YWRV8PQE&EueNjXw@e;kwNpo&cStZ z8Q5u{vh>vuO8?6l82&OdFo0I>(KkXzZ6hfJhCuoU_knpC82%j`Lg_yfQuc!m8Y3w_ zMitXH1VHT{(CKgBLKa&7Z)ad&03BgQUzgyt4Rmlm$a(*2hDiE<%E0hn8wRghAs$gQ)!v3Mf!J;t%Kq^A|%P4=jb=EC6!&C?4D)z{0@502=QB zrF$3#-Pi`o7lX*jyeTf2!KxLM!&gX$>5p) zf;qXAfq@Zp0?Ieg=_juTaUNL5zyLmh9p>;+dQgV|BQFC3<59Hs&x}FoWCeu>3=F*I znHd=WP8g&a;57roKYPfa&uHO4C=(Hiqd;RnjKT~IjObT544x%$DDDNNDNsY`k0JxZ zzjK2u5By_b_}#+5aD|D1;SGwbM>&H!1h6fE23-*|I33M$A6i%a_JLk$1Y2@E(DZf& zhX0^#%7Zi9MqPy~1hg3#7+ygW6{vs(;X$+o2Ai`%M+|-jUCC9z!0`XuAgBi}GBEtp z9c}kui>pB|2D-B03sU+Aow4aV=v~bK%BKH8TVom-82*EIv4=5N1 zf5?rx5G4e{85kIlj{yZ8iD-_ZV$gCxdEk!%1H*sNaU)*_pdJ7%^06H)_XlkPA;w{) zNM%1LK|Nw%U{EGT#;#;8`hqO(V%t{=*svT3=IF*GBDh0WMGJCWnkdi#=yYvj)8&k zDFXwGJ_ExLtRAt1(5MhY0s~_30+e7t=e2^4gd1i-1M=E`&?aIY28REt3=IDq85sUY zF);kkVqo}R#K7?$V!0;b*jh!h2!#_y|hX0_2;cg5J4Cv)UJ0l}#&EPN#qLJwx z(3Owi!~tys-DF^3kRF*qfz{Wd3=9k>pVVq}3=E@n0G%R$ zM%E;uwf(;_Ffh2&$gZJoEa*s7(85zt*$=}lL)}v&*!`f*U7$nWU~vheLF1sKZGY^E z0<=cE|ZM~`qZFfgDW91JQ~mN76e@X*m6qgD|P0Z& zz`Jf}y8twH25S3*(mo7>))jgTZEp=5*MsU}&>RgcZa_4s><=0?J{(w&fYytn@1p>P zF=$i1=D<2_)OH32hByWWh97ACG*Es3<%`h(qD}~a_EDpcD}(aD76t|eaq3txY6=-4 zAkV!@$6xgG%$4H%Ap4P-d|D~I9quX=`a|2i1X|LL3sewHcB56d0^Tu28POMj0}g~i!mJk8o+S+*EEK6f6p+S`}c<7+`r!p=l}o5 ziqHM~#c=NLYlgFbPBWbPIhEn~SAT~6uSFORzF}ZE11iGtMh56Ymjd+h9%e?C2h2?D zCd`bWX)wI{M#&HhW)P@81}vpVmSV_fZ^QV z8w}_F|06ryBj*7y``=%NGk>lyocxi=aQHnx!=?WW4Cg^rD%_(cSkCeUDF9t&rvO(q z%Aj`$fcDiL#8MA{*3N;>S{?dr(gSZ87@FoYFdX}0&T!_>4!rFfa9YOE_90I07sHu9 zn;4FNHDvhzpMil5T!etu;~vKn#-Ori^h^hOr+7^JK=HE|OZZ--pyI?+N6Qvw!a}ocs~QC?>1Oz{rGthB>I*2c7Q#suM6{V^m}yLI5-- z2bvEC&542H2N{Fv2h)LY6h(G|+B`G&vN0V0R>W}r-v_#-eONtl;r}bfl-e^4Oe~=D z8<4`2@dX0|gYW1$-4w+xEi{A9kN~a21zl8t6kp(eG3Z>;Xa)ub(0U?T*iBtCw>@BB zSbBzq;na^uzxpx+nA3)_8=v-o#K{-#*xr%|| z>VF1?<6mtU&i#8lplKhJ{y`X&2M)Y`%@C4N%Fe*R%Ekzq2B2BojT%gy5CGjT;K{(i zfPS_vC=Y<}YX$~}Rt5$J(ArY!*hmdi_Pt_Y*!e`1;qt6pe zFr4~X!EpZH-$9lBL3!Z(zuycef8>D2MSIub@yMvuAPWJ|`Eex-3=Ge)jFE!!18Dsm zXs@>IfUey=@{xgI{~Jw)vw!XmzO)Y=FFo_;D#N~43JeE9M>`C%SRQo_z7PPlhin)a z7|fd339zE?23{LjF! z`x(CQ9+exMAt1!SzyP|F9CQykdisa)LF3z?d3{T|?JGF`71ZaIV>tKk`ruCg|Nk?b z`+JVz$S3j9e!$>N;6%6zG=O2iz`y{yqXBd$AuQda)1XQobf%LZP46sEse_F9ocmGXSVI6bP6|4+Z#VjyUbKD!=7Y;h;5N|z{|u*oP6wYsvlO%qX0S%@sJn;^0ni$`7zPFg(7XWX zylM1$0LBLuc%Zh?MrJ0KOlD?g(D_7^uYCl~cb)kyH6+?Tu=IcS?>UBp?}Ql+z9TY- zN0kkM5CHWJK<5qZ!!dsh%Lo4%85luhr=am9&>jTPo=j)xTmtF)4nTe1Q$K7N&j0^B zWXgZgKAV%@4H-^>ii;r-v!hNVB?Lg@rJy!HsO=A$CxfSZw004)IA{zNbXFZ`V7!ij zfguojf0-Dxy#(rWV0Xg>(3tbnFZZzoWa1r0J=y?1$y0Q$GqAZvAIq@B>}jGxQ^S)CV|2091B^?y3jv{{!v42c6B1e&!jz zynrqTTAvEi4?5TJ3o`@5ImWeD*D`?D7|^`j#=b8YmZnLePyK9Vc>JG%!7UtTz>P`_ zyAS})qk+~8fyRwM`^!M%LZEUU)E_`k=S1)%V zKR|;gzZn^rUNFwue{9I5|I@!(8SejQVCV!56O4rD9rYubApkm)3v?DfsNDowj|$q0 z3A!%@v_9LHnSsHLantRIL$3XQ;(HOp-Tw>>p`b23nISW(Z8QX~{AXY|`8|1vjsN^% zIQuu00W@|y8iS*Nk`Ork3$*slf#KZ0FGD5|ocsHp;mAi*hGU>1EJ{2-s&~YP0B8-t zfw!^@XaC+FGU@;9pNkCpUyCw;_N$Hf2pkOnib7z=69$HL*I5|O{9ZL=(*K#?a~Y~; zFfoA6-lfR%qnby22;{aiFx>plz;NPQ>fqe>h&Cq&I$!t1x5&};&m%tNQZE2Nd&&=g zRAK;~yEfqST4?DV-92aiTxQt&LW*G@Xxfi@K`?642o8b8Cm9$d6&V@M{O%p}ZJ;y1 z8^L?vmw}c&N^*VJ!0JIO_)DQhZbM6m%xewx>`*(-o_*XTC6QHGl)b;+TX(K!Y zc0Xre2+w6?IQe6+t_A(eaO!6nxa{9Gy7zsAr&ii{A9M!F!MD;3Xa5`@bnT$CfA%pP zd?(6q6trcUHr^jKb%cjN+cL;GC?|ioF`WDNYS83?vwt5m9RF$yz89l)(Fl*EQSZ|< z1hzk7V3@KEbjMLH!?}Oo20Ra(`}c|A*q3CmpGU|0!ND*}&^QD@^QznKvooCi+sJVK zKl%52f%>eJ;B$Y!F&zI|!La8!%jkJOG>-F8<55H4_!kC-!ygnB6ciZF{AnDZs^Bxj ziEkAQt1hxLY`Tr=yiwk02+$)0wm)QG*mRqn;p7icbLb`AnnP#*K4UofBZcAkSJu(t zPI^T2s2w;%0MsQ|f1Qcp_*YkkGrvJYL$o{|8gv!pL58EB?HFoiF)}PahtpZ35~CqN zw-9Jv$iVRSKLf+wmr@KTe>5?i`}c(Eb-~#`4;W7UtYH9Ms|`BZq+>bVyfb|0Jq&06+@bK;&_4|4{@r9a^|PH}|7$IV>AM*jR)KB}A<8wQ zibg|VU_xNwF$RW-o0%96y;o#7{VSQ_%O0c-)Dw1e{M3I`Mr$c z^sjh^V_)PM_P=HvJ<9nXE^aaiQ&wjdWN%q zCNmuWI+x+Zw>b>w{!L;y^|OxQ+_to%QQ_yb66V~96EQVlr1kj;S@^B*jZ6pw!xAc+8+ z9+1R8K#chh7KbPSC!z;XH-p0zNxT877c7pHVIZ#j{~sKvh;;i0?AibS|D&bz9}Hky z{{IJsDvbUA|Njqgub}3m15j)Kg8~Sy=Kuc&xDyVb59(WpgP`X8 zVBr4)^)1Lz4UACE0|v$)P$eMo|Nk2yR09J;0o*wtA&5B0CkjX*_5XhZND|_6xHCaQ z5OHuwp%g_2K$0K_Ga!Y?|Ns9X;-DCUx(uuiCJu^eG;y%0;1mN@385Y!#KA0x=>Pu^ zac~O8C60*E|4{WH??RF?%$WZVK&l|(DEjuM_o>WLQz2QxJB zL#+J|HisJGAYWoL2cI~^zo_m9sV7eyq!a3^|NlRLcv!-p2yswMpqc|I=}^VNJj`?g z4o^_VM5|gM;-K`2nw}BjsOc6W4oc@J*$xt4;Cz9S?V#eI2tWx=s5mG-5y79@TC z2YC$a5x4{-{ewJ!lKi3h0#r0&6~|~vK+X9Nvj}Fv0ho4(2Vug{d<`n~P~8d1@Bjb* z|AFc+NV$PW{10mQK*}XhS%)eP5A}bj;fzZhp$e`74Kf3SkvJca0}e_22Xa7wizSf8|Ns9Wy!ikB0fZy|A$u6f(f^P= zjOT$A%`jc0~|u& zG6co^KVWBIivI_-zOj^P{~H*fjR&wsuv@^y0|t9Ym5UM%poR%MR`Guf4E3PkWI$8@ zzkz`ft2j7pKn?|m2-r?A0SM^}73^6EfnOXYejysb?!+aIl71lSG1Cu39GngzOcWBAI4I7*2@*v$Bt((3 z7fc!ym&n-*CJssvn8E)aOPvDM(STMZLWTc7K<~RiB|zl{l!;1xM&*w31}FqH7#JA9 zsgjX_fssK0oS;E0h7<+{FoueZQllX-8UmwWGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ON zU^E0qLtr!nMnhmU1V%$(Gz3ONU^E1%7Xr6pViXvzwGS0|FW~c{?E?-_dh%PKNSD_m!19nUv~DBzuDP4e`jYG{LIW!evqEQ z_$D)JK-@lRDfuDrKRcV@XLb(5m+V}g-`Uwof3vgC{L9Y%4UKD9oa3WG>EZ65?ChE^ znOPG5^Ya)UCnl2bpi!N43xNxTWeoqbvlxD4=cxV7&R+RHJNqXI@s6G*{{GF*KKeU5 z+wMVlD8sk(bh`Ot)FRSD;B86@!~en}hVPkK=6|!Z&r%TYDCyu|cJ|{xSy^E(l9Cv| zXJnA>uu&Z}4}l%ktqlLOG8uklWgGp?&b~~oxQChZFFX6?uk4)Q|0%Hyud}jg?w?T$ zNDF~)+1U(Vva+TBW@jIS#Tm8ff7#jheq?8x{m;#1m>fOY9w04-Xs7yhdIrP2)HLQl zS=qhRj&pSL{$yuw`;wg_@GU!sb{-lvnWPZ-o1Me(D?8ivUv~Bz^!TCy|M#z~tn~i{ z1q=`3;z)AmsAB4e!28@hhNpRjEPt}I=hGmrQQCnp^Z#aNpZJuOCGsh2bZmh7F-n?= zzp}C!er9E9{magN3X2~)&_90X_Cm3{#X|8sH}erIK;p^fKZ z&9~G{7k{#|r~Yq`Ww@U)+83Z&gpz2`tE?=B%%E_FKUrCGs2R7!nDs9^`}pUqY`)K< zeL)gq4X<(*1_lN`1_lNh1_lOQ1_lOO1_lOS1_p*`1_p*S1_p+F1_lPu#7r3j14A(b z17i+98%O%3h=|Mob8={&=aJI|XiVsPR<`nw(fG$3p*SQN85kJ285kIp85kIx85kHc z85kJ)7#JAVGcYimWME*p%fP_!nt_4g3j+hgF9rq%(5^Djra27E$jJDAzO&1}|GBwz zjsO4I+3){mXB+>^9xeZI#4)-sF9QRE9s>gdsM9osfq~&50|Ucj1_p+23=9ky@k*>1 z8xzz2Ej~W~=^g+7va>(`%+0g(iz^TrStyH$~ul*H`9atFFX7B z&#WvBQ2!IdVIxm~63 z<$qQdL)9Em5YW%D$wXGi?c&i+A7gZAHEA(iu>c)!NL zz);G-zyOj5jpqzl7<|pnX84?y&GR=qdp)h=AGEgUTUM6R2pIQcXJB9mgSOd0c^z&dmIOI4J0Up@G4FLou-{qFh{(GCZK{L8;3IrGC&n`(C7Y2c-c}S^%{H z2Bm|Eb<~TDEQW{anQY&(bL{?R=ga{0+yCL&5A-iP`_KRE?58i2lGZEo3EW|T%u9Y_ zWMFU`&<+lUhe2nKeq&}}_$SH0@ZXk!;eR{>!~aqShX1t;4FAg*82%y|Tr#@%rFuc#rXa16vE%zfQ$LCLWcIBV! zY*2bw@h>}jVHhJQ^@SO4GK!OFu z*$fQ-uQM?G|H8lkx-tQD;4B>fV_^9Im4V^^O$LVl3m6#w`Z6&5=7TJy;DALas67W- zX8|@8y5<5jrVDBV!qf~)dUs$L!wSzphM$?4Og}O*nC7^6FuqAnLiei=0|Nu-&|gsa zfDREJzzP851W-Qy@4&$De-kv$|AVfCAQFFLVEDg-f$@(!1LI@Rd|My`1H%)f^nDI8 zzBjl^IrIpjGaodr1*-!C;P~QDt!@%(W4UzGUD{Z`GV3@QFy6y&? zs2CU+4>2$>>VQ1P$cT5k9i(I=VNgB+txW)h6R5%sr@!YxR{s}aVE8wef#Lse3gaG{ zCXW4QV9)>?%)r2~n}LBz1>`R#Mo|zoicv$L2C0k$tzklyr6~`Tbr14H<82F5TG28ML-QROuC%Fs0&)CK_chd}uSH0KE#6Q{jz7{4+wFueh_ z+v^z^KsQv78_&4({rk_r@aqo)dl48Ljbr=e+sIs*g4GiVublYxOjiuT?B zB{k6W+CNtYhW~G=Q|9AJ2me1ZF#PcccXQc5%dTnfyCG(S00RR9XdDa_DWLfe(3&+G zdjXUefAKOf{NGOFxQEsme-AJ)eG+0|0xi9zvDb#4@u0EK70~<-s*poz>jhX<^AD6K zL47u89O2Br)R6zh!1yPMff1C%LBl_^_2CdS9W?d>3LFpytv{!s7eH40V`gCZzlc70 zAD*t(F);jQV_*O!aTGEt>dn zs2VEBz`y{S9|OhzOXxax>idNm)b|2e@xN(6;-B$fA2|NmZd2cHL)%2qyzfzH`44Ib z7}3r<;P}rSkoaf#R|$@Pc0byAZ)lnf+Viv@8vmfe(Sml~0LQ;K1H=EHbWQ_*7#RMC zf#aWco*SwrgBHx}hQ>c=ucI~XJOb+f|5s*U`2Ud3@&AH>;lK81{9}z-tpRR2di;Xg>;!WFdqJ%98)Y85sU=rcpB3!@%%Q1~T&pvI}DCD2WjQ<_ruBAE8wM zs3`y%6~a(NeF0FMfSQE=K&|sFR9*=RihDIkvk$bi=MkWR@K$b~@B4iDHPdWiv z30j%)M}mRj?{o%+|CFu_`oX~PZ$1OVUs*`L2c@rJ6jQYE3h2a1P(*+*=rl^&*n-zo zP(1zRU|{$k$-wacG#NVrelaloJI}!QJAr}m3l{@ul`&rXMkSF#0CXA#DDFXc0Jnkv zWnf_a!oa}zTbzOMZwdp$zwHbR|6bx*B@XJ^zh+=~cani&+7<=|-DwOA%z}_Zwvl`{ z%ET7}pcCw1GXS8|ia{q<<5N!uIZ!eLl?OjSgHw#Zbr=}_M?m-gPGMm9KZk+gKWIJ| zw0|Iqf#I431A`!F-xp~7@;d_qWANyv50WAYbWS;_;|B_M(D_HS?*NkIJfakX&hY@b zAB49+&tE4>cV1_n^t1;sz892lMGB|Zk*pgliO*n!x@ z>l{2PK_gjRNd5pFTuQ^WjDyGH6uJZyMu(u`2I?k*_5@OBKD*(U(|rP zWawB7TJP}=8fG`3chd|V*JHZ05$YFES$2Yffk6UO$*2fY2-G2k*IWh$hM|5w3g}#? zOHe<7x)9Y!ejR0^h5+cidf5I?Q2P&bJ`Spip~(a7X#)8NR0e>~fuX~`)*%^Cp!f#W z`JiwD-CY5?&t^z^1kKT)JJvQq%K(tyK>M2Lc4j-8TLvo^wC)RZ{}3pAK)7MBI+RRz zfzES>ooxVGcm_Jl8`MW2)3{-$%>`-34^&2i&Pg0jo&?1;Xg@P(Y6+CSL1$bIf|1QZ z5yX%M5e%!5ib&9PRztn%hb?SDM_j;;xdJ5vP;*BKo5B$$209`SG)x8x6HqZp<28Q+ z8oHos89`lJkY7P~B?AKkXwBPzI%)td2W{~L9eEB43lIiP_zuUsj~v9o3=9mgVGhtf z^j-!A22c|SS7-~L;%Y$?-IEv?7+`51 zRR4h%z6{QDK^GonVEFT&f#JX#ZibUTj2KS;PGUIoyNluM--Qh4|F2>=`*$hBnLm>l zPXEehIQheo;mAi(hE@=*U;l5f7ln*Z&zAm_Y46(C|N2#lut(RPQeV zZ9svxeL?$OKv(V#o~u7*AAqcdJ@H+F;qY7Fr54Mmf`&We+=jU|Hq2Y{rk#r z_V0OyGryY|j(yf<*nF3XVK?ZeVR$Tp;&}no&;J=27;mvKvFfpaj>P2xjh4XGjRXd0 zmpUDj8|9vGn-a&DXz<(Le{oT#5=LM)=m&d@s06PB=7Z_TWN+fzt`a{=MnLXy&gc8zwrM* z!-2Pt84PVfd-_0e|Br!z5j4dIsta(rW~7OO#)m-tU(j*^P#A&mRR#tI(9{Th`UaPXNSfeey5>N9seV_;Zyk(uGl?>2_>|Nl}w{z2uz#sB{qHs8L^m{6w6kk>(dKMW5O zLE{6UtDRx{U_ogB)HVQJ6$rW-f%(sy`78V!5_TKWuXG{f3UpmYFYgZc}0RJ|hd$VUc-L+^PQ&i>g-)3^u4 zKL~^R1xG*YGMoUd@uxw^4Id*x^#N$tbPP07Qev3_xW>7ZW41Gb1A-XzYNvv~cjltwI%o_Lw1f_{kusKnfx&=*fkB9YfdTtC%h^Am_%~-b z|NjGB;~!K8ocUeEaOFP(L-S(tLSkg=0+k1#^Kq&e7#Kiz%77YEpt1lNgT|9UWd$fb zfW{a>bqHuvEohTHsO@0P$iSe*xc){8!-fC9=^p=Q|4aq90cPwO*)d7E??L$+bcJ+3 z^ax$h@rlUk0G$mgbU}Ry&{)iKrlQ{64423l-v{MkO7PjgD;c)jWn$P1ng*sk5JpX4 z0M+@RrSzcdhD#V27*;VbFkEI}U;v$Y42pB~bO7T2X3TE6G$8SR{{MQ0v%gsw&Vrma z8hca=0Y*?V0GkG?^FjAb8bZgBiy0UgCNeNEfYzPuWME(b9k6qRF~4gU!^MHf|9_S; zOx?l6uxa!tY;bJRj{uD^g05KPW@2DaP*7mp{Vasx{QqC{Zvp)6Wq9%*)B~lT-$!ji z4FT}1&v!kBbN^n`J^oMqOk=qCpMl{Ts3Jmj)+mooApjbIIsAd2;q2dIbd7(|Ov8!q zW(=Sa2s(Ln)EX=yu=+9s!{`4D3}=3~&^7+g{yD&K;4Lr1LC|0}7I%#b(Jur*qi`pG zm@u6C_mWO&;4j1JUpWkS|1&U*_WtP?&$w&?t@=6io`vD;ANtG!ocni<;p7i#hEt%$ z@3XyG93OO%W(2L14B1xJ_^a&QRd(Z z0nnnbJe>G+} z`=^=V+}~r+;bQbDgFg)C{(WRP`}YdN*}t;m@-+;zvU_h||w3k>ONj*9r zw4an8NuCfNw7d`DV327HNFF!u*=d|-fx|NjAT#t#Vp0rLm2asU58_zesPpz-|h0n`~FLqHhB|HA-L2IK!= zKt%h8|NsAgKnk@F_5c4rK$geCKY(N&NIf$DM?KhA3?REe7=v$*CO-)HDDJlh#{x9i z|NsACj~X7>_{iZ0lK;WL0Ew>u5dD7`K#2zw8vp-;!s8!26+rm^8z89xqyiM4{~w_7 z!L9(w{{Ih_N9Kcte?XGM|Ns9%@%{h*A2j|yxO*V-|G=per1c<(`2QcB4@rLi|Nn1> z$TvbLOg>cM0hm10pa;-m17^_&X!`sEbqFN={*#9}6q3FlFv2|Y2W(CQLjhFZKaf5D z7#I|w=Kcpo5GcFC_zj@21|16vbqFX!d_YR459Cqf@dG=U4>AU<;}0Why#HgU2jxb% z{Qm}iP;P~%>;DHBQTZPjysJw!?7i>ENa&ZbOJdpXIY>8SN zqnQ66H6MYj1Qm!N8itY6HAoDdkDSjz@*j}$J(LfQ1`r#Bk<&GZkDRVSeB^Wu;{R_z z1Q(e902GU0)=(le7#J857#J8p!OO@1y;61p0|Nu-ymAl*#p@^@4S~@R7!85Z5Eu=C z(GVC7fzc2c4S~@R7!85Z5MX$nmCbM=C5_>GRu<3CtZc2nSy}GCva&;eXJ-ff$;z_( zk)0#|E+?1me|`bOudM9R5TjBEyvxjDxRQ~>@I5PA@ppE1>A&pk1OKwKU;NL`{_#IM z`|tnk>_7jrv%mby&c6RQJA3t?tnBdbS=s#mGcy^krjDfhw5Sx|NXGok$z}MKldGWc zCp$a!Uv~D5|Jm99{%2?ZCxCzc&CcHOD=XXPYeoj+&rHx^i%6!9GKmU-@0nQ)pEEKA z|7K@Th9-7G;f^l<^lw&H?3eU(reC9RKvbB+6#dN3VfdDv!~Hiqdm6NmK@VGE`EUPZ z=S2Najc53n1G>}#X2U3rD+FGpXEOZnX=V76on7!hJNp-bvW8$h{L9XM@H0ExIMEb)KY*_Wsh?#O2R&CZ_q zKP#T$MOF^Ts!@z31VHKYcUG4DzwGQU$YDnf_P^}x6W_A3`M-_4?iuL3e9)Q5>I@7F zpmUK!7#J8p&2rGW>Fo>*44{cX(0z-O85kJ4iVO_a{mRTFrln7?_C=1jf7#hjerIQ^ z{LUWM-DA+f!~6^k4EhWV44`{!IvE%kHZd?TfbOPv!N9-(I-?#mO%6Kl6D#&nRQ&%d zGn1sSMGj{??00{1at#0G47>6NbPoXNK4s9o@}N7VPe9N6Mh<7teUhMi zF;F}4S9T8H-|Xz3|Jm8!hzM&)9OdNv|DTidA2jm(Cp$a*O_v}I4=K{6GK@J36 zLIq0mw-^{0qCxvk=n>BN?D(9O&G0EJoBelocHCdcm>g)z7ZT1m`e@gpqW-s8Sp1LI z)T{%U{W3QfM9~3*f(}${{9|EY_|L<@@SmT7;V%~h%Nr&JhK~#kNuaJKhBJ`wya)AB zw=gg;=whfEK!HaYSq%UGA7=QHl`ZopJ0}}7y9??|{m;(+3Z0?(`7b;B82t z>>SX^v_G;oY`<9*|1#vbbsEG#h=Y0kS2GBLy5Mip5py2-x zO75-<4F5MVF#P`r+U|h^|6^eI|B`{>-#iA!{{{>U>@o}t4C|mt1#|~4=xQ!dQABlD z4uFBm3=9mQyWK%eH$U3g589pcmydzre-i`4|Mxh;{6BW_ANLs;Y)(PL{xt&wVn7@mD)VEh`xzyR863vw#e zTsasF0(Bxmp1RJ!z#u^#2Z6>IK}P=bB)!xDg*jGylY!y41_J}AM5B%?2Zoz0!C?d!bjv9y&E_>J*82)djLHPe?VEo$#4uAf= zlsk3sn9v6ef6&Anh0SYF+3=4a+OY+tSt{dyix^m*GBPlL`oe=J04a0{XpsvjQ*WiX z4a)S3fr06lFayKCi!@394F6X#F#cjsKMAkbk0R|1I{$yuh__u<};SZWPfI5`|=M6rspoPAm^naLv zfq|QH=iPvY(O)YDhX2o~69@m!Ffja;h0McH?!rN40<8VHjpFtv%wvq-85mgZFfuUw z%VS{p|DD1(c)`H%A2gQ7z`*bZX2WozL6dr*^uL(8W9%R!|FbhN{I6$V`2Us6IC#mx z@IQuu;RoX|P5Yq5kgK2t252EXmD3fdF8&Lu3(^=EKqo$e+mOVxOaC%3{5!|M02;&o zJ6yva6xN{j{&8sdgX#g07?m+-Oz0P=EYf3O_&1e-;r~NCb22{}7(P8@VA#8cf$_8o z1H)tn24>K_3zeNTc#H$h0zYP8VEDejVEVz#!0=aP`~b{PTP|zegKV!fCkt=gFF^gF>!Dj z0$Kq9YIA}TW7i;ZB4{Hbs4oI?*Fvhbi3gDfp-$0;ZWIBP{XS5!!9qncFff2NQG8-x zU>Iy}qgEJ!Hiv-H*cAo_2GBe(walf4IiN-S8=z$)Xkou7H4Gg@W`NqA`=I^<^}hy@ z)8I}4ZG?s`c&?>x-w)DIPDP`$ev z>aUFq;N^YL^R@@255yQ47(nywprR7AZ+bAhmDm6TRcW80d+q&+)ii+WKwUA=Vqs7m zfQG3+#peJzha#&%%{y3nT0e-I{e(P~1l{um>KcH~!~pHVBcx|A%9la+GrR`f+cFp( zh{H{weIhLkkTsy7^(W;FG~YXd#U=aSFfi;ALQ70PTGQjjMt-GJ%du zqu#P@d>%jYk%8gBTV{sSzl<5q{^?{m_wO{rxqq)2&j0_xaQ^=?B6wHhdnGWfZ|}|t%HmOJ)(@Ype^I%J8KZ?1hpYRU1Lz+ z3{)q8*1v$xt^!SMkk>}t^^}3(`hNz7)4x-pbt@u0!@`tQ`ojMo3}=4FG2Hmiz_9u5 zAPh!|odnvf3M!9aXV8Gk+GPw3436Y<@z4BbU^x9tgyGyj@{1m9QFQL_VTR*h1sP6& zTF?}Gc`zCPnlW=?U|;}klLvVcR4jp_0(8V4Xr-kz3llpN!6NwFKL&;~zrDfDN^Ids zw%EU~45xm&GMojq90p_XQO!-D5p2*Nb5KzPYNmqX0aP^IU}Rtfbs<1I4?#zFV6ONE zHG4qC&$)kX6c_#2;_2+4dT_I9;W4Utb`To`>au}GdqG_W&`JZ)atu&hfD+(S1_lOD zvuP^ysH^}61_o=;(Ugor;+hO6e{7;j_@Dc`oMHV9CLCSlK^&A6ISo`UfzmtZ=si$Z z2vi4xX8u5N0mGoU0p*qVj0y@oHyAhHy~A*kh`tuKM2=7F+`nB6C%&;WoB$oyMUkh5 zs%B8fA9Nn52Ll5GXdDT2#s_Ho31}-Q=Es<-u?kRM}9)rNh$@P!H+EH&O6F zGd%35P#K^5cZ}iK7a@jYpz-tJ8FW+u$GW4PNxqoLV$^&Qro?B>(30=0ucqd|rY=l&ffD-O>6J;-qKyB@gBv+}}74^h(n56%aF zL6eYj3}^mKW;p-vEm84s?%x}RvwtQqocJcgaN$402pe%AJ)8;YIQgA{;n){ehSR^S z8P5Kl%y91CC5H3=K0qf4{(|Xq|K2m4`*)Gy+~0`|r+!*8?0?P5a0IlBn2_P4@ovk#!Knc?CM z42)RW3`pXjy;%@N;Ik(nOwiIg&>=1$9;90ZV*UT$zyMl^3TFRf;0Mp-gE&7JKxd6W z?0UcsQU+lkut%^#UOK=4cH2J?33c-ikT0Nq`5^!Q|NjpRAd>w-{r~^}|A74agW&+k zj{l7O|NlRLIh_Chf2d*q|MP<^0U7iE|NnN7L!f~KX2Zq*F+eN_slmnm16OnS{~vfD z;bnuo^8f$;!~YRxVz8lZI1F+dG8^j7!yxxV+3g^IfdT;Hmj?`>V1TfHAhG{*Kodnf zD3qXVkP&~NdO_@e$ZU|n|Njsnh#QdE5N*xiaQ*`}?jV@`1I&H^W`6)X1r#rT7$NKr z_MqrW0L%W6N5sw_q}Tz)bptHq8IeMuq5eMuEZIJgM}*!7W(4~WgB?6ZK#2_&`~N|^ zgds`n|9{ZVZ5aE113cFLBgF`AHo^`6k=*tlDMtPy#mIkU^E0qLtr!n25Sg>$;@K-kdevs zD=S;;9xH1{t%JNaiu2FvfkeSsxWVGK&||8ufIm#g~z%g%m@&+kZbAOB`&C;ZRJVEB*) zx~GvSs|KYa(50Hcv$JLYWoMs5@*my{9YN~;WoO_0ot3Q(x*T{=dW;B1y~)gC_@A8( zx?DORcye4pRRwJNx9X z>}-KSdWkb=V=Cz24p5f`wD25s-X`e&GtkB5pdA6A9ZOqe80PVay!@$4*x}*m*g7T7qff00(Ea=_=(19SJ`!_aA zaB&^D5gq-FO#i?7lbxgWcVOEDpm+w|vI+7t=ztT@B6H9w1Q!_?7#1)vFn~79x-u{@ zXfiP|i!(7Xfp$uPHfn;>KlA>;K(W7BStONxur%^7JNw@6>})wu89D&*5AwGvv?~re z1RS*M4s_r`4FdxM=l~ng!3Cgvj?-yh)6*GVrKB=}E{TQti4^)@cJ{XKS=k&v=-(Cs zZTQf$p>erDd`#1eM)Cv$MJWWoIu#@*BZ$_?Ml%^H)~3 z@GttsJ?J8MP}vPy$h`x4m=4+gh6UFBxHyJ?+2Hn{^1tlt19-y#bdh*&F6bih^MA9m z4M1`KB0U3U6fJ4c*?6F`9JKMjh=Bn-22M5mK$m)h>aaiA*@}O&v*-QK&i;fg9G)g5 z{9o?o_F{KHfMv6p8N=t?d@|htO1hu{e$ZLr|5X?m{yQ)*{C8tu_-o3*@K=O^0d#&f zXdgHDz@7*O1_sd1Ns#|R2l{}@Ju>aZukBk_HpADfY>q$K*^Zz~=0R!ve|Glqf7#i4 z{^#aSNY~Ms%*4nDay$iR_<=@||Fc5R_+P`o@c%Ib!~bs#44_fu{~s6_{$F5V_}|FD z@I;A$fgu%|&q2GXL45okppxwKoIHj{Ir)s=v$EO$WM%Vy$;beeJ0R0Fq3sFK9SR^m zi5TSff4mF~|62*)Onvzd1A_!;bN@mH21e0Aavd*d&m8EiT+lg!piPY=_#d?Y=Qk+c z+Zh=C{{~HM;m5!KGcX+f$iT4Km4N|vWIqYk(N5_U=pCM*%i4*w1>|>d-1`uScPw`W z{M*dH_>-4`5p?bkk=D^x8K}SqrAq_itpSyF|3G(BfX;r$;z!WL4^I4zf#IJo3?_i@VPXNDAxXTgG*YR@z`y{Ce^5Fg z(h5-8|L;Qn-7x>QGBEyP8zgxE)D8okYyfJ1li>e<-V6-?e~}sgdl?x2a14@r7(oZD zf%?%T+}{e0e@hDPC|=3H_=}l=5p=2t5tIi0b&%ozCkzb#bs_#I-bNa#1f_q_!405$bV;xV zlm`BS+WMDB2m{c)Ky?fZKSB4|kYE>0m6B2Zflf1r9gg(hm4V^^eIml(4+F#h`3wyI z1qM;vg8~-R?g7<*X&?@17?cM+UG%U5B+0c`2B)`;ScB@F9Arq2c(ZQ zn`o#SbeJ`${R0|PCc~Pa3=E8*_V8bj-vSvJ{)6sl+R4BGT2b_W6$8W8RSXQ)Cm0w6 zH5nL4XkwG$5;C=b4sZdDDTBt<$ut6{4OAw8?&1O8+s4Jf@E_E-H-}z)2Re%$rf{&( zpz)@43=9mQaaPbtPqc6ss9y;h&jgKaQX!0~>O|0jFOa)PX#0?5zZnAq18A&k@*umD z33PWVDDQ&~dnC(aL}~)%1JI%8pm_{WBK6Q(8R+D=?FLtPC>XVQ0_^-{*`Bj7u3AnYadMEEY7kHywH@L@NUW z19|l?=t2R|Fw>bo77XY9?PNIr|0njd4gS4lxbVM+!94;r5dli~p!>=}^Q{bO0s~ep zgTerG=q>2fLeOEip!+PbMln`F(76Dh!{yKXc4RpB?+$iOP0B%SSY?$3XQt@jo&wk>00 zSPn`8n8pr(2&h~H%?W}|69uIO(A{OAaar8q06JR;bQtuxzx4#;9sT^R3;*6QocV1z z2+uGBjm3cKW6;1js9grCqmmgJ89{f8g4V(yq6T!r^O@hm4CnuyA=&?+@Bo#GH~%v* zfXWYqse^?DS`Pu5j0N>eK;ZzIF9o&LKz*n%=z0)MhK>~;4Cnt7ca{st&&U{5FC72M z&Hy?mY_Rwm(;c983aGpW)eoQ@I-mpcL2Wh=e#2<%eS_ib-|u9_|M`C>zzqaYI>NMf zFp7ZNGz{Q9I-sHgG>x9Y7?IP$aPIGCvV!2;zXJ>>e{eB?j-?rl9wy`_(9u$-f5{P5 z^&uyH82jAc*$f>k7>DdJFQ8*=4!mJzIQMS>8S(#<;q0F<@ZB^!K(jxD0(%h3&pXV( zaPBYYcriPM^WeLTh(G4@+`runr+*1DfSM44&}&3H33SXR=%}Kzf6ItI>h#>dy9{T3 zTQPu+NZki&brEgb;8X}Y;_u9F4hC=-5J(5-{+(wy^Vva|h;=QF=54MnhmU1O{CQkaP$E=%5CW2GCg^ zAR2bK0gQeCJ%KK}ox{{hna|4;o7(D5Jt|I|Z{Jwc{HCs=^^=rrh* z3xqzH{y$Lj{(-&lr~W_q+=f5({||tUwD?i~AAID+59sk2F#5xP@X;dwKSGY*`2X<# z|NqVZ{~vt(|NsAk|Nk>MgHD5hy8S~v_^6WqKjfiyvP0VhT1ZZ#ubW{Wg z54sdP>Y&jOpezLb7Z)>3h)ZYKoSMzBE-jZ~Ve-(}Dh}ET^dmcm8MMXyUv_rIzwGR} z|FW}Z{mstK`R+L{qKKv_Pu}E*(txW zvRMCQWzonL+1bCbZM}hteg2!Boe0{l{xy5>q$$w$slVCT zmjAP}UqjuAbNkr8?Ck4*va=Ni!M1VG)=1D40chrzfr0HuMrJ!9*F)w1{sV3Pr*8~^ zTnbuv=*Gan02=6@$iTp`f`Nenw22pV<=?IiUS3a#-NgGZJG=dVb{50kLePcc8Ja$DrhceADE-8N zHV-13Lj^V{MS~X4Er%}D=w@JGFd=5K05k_%{LjvQ1$84nKV1Eroh=XAFh~VA!;An; zTZEKnJYoz49(JG=5^q-r7;Xi1>Z7Ku9|6B%!e|`)M|6z-GKnpuS8!JJJ`6*s)`aLU~;d53t zs0E?_H#@uJUv~D)f7#hnzoe(%E0iKhk=3SF#xI|K{=Q}vjOb>AmX;^g4UA# zU%|leml=`^@P`bAvY?_9RJ)hqHwT=9Wf>U$UnL?AKwFU$=#v8&LE#T-{ehNr;`Ra~ zXebX92me61;5z|7Y+_*eCrF2|2e}uN{y|4iB!M{iG06SkZ4DI+4FBI^@dKz9+swcK zKK=ky)8RLcIF6&D1H(UC28REW85sT_WMKGzoPpu(4hDwWV+;(j zx(p1#^m02W1VAk

USYs{rwd#Qzx>7=JS|F#HE?G2vlgZ~&FWpmK~rJ|fb1;*~`) zFfbfsU|;|(<0M`ec9ozD-9hUSLA4@w71R|4l|6GA7#OIv^#gR2!deCf26}Dr03C}3 zN;^(e3>H%c1_n@%1r!%lG#0B-prw4985kIFFY&~xji4ZC8~bGj1_n^?mY`xX6@ZQ? z0@>kBrha_dKqDhv(5)i+_!MHufz~v9_z${jP>SK)-*|>||0Xh=`!}26+}|38vwy4@ zeu9o^VE~nxn;94wr!g=vg0^O17()dC(AJJg3=9mQo)~dQ%YfE$t-Z>~aQ3ekbUE`M zv^8Dt7*7A5$RMQvItC7O+#6^+4HcY<(+E(02Av8HT8ax=o{f8XKIob>(Di_4|2Q+8 z|MwKl-H0_==l=a?=vniGK~>j`!InUy8K<*Ik^qfZg)lHMoQ969f$B-j{03SAdgeDj z!?}NZaJn6~X7J+wZwwdz2ZOJ`1uefP$vI>z25pxDErA6c69#IpfZ_v`H$cl{nHWy} z1g(R!XE^_#xOJfC{w-xV{E?Z?*T{nE8_-N4sI3U{1L!!vBMb}-D;XFVCNfk^UBz(m zKj;cXM0$aRIVOFI0Tf@Lcq2PVDbfhqz5_Zs3Dj5uwK=*NN+)b!xbXiU!Sn<6%fEqk zKdR?J>j2OGabP(A{|iy>Klg7r!;w$)PCuw_M&^Oi!RcQD4CnssBf|aP8P5I<17Bx4 zSk^usdd~oM&)L6j4Cntp$Loi4|K>5A`pL}zTIYrw1_R9ox#!qtQ2pfxzP|K4Mqd8N zaPHp}hSR^qz*i`5dpOW8#GcYtSg787>q#>+Pax?_!0w61jKqDm}7ApM{Jl6UD{{ef5c!NBI zW`NQBVEzZt!I+a2;r67KH!*0gQ&I z`w!Lk{{u6~od5s-utVto_D~vBY=J@*>w!T1dQk=j23-aQ2GA^M zC<6mSjJup%-k0=rLRWr#`V3|b)p_Z)Q?pt%IlOeSdiGU#{@&~*f$V`UdJFff3Q zmg}>Xl3M&eHT4_5@cW;g{rg{bb|5G%P{&!|v1w5FEQPLW0L`-dGcYiKjy~XHW@2IC z;t^o@os|W;Hbd%vcJ_4w?tA|?JKKy3;RhOf1C7Uk#*9I8U_lHF44~CGSiSu_JBQ(2 zb`Ime?Cc(V?)#UWy$Q7Hg2MDC!N9-(I+g(xhk*<7jpZ`E_7%9 z%g+A)H!JJPzwB%iQ1s$IWgawj`Uf<{A;`e+Uy*^~ALz7276t~;s5b-X#8S`+0ia=0 zkW-PJfR_y_olbtR7FFU*CUv~ES|Jm7(|L5dfUhLv}!%tDEQk9E~;dwRa(gAEn zfv3ShQ$%764F9Vc82+DTVEF%tf#LrR28RE$85qi97#J8pQ!${Eu0c_Z%@BfOplaer zb`In3>};XG+1ZN!8ybWa6ckukKzrXo%ee3?$pN|WAINo}vw8o4nvtmZv|4bB zi$R2-OGs1zTIK}uJ&vU}pb0pT`~HLM2RGeN-3DpCF);k!!@%$lbeE4v72UnyAHqRz#p93vKY5 z2?`pJQ^DC8bVO+q1H;=$1_s0R3=C}c3=E*@9sDMcDXYl9zyKQGmnBm_X#NsZ#Dm&X zWEz228)(cPRK$>5If2?Jpdu7h72&mr3`tPC7Bs(yBYO~SB4|}PXpRt69TBaG2!)_) zh(Jg3P&)SrnhOHWn}F61;ixLHxCXq({Sza@nco5oXa6cPocSZkaPkK$gQhV^F)srH z1E?KIt-E_bEASw#Kv2sWzmMmGPNMh+nt2vsIQOrD;rzc-4CntpVmSZ*Cd2uE3mFc+ zvt(dktYu(eSj5P{#LEc22L2{|D_nBm;Nl?><6W~nax z|Icvv!y5+AC{T(~WMBf-!PpETTMRU#585yYT2Bp{Q3kDW0j(T^7zmoZy!@Ym;oLva z2^P3#(9Zrj#&GesFvDd~$p+C%WfGKIK&}2g3=9mQ*?UmSAJpC!U}#w+#&GW6F}yP~ z=l_3aIQQ2Re5wYO9fo8aDBM790?hz}N-~ z23pkuo}vA#4=GQO4)g=hCWHAu8P5GJW!U+caX@CMPX1tExb~lc;p|@<2FNTd=+qHV zS_G|PxzBL!Z!yEEpP&*IG*1W#clwi{`QA(5^+AFR=l(h~oco)^aPDsi!`VN|49C7O z(Rn=t{rx{`Bf$^=&!K>ZH$V(j`Uhyc9%zF;XweN4eE`%@{r~?1_zc(o{~I9mKL!V| z_z!-F{D=AnAbmd+|9=3P^MU#Q4-o$W|NsA>?NJZdA@l(x8niV6#7CxK>OVmB{g8*~ z``?eCLFWHI{{KIiKK#E4bjRBNhyNMC^oKgoloZ4NA94`-4+Dh$52GJI`QWK52GHDF zJtQ1JQ&OOrvPMv^0u(&hXwZBUXr>9Y4GDfe0BChCXub)A3C6&v0&+tDG^hv~_xzQW z%?cVc`r8?t@%&iP85ZCl_}0rsBWs?AiabvmZkD51$3i0sYC!W&;g8 z5@QxQYCvZH$<7x1mz}*Cec<(fcJ`Nl+1c@+U7bIuw@VP@Oi(`t)Vc)?tMNUKPl*1X zo&6Jw{h+y#qkpoq`9bjm(ntXYb=*PgZxb097@DCiRnYvxl!>;sNAc{qef%#wTZw{k zd(c2L$WD-5prflnx!IP1fl--(ff3Zo<9n8vnD;+B8?-kcuY=4uL4H{BL(MNy-T6qcT z6MzmY2aRmtR1Vs+3fhDHFFV`oe|Gkj|Jm7pLGJjJn)>%fbo93Wxp`v$azM>?bR)p! zDCjad4+e(+Jq!&0r!X-5k7Z!+uz+6r3|fs&)M&-`>}-ZR8Q{xL<^N@8NB+;vO`l|E zm(9z{3aVtVbnHNVsefz?4F8)L82*1j8gTo?z_8&V1A}b{1H(3S2b0PNT`n+-fq?;( zf)IMyKwIa)e)nQv`2QKnZsY;oTF5RkkUE4`5?P==6DY-^4tj%k*MZFa-;2X;kUI`D zP-iC_sP7Lt{2$?Tg#FX-*ngUV;U7O0c0++y-GXM=5cWgrb5Px$$iVO)bU6{ozvy@t z1H*qNDhy_V><5j!A=wXNg43Tc1H=CfSi%6b+sy!CHz=M!>dD5Sz38A(F*E}}_Jhm@ z?Vkqcfny8||Iaco96Zawu+^D?0Tg5)c{FXLaY1F(LkSxptQgUp6&x(rpm{_ zzz!2Q-)tO8=y41`)k5*{{It`hyBR=(n25m;(7*!> zA7nW5n}-3kM4U{E(X@doQP2QfGy?+zXfU&nVfyY93>U$JFfcoj=}Qb}{|eJ=&jfUr11+tliiK!~ z9RJ1uHvimT9fouNRzTC?D~9v`jxwD48_aOx8)SP7nl`F(k9`3Re1Qi4K}Qg)Fr59X z#c<}20K@XL3=AMSs#-RD3mjD0!01VszfBvBU|M3s?|BwH(|9}3!{Qv&{{15p5w;z!IfAE0){|5~6 z|H0=zI>>_-8vp}>Ym+1c!$a`G6y=Hybv^`H*h-|TFzf7#g?|Fg3}tD3g|%g&Dbn@xF#9@1sX1|7AO zgVu2db=X0jX@(y}9gTzHRZ#5&+E5Ov9YV#qxZVujrU=M*7U4jO6 z{$*!pAuUz~b*w=PL8Cx&Ki~^$LCsXqeT)C)7#RNBF);jBV_=X5jm3eExdH9A#M*>` zcAFLcWoM`T&(7ZdF*S8>x3%@WZd+UK&3+&+f}8?xCWk;*dVOPH`2U!JVd`cE24PbM z2K<@j8|Xm1>}-brd3o$zR#u?q9O#@fQ04;}4(ft{Flb5rb<}Ra-**fQ|E$UA{(zd! zpbe79=7W~%e?>JPRDFVvW`U|BngVsnkv8^%!(Ic+x+G9If$qsB+F+LZW!p^H-(A_v7eW2zwx*R$m)Jg*#(g4c8 z=yD9;R?+ESObloLC@@_7&&;3>stXuEQ`N-Y1q*H(gI4^4A`H}DfLH>$oC(x4KKmDR zJ4rXg`TzSFF8-g#klzVf*#Qn5uuh0-!Xzlmfbtlq(W}P5z{tX|^fc(+3m%5^|JJ}8 z-k?VC?q@Fc13+%Wya!*KrpS~T;o zFr53VLh&p;EQZhh1zkVZ$8i4tL5B1H7BigtYtC@}Kj>OA3@4E#0P47alIWS=pr*;` zUo7C=lc#@?Wf&!z{{R2a@c(~1!~g%x5PX1<;XeZ-!#{Rrh6DD@35@GyqK$fl32-a}1PL zKP^I-RJKO4ic6QUh?Cj8g+1c!%%94Olpn?XRwS`uBc$odm&b|q4 z6@XeaL7*BNhXEk3f(Euhw~~Suo_KUxSrz@w%K8q~532PmA^JfjHMquMWncgwX#pD8 z0aev5&<$QJ{|gF0tt#RF+1YFVXJ>zVpPG92OM1G=w+v9j1*}((f#Lsh28KTy85pXn z7#J8pi*n#j`<)GLm4VC<`k#|y>7$^K%*@Cr%EJmOu$$`d1Kx^_La^RxpFAKxD zf4U4;{~IzWX@UlxJs254X#=7JjRY0%pe-37Hwu8NPlogV5*W_^f5CA6|9ghHhn6r% z$_p{*g60FzbRxK*X(-UNP!NNFa4^H(7boCjA{YK&X1MfUgvcVBnSqOkkKyF^exzb^ zHN)Az3JT!6mJk+VXKlR?TFC{%q73K%U x_zy+~h69ca42=&N7?^)TaQjaNhW3vP43C=`7(OsFF#G_W uml diagrams + ] + +# note: breathe requires doxygen xml output -> must have GENERATE_XML = YES in Doxyfile.in +# match project name in Doxyfile.in +breathe_default_project = "xodoxxml" + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +pygments_style = 'sphinx' + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +#html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] +html_favicon = '_static/img/favicon.ico' diff --git a/xo-alloc/docs/glossary.rst b/xo-alloc/docs/glossary.rst new file mode 100644 index 00000000..4e50a499 --- /dev/null +++ b/xo-alloc/docs/glossary.rst @@ -0,0 +1,28 @@ +.. _glossary: + +Glossary +-------- + +.. glossary:: + GC + | garbage collector + + mlog + | mutation log. + | Remembers cross-generation and cross-checkpoint pointers + + nursery + | in garbage collector, memory region dedicated to young objects. + | These are objects that have survived less than 2 incremental collection cycles. + + tenured + | in garbage collector, memory region dedicated to older objects. + | These are defined as objects that have survived 2 or more incremental collection cycles. + + xgen + | cross-generation tenured->nursery pointer; requires special GC bookkeeping + + xckp + | cross-checkpoint pointer; requires special GC bookkeeping + +.. toctree:: diff --git a/xo-alloc/docs/implementation.rst b/xo-alloc/docs/implementation.rst new file mode 100644 index 00000000..e325f88c --- /dev/null +++ b/xo-alloc/docs/implementation.rst @@ -0,0 +1,202 @@ +.. _implementation: + +.. toctree:: + :maxdepth: 2 + +Library +======= + +Library dependency tower for *xo-alloc*: + +.. ditaa:: + + +------------------------------------------+ + | xo_alloc | + +------------------------------------------+ + | xo_indentlog | + +------------------------------------------+ + +Install instructions :doc:`here` + +Components +========== + +Abstraction tower for *xo-alloc* components: + +.. ditaa:: + :--scale: 0.85 + + +----------------+-------------+ + | IAlloc | Object | + +----------------+-------------+ + + +-------------+ +-------------+ + | GC | | Forwarding1 | + +-------------+ +-------------+ + | ListAlloc | + +-------------+ + | ArenaAlloc | + +-------------+ + +* *IAlloc* + Allocator interface. + +* *Object* + Root Object Interface for types participating in garbage collection + +* *GC* + Incremental compacting garbage collector. + +* *ListAlloc* + Auto-expanding allocator. Contains a collection of ArenaAllocs + +* *ArenaAlloc* + Arena allocator (a.k.a bump allocator). + +* *Object* + Interface for types that participate in garbage collection + +* *Forwarding1* + Forwarding pointer. Supports the Object interface; + used internally by GC during evacuation. + +Key Points +---------- + +* Allocators can be reset, but do not support freeing of individual allocs. +* GC works with types that implement auxiliary GC-support methods. + Such types must inherit Object. +* A region may uses multiple arenas, but because of allocation activity + since the last GC. If necessary, GC will allocate a new to-space with a + single arena that's large enough to accomodate all objects that might survive + from a from-space that has acquired multiple arenas. + Intent is to scale up to find application's working set size, then stabilize + +Components +========== + +Allocators +---------- + +Inheritance +^^^^^^^^^^^ + +.. uml:: + :caption: allocators + :scale: 99% + :align: center + + class IAlloc { + + alloc() + + alloc_gc_copy() + + checkpoint() + + clear() + } + + class ArenaAlloc { + + free_ptr() + - lo_ : byte* + - checkpoint_ : byte* + - limit_ : byte* + } + + IAlloc <|-- ArenaAlloc + + class ListAlloc { + + expand() + + free_ptr() + - start_z_ + - hd_ + - full_l_ + } + + IAlloc <|-- ListAlloc + + class GC { + + add_gc_root() + + request_gc() + + gc_statistics() + - gc_root_v_[] : Object** + - nursery_[2] : ListAlloc* + - tenured_[2] : ListAlloc* + } + + IAlloc <|-- GC + + +Composition +^^^^^^^^^^^ + +.. uml:: + :caption: allocator composition + :scale: 99% + :align: center + + object gc<> + gc : nursery[from] = n0 + gc : nursery[to] = n1 + gc : tenured[from] = t0 + gc : tenured[to] = t1 + + object n0<> + + object n1<> + + object t0<> + + object t1<> + + gc o-- n0 + gc o-- n1 + gc o-- t0 + gc o-- t1 + + +Each ListAlloc composes like this: + +.. uml:: + :caption: ListAlloc composition + :scale: 99% + :align: center + + object x<> + x : hd_ = a0 + x : full_l = {a1, a2} + + object a0<> + a0 : lo_ = 0 + a0 : free_ = 12345 + a0 : hi_ = 1000000 + + object a1<> + + object a2<> + + x o-- a0 + x o-- a1 + x o-- a2 + +Here *a1* and *a2* are full, while *a0* can still allocate memory. + +Objects + +.. uml:: + :caption: objects + :scale: 99% + :align: center + + class Object { + + _is_forwarded() + + _offset_destination() + + _forward_to() + + _destination() + + _shallow_size() + + _shallow_copy() + + _forward_children() + } + + class Forwarding1 { + - dest_ : Object* + } + + Object <|-- Forwarding1 diff --git a/xo-alloc/docs/index.rst b/xo-alloc/docs/index.rst new file mode 100644 index 00000000..de643009 --- /dev/null +++ b/xo-alloc/docs/index.rst @@ -0,0 +1,17 @@ +# xo-alloc documentation master file + +xo-alloc documentation +====================== + +xo-alloc provides arena allocators and a generation garbage collector + +.. toctree:: + :maxdepth: 2 + :caption: xo-alloc contents + + install + introduction + implementation + glossary + genindex + search diff --git a/xo-alloc/docs/install.rst b/xo-alloc/docs/install.rst new file mode 100644 index 00000000..a61d9eea --- /dev/null +++ b/xo-alloc/docs/install.rst @@ -0,0 +1,120 @@ +.. _install: + +.. toctree:: + :maxdepth: 2 + +Source +====== + +Source code lives on github `here`_ + +.. _here: https://github.com/rconybea/xo-alloc + +To clone from git: + +.. code-block:: bash + + git clone https://github.com/rconybea/xo-alloc + +Tested with gcc 13.3 + +Install +======= + +One-step Install +---------------- + +Install along with the rest of *XO* from `xo-umbrella2 source`_ + +.. _xo-umbrella2 source: https://github.com/rconybea/xo-umbrella2 + +Minimal Install +--------------- + +To build+install just required dependencies: +``xo-alloc`` uses several supporting libraries from the *XO* project: + +- `xo-indentlog source`_ (structured logging) +- `xo-cmake source`_ (shared cmake macros) + +.. _xo-indentlog source: https://github.com/rconybea/indentlog +.. _xo-cmake source: https://github.com/rconybea/xo-cmake + +Building from source +-------------------- + +Install scripts for XO libraries depend on helper scripts installed from `xo-cmake`. + +Preamble: + +.. code-block:: bash + + mkdir -p ~/proj/xo + cd ~/proj/xo + + git clone https://github.com/rconybea/xo-cmake + + PREFIX=/usr/local # ..or desired installation prefix + + # want PREFIX/bin in PATH to use xo-cmake helpers + PATH=$PREFIX/bin:$PATH + +Install `xo-cmake`: + +.. code-block:: bash + + cmake -B xo-cmake/.build -S xo-cmake + cmake --install xo-cmake/.build + +Install remaining dependencie(s) in topological order: + +.. code-block:: bash + + xo-build --clone --configure --build --install xo-indentlog + xo-build --clone --configure --build --install xo-alloc + +Directories under ``PREFIX`` will then contain: + +.. code-block:: + + PREFIX + +- bin + | +- xo-build + | +- xo-cmake-config + | \- xo-cmake-lcov-harness + +- include + | \- xo + | +- alloc/ + | \- indentlog/ + +- lib + | +- cmake + | | +- xo_alloc/ + | | \- indentlog/ + | +- lib*.so + +- share + +- cmake + | \- xo_macros + | +- code-coverage.cmake + | +- xo-project-macros.cmake + | \- xo_cxx.cmake + +- etc + | \- xo + | \- subsystem-list + \- xo-macros + +- Doxyfile.in + +- gen-ccov.in + \- xo-bootstrap-macros.cmake + +CMake Support +------------- + +To use built-in cmake support, when using ``xo-alloc`` from another project: + +Make sure ``PREFIX/lib/cmake`` is searched by cmake (for example include it in ``CMAKE_PREFIX_PATH``) + +Add to your ``CMakeLists.txt``: + +.. code-block:: cmake + + FindPackage(xo_alloc CONFIG REQUIRED) + target_link_libraries(mytarget INTERFACE xo_alloc) diff --git a/xo-alloc/docs/introduction.rst b/xo-alloc/docs/introduction.rst new file mode 100644 index 00000000..7b5333cf --- /dev/null +++ b/xo-alloc/docs/introduction.rst @@ -0,0 +1,268 @@ +.. _introduction: + +.. toctree + :maxdepth: 2 + +Introduction +============ + +The ``xo-alloc`` library provides a in incremental, generational collector for c++ code. + +Features: + +* *incremental* - can reasonably expect short pause times. +* *generational* - focuses effort on collecting young objects, + on the basis that they're more likely to be garbage. +* *compacting* - each garbage collection cycle evacuates survivors to contiguous memory, + so effect is to defragment. +* *collects cycles* - collection algorithm naturally collects cyclic references + +Tradeoffs: + +* Application is responsible for spilling register values and protecting hardware stack, + since garbage collector cannot indepndently distinguish collectable object pointers from + non-pointer values. + +* GC will not spontaneously run without permission. Instead will set a pending bit, with GC + occurring only when application releases it (e.g. when stack+registers are known to be empty of values + subject to GC). + +* GC implementation is single-threaded. It cannot run in parallel with the mutator (i.e. application code) + In return this allows GC to be only lightly coupled with application. + +* GC divides each generation into separate from- and to- spaces. A collection cycle copies surviving + objects out of from-space. Once complete, the entire from-space is treated as empty, and available to + become to-space on a future cycle. This means that at any time only half of allocated memory is available + to the application; the rest is waiting to receive survivors from the next GC cycle. + +Design +------ + +Garbage Collector +^^^^^^^^^^^^^^^^^ + +The garbage collector supports two generations, labelled *nursery* and *tenured*. +Nursery objects that survive two collection cycles are promoted to tenured space. +Nursery and tenured objects are kept in separate memory areas, instead of being interspersed. + +Collection cycles come in two flavors: + +1. *incremental* collections - these collect only the nursery space. + +2. *full* collections - these collect both nursery and tenured spaces. + Full collection may incur noticeable GC pauses. + +Application Interaction +^^^^^^^^^^^^^^^^^^^^^^^ + +Application code that interacts with GC has several responsibilities. + +1. application must explicitly invoke GC, when convenient. Since in general any GC-eligible object + may get moved by the collector: once a collection cycle completes, + it's up to the application to re-load pointers from memory addresses + (GC roots) that have been shared with the collector. + +2. application must identify a set of GC roots. GC preserves everything reachable from any GC root + +3. The collector needs to know how to traverse GC-managed objects. + We teach it this by requiring that such objects inherit the ``xo::Object`` interface, + and implement auxiliary function detailed below. + +4. GC also needs to know when a mutation alters a pointer from one GC-managed object to another. + In particular, GC needs to track pointers from tenured space into nursery space, + and update them when an incremental collection moves nursery objects. + We do this by requiring application code use a GC-provided assignment primitive + on GC-eligible pointers. + + +Example GC Use +-------------- + +.. code-block:: cpp + :linenos: + + #include "xo/object/List.hpp" // polymorphic List with GC support + #include "xo/object/String.hpp" // string type with GC support + #include "xo/alloc/GC.hpp" + + int main() { + using xo::gc::Config; + using xo::obj::String; + using xo::obj::List; + using xo::gp; + + Config config = { .initial_nursery_z_ = 50*1000, + .initial_tenured_z_ = 10*1000*1000, + .debug_flag_ = false }; + + up gc = GC::make(config); + + Object::mm = gc; // use GC for allocation of Object (+ derived classes) + + gc->disable_gc(); // gc forbidden + + // tiny example data structure + gp s1 = String::copy("hello"); + gp s2 = String::copy(", "); + gp s3 = String::copy("world!"); + gp list = List::cons(s1, List::cons(s2, List::cons(s3, List::nil))); + + // tell GC what to preserve + gc->add_gc_root(reinterpret_cast(list.ptr_address()); + + gc->enable_gc(); // triggers immediate gc + + // s1, s2, s3 invalid. + // list at new address + + std::cout << "list.size=" << list->size << std::endl; + } + +GC-Eligible Types +----------------- + +Or, how to inherit ``xo::Object`` and provide GC support + +A type Foo that inherits ``xo::Object`` needs to provide overrides for Object methods ``_shallow_size()``, +``_shallow_copy()`` and ``_forward_children()``: + +Typical Pattern +^^^^^^^^^^^^^^^ + +GC support methods look something like this: + +* class definition + +.. code-block:: cpp + :linenos: + + #include "xo/alloc/Object.hpp" + + namespace xo { + class Foo : public xo::Object { + public: + ... + virtual std::size_t _shallow_size() const override; + virtual Object * _shallow_copy() const override; + virtual std::size_t _forward_children() override; + }; + } + +* use overloaded ``operator new`` + +A GC-eligible class will allocate instances using the ``MMPtr`` overload. +This allocates memory in GC-owned space + +.. code-block::cpp + :linenos: + + gp Foo::make(...) { + ... + return new MMPtr(mm) Foo(...); + } + +* ``_shallow_size()`` returns the amount of memory used by the subject: + +.. code-block:: cpp + :linenos: + + std::size_t Foo::_shallow_size() const { return sizeof(Foo); } + +* ``_shallow_copy()`` is invoked during GC to create a copy of the subject + + It should use the ``xo::Cpof`` argument to ``operator new``. + +.. code-block:: cpp + :linenos: + + Object * + Foo::_shallow_copy() const; + +* ``_forward_children()`` is invoked during GC to vist child ``xo::Object`` pointers + to make sure they survive + +.. code-block:: cpp + :linenos: + + std::size_t + Foo::_forward_children(); + +Atomic Types Without Object Pointers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Plain-old-data classes without embedded pointers + +.. code-block:: cpp + :linenos: + + Object * + Foo::_shallow_copy() const { + return new (Cpof(this)) Foo(*this); + } + +.. code-block:: cpp + :linenos: + + std::size_t + Foo::_forward_children() { return Foo::_shallow_size(); } + +For example see ``xo::obj::String`` in ``xo-object`` + +Non-GC Objects +^^^^^^^^^^^^^^ + +A class *Foo* that inherits ``xo::Object`` can opt-out of garbage collection by +omitting the ``MMptr(mm)`` overload. + +In that case `Foo::_shallow_size()`, `Foo::_shallow_copy()` and `Foo::_forward_children()` +will not be called: + +.. code-block:: cpp + :linenos: + + std::size_t Foo::_shallow_size() const { return sizeof(Foo); } + Object * Foo::_shallow_copy() const { assert(false); return nullptr; } + std::size_t Foo::_forward_children() { assert(false); return 0; } + +For example see ``xo::obj::Boolean`` in ``xo-object`` + +Structs Containing Object Pointers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A class with object pointers needs to tell GC how to traverse them + +.. code-block:: cpp + :linenos: + + #include "xo/alloc/Object.hpp" + + namespace xo { + class Foo : public xo::Object { + public: + ... + virtual std::size_t _shallow_size() const override; + virtual Object * _shallow_copy() const override; + virtual std::size_t _forward_children() override; + + private: + gp bar_; + gp quux_; + }; + } + +* ``_forward_children()`` is invoked during GC to fixup child pointers + that refer to forwarding objects: + +.. code-block:: cpp + :linenos: + + std::size_t + Foo::_forward_children() + { + Object::_forward_inplace(bar_); + Object::_forward_inplace(quux_); + + return Foo::_shallow_size(); + } + +For example see ``xo::obj::List`` in ``xo-object`` diff --git a/xo-alloc/include/xo/alloc/AllocPolicy.hpp b/xo-alloc/include/xo/alloc/AllocPolicy.hpp new file mode 100644 index 00000000..53f758ee --- /dev/null +++ b/xo-alloc/include/xo/alloc/AllocPolicy.hpp @@ -0,0 +1,58 @@ +/* AllocPolicy.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include + +namespace xo { + /** Tag class, drives overload of operator new. + * See also: xoglobal, xocopy + **/ + struct xolib { + xolib() = default; + }; + + /** @brief opt-in allocator for XO libraries. + * + * By default delegates to vanilla operator new/delete, + * but can set alloc/free functions at runtime to + * adopt a different implementation. + * + * Intending this to op-in to garbage-collector? + * Not sure if we actually need this + * + * Use: + * struct Foo { .. }; + * auto p = new (xo) Foo(..); + **/ + class XoAllocPolicy { + public: + using AllocFn = void* (*)(std::size_t); + using FreeFn = void (*)(void *); + + public: + XoAllocPolicy() = default; + + static void * global_alloc(std::size_t z) { return ::operator new(z); } + static void global_free(void * x) { ::operator delete(x); } + + void * alloc(std::size_t z) { return (*alloc_)(z); } + void free(void * x) { (*free_)(x); } + + private: + AllocFn alloc_ = global_alloc; + FreeFn free_ = global_free; + }; + + /** singleton xolib instance **/ + static XoAllocPolicy xo; +} + +inline void * operator new(std::size_t z, xo::xolib) { + return xo::xo.alloc(z); +} + +void operator delete(void * ptr) noexcept; + +/* end AllocPolicy.hpp */ diff --git a/xo-alloc/include/xo/alloc/ArenaAlloc.hpp b/xo-alloc/include/xo/alloc/ArenaAlloc.hpp new file mode 100644 index 00000000..df4d2a01 --- /dev/null +++ b/xo-alloc/include/xo/alloc/ArenaAlloc.hpp @@ -0,0 +1,231 @@ +/* file ArenaAlloc.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#pragma once + +#include "xo/allocutil/IAlloc.hpp" +#include "ObjectStatistics.hpp" + +namespace xo { + namespace gc { + /** @class ArenaAlloc + * @brief Bump allocator with fixed capacity with dynamic virtual memory commitment. + * + * @text + * + * allocation order: + * -----------------------> + * + * <----------------- .size(), .reserved() ---------------------------> + * <----------------- .committed() -------------> + * + * <-------allocated------><--------free--------><-----uncommitted----> + * XXXXXXXXXXXXXXXXXXXXXXXX______________________...................... + * ^ ^ ^ ^ ^ + * lo checkpoint free limit hi + * + * +- .alloc() -> + * +-- .expand() --> + * > < .before_checkpoint() + * > < .after_checkpoint() + * + * lifetime: + * + * 1. initial state after ctor + * + * >< committed()=0 + * <---------------------------uncommitted----------------------------> + * .................................................................... + * ^ ^ + * lo hi + * checkpoint + * free + * limit + * + * 1a. one call to ::mmap() + * 1b. vm address space [lo,hi) is reserved + * 1c. address space [lo,hi) is inaccessible. no read|write|execute permission + * + * 2. after first allocation of n bytes + * + * <--committed---> + * <--free--><--------------------uncommitted--------------------> + * > <- allocated + * XXXXXX__________..................................................... + * ^ ^ ^ ^ + * lo lo+n limit hi + * ^ free + * checkpoint + * + * 2a. committed just enough hugepages (2mb each) to accomodate n, + * i.e. expand-on-demand: + * - one call to ::mprotect() + * - .limit = .lo + (k+1) * .hugepage_z for some integer k>=0 + * - k * .page_z <= n < (k+1) * .hugepage_z + * 2b. expect immediate cost 1-5us, includes: + * - TLB flush + * invalidate TLB entries for committed range on all cores that this + * process' threads have run on since process inception. + * Also, if a kernel thread has run on one of said cores, it may + * have borrowed our TLB entries + * - page table update + * write to entry for each vm page + * - kernel overhead 100-1000 cycles (< 1us) + * 2c. expect deferred cost 1us-2us per hugepage: + * - committed pages aren't backed by physical memory until + * first touched; minor page fault on first access for each page. + * - so about 256-512us for 1MB + * 3. after .expand(z) + * + * <-------------committed------------> + * <------------free------------><----------uncomitted-----------> + * > <- allocated + * XXXXXX______________________________................................. + * ^ ^ ^ ^ + * lo lo+n limit hi + * ^ free + * checkpoint + * + * 3a. same as case 2. but without advancing .free pointer. + * + * 4. after dtor + * + * 4a. all memory returned to o/s, no longer reserved. + * - one call to ::munmap() + * + * @endtext + * + * Design Notes: + * - non-copyable, non-moveable + * - @ref lo_ <= @ref checkpoint_ <= @ref free_ <= @ref limit_ <= @ref hi_ + * - memory for ArenaAlloc itself (not the memory it allocates), ~100 bytes + * always heap allocated. Use ArenaAlloc::make() + * - memory obtained from mmap(), not heap + * - memory addresses are stable. Expand storage by committing VM pages. + * - @ref lo_ is aligned on VM page size (guaranteed by mmap()) + * - @ref lo_ + @ref committed_z_ <= @ref hi_ + * - @ref limit_ <= @ref lO_ + @ref committed_z_ + * - @ref committed_z_ is always a multiple of VM page size + * - @ref limit_ is not guaranteed to be aligned with VM page size. + * - @ref expand increases @ref limit_ and @ref committed_z_ as needed. + * + **/ + class ArenaAlloc : public IAlloc { + public: + ArenaAlloc(const ArenaAlloc &) = delete; + ArenaAlloc(ArenaAlloc &&) = delete; + ~ArenaAlloc(); + + /** Create allocator with capacity @p z, + * Reserve memory addresses for @p z bytes, + * (but don't commit them until needed) + **/ + static up make(const std::string & name, + std::size_t z, + bool debug_flag); + + /** size of virtual address range reserved for this allocator **/ + std::size_t reserved() const { return hi_ - lo_; }; + + std::size_t page_size() const { return page_z_; } + std::size_t hugepage_z() const { return hugepage_z_; } + std::byte * free_ptr() const { return free_ptr_; } + void set_free_ptr(std::byte * x); + + /** if address @p x is allocated from this arena, + * return true along with offset relative to base address @ref lo_ + * otherwise return false with 0 + **/ + std::pair location_of(const void * x) const; + + /** allocated span **/ + std::pair allocated_span() const { + return std::make_pair(lo_, free_ptr_); + } + + /** Reset to empty state; provision at least @p need_z bytes of (committed) space **/ + void reset(std::size_t need_z); + + /** gc support: If used for storing xo::Object instances, scan allocated memory + * to populate @p *p_dest. + **/ + void capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const; + + /** expand available (i.e. committed) space to size at least @p z + * In practice will round up to a multiple of @ref page_z_. + **/ + bool expand(std::size_t z); + + // inherited from IAlloc... + + virtual const std::string & name() const final override; + virtual std::size_t size() const final override; + virtual std::size_t committed() const final override; + virtual std::size_t available() const final override; + virtual std::size_t allocated() const final override; + virtual bool contains(const void * x) const final override; + virtual bool is_before_checkpoint(const void * x) const final override; + virtual std::size_t before_checkpoint() const final override; + virtual std::size_t after_checkpoint() const final override; + virtual bool debug_flag() const final override; + + virtual void clear() final override; + virtual void checkpoint() final override; + virtual std::byte * alloc(std::size_t z) final override; + virtual bool check_owned(IObject * src) const final override; + + ArenaAlloc & operator=(const ArenaAlloc &) = delete; + ArenaAlloc & operator=(ArenaAlloc &&) = delete; + + private: + ArenaAlloc(const std::string & name, + std::size_t z, bool debug_flag); + + private: + /** + * Invariants: + * - @ref free_ always a multiple of word size (assumed to be sizeof(void*)) + **/ + + /** optional instance name, for diagnostics **/ + std::string name_; + + /** size of a VM page (from getpagesize()) **/ + std::size_t page_z_ = 0; + + /** size of a huge VM page. hardwiring this in ctor (to 2MB). + * larger pages relieve pressure on TLB, but suboptimal if use << 2MB + **/ + std::size_t hugepage_z_ = 0; + + /** allocator owns memory in range [@ref lo_, @ref hi_) **/ + std::byte * lo_ = nullptr; + /** prefix of this size is actually committed. + * Remainder uses uncommitted virtual address space + **/ + std::size_t committed_z_ = 0; + /** checkpoint (for GC support); divides objects into + * older (addresses below checkpoint) + * and younger (addresses above checkpoint) + **/ + std::byte * checkpoint_ = nullptr; + /** free pointer. memory in range [@ref free_, @ref limit_) available **/ + std::byte * free_ptr_ = nullptr; + /** soft limit: end of committed virtual memory + * invariant: @ref limit_ = @ref lo_ + @ref committed_z_ + **/ + std::byte * limit_ = nullptr; + /** hard limit: end of reserved virtual memory **/ + std::byte * hi_ = nullptr; + /** true to enable detailed debug logging **/ + bool debug_flag_ = false; + }; + + } /*namespace gc*/ +} /*namespace xo*/ + + +/* end ArenaAlloc.hpp */ diff --git a/xo-alloc/include/xo/alloc/Blob.hpp b/xo-alloc/include/xo/alloc/Blob.hpp new file mode 100644 index 00000000..205e984b --- /dev/null +++ b/xo-alloc/include/xo/alloc/Blob.hpp @@ -0,0 +1,40 @@ +/** @file Blob.hpp + * + * @author Roland Conybeare, Nov 2025 + **/ + +#pragma once + +#include "Object.hpp" +#include + +namespace xo { + /** Use to allocate opaque binary data, + * with object header. + * + * Not sure if we want to bother implementing reflection for this... + **/ + class Blob : public Object { + public: + Blob(std::size_t z) : z_{z} {}; + + static gp make(gc::IAlloc * mm, std::size_t z); + + std::size_t size() const { return z_; } + std::byte * data() { return data_; } + + virtual TaggedPtr self_tp() const final override; + virtual void display(std::ostream & os) const final override; + + virtual std::size_t _shallow_size() const final override; + virtual Object * _shallow_copy(gc::IAlloc * gc) const final override; + virtual std::size_t _forward_children(gc::IAlloc * gc) final override; + + private: + std::size_t z_ = 0; + /** flexible array, with @ref z_ bytes **/ + std::byte data_[]; + }; +} + +/* end Blob.hpp */ diff --git a/xo-alloc/include/xo/alloc/CircularBuffer.hpp b/xo-alloc/include/xo/alloc/CircularBuffer.hpp new file mode 100644 index 00000000..cf1729ce --- /dev/null +++ b/xo-alloc/include/xo/alloc/CircularBuffer.hpp @@ -0,0 +1,245 @@ +/* CircularBuffer.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#pragma once + +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/tostr.hpp" +#include "xo/indentlog/print/tag.hpp" +#include +#include +#include +//#include + +namespace xo { + namespace gc { + /** @class CircularBuffer + * @brief A circular buffer + * + * push operations may overwrite prior contents, + * i.e. buffer behavior on overflow + * old + * + * @tparam T is type for buffer elements. + **/ + template + class CircularBuffer { + public: + using value_type = T; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type &; + using const_reference = const value_type &; + using pointer = value_type *; + using const_pointer = const value_type *; + + template + class _iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = TT; + using difference_type = std::ptrdiff_t; + using pointer = value_type *; + using reference = value_type &; + + _iterator(Parent * p, std::int64_t ix) : parent_{p}, index_{ix} {} + + reference operator* () const { return (*parent_)[index_]; } + pointer operator->() const { return &(*parent_)[index_]; } + _iterator & operator++() { ++index_; return *this; } + _iterator operator++(int) { _iterator retval(parent_, index_); ++index_; return retval; } + + auto operator<=>(const _iterator & other) { + if (parent_ == other.parent_) + return index_ <=> other.index_; + else + return std::partial_ordering::unordered; + } + + bool operator==(const _iterator & other) const = default; + + private: + Parent * parent_ = nullptr; + /** index position + * (-1 = just before front = rend, 0 = front, z-1 = back, z = just after back = end) + **/ + std::int64_t index_ = 0; + }; + + using iterator = _iterator, T>; + using const_iterator = _iterator, const T>; + + public: + explicit CircularBuffer(std::size_t capacity = 0, bool debug_flag = false); + CircularBuffer(const CircularBuffer& other) = default; + CircularBuffer(CircularBuffer&& other) noexcept = default; + ~CircularBuffer() = default; + + static constexpr std::int64_t npos = -1; + + /** @return location of i'th element. i: 0=front, 1=second etc **/ + std::size_t location_of(std::size_t i) const; + /** @return ordinal index (relative to front) of location @p loc; + * npos if not used + **/ + //std::int64_t index_of(std::size_t loc) const; // not implemented yet + + // standard container methods + bool empty() const noexcept { return size_ == 0; } + size_type size() const noexcept { return size_; } + size_type max_size() const noexcept { return contents_.size(); } + // void reserve(size_type new_capacity); // not implemented + size_type capacity() const noexcept { return contents_.size(); } + // void shrink_to_fit(); // not implemented + + reference at(size_type pos) { + if ((pos < 0) || (pos >= size_)) { + throw std::out_of_range(tostr("CircularBuffer::at: index out of range", + xtag("pos", pos), xtag("size", size_))); + } + + return contents_[this->location_of(pos)]; + } + + const_reference at(size_type pos) const { + reference retval = const_cast(this)->at(pos); + return retval; + } + + reference operator[](size_type pos) { + return contents_[this->location_of(pos)]; + } + + const_reference operator[](size_type pos) const { + return contents_[this->location_of(pos)]; + } + + reference front() { return contents_[front_ix_]; } + const_reference front() const { + reference retval = const_cast(this)->front(); + return retval; + } + + reference back() { return contents_[location_of(size_ - 1)]; } + const_reference back() const { + reference retval = const_cast(this)->back(); + return retval; + } + + iterator begin() { return iterator(this, 0); } + iterator end() { return iterator(this, size_); } + const_iterator begin() const { return const_iterator(this, 0); } + const_iterator end() const { return const_iterator(this, size_); } + + // reverse_iterator rbegin(); + // reverse_iterator rend(); + // const_reverse_iterator rbegin() const; + // const_reverse_iterator rend() const; + + // General Methods + + void clear() { + size_ = 0; + front_ix_ = 0; + std::size_t capacity = contents_.size(); + contents_.clear(); + contents_.resize(capacity); + } + + /** push @p x on to the end of this buffer. + * If buffer is at capacity, overwrites the oldest element + **/ + CircularBuffer & push_back(const T & x); + + // template + //reference emplace_back(Args&&... args); + + CircularBuffer & pop_back(); + + // push_front(); + // pop_front(); + + CircularBuffer& operator=(const CircularBuffer& other) = default; + CircularBuffer& operator=(CircularBuffer&& other) noexcept = default; + + private: + /** number of elements in buffer. Not the same as @code contents_.size(); + * the latter represents buffer capacity. + * + * Promise: + * size_ <= contents_.size() + **/ + std::size_t size_ = 0; + /** first element is @code contents_.at(front_ix_) **/ + std::size_t front_ix_ = 0; + /** buffer contents. contents_.size() represents buffer capacity + * first element stored in @code contents_.at(front_) + * last element stored in @code contents_.at((front_ + size_ - 1) % contents_.size()) + **/ + std::vector contents_; + /** true to enable debug logging **/ + bool debug_flag_ = false; + }; + + template + CircularBuffer::CircularBuffer(std::size_t capacity, bool debug_flag) + : size_{0}, front_ix_{0}, contents_{capacity}, debug_flag_{debug_flag} + { + } + + template + std::size_t + CircularBuffer::location_of(std::size_t i) const + { + if (size_ == 0) + return 0; + else + return (front_ix_ + i) % size_; + } + + template + CircularBuffer & + CircularBuffer::push_back(const T & x) { + scope log(XO_DEBUG(debug_flag_), rtag("x", x), xrtag("size", size_)); + + if (size_ < contents_.size()) { + ++size_; + /* _after_ incr .size_ */ + std::size_t back_ix = location_of(size_ - 1); + + this->contents_[back_ix] = x; + + log && log(xtag("back_ix", back_ix), xtag("+size", size_)); + } else { + std::size_t back_ix = location_of(size_); + + this->contents_[back_ix] = x; + /* buffer was full, so oldest element replaced */ + this->front_ix_ = (this->front_ix_ + 1) % contents_.size(); + + log && log(xtag("back_ix", back_ix), xtag("+front", front_ix_)); + } + + return *this; + } + + template + CircularBuffer & + CircularBuffer::pop_back() { + if (size_ > 0) { + std::size_t back_ix = location_of(size_ - 1); + + this->contents_[back_ix] = T(); + --(this->size_); + } else { + assert(false); + } + + return *this; + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* CircularBuffer.hpp */ diff --git a/xo-alloc/include/xo/alloc/Forwarding1.hpp b/xo-alloc/include/xo/alloc/Forwarding1.hpp new file mode 100644 index 00000000..90ff0198 --- /dev/null +++ b/xo-alloc/include/xo/alloc/Forwarding1.hpp @@ -0,0 +1,56 @@ +/* file Forwarding1.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "Object.hpp" + +namespace xo { + namespace obj { + /** @class Forwarding1 + * @brief forwarding pointer for garbage collector. + * + * Used internally by garbage collector (see @ref GC). + * During evacuate phase overwrite from-space objects in-place + * with an instance of this class. + * + * This class suitable only for singly-inheriting objects, + * i.e. those that have exactly one vtable. + **/ + class Forwarding1 : public Object { + public: + explicit Forwarding1(gp dest); + + // inherited from Object.. + virtual TaggedPtr self_tp() const final override; + virtual void display(std::ostream & os) const final override; + virtual bool _is_forwarded() const final override { return true; } + virtual IObject * _offset_destination(IObject * src) const final override; + virtual IObject * _destination() final override; + /** required by Object i/face, but never called on Forwarding1 **/ + virtual std::size_t _shallow_size() const final override; + /** required by Object i/face, but never called on Forwarding1 **/ + virtual IObject * _shallow_copy(gc::IAlloc * mm) const final override; + /** required by Object i/face, but never called on Forwarding1 **/ + virtual std::size_t _forward_children(gc::IAlloc * mm) final override; + + private: + /** the object that used to be located at this address (i.e. @c this) + * has been moved to @ref destination_ , + * with original location overwritten by a forwarding pointer + * + * Require: + * - can only use Forwarding with types that have a single vtable. + * To forward a multiply-inheriting class with two vtables, use Forwarding2. + * - if you try to use Forwarding for an object with multiple vtables, + * one of the vtable pointers will be replaced by @ref destination_. + * UB revealed when GC traverses a pointer that relies on the 2nd + * vtable to index virtual methods. + **/ + gp dest_; + }; + + } /*namespace obj*/ +} /*namespace xo*/ + +/* end Forwarding1.hpp */ diff --git a/xo-alloc/include/xo/alloc/GC.hpp b/xo-alloc/include/xo/alloc/GC.hpp new file mode 100644 index 00000000..d8acc08c --- /dev/null +++ b/xo-alloc/include/xo/alloc/GC.hpp @@ -0,0 +1,496 @@ +/* GC.hpp + * + * author: Roland Conybeare, jul 2025 + */ + +#pragma once + +#include "ArenaAlloc.hpp" +#include "GcStatistics.hpp" +#include "Object.hpp" +#include "xo/callback/UpCallbackSet.hpp" +#include "xo/indentlog/print/array.hpp" +#include +#include + +namespace xo { + /** types that can participate in GC inherit from this base class. See Object.hpp in this directory **/ + class Object; + + namespace gc { + enum class role { + /** nursery: generation for new objects **/ + from_space, + /** tenured: generation for objects that have survived two collections **/ + to_space, + N, + }; + + constexpr std::size_t role2int(role x) { return static_cast(x); } + + /** @class Config + * @brief garbage collector configuration + **/ + struct Config { + /** initial size in bytes for youngest (Nursery) generation. + * GC allocates two nursery spaces of this size. + * This number represents reserved address space. + * pages are committed on demand. + * Initial committment will be up to @ref incr_gc_threshold_ + **/ + std::size_t initial_nursery_z_ = 64*1024*1024; + /** initial size in bytes for oldest (Tenured) generation. + * GC allocates two tenured spaces of this size. + * This number represents reserved address space. + * pages are committed on demand. + * Initial committment will be up to @ref full_gc_threshold_ + **/ + std::size_t initial_tenured_z_ = 128*1024*1024; + /** trigger incremental GC after this many bytes allocated in nursery **/ + std::size_t incr_gc_threshold_ = 64*1024; + /** trigger full GC after this many bytes promoted to tenured **/ + std::size_t full_gc_threshold_ = 512*1024; + + /** true to permit incremental garbage collection **/ + bool allow_incremental_gc_ = true; + /** true to report statistics **/ + bool stats_flag_ = false; + /** true to capture per-type object statistics **/ + bool object_stats_flag_ = false; + /** remember basic gc statistics for this many GC's; separately for incremental + full GCs **/ + std::size_t stats_history_z_ = 256; + /** true to enable debug logging **/ + bool debug_flag_ = false; + }; + + /** @class GCRunstate + * @brief encapsulate state needed while GC is running + * + * state pertaining to a single GC invocation. + * We stash an instance of this in @ref GC as context, + * so that per-Object-derived-type auxiliary functions can be slightly streamlined + **/ + class GCRunstate { + public: + GCRunstate() = default; + explicit GCRunstate(bool in_progress, bool full_move) + : in_progress_{in_progress}, full_move_{full_move} {} + + bool in_progress() const { return in_progress_; } + bool full_move() const { return full_move_; } + + private: + /** true when GC begins; remains true until GC cycle complete **/ + bool in_progress_ = false; + /** true for full GC; false for incremental GC **/ + bool full_move_ = false; + }; + + class MutationLogEntry { + public: + MutationLogEntry(IObject * parent, IObject ** lhs) + : parent_{parent}, lhs_{lhs} {} + + IObject * parent() const { return parent_; } + IObject ** lhs() const { return lhs_; } + + IObject * child() const { return *lhs_; } + + bool is_child_forwarded() const; + bool is_parent_forwarded() const; + + IObject * parent_destination() const; + + /** Flag obsolete mutation. + * Future proofing, never happens for regular objects + **/ + bool is_dead() const { return false; } + + MutationLogEntry update_parent_moved(IObject * parent_to) const; + void fixup_parent_child_moved(IObject * child_to); + + private: + IObject * parent_ = nullptr; + IObject ** lhs_ = nullptr; + }; + + using MutationLog = std::vector; + + /** @class GcCopyCallback + * @brief optional callback to observe individual copy operations during GC + * + * For viz + **/ + class GcCopyCallback { + public: + virtual ~GcCopyCallback() = default; + + virtual void notify_gc_copy(std::size_t z, const void * src_addr, const void * dest_addr, + generation src_gen, generation dest_gen) = 0; + /** invoked when added to callback set (i.e. @ref GC::GcCopyCallbackSet) **/ + void notify_add_callback() {} + /** invoked when removed from callback set **/ + void notify_remove_callback() {} + }; + + /** @class GC + * @brief generational garbage collector + * + * Works with objects of type @ref xo::Object + **/ + class GC : public IAlloc { + public: + using CallbackId = xo::fn::CallbackId; + using GcCopyCallbackSet = xo::fn::UpCallbackSet; + using nanos = decltype(xo::qty::qty::nanosecond); + + /** rebind is for typed allocators. since IAlloc is untyped, + * we want degenerate version + **/ + template + struct rebind { using other = GC; }; + + public: + /** create new GC instance with configuration @p config **/ + explicit GC(const Config & config); + /** noncopyable **/ + GC(const GC & other) = delete; + virtual ~GC(); + + /** create GC allocator. + * + * Initial memory consumption: + * approximately 2x @ref Config::nursery_size_ + 2x @ref Config::tenured_size_ + **/ + static up make(const Config & config); + + /** runtime downcast **/ + static GC * from(IAlloc * mm); + + const Config & config() const { return config_; } + std::uint8_t nursery_polarity() const { return nursery_polarity_; } + std::uint8_t tenured_polarity() const { return tenured_polarity_; } + const GCRunstate & runstate() const { return runstate_; } + const GcStatistics & native_gc_statistics() const { return gc_statistics_; } + GcStatisticsExt get_gc_statistics() const; + const GcStatisticsHistory & gc_history() const { return gc_history_; } + + /** true iff GC permitted in current state **/ + bool is_gc_enabled() const { return gc_enabled_ == 0; } + /** true iff GC has been requested **/ + bool is_gc_pending() const { return incr_gc_pending_ || full_gc_pending_; } + /** true iff full GC pending **/ + bool is_full_gc_pending() const { return full_gc_pending_; } + /** true during (and only during) a GC cycle **/ + bool gc_in_progress() const { return runstate_.in_progress(); } + + /** @return pagesize (will be the same for {nursery, tenured} spaces) **/ + std::size_t pagesize() const; + /** @return hugepage size (will be the same for {nursery, tenured} spaces) **/ + std::size_t hugepage_z() const; + + /** @return allocation portion of Nursery to-space **/ + std::size_t nursery_to_allocated() const; + /** @return reserved size of Nursery to-space **/ + std::size_t nursery_to_reserved() const; + /** @return committed size of Nursery to-space **/ + std::size_t nursery_to_committed() const; + /** @return nursery bytes used before checkpoint **/ + std::size_t nursery_before_checkpoint() const; + /** @return nursery bytes used after checkpoint **/ + std::size_t nursery_after_checkpoint() const; + /** @return allocated memory range for nursery **/ + std::pair nursery_span(role role) const; + /** @return nursery bytes used in from-space + * (only interesting during GC copy phase, e.g. during scope of a GcCopyCallback call) + **/ + std::size_t nursery_from_allocated() const; + /** @return reserved size of Tenured to-space **/ + std::size_t tenured_to_reserved() const; + /** @return committed size of Tenured to-space **/ + std::size_t tenured_to_committed() const; + /** @return tenured bytes used before checkpoint **/ + std::size_t tenured_before_checkpoint() const; + /** @return tenured bytes used after checkpoint = promoted since last GC **/ + std::size_t tenured_after_checkpoint() const; + + /** @return generation to which object at @p x belongs **/ + generation_result tospace_generation_of(const void * x) const; + /** @return generation to which object at @p x belongs, + * location relative to base address for that generation, + * and allocated size of that generation + * @p role chooses between to-space and from-space + **/ + std::tuple location_of(role role, const void * x) const; + /** @return generation to which object at @p x belongs, + * location relative to base address for @p x, + * and allocated size of generation + **/ + std::tuple tospace_location_of(const void * x) const; + /** @return generation that contains @p x, given it's in from-space **/ + generation_result fromspace_generation_of(const void * x) const; + /** @return generation to which object at @p x belongs, + * location relative to base address for @p x, + * and allocated size of generation + **/ + std::tuple fromspace_location_of(const void * x) const; + /** true iff from-space contains @p x **/ + bool fromspace_contains(const void * x) const; + /** @return free pointer for generation @p gen, i.e. nursery or tenured space **/ + std::byte * free_ptr(generation gen); + /** @return current size of (number of entries in) mutation log **/ + std::size_t mlog_size() const; + + /** add gc root at address @p addr . Gc will keep alive anything reachable + * from @c *addr + **/ + void add_gc_root(IObject ** addr); + /** reverse the effect of previous call to @ref add_gc_root **/ + void remove_gc_root(IObject ** addr); + + /** convenience wrapper **/ + + template + void add_gc_root_dwim(gp * p) { + static_assert(std::is_convertible_v); + this->add_gc_root(reinterpret_cast(p->ptr_address())); + } + + template + void remove_gc_root_dwim(gp * p) { + static_assert(std::is_convertible_v); + this->remove_gc_root(reinterpret_cast(p->ptr_address())); + } + + /** may optionally use this to observe GC copy phase. + * Will be invoked once _per surviving object_, so not cheap. + * Intended for GC visualization. + **/ + CallbackId add_gc_copy_callback(up fn); + /** request garbage collection. + * If GC currently disabled, collection will be deferred until the next time GC + * is in an enabled state. See @ref disable_gc and @ref enable_gc + **/ + void request_gc(generation g); + /** disable garbage collection until matching call to @ref enable_gc. + * + * GC is disabled when number of calls to @ref disable_gc exceeds number of + * calls to @ref enable_gc. + **/ + void disable_gc(); + /** enable garbage collection + * + * GC is enabled when number of calls to @ref enable_gc is at least as large + * as number of calls to @ref disable_gc. + * + * @return true iff GC performed + **/ + bool enable_gc(); + /** same as @c this->enable_gc() followed by @c this->disable_gc() + * @return true iff GC performed + **/ + bool enable_gc_once(); + + // inherited from IAlloc.. + + virtual const std::string & name() const final override; + /** capacity in bytes (counting both free+allocated) for object storage. + * only counts one of {to-space, from-space}, + * since one role is always held empty between collections. + **/ + virtual std::size_t size() const final override; + /** for committed count both to-space and from-space **/ + virtual std::size_t committed() const final override; + virtual std::size_t allocated() const final override; + virtual std::size_t available() const final override; + /** only tests to-space **/ + virtual bool contains(const void * x) const final override; + virtual bool is_before_checkpoint(const void * x) const final override; + virtual std::size_t before_checkpoint() const final override; + virtual std::size_t after_checkpoint() const final override; + virtual bool debug_flag() const final override; + + virtual void clear() final override; + virtual void checkpoint() final override; + + /** GC bookkeeping for an assignment that modifes an Object reference. + * Whenever an @ref Object instance P contains a member variable that can refer + * to another @ref Object, then we need to involve GC to perform the assignment. + * In particular a side-effect that changes the target of such reference to Q after P + * has been promoted, may lead to a tenured->nursery cross-generational pointer. + * GC needs to know about such pointers to it can update them as part of subsequent + * incremental collections. + * + * @param parent. object with member variable being modified + * @param lhs. address of a member variable within the allocation of @p parent. + * @param rhs. new target for @p *lhs + **/ + virtual void assign_member(IObject * parent, IObject ** lhs, IObject* rhs) final override; + /** evacuate @p *lhs and replace with forwarding pointer **/ + virtual void forward_inplace(IObject ** lhs) final override; + /** during GC check for source objects owned by GC. + * See Object::_shallow_move. + **/ + virtual bool check_owned(IObject * src) const final override; + /** queries during GC to determine if object at address @p src should move: + * - full GC -> always + * - incr GC -> if not tenured + **/ + virtual bool check_move(IObject * src) const final override; + /** if src is cross-generational (or cross-checkpoint), verify that it + * is recorded in mutation log, + * given an object @p parent that contains object pointer @p lhs + **/ + virtual bool check_write_barrier(const void * parent, const void * const * lhs, bool may_throw) const final; + + virtual std::byte * alloc(std::size_t z) final override; + virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; + + private: + ArenaAlloc * nursery_to() const { return nursery(role::to_space); } + ArenaAlloc * nursery_from() const { return nursery(role::from_space); } + + ArenaAlloc * tenured_to() const { return tenured(role::to_space); } + ArenaAlloc * tenured_from() const { return tenured(role::from_space); } + + ArenaAlloc * nursery(role r) const { return nursery_[role2int(r)].get(); } + ArenaAlloc * tenured(role r) const { return tenured_[role2int(r)].get(); } + + MutationLog * mutation_log(role r) const { return mutation_log_[role2int(r)].get(); } + + /** begin GC now **/ + void execute_gc(generation g); + /** cleanup phase. aux function for @ref execute_gc **/ + void cleanup_phase(generation g, nanos dt); + /** swap roles of From/To spaces for nursery generation **/ + void swap_nursery(); + /** swap roles of From/To spaces for tenured generation **/ + void swap_tenured(); + /** swap roles of From/To spaces for mutation log **/ + void swap_mutation_log(); + /** swap roles of FromSpace/ToSpace **/ + void swap_spaces(generation g); + /** scan to-space for object statistics before GC */ + void capture_object_statistics(generation upto, capture_phase phase); + /** copy object **/ + void copy_object(IObject ** addr, generation upto, ObjectStatistics * object_stats); + /** copy everything reachable from global gc roots **/ + void copy_globals(generation g); + /** review mutation log; may discover+rescue reachable objects. + **/ + void forward_mutation_log(generation upto); + /** Aux function for @ref execute_gc. Updates bookkeeping for cross-generational + * (T->N, aka xgen) and (N1->N0, aka xckp) pointers + **/ + void incremental_gc_forward_mlog(ObjectStatistics * per_type_stats); + /** + * Aux function for @ref incremental_gc_forward_mlog. Calls this function until + * fixpoint. + * + * @param from_mlog incoming mutation log. Contains {xgen,xckp} pointers before GC. + * Contents of this log is consumed (+discarded) before method returns. + * @param to_mlog outgoing mutation log. Will contain {xgen,xckp} pointers after GC. + * @param defer_mlog contains log entries associated with possible garbage. + **/ + void incremental_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * per_type_stats); + /** Aux function for @ref execute_gc. Updates bookkeeping for cross-generational + * (T->N, aka xgen) and (N1->N0, aka xckcp) pointers on full gc + **/ + void full_gc_forward_mlog(ObjectStatistics * per_type_stats); + /** + * Aux function for @ref full_gc_forward_mlog. Calls this function until fixpoint. + * + * @param from_mlog incoming mutation log. Contains {xgen,xckp} pointers before GC. + * Contents of this log is consumed (+discarded) before method returns. + * @param to_mlog outgoing mutation log. Will contain {xgen,xckp} pointers after GC. + * @param defer_mlog contains log entries associated with possible garbage. + * + **/ + void full_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * per_type_stats); + + private: + /** garbage collector configuration **/ + Config config_; + + /** keep track of the identity of from-space and to-space. + * assist for animation (see xo-imgui/example/ex2). + * polarity alternates between 0 and 1 on each GC + **/ + std::uint8_t nursery_polarity_ = 0; + std::uint8_t tenured_polarity_ = 0; + + /** contains allocated objects, along with unreachable garbage to be collected. + * roles reverse after each incremental, or full, collection. + **/ + std::array, role2int(role::N)> nursery_; + /** empty space, destination for objects that survive collection. + * roles reverse after each full collection. + **/ + std::array, role2int(role::N)> tenured_; + + /** current state of GC activity. + * @text + * in_progress full_move descr + * ----------------------------------------- + * false * gc not running + * true false incremental gc + * true true full gc + * ----------------------------------------- + * @endtext + **/ + GCRunstate runstate_; + + /** root object handles: targets of handles in this vector are always preserved by GC. + * Application can introduce new root object pointers at any time provided GC not running, + * but cannot withdraw them. + **/ + std::vector gc_root_v_; + + /** log cross-generational and cross-checkpoint mutations. + * These need to be adjusted on next incremental collection. + * + * mutation_log_[tospace] accumulates {xgen,xckp} pointers until + * the next GC. + * + * See GC aux functions + * @ref incremental_gc_forward_mlog + * @ref full_gc_forward_mlog + * + **/ + std::array, role2int(role::N)> mutation_log_; + /** temporary mutation log (for deferred entries) **/ + up defer_mutation_log_; + + /** allocation/collection counters **/ + GcStatistics gc_statistics_; + /** optional per-object-type counters. snapshot at beginning of collection cycle **/ + std::array object_statistics_sab_; + /** optional per-object-type counters. snapshot at end of collection cycle **/ + std::array object_statistics_sae_; + + /** true when GC requested, + * remains true until GC.. completes? begins? + **/ + bool incr_gc_pending_ = false; + bool full_gc_pending_ = false; + + /** enabled when 0. disabled when <0 **/ + int gc_enabled_ = 0; + + /** rotating per-gc statistics history **/ + GcStatisticsHistory gc_history_; + + /** for (optional) viz: invoke when copying individual objects **/ + GcCopyCallbackSet gc_copy_cbset_; + }; + } /*namespace gc*/ + +} /*namespace xo*/ + +/* end GC.hpp */ diff --git a/xo-alloc/include/xo/alloc/GcStatistics.hpp b/xo-alloc/include/xo/alloc/GcStatistics.hpp new file mode 100644 index 00000000..69053457 --- /dev/null +++ b/xo-alloc/include/xo/alloc/GcStatistics.hpp @@ -0,0 +1,287 @@ +/** @file GcStatistics.hpp + * + * @author Roland Conybeare, Aug 2025 + **/ + +#pragma once + +#include "generation.hpp" +#include "CircularBuffer.hpp" +#include +#include +#include +#include +#include +#include + +namespace xo { + namespace gc { + /** @class PerGenerationStatistics + * @brief garbage collection statistics for particular GC generation + **/ + class PerGenerationStatistics { + public: + /** update statistics after a GC cycle + * @param alloc_z. new allocations (since preceding GC) + * @param before_z. generation size (bytes allocated) before collection + * @param after_z. generation size after collection + * @param promote_z. bytes promoted to next generation + **/ + void include_gc(std::size_t alloc_z, std::size_t before_z, std::size_t after_z, + std::size_t promote_z); + /** update with current state (use at end of gc cycle) **/ + void update_snapshot(std::size_t after_z); + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** number of bytes currently in use **/ + std::size_t used_z_ = 0; + + /** number of collection cycles completed **/ + std::size_t n_gc_ = 0; + /** sum of new alloc bytes, sampled at start of each collection cycle **/ + std::size_t new_alloc_z_ = 0; + /** sum of allocated bytes sampled at beginning of each collection cycle **/ + std::size_t scanned_z_ = 0; + /** sum of bytes remaining after collection cycle **/ + std::size_t survive_z_ = 0; + /** sum of bytes promoted to next generation **/ + std::size_t promote_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const PerGenerationStatistics & x) { + x.display(os); + return os; + } + + /** @class GcStatistics + * @brief garbage collection statistics + **/ + class GcStatistics { + public: + GcStatistics() = default; + + /** update statistics at beginning of a GC cycle + * @param upto. nursery -> incremental collection; tenured -> full collection + * @param alloc_z. new allocations (since preceding GC) + **/ + void begin_gc(generation upto, + std::size_t alloc_z); + + /** update statistics after a GC cycle + * @param upto. nursery -> incremental collection; tenured -> full collection + * @param alloc_z. new allocations (since preceding GC) + * @param before_z. generation size (bytes allocated) before collection + * @param after_z. generation size after collection + * @param promote_z. bytes promoted to next generation + **/ + void include_gc(generation upto, std::size_t alloc_z, + std::size_t before_z, std::size_t after_z, std::size_t promote_z); + /** update snapshot for current state. + * Use with tenured stats after incremental gc + **/ + void update_snapshot(generation upto, std::size_t after_z); + + /** number of collection cycles, whether full or incremental **/ + std::size_t n_gc() const { return gen_v_[gen2int(generation::nursery)].n_gc_; } + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** statistics gathered across {incr, full} GCs respectively **/ + std::array(generation::N)> gen_v_; + /** total bytes allocated since inception **/ + std::size_t total_allocated_ = 0; + /** snapshot of total bytes promoted asof beginning of last gc cycle **/ + std::size_t total_promoted_sab_ = 0; + /** total bytes promoted from nursery->tenured since inception **/ + std::size_t total_promoted_ = 0; + + /** total number of mutations to already-allocated objects, + * whether or not GC needs to log them. + **/ + std::size_t n_mutation_ = 0; + /** total number of mutation eligible for logging (cumulative across GCs) **/ + std::size_t n_logged_mutation_ = 0; + /** total number of cross-generation mutations + * (tenured->nursery when reported; cumulative across GCs) **/ + std::size_t n_xgen_mutation_ = 0; + /** total number of cross-checkpoint mutations + * (N0 -> N1 when reported; cumulative across GCs) + **/ + std::size_t n_xckp_mutation_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatistics & x) { + x.display(os); + return os; + } + + /** @class GcStatisticsExt + * @brief extend GcStatistics for application convenience + **/ + class GcStatisticsExt : public GcStatistics { + public: + GcStatisticsExt() = default; + explicit GcStatisticsExt(const GcStatistics & x) : GcStatistics{x} {} + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** current capacity of nursery generation **/ + std::size_t nursery_z_ = 0; + /** current nursery survivor size **/ + std::size_t nursery_before_checkpoint_z_ = 0; + /** current nursery new alloc size **/ + std::size_t nursery_after_checkpoint_z_ = 0; + /** current capacity of tenured generation **/ + std::size_t tenured_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatisticsExt & x) { + x.display(os); + return os; + } + + /** @class GcStatisticsHistoryItem + * @brief info we want to record over time (won't have cumulative things in it) + **/ + class GcStatisticsHistoryItem { + public: + using nanos = xo::qty::type::nanoseconds; + + public: + GcStatisticsHistoryItem() = default; + constexpr GcStatisticsHistoryItem(std::size_t gc_seq, + generation upto, + std::size_t new_alloc_z, + std::size_t survive_z, + std::size_t promote_z, + std::size_t persist_z, + std::size_t effort_z, + std::size_t garbage0_z, + std::size_t garbage1_z, + std::size_t garbageN_z, + nanos dt, + std::size_t sum_effort_z, + std::size_t sum_garbage_z) + : gc_seq_{gc_seq}, + upto_{upto}, + new_alloc_z_{new_alloc_z}, + survive_z_{survive_z}, + promote_z_{promote_z}, + persist_z_{persist_z}, + effort_z_{effort_z}, + garbage0_z_{garbage0_z}, + garbage1_z_{garbage1_z}, + garbageN_z_{garbageN_z}, + dt_{dt}, + sum_effort_z_{sum_effort_z}, + sum_garbage_z_{sum_garbage_z} + {} + constexpr GcStatisticsHistoryItem(const GcStatisticsHistoryItem &) = default; + + std::size_t garbage_z() const { return garbage0_z_ + garbage1_z_ + garbageN_z_; } + + float efficiency() const { + std::size_t gz = this->garbage_z(); + + return gz / static_cast(effort_z_ + gz); + } + + /** lifetime byte-weighted average collection efficiency. Always in [0.0, 1.0] **/ + float average_efficiency() const { + return sum_garbage_z_ / static_cast(sum_effort_z_ + sum_garbage_z_); + } + + /** collection rate, in bytes/sec **/ + float collection_rate() const; + + GcStatisticsHistoryItem & operator=(const GcStatisticsHistoryItem & x) { + gc_seq_ = x.gc_seq_; + upto_ = x.upto_; + new_alloc_z_ = x.new_alloc_z_; + survive_z_ = x.survive_z_; + promote_z_ = x.promote_z_; + persist_z_ = x.persist_z_; + effort_z_ = x.effort_z_; + garbage0_z_ = x.garbage0_z_; + garbage1_z_ = x.garbage1_z_; + garbageN_z_ = x.garbageN_z_; + this->dt_.scale_ = x.dt_.scale_; + + sum_effort_z_ = x.sum_effort_z_; + sum_garbage_z_ = x.sum_garbage_z_; + + return *this; + } + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** sequence number for collection being reported **/ + std::size_t gc_seq_ = 0; + /** type of GC that generated this record **/ + generation upto_; + /** #of bytes new allocation **/ + std::size_t new_alloc_z_ = 0; + /** #of bytes surviving their first collection (i.e. N0->N1) **/ + std::size_t survive_z_ = 0; + /** #of bytes promoted to tenured. + * Comprises all objects surviving their 2nd collection (i.e. N1->T) + **/ + std::size_t promote_z_ = 0; + /** #of bytes surviving 3rd of later collection **/ + std::size_t persist_z_ = 0; + /** #of bytes copied **/ + std::size_t effort_z_ = 0; + /** #of bytes garbage from N0 (i.e. survived 0 GCs) **/ + std::size_t garbage0_z_ = 0; + /** #of bytes garbage from N1 (i.e. survived 1 GCs) **/ + std::size_t garbage1_z_ = 0; + /** #of bytes garbage from T (i.e. survived 2+ GCs) **/ + std::size_t garbageN_z_ = 0; + /** elapsed time for this GC (see @ref GC::execute_gc) **/ + nanos dt_; + + // ----- cumulative statistics ----- + + /** sum (in bytes) copied by collections since inception **/ + std::size_t sum_effort_z_ = 0; + /** sum (in bytes) of garbage collected since inception **/ + std::size_t sum_garbage_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatisticsHistoryItem & x) { + x.display(os); + return os; + } + + using GcStatisticsHistory = CircularBuffer; + } /*namespace gc*/ + + namespace print { + template <> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::PerGenerationStatistics &); + }; + + template<> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatistics &); + }; + + template<> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatisticsExt &); + }; + + template<> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatisticsHistoryItem &); + }; + } /*namespace print*/ +} /*namespace xo*/ + +/* end GcStatistics.hpp */ diff --git a/xo-alloc/include/xo/alloc/ListAlloc.hpp b/xo-alloc/include/xo/alloc/ListAlloc.hpp new file mode 100644 index 00000000..30d91cfb --- /dev/null +++ b/xo-alloc/include/xo/alloc/ListAlloc.hpp @@ -0,0 +1,100 @@ +/* file ListAlloc.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#pragma once + +#include "IAlloc.hpp" +#include "ObjectStatistics.hpp" +#include +#include +#include + +namespace xo { + namespace gc { + class ArenaAlloc; + + /** GC-compatible allocator using a linked list of buckets. + * + * - all allocs done from first allocator in list + * GC Support: + * - reserved memory, released after call to @ref release_redline_memory. + * + * TODO: reserve address space using mmap, + * but don't commit until alloc requires it. + **/ + class ListAlloc : public IAlloc { + public: + ListAlloc(std::unique_ptr hd, + ArenaAlloc * marked, + std::size_t cz, std::size_t nz, std::size_t tz, + bool debug_flag); + ~ListAlloc(); + + static up make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag); + + /** page size used by underlying ArenaAlloc **/ + std::size_t page_size() const; + + /** hugepage size used by underlying ArenaAlloc **/ + std::size_t hugepage_z() const; + + /** reset to have at least @p z bytes of storage **/ + bool reset(std::size_t z); + + /** expand bucket list to accomodate a request of size @p z **/ + bool expand(std::size_t z, const std::string & name); + + /** current free pointer **/ + std::byte * free_ptr() const; + + /** scan space (must not contain forwarding pointers, because loses size info) + * + gather stats by object type + * + * See @ref Object::self_tp + **/ + void capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const; + + // inherited from IAlloc.. + + virtual const std::string & name() const final override; + virtual std::size_t size() const final override; + virtual std::size_t committed() const final override; + virtual std::size_t available() const final override; + virtual std::size_t allocated() const final override; + virtual bool contains(const void * x) const final override; + virtual bool is_before_checkpoint(const void * x) const final override; + virtual std::size_t before_checkpoint() const final override; + virtual std::size_t after_checkpoint() const final override; + virtual bool debug_flag() const final override; + + virtual void clear() final override; + virtual void checkpoint() final override; + virtual std::byte * alloc(std::size_t z) final override; + + private: + /** **/ + std::size_t start_z_ = 0; + /** all new allocs from this list **/ + std::unique_ptr hd_; + /** allocator that was in @ref hd_ when @ref checkpoint last called **/ + ArenaAlloc * marked_ = nullptr; + /** overflow allocs (expect list to be short); + * from trying to converge on app working set size + **/ + std::list> full_l_; + /** size of current arena @ref hd_ **/ + std::size_t current_z_ = 0; + /** if @ref hd_ fills, size of next arena to allocate **/ + std::size_t next_z_ = 0; + /** total size of @ref hd_ + contents of @ref full_l_ **/ + std::size_t total_z_ = 0; + /** true to enable debug logging **/ + bool debug_flag_ = false; + }; + } /*namespace gc*/ +} /*namespace xo*/ + +/* end ListAlloc.hpp */ diff --git a/xo-alloc/include/xo/alloc/Object.hpp b/xo-alloc/include/xo/alloc/Object.hpp new file mode 100644 index 00000000..9665e66a --- /dev/null +++ b/xo-alloc/include/xo/alloc/Object.hpp @@ -0,0 +1,170 @@ +/* Object.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#pragma once + +#include "xo/allocutil/IObject.hpp" +#include "xo/reflect/TaggedPtr.hpp" +#include "xo/allocutil/ObjectVisitor.hpp" +#include "xo/allocutil/gc_ptr.hpp" +#include +#include + +namespace xo { + namespace gc { + class IAlloc; + class GC; + class ObjectStatistics; + }; + + /** Root class for all xo GC-collectable objects. + * + * Design notes: + * 1. xo::IObject -> xo-allocutil header-only library + * xo::Object -> xo-alloc ordinary library + * 2. relying on inheritance means we insist that GC traits + * for a type appear directly in that type's vtable, and at specific locations. + * This implies one level of indirection when GC traverses an instance. + * 3. Could adapt a gc-aware XO library (such as xo-ordinaltree) + * to a non-xo garbage collector. + * Would still need to use xo::IObject and xo::gc::gc_allocator_traits, + * but not necessarily xo::Object + * 4. Would be feasible to relax the must-inherit-from-Object constraint + * by having GC use its own wrapper, at cost of an extra layer of indirection + **/ + class Object : public IObject { + public: + using TaggedPtr = xo::reflect::TaggedPtr; + + public: + static gp from(gp x) { + return dynamic_cast(x.ptr()); + } + + virtual ~Object() noexcept = default; + + /** memory allocator for objects. Likely this will be a GC instance, + * but simple arena also supported. + * + * Load-bearing for .assign_member() + **/ + static gc::IAlloc * mm; + + /** assign value @p rhs to member @p *lhs of @p parent. + * if assignment creates a cross-generational or cross-checkpoint pointer, + * add mutation log entry. + * + * DEPRECATED. prefer IObject::_gc_assign_member, for explicit alloc + **/ + template + static void assign_member(gp parent, gp * lhs, gp rhs); + + /** use from GC aux functions **/ + static gc::GC * _gc() { return reinterpret_cast(mm); } + + /** during GC + * 1. copy destination object @p *addr to (new) to-space. + * 2. overwrite existing object @p *addr with a forwarding pointer to + * copy made in step 1. + * 3. return the location of the copy make in step 1. + * + * @p src. source object to be forwarded + * @p gc. allocator (poassibly garbage collector) + */ + static IObject * _forward(IObject * src, gc::IAlloc * gc); + + template + static void _forward_inplace(T ** src_addr, gc::IAlloc * gc) { + IObject * fwd = _forward(*src_addr, gc); + + *src_addr = reinterpret_cast(fwd); + } + + template + static void _forward_inplace(gp & src, gc::IAlloc * gc) { + _forward_inplace(src.ptr_address(), gc); + } + + /** primary workhorse for garbage collection. + * + * we assign each object one of three colors: black|gray|white. + * + * color | location | children | action | + * ------+------------+------------+-------------------------+ + * black | from-space | any | move to to-space | + * gray | to-space | any | move remaining children | + * white | to-space | white/gray | done | + * + * initially all reachable objects are black. + * GC is complete when all reachable objects are white. + * GC needs a variable amount of temporary storage to keep track of all gray objects + * + * Evacuate reachable object graph rooted at @p src to to-space. + * On return all objects reachable from @p src are white + * + * @param src address of object to evacuate + * @param gc garbage collector + * @param stats per-object-type GC statistics + **/ + static IObject * _deep_move(IObject * src, gc::GC * gc, gc::ObjectStatistics * stats); + + /** copy @p src to to-space. Overwrite original with forwarding pointer to new location. + * return the new location + **/ + static IObject * _shallow_move(IObject * src, gc::IAlloc * gc); + + // Reflection support + + /** tagged pointer with runtime type information + **/ + virtual TaggedPtr self_tp() const; + + /** print on stream @p os **/ + virtual void display(std::ostream & os) const; + + // Inherited from IObject.. + + //virtual bool _is_forwarded() const override { return false; } + //virtual IObject * _offset_destination(IObject * src) const override { return src; }; + virtual void _forward_to(IObject * dest) override; + //virtual IObject * _destination() override { return nullptr; } + + virtual std::size_t _shallow_size() const override = 0; + virtual IObject * _shallow_copy(gc::IAlloc * gc) const override = 0; + virtual std::size_t _forward_children(gc::IAlloc * gc) override = 0; + }; + + static_assert(std::is_destructible_v, "Object must be destructible"); + static_assert(std::is_nothrow_destructible_v, "Object must be noexcept destructible"); + + template + void + Object::assign_member(gp parent, gp * lhs, gp rhs) + { + Object::mm->assign_member(reinterpret_cast(parent.ptr()), + reinterpret_cast(lhs->ptr_address()), + reinterpret_cast(rhs.ptr())); + } + + namespace gc { + template + class ObjectVisitor> { + public: + static void forward_children(gp & target, + IAlloc * gc) + { + Object::_forward_inplace(target, gc); + } + }; + } + + std::ostream & + operator<< (std::ostream & os, gp x); + +} /*namespace xo*/ + +void * operator new (std::size_t z, const xo::Cpof & copy); + +/* end Object.hpp */ diff --git a/xo-alloc/include/xo/alloc/ObjectStatistics.hpp b/xo-alloc/include/xo/alloc/ObjectStatistics.hpp new file mode 100644 index 00000000..43ddd70e --- /dev/null +++ b/xo-alloc/include/xo/alloc/ObjectStatistics.hpp @@ -0,0 +1,87 @@ +/* file ObjectStatistics.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#pragma once + +#include "xo/indentlog/print/pretty.hpp" +#include +#include + +namespace xo { + namespace reflect { class TypeDescrBase; } + + namespace gc { + enum class capture_phase { + /** snapshot-at-beginning **/ + sab, + /** snapshot-at-end **/ + sae, + }; + + /** @class PerObjectTypeStatistics + * @brief statistics for a particular object type + * + * Gathered for each leaf type descended from xo::obj::Object. + * See @ref xo::obj::Object::self_tp + * + * See @ref GC::capture_object_statistics + * (gathers @ref scanned_n_, @ref scanned_z_) + **/ + struct PerObjectTypeStatistics { + using TypeDescr = xo::reflect::TypeDescrBase const *; + + void display(std::ostream & os) const; + + /** stats here are for objects of this type **/ + TypeDescr td_ = nullptr; + /** number of objects scanned **/ + std::size_t scanned_n_ = 0; + /** number of bytes scanned **/ + std::size_t scanned_z_ = 0; + /** number of objects surviving **/ + std::size_t survive_n_ = 0; + /** number of bytes from surviving objects **/ + std::size_t survive_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const PerObjectTypeStatistics & x) { + x.display(os); + return os; + } + + /** @class ObjectStatistics + * @brief placeholder for type-driven allocation statistics + * + * Passed to @ref Object::deep_move for example + **/ + class ObjectStatistics { + public: + void display(std::ostream & os) const; + + /** per-object-type statistics, indexed by TypeId **/ + std::vector per_type_stats_v_; + }; + + inline std::ostream & operator<< (std::ostream & os, const ObjectStatistics & x) { + x.display(os); + return os; + } + + } /*namespace gc*/ + + namespace print { + template <> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::PerObjectTypeStatistics &); + }; + + template <> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::ObjectStatistics &); + }; + } /*namespace print*/ +} /*namespace xo*/ + +/* end ObjectStatistics.hpp */ diff --git a/xo-alloc/include/xo/alloc/Stack.hpp b/xo-alloc/include/xo/alloc/Stack.hpp new file mode 100644 index 00000000..b894d853 --- /dev/null +++ b/xo-alloc/include/xo/alloc/Stack.hpp @@ -0,0 +1,49 @@ +/* Stack.hpp + * + * author: Roland Conybeare, jul 2025 + */ + +#pragma once + +#include + +namespace xo { + namespace gc { + /** Simple stack implementation + **/ + template + class Stack { + public: + explicit Stack(std::size_t capacity) { + this->contents_.reserve(capacity); + } + + bool is_empty() const { return contents_.empty(); } + std::size_t available() const { return contents_.capacity() - contents_.size(); } + void drop() { contents_.resize(contents_.size() - 1); } + void push(const T & x) { contents_.push_back(x); } + T pop() { + T retval = contents_[contents_.size() - 1]; + this->drop(); + return retval; + } + const T & top() const { + return this->lookup(0); + } + const T & lookup(std::size_t i) const { + return contents_.at(contents_.size() - 1 - i); + } + void clear() { contents_.clear(); } + void reset_to(std::size_t z) { contents_.resize(z); } + + std::size_t n_elements() const { return contents_.size(); } + std::size_t capacity() const { return contents_.capacity(); } + + private: + std::vector contents_; + }; + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end Stack.hpp */ diff --git a/xo-alloc/include/xo/alloc/generation.hpp b/xo-alloc/include/xo/alloc/generation.hpp new file mode 100644 index 00000000..9f122044 --- /dev/null +++ b/xo-alloc/include/xo/alloc/generation.hpp @@ -0,0 +1,54 @@ +/* generation.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#pragma once + +#include +#include +#include + +namespace xo { + namespace gc { + enum class generation { + nursery, + tenured, + N + }; + + constexpr std::size_t gen2int(generation x) { return static_cast(x); } + + const char * gen2str(generation x); + + inline std::ostream & operator<<(std::ostream & os, generation x) { + os << gen2str(x); + return os; + } + + enum class generation_result { + nursery, + tenured, + not_found + }; + + inline generation valid_genresult2gen(generation_result x) { + assert(x != generation_result::not_found); + + if (x == generation_result::nursery) + return generation::nursery; + else + return generation::tenured; + } + + const char * genresult2str(generation_result x); + + inline std::ostream & operator<<(std::ostream & os, generation_result x) { + os << genresult2str(x); + return os; + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end generation.hpp */ diff --git a/xo-alloc/src/alloc/AllocPolicy.cpp b/xo-alloc/src/alloc/AllocPolicy.cpp new file mode 100644 index 00000000..b1dc162f --- /dev/null +++ b/xo-alloc/src/alloc/AllocPolicy.cpp @@ -0,0 +1,13 @@ +/* AllocPolicy.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "AllocPolicy.hpp" + +/* note: inline/.hpp definition not allowed for operator delete */ +void operator delete(void * ptr) noexcept { + xo::xo.free(ptr); +} + +/* end AllocPolicy.cpp */ diff --git a/xo-alloc/src/alloc/ArenaAlloc.cpp b/xo-alloc/src/alloc/ArenaAlloc.cpp new file mode 100644 index 00000000..dc80eb4e --- /dev/null +++ b/xo-alloc/src/alloc/ArenaAlloc.cpp @@ -0,0 +1,426 @@ +/* file ArenaAlloc.cpp + * + * author: Roland Conybeare + */ + +#include "ArenaAlloc.hpp" +#include "Object.hpp" +#include "ObjectStatistics.hpp" +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/tag.hpp" +#include +#include // for getpagesize() on OSX +#include + +namespace xo { + using std::byte; + + namespace gc { + namespace { + /* alignment better be a power of 2 */ + std::size_t + align_lub(std::size_t x, std::size_t align) + { + /* e.g: + * align = 4096, x%align = 100 -> dx = 3996 + * align = 4096, x%align = 0 -> dx = 0 + */ + std::size_t dx = (align - (x % align)) % align; + + return x + dx; + } + } + + ArenaAlloc::ArenaAlloc(const std::string & name, + std::size_t z, + bool debug_flag) + { + scope log(XO_DEBUG(debug_flag), xtag("name", name)); + + constexpr size_t c_hugepage_z = 2 * 1024 * 1024; + + this->name_ = name; + this->page_z_ = getpagesize(); + this->hugepage_z_ = c_hugepage_z; + + // 1. need k pagetable entries where k is lub {k | k * .page_z >= z} + // 2. base will be aligned with .page_z but likely not with .hugepage_z + // 3. bad to have misalignment, because misaligned {prefix, suffix} of [base, base+z) + // will use 4k pages instead of 2mb pages + // + // strategy: + // 4. round up z to multiple of c_hugepage_z + // 5. over-request so reserved range contains an aligned subrange of size z + // 6. unmap misaligned prefix + // 7. unmap misaligned suffix. + // 8. enable huge pages for now-aligned remainder of reserved range + // + // Z. note: rejecting inferior MAP_HUGETLB|MAP_HUGE_2MB flags on ::mmap here: + // Za. requires previously-reserved memory in /proc/sys/vm/nr_hugepages + // Zb. reserved pages permenently resident in RAM, never swapped + // Zc. memory cost incurred even if no application is using said pages + + z = align_lub(z, c_hugepage_z); // 4. + + // 5. + byte * base = reinterpret_cast(::mmap(nullptr, + z + c_hugepage_z, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0)); + + log && log("acquired memory [lo,hi) using mmap", + xtag("lo", base), + xtag("z", z), + xtag("hi", reinterpret_cast(base) + z)); + + if (base == MAP_FAILED) { + throw std::runtime_error(tostr("ArenaAlloc: uncommitted allocation failed", + xtag("size", z))); + } + + byte * aligned_base = reinterpret_cast(align_lub(reinterpret_cast(base), + c_hugepage_z)); + + assert(reinterpret_cast(aligned_base) % c_hugepage_z == 0); + assert(aligned_base >= base); + assert(aligned_base < base + c_hugepage_z); + + if (base < aligned_base) { + size_t prefix = aligned_base - base; + + ::munmap(base, prefix); // 6. + } + + byte * aligned_hi = aligned_base + z; + byte * hi = base + z + c_hugepage_z; + + if (aligned_hi < hi) { + size_t suffix = hi - aligned_hi; + + ::munmap(aligned_hi, suffix); // 7. + } + +#ifdef __linux__ + ::madvise(aligned_base, z, MADV_HUGEPAGE); // 8. +#endif + // TODO: for OSX -> need something else here. + // MAP_ALIGNED_SUPER with mmap() and/or + // use mach_vm_allocate() + // + + this->lo_ = aligned_base; + this->committed_z_ = 0; + this->checkpoint_ = lo_; + this->free_ptr_ = lo_; + this->limit_ = lo_; + this->hi_ = lo_ + z; + this->debug_flag_ = debug_flag; + + if (!lo_) { + throw std::runtime_error(tostr("ArenaAlloc: allocation failed", + xtag("size", z))); + } + + log && log(xtag("lo", (void*)lo_), + xtag("page_z", page_z_), + xtag("hugepage_z", hugepage_z_)); + } + + ArenaAlloc::~ArenaAlloc() + { + scope log(XO_DEBUG(debug_flag_)); + + // hygiene.. + + if (lo_) { + log && log("unmap [lo,hi)", xtag("lo", lo_), xtag("z", hi_ - lo_), xtag("hi", hi_)); + + ::munmap(lo_, hi_ - lo_); + } + // could use this as fallback if we dropped the uncommitted technique + //delete [] this->lo_; + + this->lo_ = nullptr; + this->committed_z_ = 0; + this->checkpoint_ = nullptr; + this->free_ptr_ = nullptr; + this->limit_ = nullptr; + this->hi_ = nullptr; + this->debug_flag_ = false; + } + + up + ArenaAlloc::make(const std::string & name, + std::size_t z, bool debug_flag) + { + return up(new ArenaAlloc(name, + z, debug_flag)); + } + + bool + ArenaAlloc::expand(size_t offset_z) + { + scope log(XO_DEBUG(debug_flag_), xtag("offset_z", offset_z), xtag("committed_z", committed_z_)); + + if (offset_z <= committed_z_) { + log && log("trivial success, offset within committed range", + xtag("offset_z", offset_z), + xtag("committed_z", committed_z_)); + return true; + } + + if (lo_ + offset_z > hi_) { + throw std::runtime_error(tostr("ArenaAlloc::expand: requested size exceeds reserved size", + xtag("requested", offset_z), xtag("reserved", reserved()))); + } + + /* + * pre: + * + * _______________................................... + * ^ ^ ^ + * lo limit hi + * + * < committed_z > + * <----------offset_z-----------> + * > <- z: 0 <= z < hugepage_z + * <---------aligned_offset_z---------> + * <--- add_commit_z --> + * + * post: + * ____________________________________.............. + * ^ ^ ^ + * lo limit hi + * + */ + + std::size_t aligned_offset_z = align_lub(offset_z, hugepage_z_); + std::byte * commit_start = lo_ + committed_z_; + std::size_t add_commit_z = aligned_offset_z - committed_z_; + + assert(limit_ == lo_ + committed_z_); + + log && log(xtag("aligned_offset_z", aligned_offset_z), + xtag("add_commit_z", add_commit_z)); + + log && log("expand committed range", + xtag("commit_start", commit_start), + xtag("add_commit_z", add_commit_z), + xtag("commit_end", commit_start + add_commit_z)); + + if (::mprotect(commit_start, add_commit_z, PROT_READ | PROT_WRITE) != 0) { + throw std::runtime_error(tostr("ArenaAlloc::expand: commit failure", + xtag("committed_z", committed_z_), + xtag("add_commit_z", add_commit_z))); + } + + this->committed_z_ = aligned_offset_z; + this->limit_ = this->lo_ + committed_z_; + + assert(committed_z_ % hugepage_z_ == 0); + assert(reinterpret_cast(limit_) % hugepage_z_ == 0); + + return true; + } + + void + ArenaAlloc::set_free_ptr(std::byte * x) + { + assert(lo_ <= x); + assert(x < limit_); + + if (lo_ <= x && x < limit_) { + this->free_ptr_ = x; + if (checkpoint_ > free_ptr_) + this->checkpoint_ = free_ptr_; + } else { + throw std::runtime_error(tostr("LinearAllog::set_free_ptr(x): expected lo <= x < limit", + xtag("lo", lo_), xtag("x", x), xtag("limit", limit_))); + } + } + + std::pair + ArenaAlloc::location_of(const void * x) const + { + if ((lo_ <= x) && (x < hi_)) { + return std::make_pair(true, reinterpret_cast(x) - lo_); + } else { + return std::make_pair(false, 0); + } + } + + void + ArenaAlloc::reset(std::size_t need_z) { + this->clear(); + this->expand(need_z); + } + + void + ArenaAlloc::capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const + { + scope log(XO_DEBUG(debug_flag_), + xtag("name", name_), + xtag("capacity", limit_ - lo_), + xtag("alloc", free_ptr_ - lo_), + xtag("lo", (void*)lo_), + xtag("free_ptr", (void*)free_ptr_)); + + using xo::reflect::TaggedPtr; + + std::byte * p = lo_; + + while (p < free_ptr_) { + log && log(xtag("p", (void *)p)); + + Object * obj = reinterpret_cast(p); + TaggedPtr tp = obj->self_tp(); + std::size_t z = obj->_shallow_size(); + std::uint32_t id = tp.td()->id().id(); + + log && log(xtag("obj", (void*)obj), + xtag("z", z), + xtag("typeid", id)); + + if (p_dest->per_type_stats_v_.size() < id + 1) + p_dest->per_type_stats_v_.resize(id + 1); + + PerObjectTypeStatistics & dest = p_dest->per_type_stats_v_.at(id); + + dest.td_ = tp.td(); + + log && log(xtag("td", tp.td()->short_name())); + + switch (phase) { + case capture_phase::sab: + ++dest.scanned_n_; + dest.scanned_z_ += z; + break; + case capture_phase::sae: + ++dest.survive_n_; + dest.survive_z_ += z; + break; + } + + p += z; + } + + assert(p == free_ptr_); + } + + const std::string & + ArenaAlloc::name() const { + return name_; + } + + std::size_t + ArenaAlloc::size() const { + return limit_ - lo_; + } + + std::size_t + ArenaAlloc::committed() const { + return committed_z_; + } + + std::size_t + ArenaAlloc::available() const { + return limit_ - free_ptr_; + } + + std::size_t + ArenaAlloc::allocated() const { + return free_ptr_ - lo_; + } + + bool + ArenaAlloc::contains(const void * x) const { + return (lo_ <= x) && (x < hi_); + } + + bool + ArenaAlloc::is_before_checkpoint(const void * x) const { + return (lo_ <= x) && (x < checkpoint_); + } + + std::size_t + ArenaAlloc::before_checkpoint() const + { + return checkpoint_ - lo_; + } + + std::size_t + ArenaAlloc::after_checkpoint() const + { + return free_ptr_ - checkpoint_; + } + + bool + ArenaAlloc::check_owned(IObject * src) const + { + byte * addr = reinterpret_cast(src); + + return (lo_ <= addr) && (addr < hi_); + } + + bool + ArenaAlloc::debug_flag() const + { + return debug_flag_; + } + + void + ArenaAlloc::clear() + { + this->set_free_ptr(lo_); + //this->limit_ = hi_; + } + + void + ArenaAlloc::checkpoint() + { + this->checkpoint_ = this->free_ptr_; + } + + std::byte * + ArenaAlloc::alloc(std::size_t z0) + { + scope log(XO_DEBUG(debug_flag_)); + + /* word size for alignment */ + constexpr uint32_t c_bpw = sizeof(std::uintptr_t); + (void)c_bpw; + + std::uintptr_t free_u64 = reinterpret_cast(free_ptr_); + (void)free_u64; + + assert(free_u64 % c_bpw == 0ul); + + std::uint32_t dz = alloc_padding(z0); + + std::size_t z1 = z0 + dz; + + assert(z1 % c_bpw == 0ul); + + this->expand(this->allocated() + z1); + + std::byte * retval = this->free_ptr_; + + log && log(xtag("self", name_), + xtag("z0", z0), + xtag("+pad", dz), + xtag("z1", z1), + xtag("size", this->size()), + xtag("avail", this->available())); + + this->free_ptr_ += z1; + + return retval; + } + + } /*namespace gc*/ +} /*namespace xo*/ + + +/* end ArenaAlloc.cpp */ diff --git a/xo-alloc/src/alloc/Blob.cpp b/xo-alloc/src/alloc/Blob.cpp new file mode 100644 index 00000000..83e4121b --- /dev/null +++ b/xo-alloc/src/alloc/Blob.cpp @@ -0,0 +1,57 @@ +/** @file Blob.cpp + * + * @author Roland Conybeare, Nov 2025 + **/ + +#include "Blob.hpp" +#include "xo/reflect/Reflect.hpp" +#include "xo/allocutil/IAlloc.hpp" + +namespace xo { + using xo::reflect::Reflect; + using xo::reflect::TaggedPtr; + + gp + Blob::make(gc::IAlloc * mm, std::size_t z) { + std::byte * mem = mm->alloc(sizeof(Blob) + z); + + return new (mem) Blob(z); + } + + TaggedPtr + Blob::self_tp() const + { + return Reflect::make_tp(const_cast(this)); + } + + void + Blob::display(std::ostream & os) const + { + os << ""; + } + + std::size_t + Blob::_shallow_size() const { + return sizeof(Blob) + z_; + } + + Object * + Blob::_shallow_copy(gc::IAlloc * mm) const { + Cpof cpof(mm, this); + std::byte * cp_mem = mm->alloc_gc_copy(sizeof(Blob) + z_, this); + + gp copy = new (cp_mem) Blob(z_); + + ::memcpy(copy->data(), data_, z_); + + return copy.get(); + } + + std::size_t + Blob::_forward_children(gc::IAlloc *) + { + return this->_shallow_size(); + } +} + +/* end Blob.cpp */ diff --git a/xo-alloc/src/alloc/CMakeLists.txt b/xo-alloc/src/alloc/CMakeLists.txt new file mode 100644 index 00000000..67e9759b --- /dev/null +++ b/xo-alloc/src/alloc/CMakeLists.txt @@ -0,0 +1,24 @@ +# alloc/CMakeLists.txt + +set(SELF_LIB xo_alloc) +set(SELF_SRCS + ArenaAlloc.cpp + ListAlloc.cpp + GC.cpp + GcStatistics.cpp + ObjectStatistics.cpp + Object.cpp + Blob.cpp + Forwarding1.cpp + generation.cpp +) + +xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) +xo_headeronly_dependency(${SELF_LIB} xo_allocutil) +# xo-unit used for time measurement +xo_headeronly_dependency(${SELF_LIB} xo_unit) +xo_dependency(${SELF_LIB} indentlog) +xo_dependency(${SELF_LIB} reflect) +xo_headeronly_dependency(${SELF_LIB} callback) + +#end CMakeLists.txt diff --git a/xo-alloc/src/alloc/Forwarding1.cpp b/xo-alloc/src/alloc/Forwarding1.cpp new file mode 100644 index 00000000..a42a3f57 --- /dev/null +++ b/xo-alloc/src/alloc/Forwarding1.cpp @@ -0,0 +1,79 @@ +/* file Forwarding1.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "Forwarding1.hpp" +#include "xo/reflect/Reflect.hpp" +#include +#include + +namespace xo { + using xo::reflect::Reflect; + using xo::reflect::TaggedPtr; + + namespace obj { + Forwarding1::Forwarding1(gp dest) + : dest_{dest} + {} + + TaggedPtr + Forwarding1::self_tp() const + { + return Reflect::make_tp(const_cast(this)); + } + + void + Forwarding1::display(std::ostream & os) const + { + os << "self_tp().td()->short_name()) + << ">"; + } + + IObject * + Forwarding1::_offset_destination(IObject * src) const + { + intptr_t offset = src - static_cast(this); + + return dest_.ptr() + offset; + } + + IObject * + Forwarding1::_destination() { + return dest_.ptr(); + } + + // LCOV_EXCL_START + std::size_t + Forwarding1::_shallow_size() const { + assert(false); + return 0; + } + // LCOV_EXCL_STOP + + // LCOV_EXCL_START + IObject * + Forwarding1::_shallow_copy(gc::IAlloc *) const { + /* forwarding objects are never copied */ + + assert(false); + return nullptr; + } + // LCOV_EXCL_STOP + + // LCOV_EXCL_START + std::size_t + Forwarding1::_forward_children(gc::IAlloc *) { + /* forwarding objects are never traced */ + + assert(false); + return 0; + } + // LCOV_EXCL_STOP + + } /*namespace obj*/ +} /*namespace xo*/ + +/* end Forwarding1.cpp */ diff --git a/xo-alloc/src/alloc/GC.cpp b/xo-alloc/src/alloc/GC.cpp new file mode 100644 index 00000000..f1c52d1b --- /dev/null +++ b/xo-alloc/src/alloc/GC.cpp @@ -0,0 +1,1526 @@ +/* GC.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "GcStatistics.hpp" +#include "GC.hpp" +#include "Object.hpp" +#include "xo/indentlog/scope.hpp" +#include +#include +#include + +namespace xo { + namespace gc { + bool + MutationLogEntry::is_child_forwarded() const + { + assert(!parent_->_is_forwarded()); + + return (*lhs_)->_is_forwarded(); + } + + bool + MutationLogEntry::is_parent_forwarded() const + { + return parent_->_is_forwarded(); + } + + IObject * + MutationLogEntry::parent_destination() const + { + //const bool c_debug_flag = true; + //scope log(XO_DEBUG(c_debug_flag)); + + if (parent_->_is_forwarded()) { + //log && log("parent is forwarded", xtag("parent", (void*)parent_)); + + return parent_->_destination(); + } else { + //log && log("parent is ordinary", xtag("parent", (void*)parent_)); + + return parent_; + } + } + + MutationLogEntry + MutationLogEntry::update_parent_moved(IObject * parent_to) const + { + std::byte * parent_from = reinterpret_cast(parent_); + std::byte * lhs_from = reinterpret_cast(lhs_); + + std::ptrdiff_t offset = (lhs_from - parent_from); + + std::byte * lhs_to = reinterpret_cast(parent_to) + offset; + + return MutationLogEntry(parent_to, + reinterpret_cast(lhs_to)); + } + + void + MutationLogEntry::fixup_parent_child_moved(IObject * child_to) + { + *(this->lhs_) = child_to; + } + + GC::GC(const Config & config) + : config_{config} + { + enum { NurseryFrom, NurseryTo, TenuredFrom, TenuredTo }; + + std::size_t nursery_size = config.initial_nursery_z_; + std::size_t tenured_size = config.initial_tenured_z_; + + if (config_.incr_gc_threshold_ > nursery_size) { + throw std::runtime_error(tostr("GC::ctor: expected nursery gc threshold < nursery size", + xtag("nursery-gc-threshold", config_.incr_gc_threshold_), + xtag("nursery-size", nursery_size))); + } + + if (nursery_size + config_.full_gc_threshold_ > tenured_size) { + throw std::runtime_error(tostr("GC::ctor: expected nursery size + tenured gc threshold < tenured size", + xtag("nursery-size", nursery_size), + xtag("tenured-size", tenured_size), + xtag("full-gc-threshold", config_.full_gc_threshold_) + )); + } + + if (config_.incr_gc_threshold_ > nursery_size) + this->config_.incr_gc_threshold_ = nursery_size; + + if (config_.full_gc_threshold_ > tenured_size) + this->config_.full_gc_threshold_ = tenured_size; + + this->nursery_[role2int(role::from_space)] + = ArenaAlloc::make("NA", nursery_size, config.debug_flag_); + + this->nursery_[role2int(role::to_space) ] + = ArenaAlloc::make("NB", nursery_size, config.debug_flag_); + + this->tenured_[role2int(role::from_space)] + = ArenaAlloc::make("TA", tenured_size, config.debug_flag_); + + this->tenured_[role2int(role::to_space) ] + = ArenaAlloc::make("TB", tenured_size, config.debug_flag_); + + nursery_[role2int(role::from_space)]->expand(config.incr_gc_threshold_); + nursery_[role2int(role::to_space) ]->expand(config.incr_gc_threshold_); + tenured_[role2int(role::from_space)]->expand(config.full_gc_threshold_); + tenured_[role2int(role::to_space) ]->expand(config.full_gc_threshold_); + + this->mutation_log_[role2int(role::from_space)] = std::make_unique(); + this->mutation_log_[role2int(role::to_space )] = std::make_unique(); + this->defer_mutation_log_ = std::make_unique(); + + this->gc_history_ = CircularBuffer(config.stats_history_z_); + + this->checkpoint(); + } + + GC::~GC() { + /* hygiene */ + this->clear(); + + this->nursery_[role2int(role::from_space)].reset(); + this->nursery_[role2int(role::to_space) ].reset(); + + this->tenured_[role2int(role::from_space)].reset(); + this->tenured_[role2int(role::to_space) ].reset(); + + this->gc_root_v_.clear(); + + this->mutation_log_[role2int(role::from_space)].reset(); + this->mutation_log_[role2int(role::to_space) ].reset(); + this->defer_mutation_log_.reset(); + } + + up + GC::make(const Config & config) + { + return std::make_unique(config); + } + + GC * + GC::from(IAlloc * mm) + { + return dynamic_cast(mm); + } + + const std::string & + GC::name() const + { + static std::string s_default_name = "GC"; + return s_default_name; + } + + std::size_t + GC::size() const + { + return nursery_to()->size() + tenured_to()->size(); + } + + std::size_t + GC::committed() const + { + return (nursery_to()->committed() + + nursery_from()->committed() + + tenured_to()->committed() + + tenured_from()->committed()); + } + + std::size_t + GC::allocated() const + { + return (nursery_[role2int(role::to_space)]->allocated() + + tenured_[role2int(role::to_space)]->allocated()); + } + + std::size_t + GC::available() const + { + return nursery_[role2int(role::to_space)]->available(); + } + + bool + GC::fromspace_contains(const void * x) const + { + return (nursery_[role2int(role::from_space)]->contains(x) + || tenured_[role2int(role::from_space)]->contains(x)); + } + + bool + GC::contains(const void * x) const + { + return (nursery_[role2int(role::to_space)]->contains(x) + || tenured_[role2int(role::to_space)]->contains(x)); + } + + bool + GC::is_before_checkpoint(const void * x) const + { + return nursery_[role2int(role::to_space)]->is_before_checkpoint(x); + } + + std::size_t + GC::before_checkpoint() const + { + return this->nursery_to()->before_checkpoint(); + } + + std::size_t + GC::after_checkpoint() const + { + return this->nursery_to()->after_checkpoint(); + } + + bool + GC::debug_flag() const + { + return config_.debug_flag_; + } + + GcStatisticsExt + GC::get_gc_statistics() const + { + GcStatisticsExt retval = GcStatisticsExt(this->native_gc_statistics()); + + retval.nursery_z_ = nursery_[role2int(role::to_space)]->size(); + retval.nursery_before_checkpoint_z_ = this->nursery_to()->before_checkpoint(); + retval.nursery_after_checkpoint_z_ = this->nursery_to()->after_checkpoint(); + retval.tenured_z_ = tenured_[role2int(role::to_space)]->size(); + + return retval; + } + + std::size_t + GC::pagesize() const + { + return nursery_to()->page_size(); + } + + std::size_t + GC::hugepage_z() const + { + return nursery_to()->hugepage_z(); + } + + std::size_t + GC::nursery_from_allocated() const + { + return nursery_from()->allocated(); + } + + std::size_t + GC::nursery_to_allocated() const + { + return nursery_to()->allocated(); + } + + std::size_t + GC::nursery_to_reserved() const + { + return nursery_to()->reserved(); + } + + std::size_t + GC::nursery_to_committed() const + { + return nursery_to()->committed(); + } + + std::size_t + GC::nursery_before_checkpoint() const + { + return nursery_to()->before_checkpoint(); + } + + std::size_t + GC::nursery_after_checkpoint() const + { + return nursery_to()->after_checkpoint(); + } + + std::pair + GC::nursery_span(role role) const { + return nursery(role)->allocated_span(); + } + + std::size_t + GC::tenured_to_reserved() const + { + return tenured_to()->reserved(); + } + + std::size_t + GC::tenured_to_committed() const + { + return tenured_to()->committed(); + } + + std::size_t + GC::tenured_before_checkpoint() const + { + return tenured_to()->before_checkpoint(); + } + + std::size_t + GC::tenured_after_checkpoint() const + { + return tenured_to()->after_checkpoint(); + } + + generation_result + GC::tospace_generation_of(const void * x) const + { + if (tenured_[role2int(role::to_space)]->contains(x)) + return generation_result::tenured; + + if (nursery_[role2int(role::to_space)]->contains(x)) + return generation_result::nursery; + + return generation_result::not_found; + } + + std::tuple + GC::location_of(role role, const void *x) const + { + { + auto space = this->tenured(role); + auto [is_tenured, offset] = space->location_of(x); + + if (is_tenured) + return std::make_tuple(generation_result::tenured, offset, space->allocated(), space->committed()); + } + + { + auto space = this->nursery(role); + auto [is_nursery, offset] = nursery(role)->location_of(x); + + if (is_nursery) + return std::make_tuple(generation_result::nursery, offset, space->allocated(), space->committed()); + } + + return std::make_tuple(generation_result::not_found, 0, 0, 0); + } + + std::tuple + GC::tospace_location_of(const void * x) const + { + return location_of(role::to_space, x); + } + + std::tuple + GC::fromspace_location_of(const void * x) const + { + return location_of(role::from_space, x); + } + + generation_result + GC::fromspace_generation_of(const void * x) const + { + if (tenured_[role2int(role::from_space)]->contains(x)) + return generation_result::tenured; + + if (nursery_[role2int(role::from_space)]->contains(x)) + return generation_result::nursery; + + return generation_result::not_found; + } + + std::byte * + GC::free_ptr(generation gen) + { + switch(gen) { + case generation::nursery: + return nursery_[role2int(role::to_space)]->free_ptr(); + case generation::tenured: + return tenured_[role2int(role::to_space)]->free_ptr(); + // LCOV_EXCL_START + case generation::N: + assert(false); + // LCOV_EXCL_STOP + } + + return nullptr; + } + + std::size_t + GC::mlog_size() const { + return mutation_log_[role2int(role::to_space)]->size(); + } + + void + GC::clear() + { + nursery_[role2int(role::from_space)]->clear(); + nursery_[role2int(role::to_space) ]->clear(); + + tenured_[role2int(role::from_space)]->clear(); + tenured_[role2int(role::to_space) ]->clear(); + } + + void + GC::add_gc_root(IObject ** addr) + { + gc_root_v_.push_back(addr); + } + + void + GC::remove_gc_root(IObject ** addr) + { + /* Multithreaded GC not supported */ + + assert(!this->gc_in_progress()); + + auto new_end_ix = std::remove(gc_root_v_.begin(), gc_root_v_.end(), addr); + + /* erase now-unused slots */ + gc_root_v_.erase(new_end_ix, gc_root_v_.end()); + } + + auto + GC::add_gc_copy_callback(up fn) -> CallbackId + { + return gc_copy_cbset_.add_callback(std::move(fn)); + } + + void + GC::checkpoint() + { + nursery_to()->checkpoint(); + /* checkpoint T generation so we can trigger GC based on new T objects rather than + * overall T size + */ + tenured_to()->checkpoint(); + } + + std::byte * + GC::alloc(std::size_t z) + { + auto N_to = this->nursery_to(); + + if (!incr_gc_pending_ && (N_to->after_checkpoint() > config_.incr_gc_threshold_)) { + /* automatically ups to generation::tenured */ + this->request_gc(generation::nursery); + } + + std::byte * x = N_to->alloc(z); + + /* ListAlloc won't fail unless we exhaust memory -- instead will increase heap size */ + assert(x); + + return x; + } + + std::byte * + GC::alloc_gc_copy(std::size_t z, const void * src) + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("z", z), + xtag("+pad", IAlloc::alloc_padding(z))); + + generation_result src_gr = this->fromspace_generation_of(src); + + std::byte * retval = nullptr; + + switch (src_gr) { + case generation_result::tenured: + { + log && log("tenured"); + + retval = this->tenured_to()->alloc(z); + + gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, + z, src, retval, generation::tenured, generation::tenured); + } + break; + case generation_result::nursery: + { + if (this->nursery_from()->is_before_checkpoint(src)) + { + /* nursery object has survived 2nd collection cycle + * -> promote into tenured generation + */ + retval = this->tenured_to()->alloc(z); + + log && log("promote", xtag("addr", (void*)retval)); + + assert(this->tospace_generation_of(retval) == generation_result::tenured); + + gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, + z, src, retval, generation::nursery, generation::tenured); + + this->gc_statistics_.total_promoted_ + += IAlloc::with_padding(z); + + } else { + log && log("nursery"); + + retval = this->nursery_to()->alloc(z); + + gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, + z, src, retval, generation::nursery, generation::nursery); + } + } + break; + case generation_result::not_found: + /* something wrong -- we only copy objects that are known to be in from-space + */ + + assert(false); + break; + } + + assert(retval); + + log && log(xtag("retval", retval)); + + return retval; + } + + void + GC::assign_member(IObject * parent, IObject ** lhs, IObject * rhs) + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("parent", parent), xtag("lhs", lhs), xtag("rhs", rhs)); + + ++gc_statistics_.n_mutation_; + + *lhs = rhs; + + if (runstate_.in_progress()) { + /* don't log mutations (if any) during GC */ + return; + } + + if (!config_.allow_incremental_gc_) { + /* full GCs don't need mutation log, since no cross-generational pointers */ + return; + } + + switch (tospace_generation_of(rhs)) + { + case generation_result::tenured: + /* only need to log mutations that create tenured->nursery pointers */ + log && log(xtag("act", "any->T no log")); + return; + + case generation_result::nursery: + switch (tospace_generation_of(parent)) { + case generation_result::nursery: + if (is_before_checkpoint(parent)) { + log && log(xtag("act", "N1->N0 must mlog")); + + // N1->N0, so must log + this->mutation_log_[role2int(role::to_space)]->push_back(MutationLogEntry(parent, lhs)); + ++(this->gc_statistics_.n_logged_mutation_); + ++(this->gc_statistics_.n_xckp_mutation_); + } else { + // parent in N0, not an xckp mutation + log && log(xtag("act", "N0->any no long")); + } + break; + case generation_result::tenured: + // T->N, so must log + log && log(xtag("act", "T->N must mlog")); + + this->mutation_log_[role2int(role::to_space)]->push_back(MutationLogEntry(parent, lhs)); + ++(this->gc_statistics_.n_logged_mutation_); + ++(this->gc_statistics_.n_xgen_mutation_); + break; + case generation_result::not_found: + // parent is global + // This may be ok (provided lhs is a gc root) + log && log(xtag("act", "root->any no log")); + break; + } + break; + + case generation_result::not_found: + log && log(xtag("act", "any->root no log")); + + // child is global; + // logging not required + break; + } + } + + void + GC::forward_inplace(IObject ** lhs) + { + scope log(XO_DEBUG(config_.debug_flag_), xtag("lhs", lhs)); + + Object::_forward_inplace(lhs, this); + } + + bool + GC::check_owned(IObject * src) const + { + return this->fromspace_contains(src); + } + + bool + GC::check_move(IObject * src) const + { + return (this->runstate().full_move() + || (this->tospace_generation_of(src) != gc::generation_result::tenured)); + } + + bool + GC::check_write_barrier(const void * parent, + const void * const * lhs, + bool may_throw_flag) const + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("P", parent), xtag("L", lhs)); + + if (!this->contains(parent)) { + if (may_throw_flag) { + throw std::runtime_error(tostr("GC::check_write_barrier", + ": expected parent object P in GC to-space", + xtag("P", parent))); + } + return false; + } + +#ifdef NOPE // don't want to assume IObject* + std::size_t parent_z = parent->_shallow_size(); + + const std::byte * parent_addr = reinterpret_cast(parent); + const std::byte * lhs_addr = reinterpret_cast(lhs); + + if ((lhs_addr < parent_addr) || (parent_addr + parent_z < lhs_addr)) { + if (may_throw_flag) { + throw std::runtime_error + (tostr("GC::check_write_barrier", + ": expected lhs address L within address extent z of parent P", + xtag("P", parent), xtag("z", parent_z), + xtag("P+z", parent_addr + parent_z), + xtag("L", lhs))); + } + return false; + } +#endif + + const void * rhs = *lhs; + + if (!rhs) + return true; + + auto parent_gen = tospace_generation_of(parent); + auto rhs_gen = tospace_generation_of(rhs); + + log && log(xtag("C", rhs), + xtag("gen(P)", parent_gen), xtag("gen(C)", rhs_gen), + xtag("P.before-ckp", is_before_checkpoint(parent)), + xtag("C.before-ckp", is_before_checkpoint(rhs))); + + switch(parent_gen) { + case generation_result::nursery: + if (is_before_checkpoint(parent)) { + switch(rhs_gen) { + case generation_result::nursery: + if (is_before_checkpoint(rhs)) { + /* no mlog entry needed */ + log && log(xtag("msg", "N1->N1 - trivial")); + return true; + } else { + /* need to check mlog */ + log && log(xtag("msg", "N1->N0 - xgen")); + } + break; + case generation_result::tenured: + /* no mlog entry needed */ + log && log(xtag("msg", "N1->T - trivial")); + return true; + case generation_result::not_found: + /* possible non-gc rhs */ + log && log(xtag("msg", "non-gc rhs - trivial")); + return true; + } + } else { + /* no mlog entry needed */ + log && log(xtag("msg", "N0->any - trivial")); + return true; + } + break; + case generation_result::tenured: + switch(rhs_gen) { + case generation_result::nursery: + /* need to check mlog */ + log && log(xtag("msg", "T->N - xgen")); + break; + case generation_result::tenured: + /* no mlog entry needed */ + log && log(xtag("msg", "T->T - trivial")); + return true; + case generation_result::not_found: + /* possible non-gc rhs */ + log && log(xtag("msg", "non-gc rhs - trivial")); + return true; + } + break; + case generation_result::not_found: + /* already excluded -> impossible */ + log && log(xtag("msg", "assert")); + assert(false && "already verified parent owned by GC"); + } + + /* control here -> expect mutation log entry. + * search mutation log + verify such entry exists + */ + for (MutationLogEntry & mlog : *(mutation_log_[role2int(role::to_space)])) { + if ((mlog.parent() == parent) && ((const void * const *)mlog.lhs() == lhs)) { + return true; + } + mlog.lhs(); + } + + if (may_throw_flag) { + throw std::runtime_error + (tostr("GC::check_write_barrier", + ": expected mlog entry for xgen pointer L->C within parent P", + xtag("P", parent), xtag("L", lhs), xtag("C", rhs), + xtag("gen(P)", parent_gen), xtag("gen(C)", rhs_gen))); + } + return false; + } + + void + GC::swap_nursery() + { + up tmp = std::move(nursery_[role2int(role::to_space)]); + nursery_[role2int(role::to_space)] = std::move(nursery_[role2int(role::from_space)]); + nursery_[role2int(role::from_space)] = std::move(tmp); + nursery_polarity_ = 1 - nursery_polarity_; + } + + void + GC::swap_tenured() + { + up tmp = std::move(tenured_[role2int(role::to_space)]); + tenured_[role2int(role::to_space)] = std::move(tenured_[role2int(role::from_space)]); + tenured_[role2int(role::from_space)] = std::move(tmp); + tenured_polarity_ = 1 - tenured_polarity_; + } + + void + GC::swap_mutation_log() + { + up tmp = std::move(mutation_log_[role2int(role::to_space)]); + mutation_log_[role2int(role::to_space)] = std::move(mutation_log_[role2int(role::from_space)]); + mutation_log_[role2int(role::from_space)] = std::move(tmp); + } + + void + GC::swap_spaces(generation target) + { + scope log(XO_DEBUG(this->debug_flag()), xtag("upto", target)); + + // will be copying into the memory regions currently labelled FromSpace + + /* gc will copy some to-be-determined amount in [0..promote_z] + from nursery->tenured generation. + */ + std::size_t max_promote_z = nursery_[role2int(role::to_space)]->before_checkpoint(); + + ArenaAlloc * tenured_to = this->tenured_to(); + + /* tenured generation may need this much space */ + std::size_t need_tenured_z = (tenured_to->allocated() + + max_promote_z + + config_.full_gc_threshold_); + + log && log(xtag("alloc_tenured_z", tenured_to->allocated()), + xtag("max_promote_z", max_promote_z), + xtag("full_gc_threshold", config_.full_gc_threshold_), + xtag("need_tenured_z", need_tenured_z)); + + tenured_to->expand(tenured_to->allocated() + + max_promote_z + + config_.full_gc_threshold_); + + if (target == generation::tenured) { + tenured_from()->clear(); + + this->swap_tenured(); + } + + /* subtracting max_promote_z is correct here, since anything not promoted is garbage */ + std::size_t need_nursery_z = (nursery(role::to_space)->allocated() + - max_promote_z + + config_.incr_gc_threshold_); + + log && log(xtag("need_nursery_z", need_nursery_z)); + + /* (from-space is about to become to-space, to receive surviving nursery objects) */ + nursery(role::from_space)->reset(need_nursery_z); + + this->swap_nursery(); + + this->swap_mutation_log(); + + ArenaAlloc * N_from = nursery(role::from_space); + log && log(xtag("nursery.from", N_from->name()), xtag("size", N_from->size())); + ArenaAlloc * N_to = nursery(role::to_space); + log && log(xtag("nursery.to", N_to->name()), xtag("size", N_to->size())); + ArenaAlloc * T_from = tenured(role::from_space); + log && log(xtag("tenured.from", T_from->name()), xtag("size", T_from->size())); + ArenaAlloc * T_to = tenured(role::to_space); + log && log(xtag("tenured.to", T_to->name()), xtag("size", T_to->size())); + + } /*swap_spaces*/ + + void + GC::capture_object_statistics(generation upto, capture_phase phase) + { + if (config_.object_stats_flag_) { + /* scan nursery */ + this->nursery_[role2int(role::to_space)]->capture_object_statistics + (phase, + &object_statistics_sab_[gen2int(generation::nursery)]); + + if (upto == generation::tenured) { + /* scan tenured */ + this->tenured_[role2int(role::to_space)]->capture_object_statistics + (phase, + &object_statistics_sab_[gen2int(generation::tenured)]); + } + } + } + + void + GC::copy_object(IObject ** pp_object, + generation upto, + ObjectStatistics * object_stats) + { + void * object_address = *pp_object; + + if (nursery_[role2int(role::to_space)]->contains(object_address) + || ((upto == generation::tenured) + && tenured_[role2int(role::to_space)]->contains(object_address))) + { + /* global is already in to-space */ + ; + } else if((upto == generation::nursery) && tenured_[role2int(role::to_space)]->contains(object_address)) + { + /* skip tenured objects when incremental collection */ + ; + } else { + *pp_object = Object::_deep_move(*pp_object, this, object_stats); + } + } + + void + GC::copy_globals(generation upto) + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("roots", gc_root_v_.size())); + + for (IObject ** pp_root : gc_root_v_) { + this->copy_object(pp_root, upto, + &object_statistics_sae_[gen2int(upto)]); + } + } + + void + GC::incremental_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * per_type_stats) + { + scope log(XO_DEBUG(config_.debug_flag_), xtag("from_mlog.size", from_mlog->size())); + + /* categorize pointers based on combination of {source address, destination address}, + * only care about the generation associated with an address. + * + * N0 : nursery(from), before checkpoint + * N0': nursery(to), before checkpoint + * N1 : nursery(from), after checkpoint + * N1': nursery(to), after checkpoint + * T : tenured(to) + * + * loc(P): parent region before GC + * loc(C): child region before GC + * + * | | forwarded | loc now post | loc after | + * | | already? | root copy | action | + * | loc(P) loc(C) | P C | P' C' | P' C' | defer | action + * ----|---------------+--------------+---------------+---------------+-------+--------------- + * (a) | T N0 | no no | T N0 | T N1' | | C->N1', +mlog + * (b) | | yes | N1' | N1' | | +mlog + * (c) | T N1 | no no | T N1 | T T | | C->T, -mlog + * (d) | | yes | T T | T T | | -mlog + * (e) | N1 N0 | no no | N1 N0 | N1 N0 | P ->C | defer + * (f) | | yes | N1 N1' | N1 N1' | P ->C'| defer + * (g) | | yes yes | T N1' | T N1' | | +mlog + * + * notes: + * (a) C survives due to xgen ptr {T -> N0}; after collection have xgen ptr {T -> N1}. + * (b) C already evac'd; after collection stil have xgen ptr {T -> N1} + * (c) C survives due to xgen ptr (T -> N1): promote to T, so no longer xgen + * (d) C already evac'd: after collection no longer xgen (T -> T) + * (e) P,C maybe garbage. don't move either, but defer mlog incase P saved by a subsequent mutation. + * in that case C saved alto, + will still have an xgen ptr, so still need an mlog entry + * (f) P maybe garbage, C survives. defer mlog incase P saved+promoted by a subsequent mutation; + * in that case will still have an xgen (T -> N) ptr, so still need an mlog entry. + */ + + std::size_t i_from = 0; + // number of rescued subgraphs via mutation log entries + std::size_t n_rescue = 0; + + for (MutationLogEntry & from_entry : *from_mlog) + { + if (log) { + if (i_from % 10000 == 0 || true) + log(xtag("i_from", i_from)); + } + + void * parent = from_entry.parent(); + + if (tospace_generation_of(parent) == generation_result::tenured) + { + // cases (a)(b)(c)(d) + // loc(P) is T. T didn't move b/c incremental gc. + + if (from_entry.is_dead()) { + // obsolete mutation -- no longer belongs to parent, discard + } else { + // note: child obtained (as it must be) by reading from parent's memory _now_. + IObject * child_from = from_entry.child(); + + if (child_from) { + if (!child_from->_is_forwarded()) { + // P->C*. + // either: + // - C*=C in from-space, so needs evac + // - C*=C' in to-space, P already updated b/c of another mutation + // + if (fromspace_generation_of(child_from) != generation_result::not_found) { + // C*=C in from-space. needs evac, along with reachable descendants + // + // Includes cases: + // (a) T->N0 + // (c) T->N1 + + ++n_rescue; + + log && log(xtag("parent", parent), xtag("act", "move child"), xtag("child.from", child_from)); + + Object::_deep_move(child_from, this, per_type_stats); + + log && log(xtag("child.to", child_from->_destination())); + + // C forwards to C', fall thru to parent fixup below + // (a) T->N1' + // (c) T->T + } else { + // P updated via some other mutation + // so don't need this mlog + ; + } + } + + // re-test, state may have changed above + if (from_entry.is_child_forwarded()) { + // P->C, C moved to C' + // Includes cases (a),(c) from above + + IObject * child_to = child_from->_destination(); + + log && log(xtag("act", "fixup parent"), xtag("parent", parent), xtag("lhs", from_entry.lhs()), xtag("child.from", child_from), xtag("child.to", child_to)); + + from_entry.fixup_parent_child_moved(child_to); + +#ifndef NDEBUG + { + // verify fixup was effective + IObject * child_from2 = from_entry.child(); + assert(child_from2 == child_to); + } +#endif + + + // P->C', loc(C') in {N1', T'} + + if (tospace_generation_of(child_to) == generation_result::nursery) { + // (b) loc(P)=T, loc(C')=N1'; also case (a) + + log && log(xtag("act", "still xgen -> keep mlog entry")); + + // still have xgen pointer, so need mlog for it + to_mlog->push_back(from_entry); + } else { + // (d) loc(P)=T, loc(C')=T; also case (c) + // no longer xgen, so does not require mlog + } + } + + } else { + // nullptr child, discard + } + } + } else if (from_entry.is_parent_forwarded()) { + // Must have: + // loc(P) = N1, because: + // loc(P)=N0 -> ineligible for mlog; + // loc(P)=T -> not moved on incr GC + // + // follows that loc(P') = T + // already have P'->C' when parent moved separately + + IObject * parent_to = from_entry.parent_destination(); + + log && log(xtag("parent_to", (void*)parent_to)); + + assert(tospace_generation_of(parent_to) == generation_result::tenured); + + MutationLogEntry to_entry = from_entry.update_parent_moved(parent_to); + + IObject * child_to = to_entry.child(); // after moving + + if (tospace_generation_of(child_to) == generation_result::nursery) { + if (to_entry.is_dead()) { + ; + } else { + // (g) loc(P)=N1, loc(C)=N0, loc(P')=T, loc(C')=N1 + to_mlog->push_back(to_entry); + } + + } + } else { + log && log("defer"); + + // loc(P) = N1, loc(C) = N0, P may be garbage + // Includes cases: + // (e) P->C, C not moved + // (f) P->C, C moved to C' + // + // P may yet be rescued by another mlog entry, so defer + + if (!from_entry.is_dead()) { + defer_mlog->push_back(from_entry); + } + } + + ++i_from; + } + + from_mlog->clear(); + + if (n_rescue == 0) { + // if we didn't rescue any objects + // then we now confirm that otherwise-unreachable parents in defer_mlog + // are garbage + + defer_mlog->clear(); + } + } + + void + GC::full_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * /*per_type_stats*/) + { + scope log(XO_DEBUG(config_.debug_flag_), xtag("from_mlog.size", from_mlog->size())); + + /* categorize pointers based on combination of {source address, destination address}, + * only care about the generation associated with an address. + * + * N0 : nursery(from), before checkpoint + * N0': nursery(to), before checkpoint + * N1 : nursery(from), after checkpoint + * N1': nursery(to), after checkpoint + * T : tenured(from) + * T': tenured(to) + * + * loc(P): parent region before GC + * loc(C): child region before GC + * + * | | forwarded | loc now post | loc after | + * | | already? | root copy | action | + * | loc(P) loc(C) | P C | P' C' | P' C' | defer | action + * ----|---------------+--------------+---------------+---------------+-------+--------------- + * (a) | T N0 | no no | T N0 | T N0 | P ->C | defer + * (b) | | yes | N1' | N1' | P ->C'| defer + * | | yes no | impossible + * (b2)| | yes | T' N1' | T' N1' | | +mlog + * (c) | T N1 | no no | T N1 | T T | P ->C | defer + * (d) | | yes | T T' | T T' | P ->C'| defer + * | | yes no | impossible + * (d2)| | yes | T' T' | T' T' | | -mlog + * (e) | N1 N0 | no no | N1 N0 | N1 N0 | P ->C | defer + * (f) | | yes | N1 N1' | N1 N1' | P ->C'| defer + * | | yes no | impossible + * (g) | | yes yes | T' N1' | T' N1' | | +mlog + * + * notes: + * (a) P,C maybe garbage. don't move either, but defer mlog incase P saved by a subsequent mutation; + * in that case C saved also, + will still have an xgen ptr, and still need an mlog entry. + * (b) C already evac'd, but P maybe garbage. defer mlog incase P rescued by a subsequent mutation; + * in that case will still have an xgen (T -> N) ptre, and still need an mlog entry. + * (b2) P,C already evac'd. Must update+rembexember xgen ptr {T -> N1} + * (c) P,C maybe garbage. don't move either, but defer mlog in case P saved by a subsequent mutation; + * in that case C promoted, no longer xgen + * (d) P maybe garbage. defer in case P saved by a subsequent mutation. + * C now tenured, so will no longer have an xgen pointer. + * (d2) P,C already evac'd. After collection no longer have xgen pointer, so no mlog. + * (e) P,C maybe garbage. don't move either, but defer mlog incase P saved by a subsequent mutation. + * in that case C saved alto, + will still have an xgen ptr, so still need an mlog entry + * (f) P maybe garbage, C survives. defer mlog incase P saved+promoted by a subsequent mutation; + * in that case will still have an xgen (T -> N) ptr, so still need an mlog entry. + * (g) P,C already evac'd. Still have xgen pointer, must mlog + */ + + std::size_t i_from = 0; + // number of rescued subgraphs via mutation log entries + std::size_t n_rescue = 0; + + for (MutationLogEntry & from_entry : *from_mlog) + { + log && (i_from % 10000 == 0) && log(xtag("i_from", i_from)); + + if (from_entry.is_parent_forwarded()) { + IObject * parent_to = from_entry.parent_destination(); + + log && log(xtag("parent_to", (void*)parent_to)); + + assert(tospace_generation_of(parent_to) == generation_result::tenured); + + MutationLogEntry to_entry = from_entry.update_parent_moved(parent_to); + + // note: child obtained (as it must be) by reading from prarent's memory _now_. + // Since parent has moved, child has too + IObject * child_to = to_entry.child(); // after moveing + + if (tospace_generation_of(parent_to) == generation_result::tenured) + { + // cases (b2)(d2)(g), loc(P) is T' + // In all these cases parent has already been moved; + // therefore child has also been moved. + // Just need to decide whether to keep mlog entry + + if (from_entry.is_dead()) { + // obsolete mutation -- no longer belongs to parent, discard + } else if (child_to) { + assert(!child_to->_is_forwarded()); + + if (tospace_generation_of(child_to) == generation_result::nursery) { + // case + // (b2) loc(P')=T', loc(C')=N1' --> +mlog + // (g) loc(P')=T', loc(C')=N1' --> +mlog + // + to_mlog->push_back(to_entry); + } else { + // case + // (d2) loc(P')=T', loc(C')=T' --> -mlog + } + } + } else { + // impossible - wouldn't have made mlog entry + + + assert(false); + } + } else { + // case + // (a) defer + // (b) defer + // (c) defer + // (d) defer + // (e) defer + // (f) defer + + defer_mlog->push_back(from_entry); + } + + ++i_from; + } + + from_mlog->clear(); + + if (n_rescue == 0) { + // if we didn't rescue any objects + // then we now confirm that otherwise-unreachable parents in defer_mlog + // are garbage + + defer_mlog->clear(); + } + } + + + void + GC::incremental_gc_forward_mlog(ObjectStatistics * per_type_stats) + { + /* control here: + * - incremental gc. + * - gc roots have been copied, along with everything reachable from them. + * + * plan: + * - forward mutations in *from_mutation_log, writing them to + * *to_mutationlog and/or *defer_mutation_log. + * Use defer when mutation P->C encountered, but P was not copied. + * P appears to be garbage, but may turn out to be live if encountered + * in another mutation. + * + */ + + MutationLog * to_mlog = mutation_log_[role2int(role::to_space)].get(); + + for (;;) { + MutationLog * from_mlog = mutation_log_[role2int(role::from_space)].get(); + MutationLog * defer_mlog = defer_mutation_log_.get(); + + this->incremental_gc_forward_mlog_phase(from_mlog, + to_mlog, + defer_mlog, + per_type_stats); + + assert(from_mlog->empty()); + + if (defer_mlog->empty()) { + /* fixpoint reached */ + break; + } + + /* control here: + * 1. at least one mlog triggered a rescue + * 2. at least one mlog was deferred (b/c otherwise-unreachable parent) + * + * it's conceivable deferred parent now reachable thanks to rescues; + * revisit entries in defer_mlog, + * + * using now-empty from_mlog as scratch for any remaining deferred entries + */ + + std::swap(mutation_log_[role2int(role::from_space)], defer_mutation_log_); + } + } + + void + GC::full_gc_forward_mlog(ObjectStatistics * per_type_stats) + { + /* control here: + * - full gc. + * - gc roots have been copied, along with everything reachable + * from them. + * + * plan: + * - forward mutations in *from_mutation_log, writing them to + * *to_mutation_log and/or *defer_mutation_log. + */ + + MutationLog * to_mlog = this->mutation_log(role::to_space); + + for (;;) { + MutationLog * from_mlog = this->mutation_log(role::from_space); + MutationLog * defer_mlog = defer_mutation_log_.get(); + + this->full_gc_forward_mlog_phase(from_mlog, + to_mlog, + defer_mlog, + per_type_stats); + assert(from_mlog->empty()); + + if (defer_mlog->empty()) + break; + + /* control here: + * 1. at least one mlog triggered a rescue + * 2. at least one mlog was deferred (had otherwise-unreachable parent) + * + * possible that deferred parent is now reachable thanks to a rescue; + * to confirm/refute this need to revisit entries in defer_mlog. + */ + std::swap(mutation_log_[role2int(role::from_space)], defer_mutation_log_); + } + } + + void + GC::forward_mutation_log(generation upto) + { + scope log(XO_DEBUG(config_.debug_flag_)); + + if (upto == generation::tenured) { + this->full_gc_forward_mlog + (&object_statistics_sae_[gen2int(generation::tenured)]); + } else { + this->incremental_gc_forward_mlog + (&object_statistics_sae_[gen2int(generation::nursery)]); + } + } + + void + GC::cleanup_phase(generation upto, nanos dt) + { + scope log(XO_DEBUG(config_.stats_flag_)); + + std::size_t N0_before_gc = nursery_from()->after_checkpoint(); + std::size_t N1_before_gc = nursery_from()->before_checkpoint(); + + std::size_t T0_before_gc = tenured_from()->after_checkpoint(); + std::size_t T1_before_gc = tenured_from()->before_checkpoint(); + + std::size_t N_before_gc = nursery_from()->allocated(); + std::size_t T_before_gc = tenured_from()->allocated(); + + std::size_t N_after_gc = nursery_to()->allocated(); + std::size_t T_after_gc = tenured_to()->allocated(); + //std::byte * N_free_ptr = nursery_[role2int(role::to_space)]->free_ptr(); + + std::size_t new_alloc_z = N0_before_gc; + /* survive_z: bytes surviving first collection */ + std::size_t survive_z = N_after_gc; + /* promote_z: bytes surviving 2nd collection */ + std::size_t promote_z = (gc_statistics_.total_promoted_ + - gc_statistics_.total_promoted_sab_); + + /* #of bytes copied by this collection cycle */ + std::size_t effort_z = 0; + if (upto == generation::nursery) { + effort_z = N_after_gc + promote_z; + } else { + effort_z += N_after_gc + T_after_gc; + } + + /* persist_z: bytes surviving 3rd or later collection */ + std::size_t persist_z = 0; + if (upto == generation::tenured) + persist_z = T_after_gc - promote_z; + /* #of bytes found to be garbage on first collection + * (reminder: N_after_gc consists *entirely* of survives from N0_before_gc; + * + all such survivors are in N_after_gc) + */ + std::size_t garbage0_z = (N0_before_gc - N_after_gc); + /* #of bytes found to be garbage on 2nd collection */ + std::size_t garbage1_z = (N1_before_gc - promote_z); + /* #of bytes found to be garbage on 3rd or later collection */ + std::size_t garbageN_z = 0; + if (upto == generation::tenured) + garbageN_z = (T_before_gc - T_after_gc + promote_z); + + /* Don't reset from-space here, it's unnecessary. + * Would be permissible, but interferes with GC object modelling in + * xo-object/utest/GC.test.cpp + */ + //this->nursery_[role2int(role::from_space)]->reset(0); + //this->tenured_[role2int(role::from_space)]->reset(0); + + /* objects currenty in to-space nursery have survived one collection */ + this->nursery_to()->checkpoint(); + if (upto == generation::tenured) + this->tenured_to()->checkpoint(); + + if (log) { + log(xtag("gcseq_before_gc", gc_statistics_.n_gc())); + log(xtag("N0_before_gc", N0_before_gc)); + log(xtag("N1_before_gc", N1_before_gc)); + log(xtag("N_after_gc", N_after_gc)); + + log(xtag("T0_before_gc", T0_before_gc)); + log(xtag("T1_before_gc", T1_before_gc)); + log(xtag("T_after_gc", T_after_gc)); + } + + this->incr_gc_pending_ = false; + this->gc_statistics_.include_gc(generation::nursery, N0_before_gc, N_before_gc, N_after_gc, promote_z); + + if (upto == generation::tenured) { + this->full_gc_pending_ = false; + this->gc_statistics_.include_gc(generation::tenured, T0_before_gc, T_before_gc, T_after_gc, 0); + } else { + // still want to update tenured stats for current alloc size + this->gc_statistics_.update_snapshot(generation::tenured, T_after_gc); + } + + std::size_t sum_effort_z = effort_z; + std::size_t sum_garbage_z = garbage0_z + garbage1_z + garbageN_z; + + if (gc_history_.size() > 0) { + sum_effort_z += gc_history_.back().sum_effort_z_; + sum_garbage_z += gc_history_.back().sum_garbage_z_; + } + + GcStatisticsHistoryItem item(gc_statistics_.n_gc(), + upto, + new_alloc_z, + survive_z, + promote_z, + persist_z, + effort_z, + garbage0_z, + garbage1_z, + garbageN_z, + dt, + sum_effort_z, + sum_garbage_z); + + log && log(xtag("gcseq_after_gc", gc_statistics_.n_gc()), + xtag("item", item)); + + this->gc_history_.push_back(item); + + } /*cleanup_phase*/ + + void + GC::execute_gc(generation upto) + { + scope log(XO_DEBUG(config_.stats_flag_ || config_.debug_flag_)); + + auto t0 = std::chrono::steady_clock::now(); + + bool full_move = (upto == generation::tenured); + + // TODO: RAII version in case of exceptions + this->runstate_ = GCRunstate(true /*in_progress*/, full_move); + + log && log("step 0: snapshot alloc stats"); + + /* new allocation since last GC */ + std::size_t new_alloc = this->after_checkpoint(); + + gc_statistics_.begin_gc(upto, new_alloc); + + log && log(xtag("new_alloc", new_alloc)); + + log && log("step 0: (optional) scan for object statistics"); + + this->capture_object_statistics(upto, capture_phase::sab); + + log && log("step 1 : swap to/from roles"); + + this->swap_spaces(upto); + + log && log("step 2a: copy globals"); + + this->copy_globals(upto); + + log && log("step 2b: TODO: copy pinned"); + + log && log("step 3 : forward mutation log"); + + this->forward_mutation_log(upto); + + log && log("step 4 : TODO: notify destructor log"); + + log && log("step 5 : TODO: keep reachable weak pointers"); + + log && log("step 6 : cleanup"); + + this->capture_object_statistics(upto, capture_phase::sae); + + auto t1 = std::chrono::steady_clock::now(); + auto dt = std::chrono::duration_cast(t1 - t0); + + this->cleanup_phase(upto, xo::qty::qty::nanoseconds(dt.count())); + + log && log("object statistics [nursery]:"); + log && log(refrtag("stats", object_statistics_sab_[gen2int(generation::nursery)])); + log && log("object statistics [tenured]:"); + log && log(refrtag("stats", object_statistics_sab_[gen2int(generation::tenured)])); + + this->runstate_ = GCRunstate(); + + // not this way.. reports cumulative stats + // this->gc_history_.push_back(this->get_gc_statistics()); + + log && log("statistics:"); + log && log(gc_statistics_); + } + + void + GC::request_gc(generation target) + { + /** full collection when >= @ref full_gc_threshold_ bytes added to tenured + * generation, since last full collection + **/ + bool need_full_gc + = ((target == generation::tenured) + || (this->tenured_to()->after_checkpoint() > config_.full_gc_threshold_) + || !config_.allow_incremental_gc_); + + if (need_full_gc) + target = generation::tenured; + + if (!runstate_.in_progress() && (gc_enabled_ == 0)) { + this->execute_gc(target); + } else { + this->incr_gc_pending_ = true; + this->full_gc_pending_ |= need_full_gc; + } + } + + void + GC::disable_gc() { + --gc_enabled_; + } + + bool + GC::enable_gc() { + ++gc_enabled_; + + if (gc_enabled_ == 0) { + /* unblock gc */ + if (incr_gc_pending_) { + this->request_gc(full_gc_pending_ ? generation::tenured : generation::nursery); + return true; + } + } + + return false; + } + + bool + GC::enable_gc_once() { + bool retval = this->enable_gc(); + this->disable_gc(); + return retval; + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end GC.cpp */ diff --git a/xo-alloc/src/alloc/GcStatistics.cpp b/xo-alloc/src/alloc/GcStatistics.cpp new file mode 100644 index 00000000..deb30685 --- /dev/null +++ b/xo-alloc/src/alloc/GcStatistics.cpp @@ -0,0 +1,214 @@ +/* GcStatistics.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "GcStatistics.hpp" +#include "xo/indentlog/print/pretty_vector.hpp" + +namespace xo { + namespace gc { + void + PerGenerationStatistics::include_gc(std::size_t alloc_z, + std::size_t before_z, + std::size_t after_z, + std::size_t promote_z) + { + this->update_snapshot(after_z); + + //++n_gc_; + new_alloc_z_ += alloc_z; + scanned_z_ += before_z; + survive_z_ += after_z; + promote_z_ += promote_z; + } + + void + PerGenerationStatistics::update_snapshot(std::size_t after_z) + { + used_z_ = after_z; + } + + void + PerGenerationStatistics::display(std::ostream & os) const + { + os << ""; + } + + void + GcStatistics::begin_gc(generation upto, + std::size_t new_alloc) + { + ++(this->gen_v_[static_cast(upto)].n_gc_); + this->total_allocated_ += new_alloc; + this->total_promoted_sab_ = total_promoted_; + } + + void + GcStatistics::include_gc(generation upto, + std::size_t alloc_z, + std::size_t before_z, + std::size_t after_z, + std::size_t promote_z) + { + gen_v_[static_cast(upto)].include_gc(alloc_z, before_z, after_z, promote_z); + } + + void + GcStatistics::update_snapshot(generation upto, + std::size_t after_z) + { + gen_v_[static_cast(upto)].update_snapshot(after_z); + } + + void + GcStatistics::display(std::ostream & os) const + { + os << ""; + } + + void + GcStatisticsExt::display(std::ostream & os) const + { + os << ""; + } + + float + GcStatisticsHistoryItem::collection_rate() const { + using namespace xo::qty::qty; + + float gz = this->garbage_z(); + + auto dt_nanos = this->dt_.with_repr(); + auto dt_sec = dt_nanos.rescale_ext(); + auto rate = gz / dt_sec; + float retval = rate.scale(); + + //scope log(XO_DEBUG(true)); + //log && log(xtag("gz", gz), xtag("dt_sec", dt_sec), xtag("rate", rate), xtag("rate/sec", retval)); + + return retval; + } + + void + GcStatisticsHistoryItem::display(std::ostream & os) const + { + os << ""; + } + + } /*namespace gc*/ + + namespace print { + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::PerGenerationStatistics & x) + { + return ppii.pps()->pretty_struct(ppii, + "PerGenerationStatistics", + refrtag("used_z", x.used_z_), + refrtag("n_gc", x.n_gc_), + refrtag("new_alloc_z", x.new_alloc_z_), + refrtag("scanned_z", x.scanned_z_), + refrtag("survive_z", x.survive_z_), + refrtag("promote_z", x.promote_z_) + ); + } + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::GcStatistics & x) + { + return ppii.pps()->pretty_struct(ppii, + "GcStatistics", + refrtag("gen_v", x.gen_v_), + refrtag("total_allocated", x.total_allocated_), + refrtag("total_promoted_sab", x.total_promoted_sab_), + refrtag("total_promoted", x.total_promoted_), + refrtag("n_mutation", x.n_mutation_), + refrtag("n_logged_mutation", x.n_logged_mutation_), + refrtag("n_xgen_mutation", x.n_xgen_mutation_), + refrtag("n_xckp_mutation", x.n_xckp_mutation_) + ); + } + + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::GcStatisticsExt & x) + { + return ppii.pps()->pretty_struct(ppii, + "GcStatisticsExt", + refrtag("gen_v", x.gen_v_), + refrtag("total_allocated", x.total_allocated_), + refrtag("total_promoted_sab", x.total_promoted_sab_), + refrtag("total_promoted", x.total_promoted_), + refrtag("n_mutation", x.n_mutation_), + refrtag("n_logged_mutation", x.n_logged_mutation_), + refrtag("n_xgen_mutation", x.n_xgen_mutation_), + refrtag("n_xckp_mutation", x.n_xckp_mutation_), + refrtag("nursery_z", x.nursery_z_), + refrtag("nursery_before_checkpoint_z", x.nursery_before_checkpoint_z_), + refrtag("nursery_after_checkpoint_z", x.nursery_after_checkpoint_z_), + refrtag("tenured_z", x.tenured_z_)); + } + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::GcStatisticsHistoryItem & x) + { + return ppii.pps()->pretty_struct(ppii, + "GcStatisticsHistoryItem", + refrtag("upto", gen2str(x.upto_)), + refrtag("survive_z", x.survive_z_), + refrtag("promote_z", x.promote_z_), + refrtag("persist_z", x.persist_z_), + refrtag("effort_z", x.effort_z_), + refrtag("garbage0_z", x.garbage0_z_), + refrtag("garbage1_z", x.garbage1_z_), + refrtag("garbageN_z", x.garbageN_z_), + refrtag("dt", x.dt_)); + } + } /*namespace print*/ +} /*namespace xo*/ + +/* end GcStatistics.cpp */ diff --git a/xo-alloc/src/alloc/ListAlloc.cpp b/xo-alloc/src/alloc/ListAlloc.cpp new file mode 100644 index 00000000..9961d59b --- /dev/null +++ b/xo-alloc/src/alloc/ListAlloc.cpp @@ -0,0 +1,400 @@ +/* file ListAlloc.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "ListAlloc.hpp" +#include "ArenaAlloc.hpp" +#include "xo/indentlog/scope.hpp" +#include +#include + +namespace xo { + namespace gc { + ListAlloc::ListAlloc(std::unique_ptr hd, + ArenaAlloc * marked, + std::size_t cz, std::size_t nz, std::size_t tz, + bool debug_flag) + : start_z_{cz}, + hd_{std::move(hd)}, + marked_{marked}, + full_l_{}, + current_z_{cz}, + next_z_{nz}, + total_z_{tz}, + debug_flag_{debug_flag} + {} + + ListAlloc::~ListAlloc() + { + this->clear(); + } + + up + ListAlloc::make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag) + { + std::unique_ptr hd{ArenaAlloc::make(name, + cz, debug_flag)}; + + if (!hd) + return nullptr; + + ArenaAlloc * marked = nullptr; + + up retval{new ListAlloc(std::move(hd), + marked, + cz, nz, cz, + debug_flag)}; + + return retval; + } + + void + ListAlloc::capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const + { + hd_->capture_object_statistics(phase, p_dest); + + for (const auto & arena : full_l_) + arena->capture_object_statistics(phase, p_dest); + + } + + const std::string & + ListAlloc::name() const { + if (hd_) { + return hd_->name(); + } + + static std::string s_default_name = "ListAlloc"; + return s_default_name; + } + + std::size_t + ListAlloc::page_size() const { + return hd_->page_size(); + } + + std::size_t + ListAlloc::hugepage_z() const { + return hd_->hugepage_z(); + } + + std::size_t + ListAlloc::size() const { + return total_z_; + } + + std::size_t + ListAlloc::committed() const { + std::size_t z = 0; + if (hd_) + z += hd_->committed(); + for (const auto & a : full_l_) + z += a->committed(); + + return z; + } + + std::byte * + ListAlloc::free_ptr() const { + return hd_->free_ptr(); + } + + std::size_t + ListAlloc::available() const { + if (hd_) { + /* can only allocate from @ref hd_, + * so even if there were available space in @ref full_l_, + * it's not accessible to ListAlloc. + */ + + return hd_->available(); + } + + return 0; + } + + std::size_t + ListAlloc::allocated() const { + std::size_t total = 0; + + if (hd_) { + total += hd_->allocated(); + } + + for (const auto & alloc : full_l_) + total += alloc->allocated(); + + return total; + } + + bool + ListAlloc::contains(const void * x) const { + if (hd_ && hd_->contains(x)) + return true; + + for (const auto & alloc : full_l_) { + if (alloc->contains(x)) + return true; + } + + return false; + } + + bool + ListAlloc::is_before_checkpoint(const void * x) const { + if (!marked_) + return true; + + if (marked_ && marked_->contains(x)) + return marked_->is_before_checkpoint(x); + + /* + * 1. allocs in full_l_ appear in oldest-to-youngest order + * 2. allocators that appear before marked_ in full_l_ count as 'before checkpoint' + * 3. allocators that appear after marked_ in full_l_ count as 'after checkpoint' + */ + + bool older_than_marked = true; + + for (const auto & alloc : full_l_) { + if (older_than_marked) { + if (alloc.get() == marked_) { + /* nothing else to test on this iteration, + * already checked .marked_ specifically + */ + break; + } else { + /* before checkpoint */ + if (alloc->contains(x)) + return true; + } + } + } + + return false; + } + + std::size_t + ListAlloc::before_checkpoint() const + { + scope log(XO_DEBUG(false && debug_flag_), xtag("marked", marked_ ? marked_->name() : "")); + + if (marked_) { + if (full_l_.empty()) { + assert(marked_ == hd_.get()); + + return marked_->before_checkpoint(); + } else { + std::size_t z = 0; + + /* control here: .marked & .full_l non-empty. */ + if (hd_.get() == marked_) { + z += hd_->before_checkpoint(); + + /* anything in .full_l is older than marked .hd */ + for (const auto & alloc : full_l_) { + z += alloc->allocated(); + } + + return z; + } else { + /* messiest case: .marked is true, + * and not the youngest arena + */ + + /* full_l always in increasing time order: oldest-to-youngest order */ + size_t i_alloc = 0; + for (const auto & alloc : full_l_) { + log && log(xtag("i_alloc", i_alloc), + xtag("alloc", alloc->name()), + xtag("z", z)); + + if (alloc.get() == marked_) { + log && log("marked", xtag("+z", marked_->before_checkpoint())); + z += marked_->before_checkpoint(); + break; + } else { + log && log("older than marked", xtag("+z", alloc->allocated())); + z += alloc->allocated(); + } + ++i_alloc; + } + } + + return z; + } + } else { + /* count *everything* allocated */ + return this->allocated(); + } + } + + std::size_t + ListAlloc::after_checkpoint() const + { + scope log(XO_DEBUG(false && debug_flag_), xtag("marked", marked_ ? marked_->name() : "")); + + if (!marked_) + return 0; + + if (full_l_.empty()) { + assert(marked_ == hd_.get()); + + return marked_->after_checkpoint(); + } + + bool older_than_marked = true; + + std::size_t z = 0; + + std::size_t i_alloc = 0; + for (const auto & alloc : full_l_) { + log && log(xtag("i_alloc", i_alloc), + xtag("alloc", alloc->name()), + xtag("z", z)); + + if (older_than_marked) { + if (alloc.get() == marked_) { + log && log("marked", xtag("+z", marked_->after_checkpoint())); + older_than_marked = false; + z += marked_->after_checkpoint(); + } + } else { + /* younger than marked */ + log && log("younger", xtag("+z", alloc->allocated())); + z += alloc->allocated(); + } + + ++i_alloc; + } + + /** head must be included, since it's always the youngest bucket **/ + z += hd_->after_checkpoint(); + + log && log("z", z); + + return z; + } + + bool + ListAlloc::debug_flag() const { + return debug_flag_; + } + + void + ListAlloc::clear() { + // general hygiene + start_z_ = 0; + hd_.reset(); + marked_ = nullptr; + full_l_.clear(); + current_z_ = 0; + next_z_ = 0; + total_z_ = 0; + } + + bool + ListAlloc::reset(std::size_t z) + { + scope log(XO_DEBUG(debug_flag_), xtag("z", z)); + + bool recycle_head_bucket = hd_ && (z <= hd_->size()); + + this->full_l_.clear(); + this->marked_ = nullptr; + + if (recycle_head_bucket) { + this->hd_->clear(); + this->total_z_ = hd_->size(); + + return true; + } else { + std::string old_name = this->hd_->name(); + + this->hd_.reset(nullptr); + this->total_z_ = 0; + + return this->expand(z, old_name + "+"); + } + } + + bool + ListAlloc::expand(std::size_t z, const std::string & name) + { + scope log(XO_DEBUG(debug_flag_), xtag("name", name)); + + //log && log("before", xtag("before_ckp", this->before_checkpoint())); + + std::size_t cz = current_z_; + std::size_t nz = next_z_; + std::size_t tz; + + do { + tz = cz + nz; + cz = nz; + nz = tz; + } while (cz < z); + + log && log("expand to", xtag("cz", cz)); + + std::unique_ptr new_alloc = ArenaAlloc::make(name, + cz, debug_flag_); + cz = new_alloc->size(); + + if (!new_alloc) + return false; + + this->current_z_ = cz; + this->next_z_ = nz; + this->total_z_ += cz; + + if (hd_) + this->full_l_.push_back(std::move(hd_)); + + this->hd_ = std::move(new_alloc); + + //log && log("after", xtag("before_ckp", this->before_checkpoint())); + + return true; + } + + void + ListAlloc::checkpoint() { + scope log(XO_DEBUG(debug_flag_)); + + hd_->checkpoint(); + + this->marked_ = hd_.get(); + + log && log(xtag("hd", (void*)hd_.get()), xtag("marked", (void*)marked_)); + } + + std::byte * + ListAlloc::alloc(std::size_t z) { + scope log(XO_DEBUG(debug_flag_)); + + /* ArenaAlloc::alloc() may modify its own size */ + + std::size_t z_pre = hd_->size(); + std::byte * retval = hd_->alloc(z); + + if (retval) { + std::size_t z_post = hd_->size(); + this->total_z_ += (z_post - z_pre); + + return retval; + } + + log && log("space exhausted -> expand"); + + if (this->expand(z, hd_->name() + "+")) + return hd_->alloc(z); + + return nullptr; + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* end ListAlloc.cpp */ diff --git a/xo-alloc/src/alloc/Object.cpp b/xo-alloc/src/alloc/Object.cpp new file mode 100644 index 00000000..b0a52f59 --- /dev/null +++ b/xo-alloc/src/alloc/Object.cpp @@ -0,0 +1,230 @@ +/* Object.cpp + * + * author: Roalnd Conybeare, Jul 2025 + */ + +#include "Object.hpp" +#include "GC.hpp" +#include "Forwarding1.hpp" + +using xo::obj::Forwarding1; + +void * +operator new (std::size_t z, const xo::Cpof & cpof) +{ + using xo::gc::GC; + + //GC * gc = reinterpret_cast(cpof.mm_); + + return cpof.mm_->alloc_gc_copy(z, cpof.src_); +} + +namespace xo { + using xo::reflect::TaggedPtr; + + gc::IAlloc * + Object::mm = nullptr; + + TaggedPtr + Object::self_tp() const + { + assert(false); + return TaggedPtr::universal_null(); + } + + void + Object::display(std::ostream & os) const + { + os << ""; + } + + IObject * + Object::_forward(IObject * src, + gc::IAlloc * gc) + { + scope log(XO_DEBUG(gc->debug_flag()), xtag("src", src)); + + if (!src) + return src; + + if (src->_is_forwarded()) { + log && log("already forwarded", xtag("dest", src->_offset_destination(src))); + return src->_offset_destination(src); + } + + if (gc->check_move(src)) { + log && log("needs forwarding"); + Object::_shallow_move(src, gc); + + /* *src is now a forwarding pointer to a copy in to-space */ + return src->_offset_destination(src); + } else { + log && log("already tenured + incr collection"); + /* don't move tenured objects during incremental collection */ + return src; + } + } + + IObject * + Object::_deep_move(IObject * from_src, gc::GC * gc, gc::ObjectStatistics * /*stats*/) + { + scope log(XO_DEBUG(gc->config().debug_flag_)); + + using gc::generation; + + if (!from_src) + return nullptr; + + IObject * retval = from_src->_destination(); + + if (retval) + return retval; + + if (!gc->check_move(from_src)) { + /** incremental collection does not move already-tenured objects **/ + return from_src; + } + + /** + * To-space: + * + * to_lo = start of to-space + * w,W = white objects. An object x is white if x + * + all immediate children of x are in to-space + * (also implies this GC cycle put it there) + * g,G = grey objects. An object x is gray if it's in to-space, + * but possibly has >0 black children + * _ = free to-space memory + * N = nursery space + * T = tenured space + * + * wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________... + * ^ ^ ^ + * to_lo grey_lo(N) free_ptr(N) + * + * After moving children of one object, advancing {nursery_grey_lo, nursery_free_ptr} + * + * wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG______... + * ^ ^ ^ + * to_lo grey_lo(N) free_ptr(N) + * + * Invariant: + * + * objects in [to_lo, gray_lo) are white. + * all gray objects are in [gray_lo, free_ptr) + * memory starting at free_ptr is free. + * + * deep_move terminates when gray_lo catches up to free_ptr + * + * Above is simplified. Complication is that GC (including incremental) may + * promote objects from nursery (N) to tenured (T) + * + * So more accurate before/after picture + * + * N wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________... + * ^ ^ ^ + * to_lo(N) grey_lo(N) free_ptr(N) + * + * T wwwwwwwwwwwwwwgggggggggggg_______________________________... + * ^ ^ ^ + * to_lo(T) grey_lo(T) free_ptr(N) + * + * After moving children of one object, advancing {nursery_grey_lo, nursery_free_ptr} + * + * N wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG_____... + * ^ ^ ^ + * to_lo(N) grey_lo(N) free_ptr(N) + * + * T wwwwwwwwwwwwwwggggggggggggGGGGG_________________________... + * ^ ^ ^ + * to_lo(T) grey_lo(T) free_ptr(T) + * + * deep_move terminates when both: + * - gray_lo(N) catches up with free_ptr(N) + * - gray_lo(T) catches up with free_ptr(T) + * + **/ + + std::array gray_lo_v + = { gc->free_ptr(generation::nursery), gc->free_ptr(generation::tenured) }; + + IObject * to_src = Object::_shallow_move(from_src, gc); + + std::size_t fixup_work = 0; + do { + fixup_work = 0; + + auto fixup_generation = [gc, &log, &gray_lo_v](generation gen) { + std::size_t work = 0; + while(gray_lo_v[gen2int(gen)] < gc->free_ptr(gen)) { + Object * x = reinterpret_cast(gray_lo_v[gen2int(gen)]); + + // update per-class stats here + + log && log("fwd children", xtag("x", x)); + + std::size_t xz = x->_forward_children(gc); + + // must pad xz to multiple of word size, + // to match behavior of LinearAlloc::alloc() + // + xz += gc::IAlloc::alloc_padding(xz); + + gray_lo_v[gen2int(gen)] += xz; + ++work; + } + + return work; + }; + + fixup_work += fixup_generation(generation::nursery); + fixup_work += fixup_generation(generation::tenured); + } while (fixup_work > 0); + + return to_src; + } /*deep_move*/ + + IObject * + Object::_shallow_move(IObject * src, gc::IAlloc * gc) + { + /* filter for source objects that are owned by GC. + * Care required though -- during GC from/to spaces have been swapped already + */ + if (gc->check_owned(src)) + { + IObject * dest = src->_shallow_copy(gc); + + if (dest != src) + src->_forward_to(dest); + + return dest; + } else { + return src; + } + } + + void + Object::_forward_to(IObject * dest) + { + char * mem = reinterpret_cast(this); + + Forwarding1 * fwd = new (mem) Forwarding1(dest); + + (void)fwd; + } + + std::ostream & + operator<< (std::ostream & os, gp x) + { + if (x.ptr()) { + x->display(os); + } else { + os << ""; + } + + return os; + } + +} /*namespace xo*/ + +/* end Object.cpp*/ diff --git a/xo-alloc/src/alloc/ObjectStatistics.cpp b/xo-alloc/src/alloc/ObjectStatistics.cpp new file mode 100644 index 00000000..863a02cb --- /dev/null +++ b/xo-alloc/src/alloc/ObjectStatistics.cpp @@ -0,0 +1,73 @@ +/* file ObjectStatistics.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "ObjectStatistics.hpp" +#include "xo/reflect/TypeDescr.hpp" +#include "xo/indentlog/print/pretty_vector.hpp" + +namespace xo { + namespace gc { + void + PerObjectTypeStatistics::display(std::ostream & os) const + { + os << "short_name()); + else + os << xrtag("td", "nullptr"); + os << xrtag("scanned_n", scanned_n_) + << xrtag("scanned_z", scanned_z_) + << xrtag("survive_n", survive_n_) + << xrtag("survive_z", survive_z_) + << ">"; + } + + void + ObjectStatistics::display(std::ostream & os) const + { + os << ""; + } + } /*namespace gc*/ + + namespace print { + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::PerObjectTypeStatistics & x) + { + static constexpr std::string_view c_nullptr_str = "nullptr"; + + if (x.td_) { + return ppii.pps()->pretty_struct(ppii, + "PerObjectTypeStatistics", + refrtag("td", x.td_ ? x.td_->short_name() : c_nullptr_str), + refrtag("scanned_n", x.scanned_n_), + refrtag("scanned_z", x.scanned_z_), + refrtag("survive_n", x.survive_n_), + refrtag("survive_z", x.survive_z_)); + } else { + /* print nothing -- empty struct */ + return true; + } + } + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::ObjectStatistics & x) + { + return ppii.pps()->pretty_struct(ppii, + "ObjectTypeStatistics", + refrtag("per_type_stats_v", x.per_type_stats_v_)); + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* end ObjectStatistics.cpp */ diff --git a/xo-alloc/src/alloc/generation.cpp b/xo-alloc/src/alloc/generation.cpp new file mode 100644 index 00000000..a0ae9e65 --- /dev/null +++ b/xo-alloc/src/alloc/generation.cpp @@ -0,0 +1,31 @@ +/* generation.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "generation.hpp" + +namespace xo { + namespace gc { + const char * gen2str(generation x) { + switch (x) { + case generation::nursery: return "nursery"; + case generation::tenured: return "tenured"; + case generation::N: break; + } + return "?generation"; + } + + const char * genresult2str(generation_result x) { + switch (x) { + case generation_result::nursery: return "nursery"; + case generation_result::tenured: return "tenured"; + case generation_result::not_found: return "not-found"; + } + return "?generation_result"; + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* generation.cpp */ diff --git a/xo-alloc/utest/ArenaAlloc.test.cpp b/xo-alloc/utest/ArenaAlloc.test.cpp new file mode 100644 index 00000000..78055eed --- /dev/null +++ b/xo-alloc/utest/ArenaAlloc.test.cpp @@ -0,0 +1,88 @@ +/* @file ArenaAlloc.test.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "xo/alloc/ArenaAlloc.hpp" +#include + +namespace xo { + using xo::gc::ArenaAlloc; + + namespace ut { + + namespace { + struct testcase_alloc { + explicit testcase_alloc(std::size_t z) + : + arena_z_{z} {} + + std::size_t arena_z_; + }; + + std::vector + s_testcase_v = { + testcase_alloc(4096) + }; + } + + + TEST_CASE("linearalloc", "[alloc]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + const testcase_alloc & tc = s_testcase_v[i_tc]; + + constexpr bool c_debug_flag = false; + + auto alloc = ArenaAlloc::make("linearalloc", + tc.arena_z_, c_debug_flag); + alloc->expand(tc.arena_z_); + + REQUIRE(alloc.get()); + REQUIRE(alloc->name() == "linearalloc"); + REQUIRE(alloc->size() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->available() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->allocated() == 0); + REQUIRE(alloc->is_before_checkpoint(alloc->free_ptr()) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + auto free0 = alloc->free_ptr(); + + auto mem = alloc->alloc(std::max(tc.arena_z_, alloc->hugepage_z())); + + REQUIRE(mem != nullptr); + + REQUIRE(mem == free0); + + REQUIRE(alloc->size() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->available() == 0); + REQUIRE(alloc->allocated() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->is_before_checkpoint(mem) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == std::max(tc.arena_z_, alloc->hugepage_z())); + + alloc->clear(); + + REQUIRE(alloc->free_ptr() == free0); + REQUIRE(alloc->available() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->allocated() == 0); + REQUIRE(alloc->is_before_checkpoint(free0) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + mem = alloc->alloc(1); + + auto used = sizeof(void*); + REQUIRE(alloc->size() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->available() == std::max(tc.arena_z_, alloc->hugepage_z()) - used); + REQUIRE(alloc->allocated() == used); + REQUIRE(alloc->is_before_checkpoint(free0) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == used); + + } + } + + } /*namespace ut */ +} /*namespace xo*/ diff --git a/xo-alloc/utest/CMakeLists.txt b/xo-alloc/utest/CMakeLists.txt new file mode 100644 index 00000000..366cf664 --- /dev/null +++ b/xo-alloc/utest/CMakeLists.txt @@ -0,0 +1,26 @@ +# xo-alloc/utest/CMakeLists.txt +# +# NOTE: more GC tests in xo-object/utest + +set(UTEST_EXE utest.alloc) +set(UTEST_SRCS + alloc_utest_main.cpp + IAlloc.test.cpp + ArenaAlloc.test.cpp + ListAlloc.test.cpp + GC.test.cpp + GcStatistics.test.cpp + ObjectStatistics.test.cpp + Forwarding1.test.cpp + CircularBuffer.test.cpp + generation.test.cpp +) + +if (ENABLE_TESTING) + xo_add_utest_executable(${UTEST_EXE} ${UTEST_SRCS}) + xo_self_dependency(${UTEST_EXE} xo_alloc) + xo_dependency(${UTEST_EXE} reflect) + xo_external_target_dependency(${UTEST_EXE} Catch2 Catch2::Catch2) +endif() + +# end CmakeLists.txt diff --git a/xo-alloc/utest/CircularBuffer.test.cpp b/xo-alloc/utest/CircularBuffer.test.cpp new file mode 100644 index 00000000..f3d63c7e --- /dev/null +++ b/xo-alloc/utest/CircularBuffer.test.cpp @@ -0,0 +1,174 @@ +/* CircularBuffer.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/CircularBuffer.hpp" +#include "xo/indentlog/print/vector.hpp" +#include + +namespace xo { + using xo::gc::CircularBuffer; + + namespace ut { + TEST_CASE("circular_buffer_0", "[circular_buffer]") + { + CircularBuffer q(10, false /*debug_flag*/); + q.push_back("a"); + REQUIRE(q.back() == "a"); + q.push_back("b"); + REQUIRE(q.back() == "b"); + q.push_back("c"); + REQUIRE(q.back() == "c"); + REQUIRE(q.location_of(0) == 0); + REQUIRE(q.location_of(1) == 1); + REQUIRE(q.location_of(2) == 2); + //REQUIRE(q.index_of(0) == 0); + + REQUIRE(q.size() == 3); + REQUIRE(q.front() == "a"); + REQUIRE(q.at(0) == "a"); + REQUIRE(q.at(1) == "b"); + REQUIRE(q.at(2) == "c"); + + CircularBuffer q2; + + q2 = q; + + q.clear(); + + REQUIRE(q2.size() == 3); + REQUIRE(q2.front() == "a"); + REQUIRE(q2.at(0) == "a"); + REQUIRE(q2.at(1) == "b"); + REQUIRE(q2.at(2) == "c"); + } + + TEST_CASE("circular_buffer_1", "[circular_buffer]") + { + CircularBuffer q(2, false /*debug_flag*/); + q.push_back("a"); + REQUIRE(q.back() == "a"); + q.push_back("b"); + REQUIRE(q.back() == "b"); + q.push_back("c"); + REQUIRE(q.back() == "c"); + REQUIRE(q.location_of(0) == 1); + REQUIRE(q.location_of(1) == 0); + //REQUIRE(q.index_of(0) == 0); + + REQUIRE(q.size() == 2); + REQUIRE(q.front() == "b"); + REQUIRE(q.at(0) == "b"); + REQUIRE(q.at(1) == "c"); + + { + std::size_t i = 0; + for (const auto & qi : q) { + REQUIRE(qi == q.at(i)); + ++i; + } + } + + CircularBuffer q2 = q; + + q.clear(); + + REQUIRE(q2.size() == 2); + REQUIRE(q2.front() == "b"); + REQUIRE(q2.at(0) == "b"); + REQUIRE(q2.at(1) == "c"); + + { + std::size_t i = 0; + for (const auto & qi : q) { + REQUIRE(qi == q2.at(i)); + ++i; + } + } + } + } + + namespace { + struct Testcase_CircularBuffer { + explicit Testcase_CircularBuffer(std::size_t capacity, + const std::vector & contents) + : capacity_{capacity}, + contents_{contents} {} + + std::size_t capacity_ = 0; + std::vector contents_; + }; + + std::vector + s_testcase_v = { + Testcase_CircularBuffer(0, {}), + Testcase_CircularBuffer(1, {"a"}), + Testcase_CircularBuffer(2, {"a", "b"}), + Testcase_CircularBuffer(2, {"a", "b", "c", "d"}) + }; + } + + namespace ut { + TEST_CASE("circular_buffer_2", "[circular_buffer]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + const Testcase_CircularBuffer & tc = s_testcase_v[i_tc]; + + INFO(tostr(xtag("i_tc", i_tc), + xtag("capacity", tc.capacity_), + xrtag("contents", tc.contents_))); + + for (std::size_t j_phase = 0; j_phase < 2; ++j_phase) { + constexpr bool c_debug_flag = false; + + CircularBuffer q(tc.capacity_, false /*debug_flag*/); + + REQUIRE(q.empty()); + REQUIRE(q.size() == 0); + REQUIRE(q.begin() == q.end()); + REQUIRE(q.capacity() == tc.capacity_); + + std::size_t n = 0; + for (const auto & s : tc.contents_) { + INFO(tostr(xtag("n0", n), xtag("s", s))); + ++n; + INFO(xtag("n1", n)); + + q.push_back(s); + + REQUIRE(q.back() == s); + REQUIRE(q.capacity() == tc.capacity_); + REQUIRE(q.size() == std::min(n, tc.capacity_)); + + std::size_t i = 0; + for (const auto & qi : q) { + INFO(tostr(xtag("i", i), xtag("qi", qi))); + + if (n <= tc.capacity_) { + REQUIRE(qi == tc.contents_.at(i)); + REQUIRE(qi == tc.contents_[i]); + } else { + REQUIRE(qi == tc.contents_.at(n - tc.capacity_ + i)); + REQUIRE(qi == tc.contents_[n - tc.capacity_ + i]); + } + ++i; + } + + REQUIRE(i == std::min(n, tc.capacity_)); + + if (tc.contents_.size() <= tc.capacity_) + REQUIRE(q.front() == tc.contents_.at(0)); + } + + q.clear(); + + REQUIRE(q.size() == 0); + REQUIRE(q.capacity() == tc.capacity_); + } + } + } + } +} + +/* CircularBuffer.test.cpp */ diff --git a/xo-alloc/utest/Forwarding1.test.cpp b/xo-alloc/utest/Forwarding1.test.cpp new file mode 100644 index 00000000..64c9f3f8 --- /dev/null +++ b/xo-alloc/utest/Forwarding1.test.cpp @@ -0,0 +1,93 @@ +/* Forwarding1.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "Forwarding1.hpp" +#include "ArenaAlloc.hpp" +#include "xo/reflect/Reflect.hpp" +#include +#include +#include + +namespace xo { + using xo::reflect::Reflect; + using xo::obj::Forwarding1; + + namespace gc { + namespace { + class DummyObject : public Object { + public: + explicit DummyObject(const char * data) { + ::strncpy(data_.data(), data, 128); + } + + gp member() const { return member_; } + void assign_member(Object * x) { + Object::mm->assign_member(this, reinterpret_cast(member_.ptr_address()), x); + } + + TaggedPtr self_tp() const final override { + return Reflect::make_tp(const_cast(this)); + } + + void display(std::ostream & os) const final override { os << data_; } + + virtual std::size_t _shallow_size() const final override { return sizeof(*this); } + virtual IObject * _shallow_copy(gc::IAlloc * mm) const final override { return new (Cpof(mm, this)) DummyObject(*this); } + virtual std::size_t _forward_children(gc::IAlloc * gc) final override { return _shallow_size(); } + + private: + std::array data_; + gp member_; + }; + } + + TEST_CASE("Forwarding1", "[gc][alloc]") + { + bool saved = tag_config::tag_color_enabled; + tag_config::tag_color_enabled = false; + + gp obj = new DummyObject("Well, I wasn't expecting that!"); + gp fwd = new Forwarding1(obj); + + REQUIRE(fwd->_destination() == obj.ptr()); + REQUIRE(fwd->_offset_destination(fwd.ptr()) == obj.ptr()); + + REQUIRE(fwd->self_tp().td()->short_name() == "Forwarding1"); + + std::stringstream ss; + ss << fwd; + + // forwarding printer looks like + // "" + // + + std::regex pattern(R"()"); + REQUIRE(std::regex_match(ss.str(), pattern)); + + //REQUIRE(ss.str() == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("IAlloc.assign_member", "[gc][alloc]") + { + /* not giving this nit it's own translation unit. + */ + + gp obj = new DummyObject("This also a surprise.."); + + up arena = ArenaAlloc::make("test", 1024, false); + + Object::mm = arena.get(); + + obj->assign_member(obj.ptr()); + + REQUIRE(obj->member().ptr() == obj.ptr()); + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end Forwarding1.test.cpp */ diff --git a/xo-alloc/utest/GC.test.cpp b/xo-alloc/utest/GC.test.cpp new file mode 100644 index 00000000..f178f4a8 --- /dev/null +++ b/xo-alloc/utest/GC.test.cpp @@ -0,0 +1,395 @@ +/* @file GC.test.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "xo/alloc/GC.hpp" +#include "xo/allocutil/gc_allocator_traits.hpp" +#include + +namespace xo { + using xo::gc::IAlloc; + using xo::gc::GC; + using xo::gc::gc_allocator_traits; + using xo::gc::generation; + using xo::gc::Config; + using xo::reflect::TaggedPtr; + + namespace ut { + + namespace { + struct testcase_gc { + testcase_gc(std::size_t nz, std::size_t tz, std::size_t n_gct, std::size_t t_gct) + : nursery_z_{nz}, tenured_z_{tz}, incr_gc_threshold_{n_gct}, full_gc_threshold_{t_gct} {} + + std::size_t nursery_z_; + std::size_t tenured_z_; + std::size_t incr_gc_threshold_; + std::size_t full_gc_threshold_; + }; + + std::vector + s_testcase_v = { + // n_gct: nursery gc threshold + // t_gct: tenured gc threshold + // + // nz tz n_gct t_gct + testcase_gc(1024, 4096, 1024, 1024) + }; + } + + TEST_CASE("gc", "[alloc][gc]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + try { + const testcase_gc & tc = s_testcase_v[i_tc]; + + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_, + .incr_gc_threshold_ = tc.incr_gc_threshold_, + .full_gc_threshold_ = tc.full_gc_threshold_, + }); + + REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); + REQUIRE(gc->nursery_to_allocated() == 0); + REQUIRE(gc->nursery_to_committed() >= tc.nursery_z_); + REQUIRE(gc->nursery_to_reserved() >= tc.nursery_z_); + REQUIRE(gc->nursery_to_reserved() < tc.nursery_z_ + gc->hugepage_z()); + REQUIRE(gc->size() >= tc.nursery_z_ + tc.tenured_z_); + REQUIRE(gc->size() < tc.nursery_z_ + gc->hugepage_z() + tc.tenured_z_ + gc->hugepage_z()); + REQUIRE(gc->allocated() == 0); + REQUIRE(gc->available() == gc->nursery_to_reserved()); + REQUIRE(gc->before_checkpoint() == 0); + // ListAlloc model is that nothing is before checkpoint + // until it's first established + REQUIRE(gc->after_checkpoint() == 0); + + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->is_gc_enabled() == true); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 0); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + + /* gc with empty state */ + gc->request_gc(generation::nursery); + + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + + /* still empty state */ + gc->request_gc(generation::tenured); + + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 1); + } catch (std::exception & ex) { + std::cerr << "caught exception: " << ex.what() << std::endl; + REQUIRE(false); + } + } + } + + /** gc-enabled allocator **/ + namespace { + /** Setup test with custom allocator + * + **/ + template + struct TestClass : public GcObjectInterface { + TestClass() = default; + explicit TestClass(const Nested & member1) : member1_{member1} {} + + // using allocator_type = Allocator; + // using allocator_traits = xo::gc::gc_allocator_traits; + + /** stage1 - just allocates some memory using allocator **/ + template + static TestClass * make_0(Allocator & alloc) { + TestClass * mem = alloc.allocate(sizeof(TestClass)); + + /* but ctor will not have run, so ub to visit object */ + + return mem; + } + + /** stage2 - use allocator_traits construct **/ + template + static TestClass * make_1(Allocator & alloc) { + using traits = gc_allocator_traits; + + TestClass * mem = traits::allocate(alloc, 1); + + /* ctor will not have run here either */ + return mem; + } + + /** stage3 - invoke construct **/ + template + static TestClass * make_2(Allocator & alloc) { + using traits = gc_allocator_traits; + + TestClass * obj = traits::allocate(alloc, 1); + try { + // placement new + traits::construct(alloc, obj); + + return obj; + } catch(...) { + traits::deallocate(alloc, obj, 1); + throw; + } + } + + /** stage4 - init nested type **/ + template + static TestClass * make_3(Allocator & alloc) { + using traits = gc_allocator_traits; + + TestClass * obj = traits::allocate(alloc, 1); + try { + Nested nested; + + // placemenet new + traits::construct(alloc, obj); + + return obj; + } catch(...) { + traits::deallocate(alloc, obj, 1); + throw; + } + } + + // ----- inherited from Object ----- + + virtual TaggedPtr self_tp() const final override { + assert(false); return TaggedPtr::universal_null(); + } + virtual void display(std::ostream & os) const final override { + os << ""; + } + virtual std::size_t _shallow_size() const final override { + assert(false); return sizeof(*this); + } + virtual IObject * _shallow_copy(IAlloc * gc) const final override { + assert(false); return nullptr; + } + virtual std::size_t _forward_children(IAlloc * gc) final override { + assert(false); return _shallow_size(); + } + + Nested member1_; + }; + + //template + struct MemberType { + public: + //using allocator_type = Allocator; + //using vector_allocator_type = typename std::allocator_traits::template rebind_alloc>; + using vector_type = std::vector>; + //using vector_type = std::vector, vector_allocator_type>; + + public: + MemberType() : ctor_ran_{true} {} + //explicit MemberType(const Allocator & alloc) + //: member2_{vector_allocator_type(alloc)}, ctor_ran_{true} {} + + explicit MemberType(const vector_type & mem2) : member2_{mem2}, ctor_ran_{true} {} + //MemberType(const vector_type & mem2, const Allocator & alloc) + //: member2_{mem2, vector_allocator_type(alloc)}, ctor_ran_{true} {} + + vector_type member2_; + bool ctor_ran_ = false; + }; + +#ifdef NOT_YET + struct MemberType2 { + public: + MemberType2() = default; + /** GC hooks rely on copy constructor. But can't write it without allocator state. + * Therefore: need copy-like constructor that takes allocator argument + **/ + + template + explicit MemberType2(Allocator & alloc, uint64 payload) { + using traits = gc_allocator_traits; + + uint64_t * ptr = traits::allocate(alloc, 1); + + this->payload_ = payload; + this->ctor_ran_ = true; + } + + uint64_t * payload_ = nullptr; + bool ctor_ran_ = false; + } +#endif + } + + TEST_CASE("vector_custom_allocator", "[alloc][vector]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + try { + const testcase_gc & tc = s_testcase_v[i_tc]; + + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_, + .incr_gc_threshold_ = tc.incr_gc_threshold_, + .full_gc_threshold_ = tc.full_gc_threshold_, + }); + + REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); + + using NestedElementAllocator = xo::gc::allocator>; + + NestedElementAllocator alloc(gc.get()); + + /** testv will use GC to allocaate element storage + * Attempt to gc will fail, because memory iteration + * won't work. + **/ + std::vector, + NestedElementAllocator> testv(alloc); + + testv.push_back(gp()); + +#ifdef NOPE + using ex_allocator = xo::gc::allocator; + using MyObjectInterface = gc_allocator_traits::template object_interface; + using NestedType = MemberType; + //using NestedType = MemberType; + using MyType = TestClass; + using MyAllocator = xo::gc::allocator; + + MyAllocator alloc(gc.get()); + + { + /* verify that MyType is constructible */ + MyType obj0; + + REQUIRE(obj0.member1_.ctor_ran_ == true); + } + + { + MyType * mem0 = MyType::make_0(alloc); + + REQUIRE(mem0 != nullptr); + REQUIRE(mem0->member1_.ctor_ran_ == false); + } + + { + MyType * mem1 = MyType::make_1(alloc); + + REQUIRE(mem1 != nullptr); + REQUIRE(mem1->member1_.ctor_ran_ == false); + } + + { + MyType * mem2 = MyType::make_2(alloc); + + REQUIRE(mem2 != nullptr); + REQUIRE(mem2->member1_.ctor_ran_ == true); + } + + { + MyType * mem3 = MyType::make_3(alloc); + + REQUIRE(mem3 != nullptr); + REQUIRE(mem3->member1_.ctor_ran_ == true); + } + + gp ptr; + { + REQUIRE(ptr.is_null()); + //ptr = MyType::make_0(); + } +#endif + } catch (std::exception & ex) { + std::cerr << "caught exception: " << ex.what() << std::endl; + REQUIRE(false); + } + } + + } + + TEST_CASE("gc_allocator_traits", "[alloc][gc]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + try { + const testcase_gc & tc = s_testcase_v[i_tc]; + + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_, + .incr_gc_threshold_ = tc.incr_gc_threshold_, + .full_gc_threshold_ = tc.full_gc_threshold_, + }); + + REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); + + using ex_allocator = xo::gc::allocator; + using MyObjectInterface = gc_allocator_traits::template object_interface; + using NestedElementAllocator = xo::gc::allocator>; + using NestedType = MemberType; + //using NestedType = MemberType; + using MyType = TestClass; + using MyAllocator = xo::gc::allocator; + + MyAllocator alloc(gc.get()); + + { + /* verify that MyType is constructible */ + MyType obj0; + + REQUIRE(obj0.member1_.ctor_ran_ == true); + } + + { + MyType * mem0 = MyType::make_0(alloc); + + REQUIRE(mem0 != nullptr); + REQUIRE(mem0->member1_.ctor_ran_ == false); + } + + { + MyType * mem1 = MyType::make_1(alloc); + + REQUIRE(mem1 != nullptr); + REQUIRE(mem1->member1_.ctor_ran_ == false); + } + + { + MyType * mem2 = MyType::make_2(alloc); + + REQUIRE(mem2 != nullptr); + REQUIRE(mem2->member1_.ctor_ran_ == true); + } + + { + MyType * mem3 = MyType::make_3(alloc); + + REQUIRE(mem3 != nullptr); + REQUIRE(mem3->member1_.ctor_ran_ == true); + } + + gp ptr; + { + REQUIRE(ptr.is_null()); + //ptr = MyType::make_0(); + } + } catch (std::exception & ex) { + std::cerr << "caught exception: " << ex.what() << std::endl; + REQUIRE(false); + } + } + + } + + } /*namespace ut*/ +} /*namespace xo*/ + +/* GC.test.cpp */ diff --git a/xo-alloc/utest/GcStatistics.test.cpp b/xo-alloc/utest/GcStatistics.test.cpp new file mode 100644 index 00000000..f5700f25 --- /dev/null +++ b/xo-alloc/utest/GcStatistics.test.cpp @@ -0,0 +1,216 @@ +/* @file GcStatistics.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/GcStatistics.hpp" +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/tostr.hpp" +#include "xo/indentlog/print/hex.hpp" +#include +#include + +namespace xo { + using xo::gc::GcStatistics; + using xo::gc::GcStatisticsExt; + using xo::gc::PerGenerationStatistics; + using xo::print::ppconfig; + + namespace ut { + TEST_CASE("PerGenerationStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerGenerationStatistics stats; + + std::string s = tostr(stats); + + //std::cerr << hex_view(s.c_str(), s.c_str() + s.length(), true /*as_text*/) << std::endl; + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + GcStatistics stats; + + std::string s = tostr(stats); + + REQUIRE(s == + "" + /***/ " ]" + /**/ " :total_allocated 0" + /**/ " :total_promoted_sab 0" + ">"); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatisticsExt", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + GcStatisticsExt stats; + + std::string s = tostr(stats); + + REQUIRE(s == " ] :total_allocated 0 :total_promoted_sab 0 :nursery_z 0 :nursery_before_ckp_z 0 :nursery_after_ckp_z 0 :tenured_z 0 :n_mutation 0 :n_logged_mutation 0 :n_xgen_mutation 0 :n_xckp_mutation 0>"); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatistics-pretty", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + std::stringstream ss; + ppconfig ppc; + GcStatistics stats; + + std::string actual = toppstr2(ppc, stats); + std::string expected + = (",\n" + " ]\n" + " :total_allocated 0\n" + " :total_promoted_sab 0\n" + " :total_promoted 0\n" + " :n_mutation 0\n" + " :n_logged_mutation 0\n" + " :n_xgen_mutation 0\n" + " :n_xckp_mutation 0>"); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + REQUIRE(actual == expected); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatisticsExt-pretty", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + std::stringstream ss; + ppconfig ppc; + GcStatisticsExt stats; + + std::string actual = toppstr2(ppc, stats); + std::string expected + = (",\n" + " ]\n" + " :total_allocated 0\n" + " :total_promoted_sab 0\n" + " :total_promoted 0\n" + " :n_mutation 0\n" + " :n_logged_mutation 0\n" + " :n_xgen_mutation 0\n" + " :n_xckp_mutation 0\n" + " :nursery_z 0\n" + " :nursery_before_checkpoint_z 0\n" + " :nursery_after_checkpoint_z 0\n" + " :tenured_z 0>"); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + REQUIRE(actual == expected); + + tag_config::tag_color_enabled = saved; + } + } +} /*namespace xo*/ + +/* GcStatistics.test.cpp */ diff --git a/xo-alloc/utest/IAlloc.test.cpp b/xo-alloc/utest/IAlloc.test.cpp new file mode 100644 index 00000000..7c1fdc56 --- /dev/null +++ b/xo-alloc/utest/IAlloc.test.cpp @@ -0,0 +1,127 @@ +/* @file IAlloc.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +//#include "xo/allocutil/IAlloc.hpp" +#include "xo/alloc/ArenaAlloc.hpp" +#include "xo/indentlog/print/tag.hpp" +#include + +namespace xo { + using xo::gc::IAlloc; + using xo::gc::ArenaAlloc; + + namespace ut { + TEST_CASE("ialloc", "[alloc]") + { + static_assert((sizeof(std::uintptr_t) == 8) && "possibly fine if this fails, but would want to know"); + + REQUIRE(IAlloc::alloc_padding(0) == 0); + + for (std::size_t i = 1; i < sizeof(std::uintptr_t); ++i) { + REQUIRE(IAlloc::alloc_padding(i) + i == IAlloc::c_alloc_alignment); + } + + REQUIRE(IAlloc::alloc_padding(IAlloc::c_alloc_alignment) == 0); + + for (std::size_t i = 1; i < sizeof(std::uintptr_t); ++i) { + REQUIRE(IAlloc::alloc_padding(IAlloc::c_alloc_alignment + i) + i == IAlloc::c_alloc_alignment); + } + } + + /* although xo::gc::allocator is intended for + * IAlloc derivatives (so T is ArenaAlloc | GC), + * + * it only relies on allocate() and deallocate() methods + */ + + namespace { + struct TestCase { + explicit TestCase(size_t arena_z, size_t n, size_t n2) : arena_z_{arena_z}, n_{n}, n2_{n2} {} + + size_t arena_z_ = 0; + size_t n_ = 0; + size_t n2_ = 0; + }; + + std::vector s_testcase_v = { TestCase{1024*1024, 9, 13} }; + } + + TEST_CASE("gc.allocator", "[alloc]") + { + using xo::gc::allocator; + + constexpr bool c_debug_flag = false; + + for (size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + INFO(xtag("i_tc", i_tc)); + + const TestCase & tc = s_testcase_v[i_tc]; + + up mm1 = ArenaAlloc::make("arena1", + tc.arena_z_, + c_debug_flag); + up mm2 = ArenaAlloc::make("arena2", + tc.arena_z_, + c_debug_flag); + + REQUIRE(mm1.get()); + REQUIRE(mm1->allocated() == 0); + + allocator alloc1(mm1.get()); + allocator alloc1a(mm1.get()); + + REQUIRE(mm2.get()); + REQUIRE(mm2->allocated() == 0); + + allocator alloc2(mm2.get()); + + SECTION("IAlloc identity determines allocator equality") { + REQUIRE(alloc1 == alloc1a); + REQUIRE(alloc1 != alloc2); + } + + int * p1 = nullptr; + size_t z1 = 0; + + SECTION("alloc space for ints") { + p1 = alloc1.allocate(tc.n_); + + REQUIRE(p1 != nullptr); + + // note: allowing for alignment + REQUIRE(mm1->allocated() >= sizeof(int32_t) * tc.n_); + REQUIRE(mm1->allocated() < sizeof(int32_t) * tc.n_ + IAlloc::c_alloc_alignment); + z1 = mm1->allocated(); + + // deallocate exists.. + alloc1.deallocate(p1, tc.n_); + + // ..but is a no-op + REQUIRE(mm1->allocated() == z1); + } + + int * p2 = nullptr; + + SECTION("allocator independence") { + REQUIRE(mm2->allocated() == 0); + + p2 = alloc2.allocate(tc.n2_); + + REQUIRE(p2 != nullptr); + REQUIRE(p1 != p2); + + REQUIRE(mm2->allocated() >= sizeof(int32_t) * tc.n2_); + REQUIRE(mm2->allocated() < sizeof(int32_t) * tc.n2_ + IAlloc::c_alloc_alignment); + + // mm1 unaffected by mm2 allocation + REQUIRE(mm1->allocated() == z1); + } + } + } + + } /*namespace ut*/ +} /*namespace xo*/ + +/* end IAlloc.test.cpp */ diff --git a/xo-alloc/utest/ListAlloc.test.cpp b/xo-alloc/utest/ListAlloc.test.cpp new file mode 100644 index 00000000..5f425568 --- /dev/null +++ b/xo-alloc/utest/ListAlloc.test.cpp @@ -0,0 +1,64 @@ +/* ListAlloc.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/ListAlloc.hpp" +#include + +namespace xo { + using xo::gc::ListAlloc; + + namespace ut { +#ifdef NOT_USING // ListAlloc probably permanently retired. Not maintaining + + TEST_CASE("ListAlloc", "[alloc][gc]") + { + /** teeny weeny allocator. + * but underlying ArenaAlloc works in multiples of VM page size + * (most likely 4k) + **/ + up alloc = ListAlloc::make("test", 16, 32, false); + + REQUIRE(alloc->name() == "test"); + REQUIRE(alloc->size() == 16); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + /* will expand */ + std::byte * mem1 = alloc->alloc(20); + + REQUIRE(mem1); + REQUIRE(alloc->size() == std::max(alloc->page_size(), alloc->hugepage_z())); + /* round up to multiple of 8 */ + REQUIRE(alloc->before_checkpoint() == 24); + REQUIRE(alloc->after_checkpoint() == 0); + + alloc->checkpoint(); + + std::byte * mem2 = alloc->alloc(30); + + REQUIRE(mem2); + REQUIRE(alloc->size() == alloc->page_size()); + REQUIRE(alloc->before_checkpoint() == 24); + /* round up to multiple of 8 */ + REQUIRE(alloc->after_checkpoint() == 32); + + std::byte * mem3 = alloc->alloc(40); + + REQUIRE(mem3); + REQUIRE(alloc->size() == alloc->page_size()); + REQUIRE(alloc->before_checkpoint() == 24); + /* already multiple of 8 */ + REQUIRE(alloc->after_checkpoint() == 32 + 40); + + REQUIRE(alloc->is_before_checkpoint(mem1) == true); + REQUIRE(alloc->is_before_checkpoint(mem2) == false); + REQUIRE(alloc->is_before_checkpoint(mem3) == false); + } +#endif + + } /*namespace ut*/ +} /*namespace xo*/ + +/* ListAlloc.test.cpp */ diff --git a/xo-alloc/utest/ObjectStatistics.test.cpp b/xo-alloc/utest/ObjectStatistics.test.cpp new file mode 100644 index 00000000..7530450d --- /dev/null +++ b/xo-alloc/utest/ObjectStatistics.test.cpp @@ -0,0 +1,185 @@ +/* @file ObjectStatistics.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/ObjectStatistics.hpp" +#include "xo/reflect/Reflect.hpp" +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/ppstr.hpp" +#include "xo/indentlog/print/tostr.hpp" +#include "xo/indentlog/print/hex.hpp" +#include + +namespace xo { + using xo::gc::ObjectStatistics; + using xo::gc::PerObjectTypeStatistics; + using xo::reflect::Reflect; + using xo::print::ppconfig; + + namespace ut { + TEST_CASE("PerObjectTypeStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerObjectTypeStatistics stats; + + std::string s = tostr(stats); + + //std::cerr << hex_view(s.c_str(), s.c_str() + s.length(), true /*as_text*/) << std::endl; + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("PerObjectTypeStatistics-1", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerObjectTypeStatistics stats; + stats.td_ = Reflect::require(); + stats.scanned_n_ = 4; + stats.scanned_z_ = 16; + stats.survive_n_ = 2; + stats.survive_z_ = 8; + + std::string s = tostr(stats); + + //std::cerr << hex_view(s.c_str(), s.c_str() + s.length(), true /*as_text*/) << std::endl; + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + ObjectStatistics stats; + + std::string s = tostr(stats); + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics-1", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + ObjectStatistics stats; + stats.per_type_stats_v_.push_back(PerObjectTypeStatistics()); + + std::string s = tostr(stats); + + REQUIRE(s == ">"); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics-pretty", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + std::stringstream ss; + ppconfig ppc; + ObjectStatistics stats; + + std::string actual = toppstr2(ppc, stats); + std::string expected + = (""); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + REQUIRE(actual == expected); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics-pretty-1", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerObjectTypeStatistics objstats; + objstats.td_ = Reflect::require(); + objstats.scanned_n_ = 4; + objstats.scanned_z_ = 16; + objstats.survive_n_ = 2; + objstats.survive_z_ = 8; + + std::stringstream ss; + ppconfig ppc; + ObjectStatistics stats; + stats.per_type_stats_v_.push_back(objstats); + + std::string actual = toppstr2(ppc, stats); + + std::string expected + = (" ]>"); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + tag_config::tag_color_enabled = saved; + } + } +} /*namespace xo*/ + +/* ObjectStatistics.test.cpp */ diff --git a/xo-alloc/utest/alloc_utest_main.cpp b/xo-alloc/utest/alloc_utest_main.cpp new file mode 100644 index 00000000..fa384613 --- /dev/null +++ b/xo-alloc/utest/alloc_utest_main.cpp @@ -0,0 +1,6 @@ +/* file alloc_utest_main.cpp */ + +#define CATCH_CONFIG_MAIN +#include "catch2/catch.hpp" + +/* end alloc_utest_main.cpp */ diff --git a/xo-alloc/utest/generation.test.cpp b/xo-alloc/utest/generation.test.cpp new file mode 100644 index 00000000..edbebffa --- /dev/null +++ b/xo-alloc/utest/generation.test.cpp @@ -0,0 +1,39 @@ +/* generation.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/generation.hpp" +#include +#include +#include + +namespace xo { + namespace gc { + TEST_CASE("generation", "[gc]") { + REQUIRE(::strcmp(gen2str(generation::nursery), "nursery") == 0); + REQUIRE(::strcmp(gen2str(generation::tenured), "tenured") == 0); + REQUIRE(::strcmp(gen2str(generation::N), "?generation") == 0); + + { + std::stringstream ss; + ss << generation::nursery; + REQUIRE(ss.str() == "nursery"); + } + + { + std::stringstream ss; + ss << generation::tenured; + REQUIRE(ss.str() == "tenured"); + } + + { + std::stringstream ss; + ss << generation::N; + REQUIRE(ss.str() == "?generation"); + } + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* generation.test.cpp */