diff --git a/xo-alloc/.gitrepo b/xo-alloc/.gitrepo new file mode 100644 index 00000000..e9c6a776 --- /dev/null +++ b/xo-alloc/.gitrepo @@ -0,0 +1,12 @@ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/ingydotnet/git-subrepo#readme +; +[subrepo] + remote = git@github.com:Rconybea/xo-alloc.git + branch = main + commit = fc656313e9582957f13446364299a8e79cbd51f0 + parent = d16545d815d055837e0973cca8483277a925d7fb + method = merge + cmdver = 0.4.9 diff --git a/xo-alloc/CMakeLists.txt b/xo-alloc/CMakeLists.txt new file mode 100644 index 00000000..0e9de5c4 --- /dev/null +++ b/xo-alloc/CMakeLists.txt @@ -0,0 +1,32 @@ +# xo-alloc/CMakeLists.txt + +cmake_minimum_required(VERSION 3.10) + +project(xo_alloc VERSION 0.1) + +include(GNUInstallDirs) +include(cmake/xo-bootstrap-macros.cmake) + +xo_cxx_toplevel_options3() + +# ---------------------------------------------------------------- +# c++ settings + +set(PROJECT_CXX_FLAGS "") +#set(PROJECT_CXX_FLAGS "-fconcepts-diagnostics-depth=2") # gcc-only! +add_definitions(${PROJECT_CXX_FLAGS}) + +# ---------------------------------------------------------------- + +# must complete definition of expression lib before configuring examples +add_subdirectory(src/alloc) +add_subdirectory(utest) +xo_export_cmake_config(${PROJECT_NAME} ${PROJECT_VERSION} ${PROJECT_NAME}Targets) + +# ---------------------------------------------------------------- +# docs targets depend on other library/utest/exec targets above, +# --> must come after them. +# +add_subdirectory(docs) + +# end CmakeLists.txt diff --git a/xo-alloc/README.md b/xo-alloc/README.md new file mode 100644 index 00000000..5a42f922 --- /dev/null +++ b/xo-alloc/README.md @@ -0,0 +1,16 @@ +# xo-alloc -- arena allocator and incremental garbage collector + +# Rules for writing garbage-collected classes. + +Topics +* allocation - allocate Objects (inheriting xo::Object) before owned scratch space. + Can relax this if/when abandon the bad-for-locality use of two pointers + into to-space to keep track of grey objects. Want to use stack anyway + so we can do depth-first search. +* destructors - can omit except for finalization +* assignment - MUST USE Object::assign_member() to assign pointers to gc-owned memory. + Only necessary for old->new pointers, so don't need to worry about this + for initialization. +* finalization - not supported (yet) + +- padding - use IAlloc::with_padding(z) for hand-allocated objects. diff --git a/xo-alloc/cmake/xo-bootstrap-macros.cmake b/xo-alloc/cmake/xo-bootstrap-macros.cmake new file mode 100755 index 00000000..592272c0 --- /dev/null +++ b/xo-alloc/cmake/xo-bootstrap-macros.cmake @@ -0,0 +1,41 @@ +# ---------------------------------------------------------------- +# for example: +# $ PREFIX=/usr/local # for example +# $ cmake -DCMAKE_MODULE_PATH=prefix -DCMAKE_INSTALL_PREFIX=$PREFIX -B .build +# +# will get +# CMAKE_MODULE_PATH +# from xo-cmake-config --cmake-module-path +# +# and expect .cmake macros in +# CMAKE_MODULE_PATH/xo_macros/xo_cxx.cmake +# ---------------------------------------------------------------- + +find_program(XO_CMAKE_CONFIG_EXECUTABLE NAMES xo-cmake-config REQUIRED) + +if ("${XO_CMAKE_CONFIG_EXECUTABLE}" STREQUAL "XO_CMAKE_CONFIG_EXECUTABLE-NOT_FOUND") + message(FATAL "could not find xo-cmake-config executable") +endif() + +message(STATUS "XO_CMAKE_CONFIG_EXECUTABLE=${XO_CMAKE_CONFIG_EXECUTABLE}") + +if (XO_SUBMODULE_BUILD) + if (("${CMAKE_MODULE_PATH}" STREQUAL "") OR ("${CMAKE_MODULE_PATH}" STREQUAL prefix)) + # local version of xo-cmake macros + set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/xo-cmake/cmake") + message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + endif() +else() + if (("${CMAKE_MODULE_PATH}" STREQUAL "") OR ("${CMAKE_MODULE_PATH}" STREQUAL prefix)) + # default to typical install location for xo-project-macros + execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --cmake-module-path OUTPUT_VARIABLE CMAKE_MODULE_PATH) + message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + endif() +endif() + +# needs to have been installed somewhere on CMAKE_MODULE_PATH, +# (e.g. from xo-cmake with the same value for CMAKE_INSTALL_PREFIX) +# +include(xo_macros/xo_cxx) + +xo_cxx_bootstrap_message() diff --git a/xo-alloc/cmake/xo_allocConfig.cmake.in b/xo-alloc/cmake/xo_allocConfig.cmake.in new file mode 100644 index 00000000..e627df64 --- /dev/null +++ b/xo-alloc/cmake/xo_allocConfig.cmake.in @@ -0,0 +1,11 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(xo_allocutil) +find_dependency(xo_unit) +find_dependency(indentlog) +find_dependency(reflect) +find_dependency(callback) +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Share.cmake") +check_required_components("@PROJECT_NAME@") diff --git a/xo-alloc/docs/CMakeLists.txt b/xo-alloc/docs/CMakeLists.txt new file mode 100644 index 00000000..e13b26a0 --- /dev/null +++ b/xo-alloc/docs/CMakeLists.txt @@ -0,0 +1,9 @@ +# xo-alloc/docs/CMakeLists.txt + +xo_doxygen_collect_deps() +xo_docdir_doxygen_config() +xo_docdir_sphinx_config( + index.rst install.rst introduction.rst implementation.rst) + +# see xo-reader/doc or xo-unit/doc for working examples +# example.rst install.rst implementation.rst diff --git a/xo-alloc/docs/README b/xo-alloc/docs/README new file mode 100644 index 00000000..6aff5d41 --- /dev/null +++ b/xo-alloc/docs/README @@ -0,0 +1,41 @@ +standalone build + + +-----------------------------------------------+ + | cmake | + | CMakeLists.txt | + | $PREFIX/share/cmake/xo_macros/xo_cxx.cmake | + +-----------------------------------------------+ + | + | +----------------------+ + +------------------------------------------------->| .build/docs/Doxyfile | + | +----------------------+ + | ^ + | (cmake) | + | /------------/ + | | + | +---------------------------------------+ +-----------------+ + +---->| doxygen |--------->| .build/docs/dox | + | | $PREFIX/share/xo-macros/Doxyfile.in | (doxygen)| +- html/ | + | +---------------------------------------+ | +- xml/ | + | +-----------------+ + | | + | |(sphinx) + | | + | v + | +---------------------------------------+ +--------------------+ + \---->| sphinx |------->| .build/docs/sphinx | + | +- conf.py | | +- html/ | + | +- _static/ | +--------------------+ + | +- *.rst | + +---------------------------------------+ + +umbrella build relies on top-level cmake macros + +files + + README this file + CMakeLists.txt build entry point + conf.py sphinx config + _static static files for sphinx + + index.rst toplevel sphinx document; entry point diff --git a/xo-alloc/docs/_static/README b/xo-alloc/docs/_static/README new file mode 100644 index 00000000..7297d046 --- /dev/null +++ b/xo-alloc/docs/_static/README @@ -0,0 +1 @@ +add any static {.html, .js, ..} files for sphinx to pickup here diff --git a/xo-alloc/docs/_static/img/favicon.ico b/xo-alloc/docs/_static/img/favicon.ico new file mode 100644 index 00000000..4163dd69 Binary files /dev/null and b/xo-alloc/docs/_static/img/favicon.ico differ diff --git a/xo-alloc/docs/conf.py b/xo-alloc/docs/conf.py new file mode 100644 index 00000000..1ca83ac9 --- /dev/null +++ b/xo-alloc/docs/conf.py @@ -0,0 +1,39 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'xo alloc documentation' +copyright = '2025, Roland Conybeare' +author = 'Roland Conybeare' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +#extensions = [] +extensions = [ "breathe", + "sphinx.ext.mathjax", # inline math + "sphinx.ext.autodoc", # generate info from docstrings + "sphinxcontrib.ditaa", # diagrams-through-ascii-art + "sphinxcontrib.plantuml" # text -> uml diagrams + ] + +# note: breathe requires doxygen xml output -> must have GENERATE_XML = YES in Doxyfile.in +# match project name in Doxyfile.in +breathe_default_project = "xodoxxml" + +templates_path = ['_templates'] +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +pygments_style = 'sphinx' + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +#html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] +html_favicon = '_static/img/favicon.ico' diff --git a/xo-alloc/docs/glossary.rst b/xo-alloc/docs/glossary.rst new file mode 100644 index 00000000..4e50a499 --- /dev/null +++ b/xo-alloc/docs/glossary.rst @@ -0,0 +1,28 @@ +.. _glossary: + +Glossary +-------- + +.. glossary:: + GC + | garbage collector + + mlog + | mutation log. + | Remembers cross-generation and cross-checkpoint pointers + + nursery + | in garbage collector, memory region dedicated to young objects. + | These are objects that have survived less than 2 incremental collection cycles. + + tenured + | in garbage collector, memory region dedicated to older objects. + | These are defined as objects that have survived 2 or more incremental collection cycles. + + xgen + | cross-generation tenured->nursery pointer; requires special GC bookkeeping + + xckp + | cross-checkpoint pointer; requires special GC bookkeeping + +.. toctree:: diff --git a/xo-alloc/docs/implementation.rst b/xo-alloc/docs/implementation.rst new file mode 100644 index 00000000..e325f88c --- /dev/null +++ b/xo-alloc/docs/implementation.rst @@ -0,0 +1,202 @@ +.. _implementation: + +.. toctree:: + :maxdepth: 2 + +Library +======= + +Library dependency tower for *xo-alloc*: + +.. ditaa:: + + +------------------------------------------+ + | xo_alloc | + +------------------------------------------+ + | xo_indentlog | + +------------------------------------------+ + +Install instructions :doc:`here` + +Components +========== + +Abstraction tower for *xo-alloc* components: + +.. ditaa:: + :--scale: 0.85 + + +----------------+-------------+ + | IAlloc | Object | + +----------------+-------------+ + + +-------------+ +-------------+ + | GC | | Forwarding1 | + +-------------+ +-------------+ + | ListAlloc | + +-------------+ + | ArenaAlloc | + +-------------+ + +* *IAlloc* + Allocator interface. + +* *Object* + Root Object Interface for types participating in garbage collection + +* *GC* + Incremental compacting garbage collector. + +* *ListAlloc* + Auto-expanding allocator. Contains a collection of ArenaAllocs + +* *ArenaAlloc* + Arena allocator (a.k.a bump allocator). + +* *Object* + Interface for types that participate in garbage collection + +* *Forwarding1* + Forwarding pointer. Supports the Object interface; + used internally by GC during evacuation. + +Key Points +---------- + +* Allocators can be reset, but do not support freeing of individual allocs. +* GC works with types that implement auxiliary GC-support methods. + Such types must inherit Object. +* A region may uses multiple arenas, but because of allocation activity + since the last GC. If necessary, GC will allocate a new to-space with a + single arena that's large enough to accomodate all objects that might survive + from a from-space that has acquired multiple arenas. + Intent is to scale up to find application's working set size, then stabilize + +Components +========== + +Allocators +---------- + +Inheritance +^^^^^^^^^^^ + +.. uml:: + :caption: allocators + :scale: 99% + :align: center + + class IAlloc { + + alloc() + + alloc_gc_copy() + + checkpoint() + + clear() + } + + class ArenaAlloc { + + free_ptr() + - lo_ : byte* + - checkpoint_ : byte* + - limit_ : byte* + } + + IAlloc <|-- ArenaAlloc + + class ListAlloc { + + expand() + + free_ptr() + - start_z_ + - hd_ + - full_l_ + } + + IAlloc <|-- ListAlloc + + class GC { + + add_gc_root() + + request_gc() + + gc_statistics() + - gc_root_v_[] : Object** + - nursery_[2] : ListAlloc* + - tenured_[2] : ListAlloc* + } + + IAlloc <|-- GC + + +Composition +^^^^^^^^^^^ + +.. uml:: + :caption: allocator composition + :scale: 99% + :align: center + + object gc<> + gc : nursery[from] = n0 + gc : nursery[to] = n1 + gc : tenured[from] = t0 + gc : tenured[to] = t1 + + object n0<> + + object n1<> + + object t0<> + + object t1<> + + gc o-- n0 + gc o-- n1 + gc o-- t0 + gc o-- t1 + + +Each ListAlloc composes like this: + +.. uml:: + :caption: ListAlloc composition + :scale: 99% + :align: center + + object x<> + x : hd_ = a0 + x : full_l = {a1, a2} + + object a0<> + a0 : lo_ = 0 + a0 : free_ = 12345 + a0 : hi_ = 1000000 + + object a1<> + + object a2<> + + x o-- a0 + x o-- a1 + x o-- a2 + +Here *a1* and *a2* are full, while *a0* can still allocate memory. + +Objects + +.. uml:: + :caption: objects + :scale: 99% + :align: center + + class Object { + + _is_forwarded() + + _offset_destination() + + _forward_to() + + _destination() + + _shallow_size() + + _shallow_copy() + + _forward_children() + } + + class Forwarding1 { + - dest_ : Object* + } + + Object <|-- Forwarding1 diff --git a/xo-alloc/docs/index.rst b/xo-alloc/docs/index.rst new file mode 100644 index 00000000..de643009 --- /dev/null +++ b/xo-alloc/docs/index.rst @@ -0,0 +1,17 @@ +# xo-alloc documentation master file + +xo-alloc documentation +====================== + +xo-alloc provides arena allocators and a generation garbage collector + +.. toctree:: + :maxdepth: 2 + :caption: xo-alloc contents + + install + introduction + implementation + glossary + genindex + search diff --git a/xo-alloc/docs/install.rst b/xo-alloc/docs/install.rst new file mode 100644 index 00000000..a61d9eea --- /dev/null +++ b/xo-alloc/docs/install.rst @@ -0,0 +1,120 @@ +.. _install: + +.. toctree:: + :maxdepth: 2 + +Source +====== + +Source code lives on github `here`_ + +.. _here: https://github.com/rconybea/xo-alloc + +To clone from git: + +.. code-block:: bash + + git clone https://github.com/rconybea/xo-alloc + +Tested with gcc 13.3 + +Install +======= + +One-step Install +---------------- + +Install along with the rest of *XO* from `xo-umbrella2 source`_ + +.. _xo-umbrella2 source: https://github.com/rconybea/xo-umbrella2 + +Minimal Install +--------------- + +To build+install just required dependencies: +``xo-alloc`` uses several supporting libraries from the *XO* project: + +- `xo-indentlog source`_ (structured logging) +- `xo-cmake source`_ (shared cmake macros) + +.. _xo-indentlog source: https://github.com/rconybea/indentlog +.. _xo-cmake source: https://github.com/rconybea/xo-cmake + +Building from source +-------------------- + +Install scripts for XO libraries depend on helper scripts installed from `xo-cmake`. + +Preamble: + +.. code-block:: bash + + mkdir -p ~/proj/xo + cd ~/proj/xo + + git clone https://github.com/rconybea/xo-cmake + + PREFIX=/usr/local # ..or desired installation prefix + + # want PREFIX/bin in PATH to use xo-cmake helpers + PATH=$PREFIX/bin:$PATH + +Install `xo-cmake`: + +.. code-block:: bash + + cmake -B xo-cmake/.build -S xo-cmake + cmake --install xo-cmake/.build + +Install remaining dependencie(s) in topological order: + +.. code-block:: bash + + xo-build --clone --configure --build --install xo-indentlog + xo-build --clone --configure --build --install xo-alloc + +Directories under ``PREFIX`` will then contain: + +.. code-block:: + + PREFIX + +- bin + | +- xo-build + | +- xo-cmake-config + | \- xo-cmake-lcov-harness + +- include + | \- xo + | +- alloc/ + | \- indentlog/ + +- lib + | +- cmake + | | +- xo_alloc/ + | | \- indentlog/ + | +- lib*.so + +- share + +- cmake + | \- xo_macros + | +- code-coverage.cmake + | +- xo-project-macros.cmake + | \- xo_cxx.cmake + +- etc + | \- xo + | \- subsystem-list + \- xo-macros + +- Doxyfile.in + +- gen-ccov.in + \- xo-bootstrap-macros.cmake + +CMake Support +------------- + +To use built-in cmake support, when using ``xo-alloc`` from another project: + +Make sure ``PREFIX/lib/cmake`` is searched by cmake (for example include it in ``CMAKE_PREFIX_PATH``) + +Add to your ``CMakeLists.txt``: + +.. code-block:: cmake + + FindPackage(xo_alloc CONFIG REQUIRED) + target_link_libraries(mytarget INTERFACE xo_alloc) diff --git a/xo-alloc/docs/introduction.rst b/xo-alloc/docs/introduction.rst new file mode 100644 index 00000000..7b5333cf --- /dev/null +++ b/xo-alloc/docs/introduction.rst @@ -0,0 +1,268 @@ +.. _introduction: + +.. toctree + :maxdepth: 2 + +Introduction +============ + +The ``xo-alloc`` library provides a in incremental, generational collector for c++ code. + +Features: + +* *incremental* - can reasonably expect short pause times. +* *generational* - focuses effort on collecting young objects, + on the basis that they're more likely to be garbage. +* *compacting* - each garbage collection cycle evacuates survivors to contiguous memory, + so effect is to defragment. +* *collects cycles* - collection algorithm naturally collects cyclic references + +Tradeoffs: + +* Application is responsible for spilling register values and protecting hardware stack, + since garbage collector cannot indepndently distinguish collectable object pointers from + non-pointer values. + +* GC will not spontaneously run without permission. Instead will set a pending bit, with GC + occurring only when application releases it (e.g. when stack+registers are known to be empty of values + subject to GC). + +* GC implementation is single-threaded. It cannot run in parallel with the mutator (i.e. application code) + In return this allows GC to be only lightly coupled with application. + +* GC divides each generation into separate from- and to- spaces. A collection cycle copies surviving + objects out of from-space. Once complete, the entire from-space is treated as empty, and available to + become to-space on a future cycle. This means that at any time only half of allocated memory is available + to the application; the rest is waiting to receive survivors from the next GC cycle. + +Design +------ + +Garbage Collector +^^^^^^^^^^^^^^^^^ + +The garbage collector supports two generations, labelled *nursery* and *tenured*. +Nursery objects that survive two collection cycles are promoted to tenured space. +Nursery and tenured objects are kept in separate memory areas, instead of being interspersed. + +Collection cycles come in two flavors: + +1. *incremental* collections - these collect only the nursery space. + +2. *full* collections - these collect both nursery and tenured spaces. + Full collection may incur noticeable GC pauses. + +Application Interaction +^^^^^^^^^^^^^^^^^^^^^^^ + +Application code that interacts with GC has several responsibilities. + +1. application must explicitly invoke GC, when convenient. Since in general any GC-eligible object + may get moved by the collector: once a collection cycle completes, + it's up to the application to re-load pointers from memory addresses + (GC roots) that have been shared with the collector. + +2. application must identify a set of GC roots. GC preserves everything reachable from any GC root + +3. The collector needs to know how to traverse GC-managed objects. + We teach it this by requiring that such objects inherit the ``xo::Object`` interface, + and implement auxiliary function detailed below. + +4. GC also needs to know when a mutation alters a pointer from one GC-managed object to another. + In particular, GC needs to track pointers from tenured space into nursery space, + and update them when an incremental collection moves nursery objects. + We do this by requiring application code use a GC-provided assignment primitive + on GC-eligible pointers. + + +Example GC Use +-------------- + +.. code-block:: cpp + :linenos: + + #include "xo/object/List.hpp" // polymorphic List with GC support + #include "xo/object/String.hpp" // string type with GC support + #include "xo/alloc/GC.hpp" + + int main() { + using xo::gc::Config; + using xo::obj::String; + using xo::obj::List; + using xo::gp; + + Config config = { .initial_nursery_z_ = 50*1000, + .initial_tenured_z_ = 10*1000*1000, + .debug_flag_ = false }; + + up gc = GC::make(config); + + Object::mm = gc; // use GC for allocation of Object (+ derived classes) + + gc->disable_gc(); // gc forbidden + + // tiny example data structure + gp s1 = String::copy("hello"); + gp s2 = String::copy(", "); + gp s3 = String::copy("world!"); + gp list = List::cons(s1, List::cons(s2, List::cons(s3, List::nil))); + + // tell GC what to preserve + gc->add_gc_root(reinterpret_cast(list.ptr_address()); + + gc->enable_gc(); // triggers immediate gc + + // s1, s2, s3 invalid. + // list at new address + + std::cout << "list.size=" << list->size << std::endl; + } + +GC-Eligible Types +----------------- + +Or, how to inherit ``xo::Object`` and provide GC support + +A type Foo that inherits ``xo::Object`` needs to provide overrides for Object methods ``_shallow_size()``, +``_shallow_copy()`` and ``_forward_children()``: + +Typical Pattern +^^^^^^^^^^^^^^^ + +GC support methods look something like this: + +* class definition + +.. code-block:: cpp + :linenos: + + #include "xo/alloc/Object.hpp" + + namespace xo { + class Foo : public xo::Object { + public: + ... + virtual std::size_t _shallow_size() const override; + virtual Object * _shallow_copy() const override; + virtual std::size_t _forward_children() override; + }; + } + +* use overloaded ``operator new`` + +A GC-eligible class will allocate instances using the ``MMPtr`` overload. +This allocates memory in GC-owned space + +.. code-block::cpp + :linenos: + + gp Foo::make(...) { + ... + return new MMPtr(mm) Foo(...); + } + +* ``_shallow_size()`` returns the amount of memory used by the subject: + +.. code-block:: cpp + :linenos: + + std::size_t Foo::_shallow_size() const { return sizeof(Foo); } + +* ``_shallow_copy()`` is invoked during GC to create a copy of the subject + + It should use the ``xo::Cpof`` argument to ``operator new``. + +.. code-block:: cpp + :linenos: + + Object * + Foo::_shallow_copy() const; + +* ``_forward_children()`` is invoked during GC to vist child ``xo::Object`` pointers + to make sure they survive + +.. code-block:: cpp + :linenos: + + std::size_t + Foo::_forward_children(); + +Atomic Types Without Object Pointers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Plain-old-data classes without embedded pointers + +.. code-block:: cpp + :linenos: + + Object * + Foo::_shallow_copy() const { + return new (Cpof(this)) Foo(*this); + } + +.. code-block:: cpp + :linenos: + + std::size_t + Foo::_forward_children() { return Foo::_shallow_size(); } + +For example see ``xo::obj::String`` in ``xo-object`` + +Non-GC Objects +^^^^^^^^^^^^^^ + +A class *Foo* that inherits ``xo::Object`` can opt-out of garbage collection by +omitting the ``MMptr(mm)`` overload. + +In that case `Foo::_shallow_size()`, `Foo::_shallow_copy()` and `Foo::_forward_children()` +will not be called: + +.. code-block:: cpp + :linenos: + + std::size_t Foo::_shallow_size() const { return sizeof(Foo); } + Object * Foo::_shallow_copy() const { assert(false); return nullptr; } + std::size_t Foo::_forward_children() { assert(false); return 0; } + +For example see ``xo::obj::Boolean`` in ``xo-object`` + +Structs Containing Object Pointers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A class with object pointers needs to tell GC how to traverse them + +.. code-block:: cpp + :linenos: + + #include "xo/alloc/Object.hpp" + + namespace xo { + class Foo : public xo::Object { + public: + ... + virtual std::size_t _shallow_size() const override; + virtual Object * _shallow_copy() const override; + virtual std::size_t _forward_children() override; + + private: + gp bar_; + gp quux_; + }; + } + +* ``_forward_children()`` is invoked during GC to fixup child pointers + that refer to forwarding objects: + +.. code-block:: cpp + :linenos: + + std::size_t + Foo::_forward_children() + { + Object::_forward_inplace(bar_); + Object::_forward_inplace(quux_); + + return Foo::_shallow_size(); + } + +For example see ``xo::obj::List`` in ``xo-object`` diff --git a/xo-alloc/include/xo/alloc/AllocPolicy.hpp b/xo-alloc/include/xo/alloc/AllocPolicy.hpp new file mode 100644 index 00000000..53f758ee --- /dev/null +++ b/xo-alloc/include/xo/alloc/AllocPolicy.hpp @@ -0,0 +1,58 @@ +/* AllocPolicy.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include + +namespace xo { + /** Tag class, drives overload of operator new. + * See also: xoglobal, xocopy + **/ + struct xolib { + xolib() = default; + }; + + /** @brief opt-in allocator for XO libraries. + * + * By default delegates to vanilla operator new/delete, + * but can set alloc/free functions at runtime to + * adopt a different implementation. + * + * Intending this to op-in to garbage-collector? + * Not sure if we actually need this + * + * Use: + * struct Foo { .. }; + * auto p = new (xo) Foo(..); + **/ + class XoAllocPolicy { + public: + using AllocFn = void* (*)(std::size_t); + using FreeFn = void (*)(void *); + + public: + XoAllocPolicy() = default; + + static void * global_alloc(std::size_t z) { return ::operator new(z); } + static void global_free(void * x) { ::operator delete(x); } + + void * alloc(std::size_t z) { return (*alloc_)(z); } + void free(void * x) { (*free_)(x); } + + private: + AllocFn alloc_ = global_alloc; + FreeFn free_ = global_free; + }; + + /** singleton xolib instance **/ + static XoAllocPolicy xo; +} + +inline void * operator new(std::size_t z, xo::xolib) { + return xo::xo.alloc(z); +} + +void operator delete(void * ptr) noexcept; + +/* end AllocPolicy.hpp */ diff --git a/xo-alloc/include/xo/alloc/ArenaAlloc.hpp b/xo-alloc/include/xo/alloc/ArenaAlloc.hpp new file mode 100644 index 00000000..df4d2a01 --- /dev/null +++ b/xo-alloc/include/xo/alloc/ArenaAlloc.hpp @@ -0,0 +1,231 @@ +/* file ArenaAlloc.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#pragma once + +#include "xo/allocutil/IAlloc.hpp" +#include "ObjectStatistics.hpp" + +namespace xo { + namespace gc { + /** @class ArenaAlloc + * @brief Bump allocator with fixed capacity with dynamic virtual memory commitment. + * + * @text + * + * allocation order: + * -----------------------> + * + * <----------------- .size(), .reserved() ---------------------------> + * <----------------- .committed() -------------> + * + * <-------allocated------><--------free--------><-----uncommitted----> + * XXXXXXXXXXXXXXXXXXXXXXXX______________________...................... + * ^ ^ ^ ^ ^ + * lo checkpoint free limit hi + * + * +- .alloc() -> + * +-- .expand() --> + * > < .before_checkpoint() + * > < .after_checkpoint() + * + * lifetime: + * + * 1. initial state after ctor + * + * >< committed()=0 + * <---------------------------uncommitted----------------------------> + * .................................................................... + * ^ ^ + * lo hi + * checkpoint + * free + * limit + * + * 1a. one call to ::mmap() + * 1b. vm address space [lo,hi) is reserved + * 1c. address space [lo,hi) is inaccessible. no read|write|execute permission + * + * 2. after first allocation of n bytes + * + * <--committed---> + * <--free--><--------------------uncommitted--------------------> + * > <- allocated + * XXXXXX__________..................................................... + * ^ ^ ^ ^ + * lo lo+n limit hi + * ^ free + * checkpoint + * + * 2a. committed just enough hugepages (2mb each) to accomodate n, + * i.e. expand-on-demand: + * - one call to ::mprotect() + * - .limit = .lo + (k+1) * .hugepage_z for some integer k>=0 + * - k * .page_z <= n < (k+1) * .hugepage_z + * 2b. expect immediate cost 1-5us, includes: + * - TLB flush + * invalidate TLB entries for committed range on all cores that this + * process' threads have run on since process inception. + * Also, if a kernel thread has run on one of said cores, it may + * have borrowed our TLB entries + * - page table update + * write to entry for each vm page + * - kernel overhead 100-1000 cycles (< 1us) + * 2c. expect deferred cost 1us-2us per hugepage: + * - committed pages aren't backed by physical memory until + * first touched; minor page fault on first access for each page. + * - so about 256-512us for 1MB + * 3. after .expand(z) + * + * <-------------committed------------> + * <------------free------------><----------uncomitted-----------> + * > <- allocated + * XXXXXX______________________________................................. + * ^ ^ ^ ^ + * lo lo+n limit hi + * ^ free + * checkpoint + * + * 3a. same as case 2. but without advancing .free pointer. + * + * 4. after dtor + * + * 4a. all memory returned to o/s, no longer reserved. + * - one call to ::munmap() + * + * @endtext + * + * Design Notes: + * - non-copyable, non-moveable + * - @ref lo_ <= @ref checkpoint_ <= @ref free_ <= @ref limit_ <= @ref hi_ + * - memory for ArenaAlloc itself (not the memory it allocates), ~100 bytes + * always heap allocated. Use ArenaAlloc::make() + * - memory obtained from mmap(), not heap + * - memory addresses are stable. Expand storage by committing VM pages. + * - @ref lo_ is aligned on VM page size (guaranteed by mmap()) + * - @ref lo_ + @ref committed_z_ <= @ref hi_ + * - @ref limit_ <= @ref lO_ + @ref committed_z_ + * - @ref committed_z_ is always a multiple of VM page size + * - @ref limit_ is not guaranteed to be aligned with VM page size. + * - @ref expand increases @ref limit_ and @ref committed_z_ as needed. + * + **/ + class ArenaAlloc : public IAlloc { + public: + ArenaAlloc(const ArenaAlloc &) = delete; + ArenaAlloc(ArenaAlloc &&) = delete; + ~ArenaAlloc(); + + /** Create allocator with capacity @p z, + * Reserve memory addresses for @p z bytes, + * (but don't commit them until needed) + **/ + static up make(const std::string & name, + std::size_t z, + bool debug_flag); + + /** size of virtual address range reserved for this allocator **/ + std::size_t reserved() const { return hi_ - lo_; }; + + std::size_t page_size() const { return page_z_; } + std::size_t hugepage_z() const { return hugepage_z_; } + std::byte * free_ptr() const { return free_ptr_; } + void set_free_ptr(std::byte * x); + + /** if address @p x is allocated from this arena, + * return true along with offset relative to base address @ref lo_ + * otherwise return false with 0 + **/ + std::pair location_of(const void * x) const; + + /** allocated span **/ + std::pair allocated_span() const { + return std::make_pair(lo_, free_ptr_); + } + + /** Reset to empty state; provision at least @p need_z bytes of (committed) space **/ + void reset(std::size_t need_z); + + /** gc support: If used for storing xo::Object instances, scan allocated memory + * to populate @p *p_dest. + **/ + void capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const; + + /** expand available (i.e. committed) space to size at least @p z + * In practice will round up to a multiple of @ref page_z_. + **/ + bool expand(std::size_t z); + + // inherited from IAlloc... + + virtual const std::string & name() const final override; + virtual std::size_t size() const final override; + virtual std::size_t committed() const final override; + virtual std::size_t available() const final override; + virtual std::size_t allocated() const final override; + virtual bool contains(const void * x) const final override; + virtual bool is_before_checkpoint(const void * x) const final override; + virtual std::size_t before_checkpoint() const final override; + virtual std::size_t after_checkpoint() const final override; + virtual bool debug_flag() const final override; + + virtual void clear() final override; + virtual void checkpoint() final override; + virtual std::byte * alloc(std::size_t z) final override; + virtual bool check_owned(IObject * src) const final override; + + ArenaAlloc & operator=(const ArenaAlloc &) = delete; + ArenaAlloc & operator=(ArenaAlloc &&) = delete; + + private: + ArenaAlloc(const std::string & name, + std::size_t z, bool debug_flag); + + private: + /** + * Invariants: + * - @ref free_ always a multiple of word size (assumed to be sizeof(void*)) + **/ + + /** optional instance name, for diagnostics **/ + std::string name_; + + /** size of a VM page (from getpagesize()) **/ + std::size_t page_z_ = 0; + + /** size of a huge VM page. hardwiring this in ctor (to 2MB). + * larger pages relieve pressure on TLB, but suboptimal if use << 2MB + **/ + std::size_t hugepage_z_ = 0; + + /** allocator owns memory in range [@ref lo_, @ref hi_) **/ + std::byte * lo_ = nullptr; + /** prefix of this size is actually committed. + * Remainder uses uncommitted virtual address space + **/ + std::size_t committed_z_ = 0; + /** checkpoint (for GC support); divides objects into + * older (addresses below checkpoint) + * and younger (addresses above checkpoint) + **/ + std::byte * checkpoint_ = nullptr; + /** free pointer. memory in range [@ref free_, @ref limit_) available **/ + std::byte * free_ptr_ = nullptr; + /** soft limit: end of committed virtual memory + * invariant: @ref limit_ = @ref lo_ + @ref committed_z_ + **/ + std::byte * limit_ = nullptr; + /** hard limit: end of reserved virtual memory **/ + std::byte * hi_ = nullptr; + /** true to enable detailed debug logging **/ + bool debug_flag_ = false; + }; + + } /*namespace gc*/ +} /*namespace xo*/ + + +/* end ArenaAlloc.hpp */ diff --git a/xo-alloc/include/xo/alloc/Blob.hpp b/xo-alloc/include/xo/alloc/Blob.hpp new file mode 100644 index 00000000..205e984b --- /dev/null +++ b/xo-alloc/include/xo/alloc/Blob.hpp @@ -0,0 +1,40 @@ +/** @file Blob.hpp + * + * @author Roland Conybeare, Nov 2025 + **/ + +#pragma once + +#include "Object.hpp" +#include + +namespace xo { + /** Use to allocate opaque binary data, + * with object header. + * + * Not sure if we want to bother implementing reflection for this... + **/ + class Blob : public Object { + public: + Blob(std::size_t z) : z_{z} {}; + + static gp make(gc::IAlloc * mm, std::size_t z); + + std::size_t size() const { return z_; } + std::byte * data() { return data_; } + + virtual TaggedPtr self_tp() const final override; + virtual void display(std::ostream & os) const final override; + + virtual std::size_t _shallow_size() const final override; + virtual Object * _shallow_copy(gc::IAlloc * gc) const final override; + virtual std::size_t _forward_children(gc::IAlloc * gc) final override; + + private: + std::size_t z_ = 0; + /** flexible array, with @ref z_ bytes **/ + std::byte data_[]; + }; +} + +/* end Blob.hpp */ diff --git a/xo-alloc/include/xo/alloc/CircularBuffer.hpp b/xo-alloc/include/xo/alloc/CircularBuffer.hpp new file mode 100644 index 00000000..cf1729ce --- /dev/null +++ b/xo-alloc/include/xo/alloc/CircularBuffer.hpp @@ -0,0 +1,245 @@ +/* CircularBuffer.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#pragma once + +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/tostr.hpp" +#include "xo/indentlog/print/tag.hpp" +#include +#include +#include +//#include + +namespace xo { + namespace gc { + /** @class CircularBuffer + * @brief A circular buffer + * + * push operations may overwrite prior contents, + * i.e. buffer behavior on overflow + * old + * + * @tparam T is type for buffer elements. + **/ + template + class CircularBuffer { + public: + using value_type = T; + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using reference = value_type &; + using const_reference = const value_type &; + using pointer = value_type *; + using const_pointer = const value_type *; + + template + class _iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = TT; + using difference_type = std::ptrdiff_t; + using pointer = value_type *; + using reference = value_type &; + + _iterator(Parent * p, std::int64_t ix) : parent_{p}, index_{ix} {} + + reference operator* () const { return (*parent_)[index_]; } + pointer operator->() const { return &(*parent_)[index_]; } + _iterator & operator++() { ++index_; return *this; } + _iterator operator++(int) { _iterator retval(parent_, index_); ++index_; return retval; } + + auto operator<=>(const _iterator & other) { + if (parent_ == other.parent_) + return index_ <=> other.index_; + else + return std::partial_ordering::unordered; + } + + bool operator==(const _iterator & other) const = default; + + private: + Parent * parent_ = nullptr; + /** index position + * (-1 = just before front = rend, 0 = front, z-1 = back, z = just after back = end) + **/ + std::int64_t index_ = 0; + }; + + using iterator = _iterator, T>; + using const_iterator = _iterator, const T>; + + public: + explicit CircularBuffer(std::size_t capacity = 0, bool debug_flag = false); + CircularBuffer(const CircularBuffer& other) = default; + CircularBuffer(CircularBuffer&& other) noexcept = default; + ~CircularBuffer() = default; + + static constexpr std::int64_t npos = -1; + + /** @return location of i'th element. i: 0=front, 1=second etc **/ + std::size_t location_of(std::size_t i) const; + /** @return ordinal index (relative to front) of location @p loc; + * npos if not used + **/ + //std::int64_t index_of(std::size_t loc) const; // not implemented yet + + // standard container methods + bool empty() const noexcept { return size_ == 0; } + size_type size() const noexcept { return size_; } + size_type max_size() const noexcept { return contents_.size(); } + // void reserve(size_type new_capacity); // not implemented + size_type capacity() const noexcept { return contents_.size(); } + // void shrink_to_fit(); // not implemented + + reference at(size_type pos) { + if ((pos < 0) || (pos >= size_)) { + throw std::out_of_range(tostr("CircularBuffer::at: index out of range", + xtag("pos", pos), xtag("size", size_))); + } + + return contents_[this->location_of(pos)]; + } + + const_reference at(size_type pos) const { + reference retval = const_cast(this)->at(pos); + return retval; + } + + reference operator[](size_type pos) { + return contents_[this->location_of(pos)]; + } + + const_reference operator[](size_type pos) const { + return contents_[this->location_of(pos)]; + } + + reference front() { return contents_[front_ix_]; } + const_reference front() const { + reference retval = const_cast(this)->front(); + return retval; + } + + reference back() { return contents_[location_of(size_ - 1)]; } + const_reference back() const { + reference retval = const_cast(this)->back(); + return retval; + } + + iterator begin() { return iterator(this, 0); } + iterator end() { return iterator(this, size_); } + const_iterator begin() const { return const_iterator(this, 0); } + const_iterator end() const { return const_iterator(this, size_); } + + // reverse_iterator rbegin(); + // reverse_iterator rend(); + // const_reverse_iterator rbegin() const; + // const_reverse_iterator rend() const; + + // General Methods + + void clear() { + size_ = 0; + front_ix_ = 0; + std::size_t capacity = contents_.size(); + contents_.clear(); + contents_.resize(capacity); + } + + /** push @p x on to the end of this buffer. + * If buffer is at capacity, overwrites the oldest element + **/ + CircularBuffer & push_back(const T & x); + + // template + //reference emplace_back(Args&&... args); + + CircularBuffer & pop_back(); + + // push_front(); + // pop_front(); + + CircularBuffer& operator=(const CircularBuffer& other) = default; + CircularBuffer& operator=(CircularBuffer&& other) noexcept = default; + + private: + /** number of elements in buffer. Not the same as @code contents_.size(); + * the latter represents buffer capacity. + * + * Promise: + * size_ <= contents_.size() + **/ + std::size_t size_ = 0; + /** first element is @code contents_.at(front_ix_) **/ + std::size_t front_ix_ = 0; + /** buffer contents. contents_.size() represents buffer capacity + * first element stored in @code contents_.at(front_) + * last element stored in @code contents_.at((front_ + size_ - 1) % contents_.size()) + **/ + std::vector contents_; + /** true to enable debug logging **/ + bool debug_flag_ = false; + }; + + template + CircularBuffer::CircularBuffer(std::size_t capacity, bool debug_flag) + : size_{0}, front_ix_{0}, contents_{capacity}, debug_flag_{debug_flag} + { + } + + template + std::size_t + CircularBuffer::location_of(std::size_t i) const + { + if (size_ == 0) + return 0; + else + return (front_ix_ + i) % size_; + } + + template + CircularBuffer & + CircularBuffer::push_back(const T & x) { + scope log(XO_DEBUG(debug_flag_), rtag("x", x), xrtag("size", size_)); + + if (size_ < contents_.size()) { + ++size_; + /* _after_ incr .size_ */ + std::size_t back_ix = location_of(size_ - 1); + + this->contents_[back_ix] = x; + + log && log(xtag("back_ix", back_ix), xtag("+size", size_)); + } else { + std::size_t back_ix = location_of(size_); + + this->contents_[back_ix] = x; + /* buffer was full, so oldest element replaced */ + this->front_ix_ = (this->front_ix_ + 1) % contents_.size(); + + log && log(xtag("back_ix", back_ix), xtag("+front", front_ix_)); + } + + return *this; + } + + template + CircularBuffer & + CircularBuffer::pop_back() { + if (size_ > 0) { + std::size_t back_ix = location_of(size_ - 1); + + this->contents_[back_ix] = T(); + --(this->size_); + } else { + assert(false); + } + + return *this; + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* CircularBuffer.hpp */ diff --git a/xo-alloc/include/xo/alloc/Forwarding1.hpp b/xo-alloc/include/xo/alloc/Forwarding1.hpp new file mode 100644 index 00000000..90ff0198 --- /dev/null +++ b/xo-alloc/include/xo/alloc/Forwarding1.hpp @@ -0,0 +1,56 @@ +/* file Forwarding1.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "Object.hpp" + +namespace xo { + namespace obj { + /** @class Forwarding1 + * @brief forwarding pointer for garbage collector. + * + * Used internally by garbage collector (see @ref GC). + * During evacuate phase overwrite from-space objects in-place + * with an instance of this class. + * + * This class suitable only for singly-inheriting objects, + * i.e. those that have exactly one vtable. + **/ + class Forwarding1 : public Object { + public: + explicit Forwarding1(gp dest); + + // inherited from Object.. + virtual TaggedPtr self_tp() const final override; + virtual void display(std::ostream & os) const final override; + virtual bool _is_forwarded() const final override { return true; } + virtual IObject * _offset_destination(IObject * src) const final override; + virtual IObject * _destination() final override; + /** required by Object i/face, but never called on Forwarding1 **/ + virtual std::size_t _shallow_size() const final override; + /** required by Object i/face, but never called on Forwarding1 **/ + virtual IObject * _shallow_copy(gc::IAlloc * mm) const final override; + /** required by Object i/face, but never called on Forwarding1 **/ + virtual std::size_t _forward_children(gc::IAlloc * mm) final override; + + private: + /** the object that used to be located at this address (i.e. @c this) + * has been moved to @ref destination_ , + * with original location overwritten by a forwarding pointer + * + * Require: + * - can only use Forwarding with types that have a single vtable. + * To forward a multiply-inheriting class with two vtables, use Forwarding2. + * - if you try to use Forwarding for an object with multiple vtables, + * one of the vtable pointers will be replaced by @ref destination_. + * UB revealed when GC traverses a pointer that relies on the 2nd + * vtable to index virtual methods. + **/ + gp dest_; + }; + + } /*namespace obj*/ +} /*namespace xo*/ + +/* end Forwarding1.hpp */ diff --git a/xo-alloc/include/xo/alloc/GC.hpp b/xo-alloc/include/xo/alloc/GC.hpp new file mode 100644 index 00000000..d8acc08c --- /dev/null +++ b/xo-alloc/include/xo/alloc/GC.hpp @@ -0,0 +1,496 @@ +/* GC.hpp + * + * author: Roland Conybeare, jul 2025 + */ + +#pragma once + +#include "ArenaAlloc.hpp" +#include "GcStatistics.hpp" +#include "Object.hpp" +#include "xo/callback/UpCallbackSet.hpp" +#include "xo/indentlog/print/array.hpp" +#include +#include + +namespace xo { + /** types that can participate in GC inherit from this base class. See Object.hpp in this directory **/ + class Object; + + namespace gc { + enum class role { + /** nursery: generation for new objects **/ + from_space, + /** tenured: generation for objects that have survived two collections **/ + to_space, + N, + }; + + constexpr std::size_t role2int(role x) { return static_cast(x); } + + /** @class Config + * @brief garbage collector configuration + **/ + struct Config { + /** initial size in bytes for youngest (Nursery) generation. + * GC allocates two nursery spaces of this size. + * This number represents reserved address space. + * pages are committed on demand. + * Initial committment will be up to @ref incr_gc_threshold_ + **/ + std::size_t initial_nursery_z_ = 64*1024*1024; + /** initial size in bytes for oldest (Tenured) generation. + * GC allocates two tenured spaces of this size. + * This number represents reserved address space. + * pages are committed on demand. + * Initial committment will be up to @ref full_gc_threshold_ + **/ + std::size_t initial_tenured_z_ = 128*1024*1024; + /** trigger incremental GC after this many bytes allocated in nursery **/ + std::size_t incr_gc_threshold_ = 64*1024; + /** trigger full GC after this many bytes promoted to tenured **/ + std::size_t full_gc_threshold_ = 512*1024; + + /** true to permit incremental garbage collection **/ + bool allow_incremental_gc_ = true; + /** true to report statistics **/ + bool stats_flag_ = false; + /** true to capture per-type object statistics **/ + bool object_stats_flag_ = false; + /** remember basic gc statistics for this many GC's; separately for incremental + full GCs **/ + std::size_t stats_history_z_ = 256; + /** true to enable debug logging **/ + bool debug_flag_ = false; + }; + + /** @class GCRunstate + * @brief encapsulate state needed while GC is running + * + * state pertaining to a single GC invocation. + * We stash an instance of this in @ref GC as context, + * so that per-Object-derived-type auxiliary functions can be slightly streamlined + **/ + class GCRunstate { + public: + GCRunstate() = default; + explicit GCRunstate(bool in_progress, bool full_move) + : in_progress_{in_progress}, full_move_{full_move} {} + + bool in_progress() const { return in_progress_; } + bool full_move() const { return full_move_; } + + private: + /** true when GC begins; remains true until GC cycle complete **/ + bool in_progress_ = false; + /** true for full GC; false for incremental GC **/ + bool full_move_ = false; + }; + + class MutationLogEntry { + public: + MutationLogEntry(IObject * parent, IObject ** lhs) + : parent_{parent}, lhs_{lhs} {} + + IObject * parent() const { return parent_; } + IObject ** lhs() const { return lhs_; } + + IObject * child() const { return *lhs_; } + + bool is_child_forwarded() const; + bool is_parent_forwarded() const; + + IObject * parent_destination() const; + + /** Flag obsolete mutation. + * Future proofing, never happens for regular objects + **/ + bool is_dead() const { return false; } + + MutationLogEntry update_parent_moved(IObject * parent_to) const; + void fixup_parent_child_moved(IObject * child_to); + + private: + IObject * parent_ = nullptr; + IObject ** lhs_ = nullptr; + }; + + using MutationLog = std::vector; + + /** @class GcCopyCallback + * @brief optional callback to observe individual copy operations during GC + * + * For viz + **/ + class GcCopyCallback { + public: + virtual ~GcCopyCallback() = default; + + virtual void notify_gc_copy(std::size_t z, const void * src_addr, const void * dest_addr, + generation src_gen, generation dest_gen) = 0; + /** invoked when added to callback set (i.e. @ref GC::GcCopyCallbackSet) **/ + void notify_add_callback() {} + /** invoked when removed from callback set **/ + void notify_remove_callback() {} + }; + + /** @class GC + * @brief generational garbage collector + * + * Works with objects of type @ref xo::Object + **/ + class GC : public IAlloc { + public: + using CallbackId = xo::fn::CallbackId; + using GcCopyCallbackSet = xo::fn::UpCallbackSet; + using nanos = decltype(xo::qty::qty::nanosecond); + + /** rebind is for typed allocators. since IAlloc is untyped, + * we want degenerate version + **/ + template + struct rebind { using other = GC; }; + + public: + /** create new GC instance with configuration @p config **/ + explicit GC(const Config & config); + /** noncopyable **/ + GC(const GC & other) = delete; + virtual ~GC(); + + /** create GC allocator. + * + * Initial memory consumption: + * approximately 2x @ref Config::nursery_size_ + 2x @ref Config::tenured_size_ + **/ + static up make(const Config & config); + + /** runtime downcast **/ + static GC * from(IAlloc * mm); + + const Config & config() const { return config_; } + std::uint8_t nursery_polarity() const { return nursery_polarity_; } + std::uint8_t tenured_polarity() const { return tenured_polarity_; } + const GCRunstate & runstate() const { return runstate_; } + const GcStatistics & native_gc_statistics() const { return gc_statistics_; } + GcStatisticsExt get_gc_statistics() const; + const GcStatisticsHistory & gc_history() const { return gc_history_; } + + /** true iff GC permitted in current state **/ + bool is_gc_enabled() const { return gc_enabled_ == 0; } + /** true iff GC has been requested **/ + bool is_gc_pending() const { return incr_gc_pending_ || full_gc_pending_; } + /** true iff full GC pending **/ + bool is_full_gc_pending() const { return full_gc_pending_; } + /** true during (and only during) a GC cycle **/ + bool gc_in_progress() const { return runstate_.in_progress(); } + + /** @return pagesize (will be the same for {nursery, tenured} spaces) **/ + std::size_t pagesize() const; + /** @return hugepage size (will be the same for {nursery, tenured} spaces) **/ + std::size_t hugepage_z() const; + + /** @return allocation portion of Nursery to-space **/ + std::size_t nursery_to_allocated() const; + /** @return reserved size of Nursery to-space **/ + std::size_t nursery_to_reserved() const; + /** @return committed size of Nursery to-space **/ + std::size_t nursery_to_committed() const; + /** @return nursery bytes used before checkpoint **/ + std::size_t nursery_before_checkpoint() const; + /** @return nursery bytes used after checkpoint **/ + std::size_t nursery_after_checkpoint() const; + /** @return allocated memory range for nursery **/ + std::pair nursery_span(role role) const; + /** @return nursery bytes used in from-space + * (only interesting during GC copy phase, e.g. during scope of a GcCopyCallback call) + **/ + std::size_t nursery_from_allocated() const; + /** @return reserved size of Tenured to-space **/ + std::size_t tenured_to_reserved() const; + /** @return committed size of Tenured to-space **/ + std::size_t tenured_to_committed() const; + /** @return tenured bytes used before checkpoint **/ + std::size_t tenured_before_checkpoint() const; + /** @return tenured bytes used after checkpoint = promoted since last GC **/ + std::size_t tenured_after_checkpoint() const; + + /** @return generation to which object at @p x belongs **/ + generation_result tospace_generation_of(const void * x) const; + /** @return generation to which object at @p x belongs, + * location relative to base address for that generation, + * and allocated size of that generation + * @p role chooses between to-space and from-space + **/ + std::tuple location_of(role role, const void * x) const; + /** @return generation to which object at @p x belongs, + * location relative to base address for @p x, + * and allocated size of generation + **/ + std::tuple tospace_location_of(const void * x) const; + /** @return generation that contains @p x, given it's in from-space **/ + generation_result fromspace_generation_of(const void * x) const; + /** @return generation to which object at @p x belongs, + * location relative to base address for @p x, + * and allocated size of generation + **/ + std::tuple fromspace_location_of(const void * x) const; + /** true iff from-space contains @p x **/ + bool fromspace_contains(const void * x) const; + /** @return free pointer for generation @p gen, i.e. nursery or tenured space **/ + std::byte * free_ptr(generation gen); + /** @return current size of (number of entries in) mutation log **/ + std::size_t mlog_size() const; + + /** add gc root at address @p addr . Gc will keep alive anything reachable + * from @c *addr + **/ + void add_gc_root(IObject ** addr); + /** reverse the effect of previous call to @ref add_gc_root **/ + void remove_gc_root(IObject ** addr); + + /** convenience wrapper **/ + + template + void add_gc_root_dwim(gp * p) { + static_assert(std::is_convertible_v); + this->add_gc_root(reinterpret_cast(p->ptr_address())); + } + + template + void remove_gc_root_dwim(gp * p) { + static_assert(std::is_convertible_v); + this->remove_gc_root(reinterpret_cast(p->ptr_address())); + } + + /** may optionally use this to observe GC copy phase. + * Will be invoked once _per surviving object_, so not cheap. + * Intended for GC visualization. + **/ + CallbackId add_gc_copy_callback(up fn); + /** request garbage collection. + * If GC currently disabled, collection will be deferred until the next time GC + * is in an enabled state. See @ref disable_gc and @ref enable_gc + **/ + void request_gc(generation g); + /** disable garbage collection until matching call to @ref enable_gc. + * + * GC is disabled when number of calls to @ref disable_gc exceeds number of + * calls to @ref enable_gc. + **/ + void disable_gc(); + /** enable garbage collection + * + * GC is enabled when number of calls to @ref enable_gc is at least as large + * as number of calls to @ref disable_gc. + * + * @return true iff GC performed + **/ + bool enable_gc(); + /** same as @c this->enable_gc() followed by @c this->disable_gc() + * @return true iff GC performed + **/ + bool enable_gc_once(); + + // inherited from IAlloc.. + + virtual const std::string & name() const final override; + /** capacity in bytes (counting both free+allocated) for object storage. + * only counts one of {to-space, from-space}, + * since one role is always held empty between collections. + **/ + virtual std::size_t size() const final override; + /** for committed count both to-space and from-space **/ + virtual std::size_t committed() const final override; + virtual std::size_t allocated() const final override; + virtual std::size_t available() const final override; + /** only tests to-space **/ + virtual bool contains(const void * x) const final override; + virtual bool is_before_checkpoint(const void * x) const final override; + virtual std::size_t before_checkpoint() const final override; + virtual std::size_t after_checkpoint() const final override; + virtual bool debug_flag() const final override; + + virtual void clear() final override; + virtual void checkpoint() final override; + + /** GC bookkeeping for an assignment that modifes an Object reference. + * Whenever an @ref Object instance P contains a member variable that can refer + * to another @ref Object, then we need to involve GC to perform the assignment. + * In particular a side-effect that changes the target of such reference to Q after P + * has been promoted, may lead to a tenured->nursery cross-generational pointer. + * GC needs to know about such pointers to it can update them as part of subsequent + * incremental collections. + * + * @param parent. object with member variable being modified + * @param lhs. address of a member variable within the allocation of @p parent. + * @param rhs. new target for @p *lhs + **/ + virtual void assign_member(IObject * parent, IObject ** lhs, IObject* rhs) final override; + /** evacuate @p *lhs and replace with forwarding pointer **/ + virtual void forward_inplace(IObject ** lhs) final override; + /** during GC check for source objects owned by GC. + * See Object::_shallow_move. + **/ + virtual bool check_owned(IObject * src) const final override; + /** queries during GC to determine if object at address @p src should move: + * - full GC -> always + * - incr GC -> if not tenured + **/ + virtual bool check_move(IObject * src) const final override; + /** if src is cross-generational (or cross-checkpoint), verify that it + * is recorded in mutation log, + * given an object @p parent that contains object pointer @p lhs + **/ + virtual bool check_write_barrier(const void * parent, const void * const * lhs, bool may_throw) const final; + + virtual std::byte * alloc(std::size_t z) final override; + virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; + + private: + ArenaAlloc * nursery_to() const { return nursery(role::to_space); } + ArenaAlloc * nursery_from() const { return nursery(role::from_space); } + + ArenaAlloc * tenured_to() const { return tenured(role::to_space); } + ArenaAlloc * tenured_from() const { return tenured(role::from_space); } + + ArenaAlloc * nursery(role r) const { return nursery_[role2int(r)].get(); } + ArenaAlloc * tenured(role r) const { return tenured_[role2int(r)].get(); } + + MutationLog * mutation_log(role r) const { return mutation_log_[role2int(r)].get(); } + + /** begin GC now **/ + void execute_gc(generation g); + /** cleanup phase. aux function for @ref execute_gc **/ + void cleanup_phase(generation g, nanos dt); + /** swap roles of From/To spaces for nursery generation **/ + void swap_nursery(); + /** swap roles of From/To spaces for tenured generation **/ + void swap_tenured(); + /** swap roles of From/To spaces for mutation log **/ + void swap_mutation_log(); + /** swap roles of FromSpace/ToSpace **/ + void swap_spaces(generation g); + /** scan to-space for object statistics before GC */ + void capture_object_statistics(generation upto, capture_phase phase); + /** copy object **/ + void copy_object(IObject ** addr, generation upto, ObjectStatistics * object_stats); + /** copy everything reachable from global gc roots **/ + void copy_globals(generation g); + /** review mutation log; may discover+rescue reachable objects. + **/ + void forward_mutation_log(generation upto); + /** Aux function for @ref execute_gc. Updates bookkeeping for cross-generational + * (T->N, aka xgen) and (N1->N0, aka xckp) pointers + **/ + void incremental_gc_forward_mlog(ObjectStatistics * per_type_stats); + /** + * Aux function for @ref incremental_gc_forward_mlog. Calls this function until + * fixpoint. + * + * @param from_mlog incoming mutation log. Contains {xgen,xckp} pointers before GC. + * Contents of this log is consumed (+discarded) before method returns. + * @param to_mlog outgoing mutation log. Will contain {xgen,xckp} pointers after GC. + * @param defer_mlog contains log entries associated with possible garbage. + **/ + void incremental_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * per_type_stats); + /** Aux function for @ref execute_gc. Updates bookkeeping for cross-generational + * (T->N, aka xgen) and (N1->N0, aka xckcp) pointers on full gc + **/ + void full_gc_forward_mlog(ObjectStatistics * per_type_stats); + /** + * Aux function for @ref full_gc_forward_mlog. Calls this function until fixpoint. + * + * @param from_mlog incoming mutation log. Contains {xgen,xckp} pointers before GC. + * Contents of this log is consumed (+discarded) before method returns. + * @param to_mlog outgoing mutation log. Will contain {xgen,xckp} pointers after GC. + * @param defer_mlog contains log entries associated with possible garbage. + * + **/ + void full_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * per_type_stats); + + private: + /** garbage collector configuration **/ + Config config_; + + /** keep track of the identity of from-space and to-space. + * assist for animation (see xo-imgui/example/ex2). + * polarity alternates between 0 and 1 on each GC + **/ + std::uint8_t nursery_polarity_ = 0; + std::uint8_t tenured_polarity_ = 0; + + /** contains allocated objects, along with unreachable garbage to be collected. + * roles reverse after each incremental, or full, collection. + **/ + std::array, role2int(role::N)> nursery_; + /** empty space, destination for objects that survive collection. + * roles reverse after each full collection. + **/ + std::array, role2int(role::N)> tenured_; + + /** current state of GC activity. + * @text + * in_progress full_move descr + * ----------------------------------------- + * false * gc not running + * true false incremental gc + * true true full gc + * ----------------------------------------- + * @endtext + **/ + GCRunstate runstate_; + + /** root object handles: targets of handles in this vector are always preserved by GC. + * Application can introduce new root object pointers at any time provided GC not running, + * but cannot withdraw them. + **/ + std::vector gc_root_v_; + + /** log cross-generational and cross-checkpoint mutations. + * These need to be adjusted on next incremental collection. + * + * mutation_log_[tospace] accumulates {xgen,xckp} pointers until + * the next GC. + * + * See GC aux functions + * @ref incremental_gc_forward_mlog + * @ref full_gc_forward_mlog + * + **/ + std::array, role2int(role::N)> mutation_log_; + /** temporary mutation log (for deferred entries) **/ + up defer_mutation_log_; + + /** allocation/collection counters **/ + GcStatistics gc_statistics_; + /** optional per-object-type counters. snapshot at beginning of collection cycle **/ + std::array object_statistics_sab_; + /** optional per-object-type counters. snapshot at end of collection cycle **/ + std::array object_statistics_sae_; + + /** true when GC requested, + * remains true until GC.. completes? begins? + **/ + bool incr_gc_pending_ = false; + bool full_gc_pending_ = false; + + /** enabled when 0. disabled when <0 **/ + int gc_enabled_ = 0; + + /** rotating per-gc statistics history **/ + GcStatisticsHistory gc_history_; + + /** for (optional) viz: invoke when copying individual objects **/ + GcCopyCallbackSet gc_copy_cbset_; + }; + } /*namespace gc*/ + +} /*namespace xo*/ + +/* end GC.hpp */ diff --git a/xo-alloc/include/xo/alloc/GcStatistics.hpp b/xo-alloc/include/xo/alloc/GcStatistics.hpp new file mode 100644 index 00000000..69053457 --- /dev/null +++ b/xo-alloc/include/xo/alloc/GcStatistics.hpp @@ -0,0 +1,287 @@ +/** @file GcStatistics.hpp + * + * @author Roland Conybeare, Aug 2025 + **/ + +#pragma once + +#include "generation.hpp" +#include "CircularBuffer.hpp" +#include +#include +#include +#include +#include +#include + +namespace xo { + namespace gc { + /** @class PerGenerationStatistics + * @brief garbage collection statistics for particular GC generation + **/ + class PerGenerationStatistics { + public: + /** update statistics after a GC cycle + * @param alloc_z. new allocations (since preceding GC) + * @param before_z. generation size (bytes allocated) before collection + * @param after_z. generation size after collection + * @param promote_z. bytes promoted to next generation + **/ + void include_gc(std::size_t alloc_z, std::size_t before_z, std::size_t after_z, + std::size_t promote_z); + /** update with current state (use at end of gc cycle) **/ + void update_snapshot(std::size_t after_z); + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** number of bytes currently in use **/ + std::size_t used_z_ = 0; + + /** number of collection cycles completed **/ + std::size_t n_gc_ = 0; + /** sum of new alloc bytes, sampled at start of each collection cycle **/ + std::size_t new_alloc_z_ = 0; + /** sum of allocated bytes sampled at beginning of each collection cycle **/ + std::size_t scanned_z_ = 0; + /** sum of bytes remaining after collection cycle **/ + std::size_t survive_z_ = 0; + /** sum of bytes promoted to next generation **/ + std::size_t promote_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const PerGenerationStatistics & x) { + x.display(os); + return os; + } + + /** @class GcStatistics + * @brief garbage collection statistics + **/ + class GcStatistics { + public: + GcStatistics() = default; + + /** update statistics at beginning of a GC cycle + * @param upto. nursery -> incremental collection; tenured -> full collection + * @param alloc_z. new allocations (since preceding GC) + **/ + void begin_gc(generation upto, + std::size_t alloc_z); + + /** update statistics after a GC cycle + * @param upto. nursery -> incremental collection; tenured -> full collection + * @param alloc_z. new allocations (since preceding GC) + * @param before_z. generation size (bytes allocated) before collection + * @param after_z. generation size after collection + * @param promote_z. bytes promoted to next generation + **/ + void include_gc(generation upto, std::size_t alloc_z, + std::size_t before_z, std::size_t after_z, std::size_t promote_z); + /** update snapshot for current state. + * Use with tenured stats after incremental gc + **/ + void update_snapshot(generation upto, std::size_t after_z); + + /** number of collection cycles, whether full or incremental **/ + std::size_t n_gc() const { return gen_v_[gen2int(generation::nursery)].n_gc_; } + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** statistics gathered across {incr, full} GCs respectively **/ + std::array(generation::N)> gen_v_; + /** total bytes allocated since inception **/ + std::size_t total_allocated_ = 0; + /** snapshot of total bytes promoted asof beginning of last gc cycle **/ + std::size_t total_promoted_sab_ = 0; + /** total bytes promoted from nursery->tenured since inception **/ + std::size_t total_promoted_ = 0; + + /** total number of mutations to already-allocated objects, + * whether or not GC needs to log them. + **/ + std::size_t n_mutation_ = 0; + /** total number of mutation eligible for logging (cumulative across GCs) **/ + std::size_t n_logged_mutation_ = 0; + /** total number of cross-generation mutations + * (tenured->nursery when reported; cumulative across GCs) **/ + std::size_t n_xgen_mutation_ = 0; + /** total number of cross-checkpoint mutations + * (N0 -> N1 when reported; cumulative across GCs) + **/ + std::size_t n_xckp_mutation_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatistics & x) { + x.display(os); + return os; + } + + /** @class GcStatisticsExt + * @brief extend GcStatistics for application convenience + **/ + class GcStatisticsExt : public GcStatistics { + public: + GcStatisticsExt() = default; + explicit GcStatisticsExt(const GcStatistics & x) : GcStatistics{x} {} + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** current capacity of nursery generation **/ + std::size_t nursery_z_ = 0; + /** current nursery survivor size **/ + std::size_t nursery_before_checkpoint_z_ = 0; + /** current nursery new alloc size **/ + std::size_t nursery_after_checkpoint_z_ = 0; + /** current capacity of tenured generation **/ + std::size_t tenured_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatisticsExt & x) { + x.display(os); + return os; + } + + /** @class GcStatisticsHistoryItem + * @brief info we want to record over time (won't have cumulative things in it) + **/ + class GcStatisticsHistoryItem { + public: + using nanos = xo::qty::type::nanoseconds; + + public: + GcStatisticsHistoryItem() = default; + constexpr GcStatisticsHistoryItem(std::size_t gc_seq, + generation upto, + std::size_t new_alloc_z, + std::size_t survive_z, + std::size_t promote_z, + std::size_t persist_z, + std::size_t effort_z, + std::size_t garbage0_z, + std::size_t garbage1_z, + std::size_t garbageN_z, + nanos dt, + std::size_t sum_effort_z, + std::size_t sum_garbage_z) + : gc_seq_{gc_seq}, + upto_{upto}, + new_alloc_z_{new_alloc_z}, + survive_z_{survive_z}, + promote_z_{promote_z}, + persist_z_{persist_z}, + effort_z_{effort_z}, + garbage0_z_{garbage0_z}, + garbage1_z_{garbage1_z}, + garbageN_z_{garbageN_z}, + dt_{dt}, + sum_effort_z_{sum_effort_z}, + sum_garbage_z_{sum_garbage_z} + {} + constexpr GcStatisticsHistoryItem(const GcStatisticsHistoryItem &) = default; + + std::size_t garbage_z() const { return garbage0_z_ + garbage1_z_ + garbageN_z_; } + + float efficiency() const { + std::size_t gz = this->garbage_z(); + + return gz / static_cast(effort_z_ + gz); + } + + /** lifetime byte-weighted average collection efficiency. Always in [0.0, 1.0] **/ + float average_efficiency() const { + return sum_garbage_z_ / static_cast(sum_effort_z_ + sum_garbage_z_); + } + + /** collection rate, in bytes/sec **/ + float collection_rate() const; + + GcStatisticsHistoryItem & operator=(const GcStatisticsHistoryItem & x) { + gc_seq_ = x.gc_seq_; + upto_ = x.upto_; + new_alloc_z_ = x.new_alloc_z_; + survive_z_ = x.survive_z_; + promote_z_ = x.promote_z_; + persist_z_ = x.persist_z_; + effort_z_ = x.effort_z_; + garbage0_z_ = x.garbage0_z_; + garbage1_z_ = x.garbage1_z_; + garbageN_z_ = x.garbageN_z_; + this->dt_.scale_ = x.dt_.scale_; + + sum_effort_z_ = x.sum_effort_z_; + sum_garbage_z_ = x.sum_garbage_z_; + + return *this; + } + + /** @param os. write stats on this output stream **/ + void display(std::ostream & os) const; + + /** sequence number for collection being reported **/ + std::size_t gc_seq_ = 0; + /** type of GC that generated this record **/ + generation upto_; + /** #of bytes new allocation **/ + std::size_t new_alloc_z_ = 0; + /** #of bytes surviving their first collection (i.e. N0->N1) **/ + std::size_t survive_z_ = 0; + /** #of bytes promoted to tenured. + * Comprises all objects surviving their 2nd collection (i.e. N1->T) + **/ + std::size_t promote_z_ = 0; + /** #of bytes surviving 3rd of later collection **/ + std::size_t persist_z_ = 0; + /** #of bytes copied **/ + std::size_t effort_z_ = 0; + /** #of bytes garbage from N0 (i.e. survived 0 GCs) **/ + std::size_t garbage0_z_ = 0; + /** #of bytes garbage from N1 (i.e. survived 1 GCs) **/ + std::size_t garbage1_z_ = 0; + /** #of bytes garbage from T (i.e. survived 2+ GCs) **/ + std::size_t garbageN_z_ = 0; + /** elapsed time for this GC (see @ref GC::execute_gc) **/ + nanos dt_; + + // ----- cumulative statistics ----- + + /** sum (in bytes) copied by collections since inception **/ + std::size_t sum_effort_z_ = 0; + /** sum (in bytes) of garbage collected since inception **/ + std::size_t sum_garbage_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const GcStatisticsHistoryItem & x) { + x.display(os); + return os; + } + + using GcStatisticsHistory = CircularBuffer; + } /*namespace gc*/ + + namespace print { + template <> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::PerGenerationStatistics &); + }; + + template<> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatistics &); + }; + + template<> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatisticsExt &); + }; + + template<> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::GcStatisticsHistoryItem &); + }; + } /*namespace print*/ +} /*namespace xo*/ + +/* end GcStatistics.hpp */ diff --git a/xo-alloc/include/xo/alloc/ListAlloc.hpp b/xo-alloc/include/xo/alloc/ListAlloc.hpp new file mode 100644 index 00000000..30d91cfb --- /dev/null +++ b/xo-alloc/include/xo/alloc/ListAlloc.hpp @@ -0,0 +1,100 @@ +/* file ListAlloc.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#pragma once + +#include "IAlloc.hpp" +#include "ObjectStatistics.hpp" +#include +#include +#include + +namespace xo { + namespace gc { + class ArenaAlloc; + + /** GC-compatible allocator using a linked list of buckets. + * + * - all allocs done from first allocator in list + * GC Support: + * - reserved memory, released after call to @ref release_redline_memory. + * + * TODO: reserve address space using mmap, + * but don't commit until alloc requires it. + **/ + class ListAlloc : public IAlloc { + public: + ListAlloc(std::unique_ptr hd, + ArenaAlloc * marked, + std::size_t cz, std::size_t nz, std::size_t tz, + bool debug_flag); + ~ListAlloc(); + + static up make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag); + + /** page size used by underlying ArenaAlloc **/ + std::size_t page_size() const; + + /** hugepage size used by underlying ArenaAlloc **/ + std::size_t hugepage_z() const; + + /** reset to have at least @p z bytes of storage **/ + bool reset(std::size_t z); + + /** expand bucket list to accomodate a request of size @p z **/ + bool expand(std::size_t z, const std::string & name); + + /** current free pointer **/ + std::byte * free_ptr() const; + + /** scan space (must not contain forwarding pointers, because loses size info) + * + gather stats by object type + * + * See @ref Object::self_tp + **/ + void capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const; + + // inherited from IAlloc.. + + virtual const std::string & name() const final override; + virtual std::size_t size() const final override; + virtual std::size_t committed() const final override; + virtual std::size_t available() const final override; + virtual std::size_t allocated() const final override; + virtual bool contains(const void * x) const final override; + virtual bool is_before_checkpoint(const void * x) const final override; + virtual std::size_t before_checkpoint() const final override; + virtual std::size_t after_checkpoint() const final override; + virtual bool debug_flag() const final override; + + virtual void clear() final override; + virtual void checkpoint() final override; + virtual std::byte * alloc(std::size_t z) final override; + + private: + /** **/ + std::size_t start_z_ = 0; + /** all new allocs from this list **/ + std::unique_ptr hd_; + /** allocator that was in @ref hd_ when @ref checkpoint last called **/ + ArenaAlloc * marked_ = nullptr; + /** overflow allocs (expect list to be short); + * from trying to converge on app working set size + **/ + std::list> full_l_; + /** size of current arena @ref hd_ **/ + std::size_t current_z_ = 0; + /** if @ref hd_ fills, size of next arena to allocate **/ + std::size_t next_z_ = 0; + /** total size of @ref hd_ + contents of @ref full_l_ **/ + std::size_t total_z_ = 0; + /** true to enable debug logging **/ + bool debug_flag_ = false; + }; + } /*namespace gc*/ +} /*namespace xo*/ + +/* end ListAlloc.hpp */ diff --git a/xo-alloc/include/xo/alloc/Object.hpp b/xo-alloc/include/xo/alloc/Object.hpp new file mode 100644 index 00000000..9665e66a --- /dev/null +++ b/xo-alloc/include/xo/alloc/Object.hpp @@ -0,0 +1,170 @@ +/* Object.hpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#pragma once + +#include "xo/allocutil/IObject.hpp" +#include "xo/reflect/TaggedPtr.hpp" +#include "xo/allocutil/ObjectVisitor.hpp" +#include "xo/allocutil/gc_ptr.hpp" +#include +#include + +namespace xo { + namespace gc { + class IAlloc; + class GC; + class ObjectStatistics; + }; + + /** Root class for all xo GC-collectable objects. + * + * Design notes: + * 1. xo::IObject -> xo-allocutil header-only library + * xo::Object -> xo-alloc ordinary library + * 2. relying on inheritance means we insist that GC traits + * for a type appear directly in that type's vtable, and at specific locations. + * This implies one level of indirection when GC traverses an instance. + * 3. Could adapt a gc-aware XO library (such as xo-ordinaltree) + * to a non-xo garbage collector. + * Would still need to use xo::IObject and xo::gc::gc_allocator_traits, + * but not necessarily xo::Object + * 4. Would be feasible to relax the must-inherit-from-Object constraint + * by having GC use its own wrapper, at cost of an extra layer of indirection + **/ + class Object : public IObject { + public: + using TaggedPtr = xo::reflect::TaggedPtr; + + public: + static gp from(gp x) { + return dynamic_cast(x.ptr()); + } + + virtual ~Object() noexcept = default; + + /** memory allocator for objects. Likely this will be a GC instance, + * but simple arena also supported. + * + * Load-bearing for .assign_member() + **/ + static gc::IAlloc * mm; + + /** assign value @p rhs to member @p *lhs of @p parent. + * if assignment creates a cross-generational or cross-checkpoint pointer, + * add mutation log entry. + * + * DEPRECATED. prefer IObject::_gc_assign_member, for explicit alloc + **/ + template + static void assign_member(gp parent, gp * lhs, gp rhs); + + /** use from GC aux functions **/ + static gc::GC * _gc() { return reinterpret_cast(mm); } + + /** during GC + * 1. copy destination object @p *addr to (new) to-space. + * 2. overwrite existing object @p *addr with a forwarding pointer to + * copy made in step 1. + * 3. return the location of the copy make in step 1. + * + * @p src. source object to be forwarded + * @p gc. allocator (poassibly garbage collector) + */ + static IObject * _forward(IObject * src, gc::IAlloc * gc); + + template + static void _forward_inplace(T ** src_addr, gc::IAlloc * gc) { + IObject * fwd = _forward(*src_addr, gc); + + *src_addr = reinterpret_cast(fwd); + } + + template + static void _forward_inplace(gp & src, gc::IAlloc * gc) { + _forward_inplace(src.ptr_address(), gc); + } + + /** primary workhorse for garbage collection. + * + * we assign each object one of three colors: black|gray|white. + * + * color | location | children | action | + * ------+------------+------------+-------------------------+ + * black | from-space | any | move to to-space | + * gray | to-space | any | move remaining children | + * white | to-space | white/gray | done | + * + * initially all reachable objects are black. + * GC is complete when all reachable objects are white. + * GC needs a variable amount of temporary storage to keep track of all gray objects + * + * Evacuate reachable object graph rooted at @p src to to-space. + * On return all objects reachable from @p src are white + * + * @param src address of object to evacuate + * @param gc garbage collector + * @param stats per-object-type GC statistics + **/ + static IObject * _deep_move(IObject * src, gc::GC * gc, gc::ObjectStatistics * stats); + + /** copy @p src to to-space. Overwrite original with forwarding pointer to new location. + * return the new location + **/ + static IObject * _shallow_move(IObject * src, gc::IAlloc * gc); + + // Reflection support + + /** tagged pointer with runtime type information + **/ + virtual TaggedPtr self_tp() const; + + /** print on stream @p os **/ + virtual void display(std::ostream & os) const; + + // Inherited from IObject.. + + //virtual bool _is_forwarded() const override { return false; } + //virtual IObject * _offset_destination(IObject * src) const override { return src; }; + virtual void _forward_to(IObject * dest) override; + //virtual IObject * _destination() override { return nullptr; } + + virtual std::size_t _shallow_size() const override = 0; + virtual IObject * _shallow_copy(gc::IAlloc * gc) const override = 0; + virtual std::size_t _forward_children(gc::IAlloc * gc) override = 0; + }; + + static_assert(std::is_destructible_v, "Object must be destructible"); + static_assert(std::is_nothrow_destructible_v, "Object must be noexcept destructible"); + + template + void + Object::assign_member(gp parent, gp * lhs, gp rhs) + { + Object::mm->assign_member(reinterpret_cast(parent.ptr()), + reinterpret_cast(lhs->ptr_address()), + reinterpret_cast(rhs.ptr())); + } + + namespace gc { + template + class ObjectVisitor> { + public: + static void forward_children(gp & target, + IAlloc * gc) + { + Object::_forward_inplace(target, gc); + } + }; + } + + std::ostream & + operator<< (std::ostream & os, gp x); + +} /*namespace xo*/ + +void * operator new (std::size_t z, const xo::Cpof & copy); + +/* end Object.hpp */ diff --git a/xo-alloc/include/xo/alloc/ObjectStatistics.hpp b/xo-alloc/include/xo/alloc/ObjectStatistics.hpp new file mode 100644 index 00000000..43ddd70e --- /dev/null +++ b/xo-alloc/include/xo/alloc/ObjectStatistics.hpp @@ -0,0 +1,87 @@ +/* file ObjectStatistics.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#pragma once + +#include "xo/indentlog/print/pretty.hpp" +#include +#include + +namespace xo { + namespace reflect { class TypeDescrBase; } + + namespace gc { + enum class capture_phase { + /** snapshot-at-beginning **/ + sab, + /** snapshot-at-end **/ + sae, + }; + + /** @class PerObjectTypeStatistics + * @brief statistics for a particular object type + * + * Gathered for each leaf type descended from xo::obj::Object. + * See @ref xo::obj::Object::self_tp + * + * See @ref GC::capture_object_statistics + * (gathers @ref scanned_n_, @ref scanned_z_) + **/ + struct PerObjectTypeStatistics { + using TypeDescr = xo::reflect::TypeDescrBase const *; + + void display(std::ostream & os) const; + + /** stats here are for objects of this type **/ + TypeDescr td_ = nullptr; + /** number of objects scanned **/ + std::size_t scanned_n_ = 0; + /** number of bytes scanned **/ + std::size_t scanned_z_ = 0; + /** number of objects surviving **/ + std::size_t survive_n_ = 0; + /** number of bytes from surviving objects **/ + std::size_t survive_z_ = 0; + }; + + inline std::ostream & operator<< (std::ostream & os, const PerObjectTypeStatistics & x) { + x.display(os); + return os; + } + + /** @class ObjectStatistics + * @brief placeholder for type-driven allocation statistics + * + * Passed to @ref Object::deep_move for example + **/ + class ObjectStatistics { + public: + void display(std::ostream & os) const; + + /** per-object-type statistics, indexed by TypeId **/ + std::vector per_type_stats_v_; + }; + + inline std::ostream & operator<< (std::ostream & os, const ObjectStatistics & x) { + x.display(os); + return os; + } + + } /*namespace gc*/ + + namespace print { + template <> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::PerObjectTypeStatistics &); + }; + + template <> + struct ppdetail { + static bool print_pretty(const ppindentinfo &, const xo::gc::ObjectStatistics &); + }; + } /*namespace print*/ +} /*namespace xo*/ + +/* end ObjectStatistics.hpp */ diff --git a/xo-alloc/include/xo/alloc/Stack.hpp b/xo-alloc/include/xo/alloc/Stack.hpp new file mode 100644 index 00000000..b894d853 --- /dev/null +++ b/xo-alloc/include/xo/alloc/Stack.hpp @@ -0,0 +1,49 @@ +/* Stack.hpp + * + * author: Roland Conybeare, jul 2025 + */ + +#pragma once + +#include + +namespace xo { + namespace gc { + /** Simple stack implementation + **/ + template + class Stack { + public: + explicit Stack(std::size_t capacity) { + this->contents_.reserve(capacity); + } + + bool is_empty() const { return contents_.empty(); } + std::size_t available() const { return contents_.capacity() - contents_.size(); } + void drop() { contents_.resize(contents_.size() - 1); } + void push(const T & x) { contents_.push_back(x); } + T pop() { + T retval = contents_[contents_.size() - 1]; + this->drop(); + return retval; + } + const T & top() const { + return this->lookup(0); + } + const T & lookup(std::size_t i) const { + return contents_.at(contents_.size() - 1 - i); + } + void clear() { contents_.clear(); } + void reset_to(std::size_t z) { contents_.resize(z); } + + std::size_t n_elements() const { return contents_.size(); } + std::size_t capacity() const { return contents_.capacity(); } + + private: + std::vector contents_; + }; + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end Stack.hpp */ diff --git a/xo-alloc/include/xo/alloc/generation.hpp b/xo-alloc/include/xo/alloc/generation.hpp new file mode 100644 index 00000000..9f122044 --- /dev/null +++ b/xo-alloc/include/xo/alloc/generation.hpp @@ -0,0 +1,54 @@ +/* generation.hpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#pragma once + +#include +#include +#include + +namespace xo { + namespace gc { + enum class generation { + nursery, + tenured, + N + }; + + constexpr std::size_t gen2int(generation x) { return static_cast(x); } + + const char * gen2str(generation x); + + inline std::ostream & operator<<(std::ostream & os, generation x) { + os << gen2str(x); + return os; + } + + enum class generation_result { + nursery, + tenured, + not_found + }; + + inline generation valid_genresult2gen(generation_result x) { + assert(x != generation_result::not_found); + + if (x == generation_result::nursery) + return generation::nursery; + else + return generation::tenured; + } + + const char * genresult2str(generation_result x); + + inline std::ostream & operator<<(std::ostream & os, generation_result x) { + os << genresult2str(x); + return os; + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end generation.hpp */ diff --git a/xo-alloc/src/alloc/AllocPolicy.cpp b/xo-alloc/src/alloc/AllocPolicy.cpp new file mode 100644 index 00000000..b1dc162f --- /dev/null +++ b/xo-alloc/src/alloc/AllocPolicy.cpp @@ -0,0 +1,13 @@ +/* AllocPolicy.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "AllocPolicy.hpp" + +/* note: inline/.hpp definition not allowed for operator delete */ +void operator delete(void * ptr) noexcept { + xo::xo.free(ptr); +} + +/* end AllocPolicy.cpp */ diff --git a/xo-alloc/src/alloc/ArenaAlloc.cpp b/xo-alloc/src/alloc/ArenaAlloc.cpp new file mode 100644 index 00000000..dc80eb4e --- /dev/null +++ b/xo-alloc/src/alloc/ArenaAlloc.cpp @@ -0,0 +1,426 @@ +/* file ArenaAlloc.cpp + * + * author: Roland Conybeare + */ + +#include "ArenaAlloc.hpp" +#include "Object.hpp" +#include "ObjectStatistics.hpp" +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/tag.hpp" +#include +#include // for getpagesize() on OSX +#include + +namespace xo { + using std::byte; + + namespace gc { + namespace { + /* alignment better be a power of 2 */ + std::size_t + align_lub(std::size_t x, std::size_t align) + { + /* e.g: + * align = 4096, x%align = 100 -> dx = 3996 + * align = 4096, x%align = 0 -> dx = 0 + */ + std::size_t dx = (align - (x % align)) % align; + + return x + dx; + } + } + + ArenaAlloc::ArenaAlloc(const std::string & name, + std::size_t z, + bool debug_flag) + { + scope log(XO_DEBUG(debug_flag), xtag("name", name)); + + constexpr size_t c_hugepage_z = 2 * 1024 * 1024; + + this->name_ = name; + this->page_z_ = getpagesize(); + this->hugepage_z_ = c_hugepage_z; + + // 1. need k pagetable entries where k is lub {k | k * .page_z >= z} + // 2. base will be aligned with .page_z but likely not with .hugepage_z + // 3. bad to have misalignment, because misaligned {prefix, suffix} of [base, base+z) + // will use 4k pages instead of 2mb pages + // + // strategy: + // 4. round up z to multiple of c_hugepage_z + // 5. over-request so reserved range contains an aligned subrange of size z + // 6. unmap misaligned prefix + // 7. unmap misaligned suffix. + // 8. enable huge pages for now-aligned remainder of reserved range + // + // Z. note: rejecting inferior MAP_HUGETLB|MAP_HUGE_2MB flags on ::mmap here: + // Za. requires previously-reserved memory in /proc/sys/vm/nr_hugepages + // Zb. reserved pages permenently resident in RAM, never swapped + // Zc. memory cost incurred even if no application is using said pages + + z = align_lub(z, c_hugepage_z); // 4. + + // 5. + byte * base = reinterpret_cast(::mmap(nullptr, + z + c_hugepage_z, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0)); + + log && log("acquired memory [lo,hi) using mmap", + xtag("lo", base), + xtag("z", z), + xtag("hi", reinterpret_cast(base) + z)); + + if (base == MAP_FAILED) { + throw std::runtime_error(tostr("ArenaAlloc: uncommitted allocation failed", + xtag("size", z))); + } + + byte * aligned_base = reinterpret_cast(align_lub(reinterpret_cast(base), + c_hugepage_z)); + + assert(reinterpret_cast(aligned_base) % c_hugepage_z == 0); + assert(aligned_base >= base); + assert(aligned_base < base + c_hugepage_z); + + if (base < aligned_base) { + size_t prefix = aligned_base - base; + + ::munmap(base, prefix); // 6. + } + + byte * aligned_hi = aligned_base + z; + byte * hi = base + z + c_hugepage_z; + + if (aligned_hi < hi) { + size_t suffix = hi - aligned_hi; + + ::munmap(aligned_hi, suffix); // 7. + } + +#ifdef __linux__ + ::madvise(aligned_base, z, MADV_HUGEPAGE); // 8. +#endif + // TODO: for OSX -> need something else here. + // MAP_ALIGNED_SUPER with mmap() and/or + // use mach_vm_allocate() + // + + this->lo_ = aligned_base; + this->committed_z_ = 0; + this->checkpoint_ = lo_; + this->free_ptr_ = lo_; + this->limit_ = lo_; + this->hi_ = lo_ + z; + this->debug_flag_ = debug_flag; + + if (!lo_) { + throw std::runtime_error(tostr("ArenaAlloc: allocation failed", + xtag("size", z))); + } + + log && log(xtag("lo", (void*)lo_), + xtag("page_z", page_z_), + xtag("hugepage_z", hugepage_z_)); + } + + ArenaAlloc::~ArenaAlloc() + { + scope log(XO_DEBUG(debug_flag_)); + + // hygiene.. + + if (lo_) { + log && log("unmap [lo,hi)", xtag("lo", lo_), xtag("z", hi_ - lo_), xtag("hi", hi_)); + + ::munmap(lo_, hi_ - lo_); + } + // could use this as fallback if we dropped the uncommitted technique + //delete [] this->lo_; + + this->lo_ = nullptr; + this->committed_z_ = 0; + this->checkpoint_ = nullptr; + this->free_ptr_ = nullptr; + this->limit_ = nullptr; + this->hi_ = nullptr; + this->debug_flag_ = false; + } + + up + ArenaAlloc::make(const std::string & name, + std::size_t z, bool debug_flag) + { + return up(new ArenaAlloc(name, + z, debug_flag)); + } + + bool + ArenaAlloc::expand(size_t offset_z) + { + scope log(XO_DEBUG(debug_flag_), xtag("offset_z", offset_z), xtag("committed_z", committed_z_)); + + if (offset_z <= committed_z_) { + log && log("trivial success, offset within committed range", + xtag("offset_z", offset_z), + xtag("committed_z", committed_z_)); + return true; + } + + if (lo_ + offset_z > hi_) { + throw std::runtime_error(tostr("ArenaAlloc::expand: requested size exceeds reserved size", + xtag("requested", offset_z), xtag("reserved", reserved()))); + } + + /* + * pre: + * + * _______________................................... + * ^ ^ ^ + * lo limit hi + * + * < committed_z > + * <----------offset_z-----------> + * > <- z: 0 <= z < hugepage_z + * <---------aligned_offset_z---------> + * <--- add_commit_z --> + * + * post: + * ____________________________________.............. + * ^ ^ ^ + * lo limit hi + * + */ + + std::size_t aligned_offset_z = align_lub(offset_z, hugepage_z_); + std::byte * commit_start = lo_ + committed_z_; + std::size_t add_commit_z = aligned_offset_z - committed_z_; + + assert(limit_ == lo_ + committed_z_); + + log && log(xtag("aligned_offset_z", aligned_offset_z), + xtag("add_commit_z", add_commit_z)); + + log && log("expand committed range", + xtag("commit_start", commit_start), + xtag("add_commit_z", add_commit_z), + xtag("commit_end", commit_start + add_commit_z)); + + if (::mprotect(commit_start, add_commit_z, PROT_READ | PROT_WRITE) != 0) { + throw std::runtime_error(tostr("ArenaAlloc::expand: commit failure", + xtag("committed_z", committed_z_), + xtag("add_commit_z", add_commit_z))); + } + + this->committed_z_ = aligned_offset_z; + this->limit_ = this->lo_ + committed_z_; + + assert(committed_z_ % hugepage_z_ == 0); + assert(reinterpret_cast(limit_) % hugepage_z_ == 0); + + return true; + } + + void + ArenaAlloc::set_free_ptr(std::byte * x) + { + assert(lo_ <= x); + assert(x < limit_); + + if (lo_ <= x && x < limit_) { + this->free_ptr_ = x; + if (checkpoint_ > free_ptr_) + this->checkpoint_ = free_ptr_; + } else { + throw std::runtime_error(tostr("LinearAllog::set_free_ptr(x): expected lo <= x < limit", + xtag("lo", lo_), xtag("x", x), xtag("limit", limit_))); + } + } + + std::pair + ArenaAlloc::location_of(const void * x) const + { + if ((lo_ <= x) && (x < hi_)) { + return std::make_pair(true, reinterpret_cast(x) - lo_); + } else { + return std::make_pair(false, 0); + } + } + + void + ArenaAlloc::reset(std::size_t need_z) { + this->clear(); + this->expand(need_z); + } + + void + ArenaAlloc::capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const + { + scope log(XO_DEBUG(debug_flag_), + xtag("name", name_), + xtag("capacity", limit_ - lo_), + xtag("alloc", free_ptr_ - lo_), + xtag("lo", (void*)lo_), + xtag("free_ptr", (void*)free_ptr_)); + + using xo::reflect::TaggedPtr; + + std::byte * p = lo_; + + while (p < free_ptr_) { + log && log(xtag("p", (void *)p)); + + Object * obj = reinterpret_cast(p); + TaggedPtr tp = obj->self_tp(); + std::size_t z = obj->_shallow_size(); + std::uint32_t id = tp.td()->id().id(); + + log && log(xtag("obj", (void*)obj), + xtag("z", z), + xtag("typeid", id)); + + if (p_dest->per_type_stats_v_.size() < id + 1) + p_dest->per_type_stats_v_.resize(id + 1); + + PerObjectTypeStatistics & dest = p_dest->per_type_stats_v_.at(id); + + dest.td_ = tp.td(); + + log && log(xtag("td", tp.td()->short_name())); + + switch (phase) { + case capture_phase::sab: + ++dest.scanned_n_; + dest.scanned_z_ += z; + break; + case capture_phase::sae: + ++dest.survive_n_; + dest.survive_z_ += z; + break; + } + + p += z; + } + + assert(p == free_ptr_); + } + + const std::string & + ArenaAlloc::name() const { + return name_; + } + + std::size_t + ArenaAlloc::size() const { + return limit_ - lo_; + } + + std::size_t + ArenaAlloc::committed() const { + return committed_z_; + } + + std::size_t + ArenaAlloc::available() const { + return limit_ - free_ptr_; + } + + std::size_t + ArenaAlloc::allocated() const { + return free_ptr_ - lo_; + } + + bool + ArenaAlloc::contains(const void * x) const { + return (lo_ <= x) && (x < hi_); + } + + bool + ArenaAlloc::is_before_checkpoint(const void * x) const { + return (lo_ <= x) && (x < checkpoint_); + } + + std::size_t + ArenaAlloc::before_checkpoint() const + { + return checkpoint_ - lo_; + } + + std::size_t + ArenaAlloc::after_checkpoint() const + { + return free_ptr_ - checkpoint_; + } + + bool + ArenaAlloc::check_owned(IObject * src) const + { + byte * addr = reinterpret_cast(src); + + return (lo_ <= addr) && (addr < hi_); + } + + bool + ArenaAlloc::debug_flag() const + { + return debug_flag_; + } + + void + ArenaAlloc::clear() + { + this->set_free_ptr(lo_); + //this->limit_ = hi_; + } + + void + ArenaAlloc::checkpoint() + { + this->checkpoint_ = this->free_ptr_; + } + + std::byte * + ArenaAlloc::alloc(std::size_t z0) + { + scope log(XO_DEBUG(debug_flag_)); + + /* word size for alignment */ + constexpr uint32_t c_bpw = sizeof(std::uintptr_t); + (void)c_bpw; + + std::uintptr_t free_u64 = reinterpret_cast(free_ptr_); + (void)free_u64; + + assert(free_u64 % c_bpw == 0ul); + + std::uint32_t dz = alloc_padding(z0); + + std::size_t z1 = z0 + dz; + + assert(z1 % c_bpw == 0ul); + + this->expand(this->allocated() + z1); + + std::byte * retval = this->free_ptr_; + + log && log(xtag("self", name_), + xtag("z0", z0), + xtag("+pad", dz), + xtag("z1", z1), + xtag("size", this->size()), + xtag("avail", this->available())); + + this->free_ptr_ += z1; + + return retval; + } + + } /*namespace gc*/ +} /*namespace xo*/ + + +/* end ArenaAlloc.cpp */ diff --git a/xo-alloc/src/alloc/Blob.cpp b/xo-alloc/src/alloc/Blob.cpp new file mode 100644 index 00000000..83e4121b --- /dev/null +++ b/xo-alloc/src/alloc/Blob.cpp @@ -0,0 +1,57 @@ +/** @file Blob.cpp + * + * @author Roland Conybeare, Nov 2025 + **/ + +#include "Blob.hpp" +#include "xo/reflect/Reflect.hpp" +#include "xo/allocutil/IAlloc.hpp" + +namespace xo { + using xo::reflect::Reflect; + using xo::reflect::TaggedPtr; + + gp + Blob::make(gc::IAlloc * mm, std::size_t z) { + std::byte * mem = mm->alloc(sizeof(Blob) + z); + + return new (mem) Blob(z); + } + + TaggedPtr + Blob::self_tp() const + { + return Reflect::make_tp(const_cast(this)); + } + + void + Blob::display(std::ostream & os) const + { + os << ""; + } + + std::size_t + Blob::_shallow_size() const { + return sizeof(Blob) + z_; + } + + Object * + Blob::_shallow_copy(gc::IAlloc * mm) const { + Cpof cpof(mm, this); + std::byte * cp_mem = mm->alloc_gc_copy(sizeof(Blob) + z_, this); + + gp copy = new (cp_mem) Blob(z_); + + ::memcpy(copy->data(), data_, z_); + + return copy.get(); + } + + std::size_t + Blob::_forward_children(gc::IAlloc *) + { + return this->_shallow_size(); + } +} + +/* end Blob.cpp */ diff --git a/xo-alloc/src/alloc/CMakeLists.txt b/xo-alloc/src/alloc/CMakeLists.txt new file mode 100644 index 00000000..67e9759b --- /dev/null +++ b/xo-alloc/src/alloc/CMakeLists.txt @@ -0,0 +1,24 @@ +# alloc/CMakeLists.txt + +set(SELF_LIB xo_alloc) +set(SELF_SRCS + ArenaAlloc.cpp + ListAlloc.cpp + GC.cpp + GcStatistics.cpp + ObjectStatistics.cpp + Object.cpp + Blob.cpp + Forwarding1.cpp + generation.cpp +) + +xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) +xo_headeronly_dependency(${SELF_LIB} xo_allocutil) +# xo-unit used for time measurement +xo_headeronly_dependency(${SELF_LIB} xo_unit) +xo_dependency(${SELF_LIB} indentlog) +xo_dependency(${SELF_LIB} reflect) +xo_headeronly_dependency(${SELF_LIB} callback) + +#end CMakeLists.txt diff --git a/xo-alloc/src/alloc/Forwarding1.cpp b/xo-alloc/src/alloc/Forwarding1.cpp new file mode 100644 index 00000000..a42a3f57 --- /dev/null +++ b/xo-alloc/src/alloc/Forwarding1.cpp @@ -0,0 +1,79 @@ +/* file Forwarding1.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "Forwarding1.hpp" +#include "xo/reflect/Reflect.hpp" +#include +#include + +namespace xo { + using xo::reflect::Reflect; + using xo::reflect::TaggedPtr; + + namespace obj { + Forwarding1::Forwarding1(gp dest) + : dest_{dest} + {} + + TaggedPtr + Forwarding1::self_tp() const + { + return Reflect::make_tp(const_cast(this)); + } + + void + Forwarding1::display(std::ostream & os) const + { + os << "self_tp().td()->short_name()) + << ">"; + } + + IObject * + Forwarding1::_offset_destination(IObject * src) const + { + intptr_t offset = src - static_cast(this); + + return dest_.ptr() + offset; + } + + IObject * + Forwarding1::_destination() { + return dest_.ptr(); + } + + // LCOV_EXCL_START + std::size_t + Forwarding1::_shallow_size() const { + assert(false); + return 0; + } + // LCOV_EXCL_STOP + + // LCOV_EXCL_START + IObject * + Forwarding1::_shallow_copy(gc::IAlloc *) const { + /* forwarding objects are never copied */ + + assert(false); + return nullptr; + } + // LCOV_EXCL_STOP + + // LCOV_EXCL_START + std::size_t + Forwarding1::_forward_children(gc::IAlloc *) { + /* forwarding objects are never traced */ + + assert(false); + return 0; + } + // LCOV_EXCL_STOP + + } /*namespace obj*/ +} /*namespace xo*/ + +/* end Forwarding1.cpp */ diff --git a/xo-alloc/src/alloc/GC.cpp b/xo-alloc/src/alloc/GC.cpp new file mode 100644 index 00000000..f1c52d1b --- /dev/null +++ b/xo-alloc/src/alloc/GC.cpp @@ -0,0 +1,1526 @@ +/* GC.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "GcStatistics.hpp" +#include "GC.hpp" +#include "Object.hpp" +#include "xo/indentlog/scope.hpp" +#include +#include +#include + +namespace xo { + namespace gc { + bool + MutationLogEntry::is_child_forwarded() const + { + assert(!parent_->_is_forwarded()); + + return (*lhs_)->_is_forwarded(); + } + + bool + MutationLogEntry::is_parent_forwarded() const + { + return parent_->_is_forwarded(); + } + + IObject * + MutationLogEntry::parent_destination() const + { + //const bool c_debug_flag = true; + //scope log(XO_DEBUG(c_debug_flag)); + + if (parent_->_is_forwarded()) { + //log && log("parent is forwarded", xtag("parent", (void*)parent_)); + + return parent_->_destination(); + } else { + //log && log("parent is ordinary", xtag("parent", (void*)parent_)); + + return parent_; + } + } + + MutationLogEntry + MutationLogEntry::update_parent_moved(IObject * parent_to) const + { + std::byte * parent_from = reinterpret_cast(parent_); + std::byte * lhs_from = reinterpret_cast(lhs_); + + std::ptrdiff_t offset = (lhs_from - parent_from); + + std::byte * lhs_to = reinterpret_cast(parent_to) + offset; + + return MutationLogEntry(parent_to, + reinterpret_cast(lhs_to)); + } + + void + MutationLogEntry::fixup_parent_child_moved(IObject * child_to) + { + *(this->lhs_) = child_to; + } + + GC::GC(const Config & config) + : config_{config} + { + enum { NurseryFrom, NurseryTo, TenuredFrom, TenuredTo }; + + std::size_t nursery_size = config.initial_nursery_z_; + std::size_t tenured_size = config.initial_tenured_z_; + + if (config_.incr_gc_threshold_ > nursery_size) { + throw std::runtime_error(tostr("GC::ctor: expected nursery gc threshold < nursery size", + xtag("nursery-gc-threshold", config_.incr_gc_threshold_), + xtag("nursery-size", nursery_size))); + } + + if (nursery_size + config_.full_gc_threshold_ > tenured_size) { + throw std::runtime_error(tostr("GC::ctor: expected nursery size + tenured gc threshold < tenured size", + xtag("nursery-size", nursery_size), + xtag("tenured-size", tenured_size), + xtag("full-gc-threshold", config_.full_gc_threshold_) + )); + } + + if (config_.incr_gc_threshold_ > nursery_size) + this->config_.incr_gc_threshold_ = nursery_size; + + if (config_.full_gc_threshold_ > tenured_size) + this->config_.full_gc_threshold_ = tenured_size; + + this->nursery_[role2int(role::from_space)] + = ArenaAlloc::make("NA", nursery_size, config.debug_flag_); + + this->nursery_[role2int(role::to_space) ] + = ArenaAlloc::make("NB", nursery_size, config.debug_flag_); + + this->tenured_[role2int(role::from_space)] + = ArenaAlloc::make("TA", tenured_size, config.debug_flag_); + + this->tenured_[role2int(role::to_space) ] + = ArenaAlloc::make("TB", tenured_size, config.debug_flag_); + + nursery_[role2int(role::from_space)]->expand(config.incr_gc_threshold_); + nursery_[role2int(role::to_space) ]->expand(config.incr_gc_threshold_); + tenured_[role2int(role::from_space)]->expand(config.full_gc_threshold_); + tenured_[role2int(role::to_space) ]->expand(config.full_gc_threshold_); + + this->mutation_log_[role2int(role::from_space)] = std::make_unique(); + this->mutation_log_[role2int(role::to_space )] = std::make_unique(); + this->defer_mutation_log_ = std::make_unique(); + + this->gc_history_ = CircularBuffer(config.stats_history_z_); + + this->checkpoint(); + } + + GC::~GC() { + /* hygiene */ + this->clear(); + + this->nursery_[role2int(role::from_space)].reset(); + this->nursery_[role2int(role::to_space) ].reset(); + + this->tenured_[role2int(role::from_space)].reset(); + this->tenured_[role2int(role::to_space) ].reset(); + + this->gc_root_v_.clear(); + + this->mutation_log_[role2int(role::from_space)].reset(); + this->mutation_log_[role2int(role::to_space) ].reset(); + this->defer_mutation_log_.reset(); + } + + up + GC::make(const Config & config) + { + return std::make_unique(config); + } + + GC * + GC::from(IAlloc * mm) + { + return dynamic_cast(mm); + } + + const std::string & + GC::name() const + { + static std::string s_default_name = "GC"; + return s_default_name; + } + + std::size_t + GC::size() const + { + return nursery_to()->size() + tenured_to()->size(); + } + + std::size_t + GC::committed() const + { + return (nursery_to()->committed() + + nursery_from()->committed() + + tenured_to()->committed() + + tenured_from()->committed()); + } + + std::size_t + GC::allocated() const + { + return (nursery_[role2int(role::to_space)]->allocated() + + tenured_[role2int(role::to_space)]->allocated()); + } + + std::size_t + GC::available() const + { + return nursery_[role2int(role::to_space)]->available(); + } + + bool + GC::fromspace_contains(const void * x) const + { + return (nursery_[role2int(role::from_space)]->contains(x) + || tenured_[role2int(role::from_space)]->contains(x)); + } + + bool + GC::contains(const void * x) const + { + return (nursery_[role2int(role::to_space)]->contains(x) + || tenured_[role2int(role::to_space)]->contains(x)); + } + + bool + GC::is_before_checkpoint(const void * x) const + { + return nursery_[role2int(role::to_space)]->is_before_checkpoint(x); + } + + std::size_t + GC::before_checkpoint() const + { + return this->nursery_to()->before_checkpoint(); + } + + std::size_t + GC::after_checkpoint() const + { + return this->nursery_to()->after_checkpoint(); + } + + bool + GC::debug_flag() const + { + return config_.debug_flag_; + } + + GcStatisticsExt + GC::get_gc_statistics() const + { + GcStatisticsExt retval = GcStatisticsExt(this->native_gc_statistics()); + + retval.nursery_z_ = nursery_[role2int(role::to_space)]->size(); + retval.nursery_before_checkpoint_z_ = this->nursery_to()->before_checkpoint(); + retval.nursery_after_checkpoint_z_ = this->nursery_to()->after_checkpoint(); + retval.tenured_z_ = tenured_[role2int(role::to_space)]->size(); + + return retval; + } + + std::size_t + GC::pagesize() const + { + return nursery_to()->page_size(); + } + + std::size_t + GC::hugepage_z() const + { + return nursery_to()->hugepage_z(); + } + + std::size_t + GC::nursery_from_allocated() const + { + return nursery_from()->allocated(); + } + + std::size_t + GC::nursery_to_allocated() const + { + return nursery_to()->allocated(); + } + + std::size_t + GC::nursery_to_reserved() const + { + return nursery_to()->reserved(); + } + + std::size_t + GC::nursery_to_committed() const + { + return nursery_to()->committed(); + } + + std::size_t + GC::nursery_before_checkpoint() const + { + return nursery_to()->before_checkpoint(); + } + + std::size_t + GC::nursery_after_checkpoint() const + { + return nursery_to()->after_checkpoint(); + } + + std::pair + GC::nursery_span(role role) const { + return nursery(role)->allocated_span(); + } + + std::size_t + GC::tenured_to_reserved() const + { + return tenured_to()->reserved(); + } + + std::size_t + GC::tenured_to_committed() const + { + return tenured_to()->committed(); + } + + std::size_t + GC::tenured_before_checkpoint() const + { + return tenured_to()->before_checkpoint(); + } + + std::size_t + GC::tenured_after_checkpoint() const + { + return tenured_to()->after_checkpoint(); + } + + generation_result + GC::tospace_generation_of(const void * x) const + { + if (tenured_[role2int(role::to_space)]->contains(x)) + return generation_result::tenured; + + if (nursery_[role2int(role::to_space)]->contains(x)) + return generation_result::nursery; + + return generation_result::not_found; + } + + std::tuple + GC::location_of(role role, const void *x) const + { + { + auto space = this->tenured(role); + auto [is_tenured, offset] = space->location_of(x); + + if (is_tenured) + return std::make_tuple(generation_result::tenured, offset, space->allocated(), space->committed()); + } + + { + auto space = this->nursery(role); + auto [is_nursery, offset] = nursery(role)->location_of(x); + + if (is_nursery) + return std::make_tuple(generation_result::nursery, offset, space->allocated(), space->committed()); + } + + return std::make_tuple(generation_result::not_found, 0, 0, 0); + } + + std::tuple + GC::tospace_location_of(const void * x) const + { + return location_of(role::to_space, x); + } + + std::tuple + GC::fromspace_location_of(const void * x) const + { + return location_of(role::from_space, x); + } + + generation_result + GC::fromspace_generation_of(const void * x) const + { + if (tenured_[role2int(role::from_space)]->contains(x)) + return generation_result::tenured; + + if (nursery_[role2int(role::from_space)]->contains(x)) + return generation_result::nursery; + + return generation_result::not_found; + } + + std::byte * + GC::free_ptr(generation gen) + { + switch(gen) { + case generation::nursery: + return nursery_[role2int(role::to_space)]->free_ptr(); + case generation::tenured: + return tenured_[role2int(role::to_space)]->free_ptr(); + // LCOV_EXCL_START + case generation::N: + assert(false); + // LCOV_EXCL_STOP + } + + return nullptr; + } + + std::size_t + GC::mlog_size() const { + return mutation_log_[role2int(role::to_space)]->size(); + } + + void + GC::clear() + { + nursery_[role2int(role::from_space)]->clear(); + nursery_[role2int(role::to_space) ]->clear(); + + tenured_[role2int(role::from_space)]->clear(); + tenured_[role2int(role::to_space) ]->clear(); + } + + void + GC::add_gc_root(IObject ** addr) + { + gc_root_v_.push_back(addr); + } + + void + GC::remove_gc_root(IObject ** addr) + { + /* Multithreaded GC not supported */ + + assert(!this->gc_in_progress()); + + auto new_end_ix = std::remove(gc_root_v_.begin(), gc_root_v_.end(), addr); + + /* erase now-unused slots */ + gc_root_v_.erase(new_end_ix, gc_root_v_.end()); + } + + auto + GC::add_gc_copy_callback(up fn) -> CallbackId + { + return gc_copy_cbset_.add_callback(std::move(fn)); + } + + void + GC::checkpoint() + { + nursery_to()->checkpoint(); + /* checkpoint T generation so we can trigger GC based on new T objects rather than + * overall T size + */ + tenured_to()->checkpoint(); + } + + std::byte * + GC::alloc(std::size_t z) + { + auto N_to = this->nursery_to(); + + if (!incr_gc_pending_ && (N_to->after_checkpoint() > config_.incr_gc_threshold_)) { + /* automatically ups to generation::tenured */ + this->request_gc(generation::nursery); + } + + std::byte * x = N_to->alloc(z); + + /* ListAlloc won't fail unless we exhaust memory -- instead will increase heap size */ + assert(x); + + return x; + } + + std::byte * + GC::alloc_gc_copy(std::size_t z, const void * src) + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("z", z), + xtag("+pad", IAlloc::alloc_padding(z))); + + generation_result src_gr = this->fromspace_generation_of(src); + + std::byte * retval = nullptr; + + switch (src_gr) { + case generation_result::tenured: + { + log && log("tenured"); + + retval = this->tenured_to()->alloc(z); + + gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, + z, src, retval, generation::tenured, generation::tenured); + } + break; + case generation_result::nursery: + { + if (this->nursery_from()->is_before_checkpoint(src)) + { + /* nursery object has survived 2nd collection cycle + * -> promote into tenured generation + */ + retval = this->tenured_to()->alloc(z); + + log && log("promote", xtag("addr", (void*)retval)); + + assert(this->tospace_generation_of(retval) == generation_result::tenured); + + gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, + z, src, retval, generation::nursery, generation::tenured); + + this->gc_statistics_.total_promoted_ + += IAlloc::with_padding(z); + + } else { + log && log("nursery"); + + retval = this->nursery_to()->alloc(z); + + gc_copy_cbset_.invoke(&GcCopyCallback::notify_gc_copy, + z, src, retval, generation::nursery, generation::nursery); + } + } + break; + case generation_result::not_found: + /* something wrong -- we only copy objects that are known to be in from-space + */ + + assert(false); + break; + } + + assert(retval); + + log && log(xtag("retval", retval)); + + return retval; + } + + void + GC::assign_member(IObject * parent, IObject ** lhs, IObject * rhs) + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("parent", parent), xtag("lhs", lhs), xtag("rhs", rhs)); + + ++gc_statistics_.n_mutation_; + + *lhs = rhs; + + if (runstate_.in_progress()) { + /* don't log mutations (if any) during GC */ + return; + } + + if (!config_.allow_incremental_gc_) { + /* full GCs don't need mutation log, since no cross-generational pointers */ + return; + } + + switch (tospace_generation_of(rhs)) + { + case generation_result::tenured: + /* only need to log mutations that create tenured->nursery pointers */ + log && log(xtag("act", "any->T no log")); + return; + + case generation_result::nursery: + switch (tospace_generation_of(parent)) { + case generation_result::nursery: + if (is_before_checkpoint(parent)) { + log && log(xtag("act", "N1->N0 must mlog")); + + // N1->N0, so must log + this->mutation_log_[role2int(role::to_space)]->push_back(MutationLogEntry(parent, lhs)); + ++(this->gc_statistics_.n_logged_mutation_); + ++(this->gc_statistics_.n_xckp_mutation_); + } else { + // parent in N0, not an xckp mutation + log && log(xtag("act", "N0->any no long")); + } + break; + case generation_result::tenured: + // T->N, so must log + log && log(xtag("act", "T->N must mlog")); + + this->mutation_log_[role2int(role::to_space)]->push_back(MutationLogEntry(parent, lhs)); + ++(this->gc_statistics_.n_logged_mutation_); + ++(this->gc_statistics_.n_xgen_mutation_); + break; + case generation_result::not_found: + // parent is global + // This may be ok (provided lhs is a gc root) + log && log(xtag("act", "root->any no log")); + break; + } + break; + + case generation_result::not_found: + log && log(xtag("act", "any->root no log")); + + // child is global; + // logging not required + break; + } + } + + void + GC::forward_inplace(IObject ** lhs) + { + scope log(XO_DEBUG(config_.debug_flag_), xtag("lhs", lhs)); + + Object::_forward_inplace(lhs, this); + } + + bool + GC::check_owned(IObject * src) const + { + return this->fromspace_contains(src); + } + + bool + GC::check_move(IObject * src) const + { + return (this->runstate().full_move() + || (this->tospace_generation_of(src) != gc::generation_result::tenured)); + } + + bool + GC::check_write_barrier(const void * parent, + const void * const * lhs, + bool may_throw_flag) const + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("P", parent), xtag("L", lhs)); + + if (!this->contains(parent)) { + if (may_throw_flag) { + throw std::runtime_error(tostr("GC::check_write_barrier", + ": expected parent object P in GC to-space", + xtag("P", parent))); + } + return false; + } + +#ifdef NOPE // don't want to assume IObject* + std::size_t parent_z = parent->_shallow_size(); + + const std::byte * parent_addr = reinterpret_cast(parent); + const std::byte * lhs_addr = reinterpret_cast(lhs); + + if ((lhs_addr < parent_addr) || (parent_addr + parent_z < lhs_addr)) { + if (may_throw_flag) { + throw std::runtime_error + (tostr("GC::check_write_barrier", + ": expected lhs address L within address extent z of parent P", + xtag("P", parent), xtag("z", parent_z), + xtag("P+z", parent_addr + parent_z), + xtag("L", lhs))); + } + return false; + } +#endif + + const void * rhs = *lhs; + + if (!rhs) + return true; + + auto parent_gen = tospace_generation_of(parent); + auto rhs_gen = tospace_generation_of(rhs); + + log && log(xtag("C", rhs), + xtag("gen(P)", parent_gen), xtag("gen(C)", rhs_gen), + xtag("P.before-ckp", is_before_checkpoint(parent)), + xtag("C.before-ckp", is_before_checkpoint(rhs))); + + switch(parent_gen) { + case generation_result::nursery: + if (is_before_checkpoint(parent)) { + switch(rhs_gen) { + case generation_result::nursery: + if (is_before_checkpoint(rhs)) { + /* no mlog entry needed */ + log && log(xtag("msg", "N1->N1 - trivial")); + return true; + } else { + /* need to check mlog */ + log && log(xtag("msg", "N1->N0 - xgen")); + } + break; + case generation_result::tenured: + /* no mlog entry needed */ + log && log(xtag("msg", "N1->T - trivial")); + return true; + case generation_result::not_found: + /* possible non-gc rhs */ + log && log(xtag("msg", "non-gc rhs - trivial")); + return true; + } + } else { + /* no mlog entry needed */ + log && log(xtag("msg", "N0->any - trivial")); + return true; + } + break; + case generation_result::tenured: + switch(rhs_gen) { + case generation_result::nursery: + /* need to check mlog */ + log && log(xtag("msg", "T->N - xgen")); + break; + case generation_result::tenured: + /* no mlog entry needed */ + log && log(xtag("msg", "T->T - trivial")); + return true; + case generation_result::not_found: + /* possible non-gc rhs */ + log && log(xtag("msg", "non-gc rhs - trivial")); + return true; + } + break; + case generation_result::not_found: + /* already excluded -> impossible */ + log && log(xtag("msg", "assert")); + assert(false && "already verified parent owned by GC"); + } + + /* control here -> expect mutation log entry. + * search mutation log + verify such entry exists + */ + for (MutationLogEntry & mlog : *(mutation_log_[role2int(role::to_space)])) { + if ((mlog.parent() == parent) && ((const void * const *)mlog.lhs() == lhs)) { + return true; + } + mlog.lhs(); + } + + if (may_throw_flag) { + throw std::runtime_error + (tostr("GC::check_write_barrier", + ": expected mlog entry for xgen pointer L->C within parent P", + xtag("P", parent), xtag("L", lhs), xtag("C", rhs), + xtag("gen(P)", parent_gen), xtag("gen(C)", rhs_gen))); + } + return false; + } + + void + GC::swap_nursery() + { + up tmp = std::move(nursery_[role2int(role::to_space)]); + nursery_[role2int(role::to_space)] = std::move(nursery_[role2int(role::from_space)]); + nursery_[role2int(role::from_space)] = std::move(tmp); + nursery_polarity_ = 1 - nursery_polarity_; + } + + void + GC::swap_tenured() + { + up tmp = std::move(tenured_[role2int(role::to_space)]); + tenured_[role2int(role::to_space)] = std::move(tenured_[role2int(role::from_space)]); + tenured_[role2int(role::from_space)] = std::move(tmp); + tenured_polarity_ = 1 - tenured_polarity_; + } + + void + GC::swap_mutation_log() + { + up tmp = std::move(mutation_log_[role2int(role::to_space)]); + mutation_log_[role2int(role::to_space)] = std::move(mutation_log_[role2int(role::from_space)]); + mutation_log_[role2int(role::from_space)] = std::move(tmp); + } + + void + GC::swap_spaces(generation target) + { + scope log(XO_DEBUG(this->debug_flag()), xtag("upto", target)); + + // will be copying into the memory regions currently labelled FromSpace + + /* gc will copy some to-be-determined amount in [0..promote_z] + from nursery->tenured generation. + */ + std::size_t max_promote_z = nursery_[role2int(role::to_space)]->before_checkpoint(); + + ArenaAlloc * tenured_to = this->tenured_to(); + + /* tenured generation may need this much space */ + std::size_t need_tenured_z = (tenured_to->allocated() + + max_promote_z + + config_.full_gc_threshold_); + + log && log(xtag("alloc_tenured_z", tenured_to->allocated()), + xtag("max_promote_z", max_promote_z), + xtag("full_gc_threshold", config_.full_gc_threshold_), + xtag("need_tenured_z", need_tenured_z)); + + tenured_to->expand(tenured_to->allocated() + + max_promote_z + + config_.full_gc_threshold_); + + if (target == generation::tenured) { + tenured_from()->clear(); + + this->swap_tenured(); + } + + /* subtracting max_promote_z is correct here, since anything not promoted is garbage */ + std::size_t need_nursery_z = (nursery(role::to_space)->allocated() + - max_promote_z + + config_.incr_gc_threshold_); + + log && log(xtag("need_nursery_z", need_nursery_z)); + + /* (from-space is about to become to-space, to receive surviving nursery objects) */ + nursery(role::from_space)->reset(need_nursery_z); + + this->swap_nursery(); + + this->swap_mutation_log(); + + ArenaAlloc * N_from = nursery(role::from_space); + log && log(xtag("nursery.from", N_from->name()), xtag("size", N_from->size())); + ArenaAlloc * N_to = nursery(role::to_space); + log && log(xtag("nursery.to", N_to->name()), xtag("size", N_to->size())); + ArenaAlloc * T_from = tenured(role::from_space); + log && log(xtag("tenured.from", T_from->name()), xtag("size", T_from->size())); + ArenaAlloc * T_to = tenured(role::to_space); + log && log(xtag("tenured.to", T_to->name()), xtag("size", T_to->size())); + + } /*swap_spaces*/ + + void + GC::capture_object_statistics(generation upto, capture_phase phase) + { + if (config_.object_stats_flag_) { + /* scan nursery */ + this->nursery_[role2int(role::to_space)]->capture_object_statistics + (phase, + &object_statistics_sab_[gen2int(generation::nursery)]); + + if (upto == generation::tenured) { + /* scan tenured */ + this->tenured_[role2int(role::to_space)]->capture_object_statistics + (phase, + &object_statistics_sab_[gen2int(generation::tenured)]); + } + } + } + + void + GC::copy_object(IObject ** pp_object, + generation upto, + ObjectStatistics * object_stats) + { + void * object_address = *pp_object; + + if (nursery_[role2int(role::to_space)]->contains(object_address) + || ((upto == generation::tenured) + && tenured_[role2int(role::to_space)]->contains(object_address))) + { + /* global is already in to-space */ + ; + } else if((upto == generation::nursery) && tenured_[role2int(role::to_space)]->contains(object_address)) + { + /* skip tenured objects when incremental collection */ + ; + } else { + *pp_object = Object::_deep_move(*pp_object, this, object_stats); + } + } + + void + GC::copy_globals(generation upto) + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("roots", gc_root_v_.size())); + + for (IObject ** pp_root : gc_root_v_) { + this->copy_object(pp_root, upto, + &object_statistics_sae_[gen2int(upto)]); + } + } + + void + GC::incremental_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * per_type_stats) + { + scope log(XO_DEBUG(config_.debug_flag_), xtag("from_mlog.size", from_mlog->size())); + + /* categorize pointers based on combination of {source address, destination address}, + * only care about the generation associated with an address. + * + * N0 : nursery(from), before checkpoint + * N0': nursery(to), before checkpoint + * N1 : nursery(from), after checkpoint + * N1': nursery(to), after checkpoint + * T : tenured(to) + * + * loc(P): parent region before GC + * loc(C): child region before GC + * + * | | forwarded | loc now post | loc after | + * | | already? | root copy | action | + * | loc(P) loc(C) | P C | P' C' | P' C' | defer | action + * ----|---------------+--------------+---------------+---------------+-------+--------------- + * (a) | T N0 | no no | T N0 | T N1' | | C->N1', +mlog + * (b) | | yes | N1' | N1' | | +mlog + * (c) | T N1 | no no | T N1 | T T | | C->T, -mlog + * (d) | | yes | T T | T T | | -mlog + * (e) | N1 N0 | no no | N1 N0 | N1 N0 | P ->C | defer + * (f) | | yes | N1 N1' | N1 N1' | P ->C'| defer + * (g) | | yes yes | T N1' | T N1' | | +mlog + * + * notes: + * (a) C survives due to xgen ptr {T -> N0}; after collection have xgen ptr {T -> N1}. + * (b) C already evac'd; after collection stil have xgen ptr {T -> N1} + * (c) C survives due to xgen ptr (T -> N1): promote to T, so no longer xgen + * (d) C already evac'd: after collection no longer xgen (T -> T) + * (e) P,C maybe garbage. don't move either, but defer mlog incase P saved by a subsequent mutation. + * in that case C saved alto, + will still have an xgen ptr, so still need an mlog entry + * (f) P maybe garbage, C survives. defer mlog incase P saved+promoted by a subsequent mutation; + * in that case will still have an xgen (T -> N) ptr, so still need an mlog entry. + */ + + std::size_t i_from = 0; + // number of rescued subgraphs via mutation log entries + std::size_t n_rescue = 0; + + for (MutationLogEntry & from_entry : *from_mlog) + { + if (log) { + if (i_from % 10000 == 0 || true) + log(xtag("i_from", i_from)); + } + + void * parent = from_entry.parent(); + + if (tospace_generation_of(parent) == generation_result::tenured) + { + // cases (a)(b)(c)(d) + // loc(P) is T. T didn't move b/c incremental gc. + + if (from_entry.is_dead()) { + // obsolete mutation -- no longer belongs to parent, discard + } else { + // note: child obtained (as it must be) by reading from parent's memory _now_. + IObject * child_from = from_entry.child(); + + if (child_from) { + if (!child_from->_is_forwarded()) { + // P->C*. + // either: + // - C*=C in from-space, so needs evac + // - C*=C' in to-space, P already updated b/c of another mutation + // + if (fromspace_generation_of(child_from) != generation_result::not_found) { + // C*=C in from-space. needs evac, along with reachable descendants + // + // Includes cases: + // (a) T->N0 + // (c) T->N1 + + ++n_rescue; + + log && log(xtag("parent", parent), xtag("act", "move child"), xtag("child.from", child_from)); + + Object::_deep_move(child_from, this, per_type_stats); + + log && log(xtag("child.to", child_from->_destination())); + + // C forwards to C', fall thru to parent fixup below + // (a) T->N1' + // (c) T->T + } else { + // P updated via some other mutation + // so don't need this mlog + ; + } + } + + // re-test, state may have changed above + if (from_entry.is_child_forwarded()) { + // P->C, C moved to C' + // Includes cases (a),(c) from above + + IObject * child_to = child_from->_destination(); + + log && log(xtag("act", "fixup parent"), xtag("parent", parent), xtag("lhs", from_entry.lhs()), xtag("child.from", child_from), xtag("child.to", child_to)); + + from_entry.fixup_parent_child_moved(child_to); + +#ifndef NDEBUG + { + // verify fixup was effective + IObject * child_from2 = from_entry.child(); + assert(child_from2 == child_to); + } +#endif + + + // P->C', loc(C') in {N1', T'} + + if (tospace_generation_of(child_to) == generation_result::nursery) { + // (b) loc(P)=T, loc(C')=N1'; also case (a) + + log && log(xtag("act", "still xgen -> keep mlog entry")); + + // still have xgen pointer, so need mlog for it + to_mlog->push_back(from_entry); + } else { + // (d) loc(P)=T, loc(C')=T; also case (c) + // no longer xgen, so does not require mlog + } + } + + } else { + // nullptr child, discard + } + } + } else if (from_entry.is_parent_forwarded()) { + // Must have: + // loc(P) = N1, because: + // loc(P)=N0 -> ineligible for mlog; + // loc(P)=T -> not moved on incr GC + // + // follows that loc(P') = T + // already have P'->C' when parent moved separately + + IObject * parent_to = from_entry.parent_destination(); + + log && log(xtag("parent_to", (void*)parent_to)); + + assert(tospace_generation_of(parent_to) == generation_result::tenured); + + MutationLogEntry to_entry = from_entry.update_parent_moved(parent_to); + + IObject * child_to = to_entry.child(); // after moving + + if (tospace_generation_of(child_to) == generation_result::nursery) { + if (to_entry.is_dead()) { + ; + } else { + // (g) loc(P)=N1, loc(C)=N0, loc(P')=T, loc(C')=N1 + to_mlog->push_back(to_entry); + } + + } + } else { + log && log("defer"); + + // loc(P) = N1, loc(C) = N0, P may be garbage + // Includes cases: + // (e) P->C, C not moved + // (f) P->C, C moved to C' + // + // P may yet be rescued by another mlog entry, so defer + + if (!from_entry.is_dead()) { + defer_mlog->push_back(from_entry); + } + } + + ++i_from; + } + + from_mlog->clear(); + + if (n_rescue == 0) { + // if we didn't rescue any objects + // then we now confirm that otherwise-unreachable parents in defer_mlog + // are garbage + + defer_mlog->clear(); + } + } + + void + GC::full_gc_forward_mlog_phase(MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * defer_mlog, + ObjectStatistics * /*per_type_stats*/) + { + scope log(XO_DEBUG(config_.debug_flag_), xtag("from_mlog.size", from_mlog->size())); + + /* categorize pointers based on combination of {source address, destination address}, + * only care about the generation associated with an address. + * + * N0 : nursery(from), before checkpoint + * N0': nursery(to), before checkpoint + * N1 : nursery(from), after checkpoint + * N1': nursery(to), after checkpoint + * T : tenured(from) + * T': tenured(to) + * + * loc(P): parent region before GC + * loc(C): child region before GC + * + * | | forwarded | loc now post | loc after | + * | | already? | root copy | action | + * | loc(P) loc(C) | P C | P' C' | P' C' | defer | action + * ----|---------------+--------------+---------------+---------------+-------+--------------- + * (a) | T N0 | no no | T N0 | T N0 | P ->C | defer + * (b) | | yes | N1' | N1' | P ->C'| defer + * | | yes no | impossible + * (b2)| | yes | T' N1' | T' N1' | | +mlog + * (c) | T N1 | no no | T N1 | T T | P ->C | defer + * (d) | | yes | T T' | T T' | P ->C'| defer + * | | yes no | impossible + * (d2)| | yes | T' T' | T' T' | | -mlog + * (e) | N1 N0 | no no | N1 N0 | N1 N0 | P ->C | defer + * (f) | | yes | N1 N1' | N1 N1' | P ->C'| defer + * | | yes no | impossible + * (g) | | yes yes | T' N1' | T' N1' | | +mlog + * + * notes: + * (a) P,C maybe garbage. don't move either, but defer mlog incase P saved by a subsequent mutation; + * in that case C saved also, + will still have an xgen ptr, and still need an mlog entry. + * (b) C already evac'd, but P maybe garbage. defer mlog incase P rescued by a subsequent mutation; + * in that case will still have an xgen (T -> N) ptre, and still need an mlog entry. + * (b2) P,C already evac'd. Must update+rembexember xgen ptr {T -> N1} + * (c) P,C maybe garbage. don't move either, but defer mlog in case P saved by a subsequent mutation; + * in that case C promoted, no longer xgen + * (d) P maybe garbage. defer in case P saved by a subsequent mutation. + * C now tenured, so will no longer have an xgen pointer. + * (d2) P,C already evac'd. After collection no longer have xgen pointer, so no mlog. + * (e) P,C maybe garbage. don't move either, but defer mlog incase P saved by a subsequent mutation. + * in that case C saved alto, + will still have an xgen ptr, so still need an mlog entry + * (f) P maybe garbage, C survives. defer mlog incase P saved+promoted by a subsequent mutation; + * in that case will still have an xgen (T -> N) ptr, so still need an mlog entry. + * (g) P,C already evac'd. Still have xgen pointer, must mlog + */ + + std::size_t i_from = 0; + // number of rescued subgraphs via mutation log entries + std::size_t n_rescue = 0; + + for (MutationLogEntry & from_entry : *from_mlog) + { + log && (i_from % 10000 == 0) && log(xtag("i_from", i_from)); + + if (from_entry.is_parent_forwarded()) { + IObject * parent_to = from_entry.parent_destination(); + + log && log(xtag("parent_to", (void*)parent_to)); + + assert(tospace_generation_of(parent_to) == generation_result::tenured); + + MutationLogEntry to_entry = from_entry.update_parent_moved(parent_to); + + // note: child obtained (as it must be) by reading from prarent's memory _now_. + // Since parent has moved, child has too + IObject * child_to = to_entry.child(); // after moveing + + if (tospace_generation_of(parent_to) == generation_result::tenured) + { + // cases (b2)(d2)(g), loc(P) is T' + // In all these cases parent has already been moved; + // therefore child has also been moved. + // Just need to decide whether to keep mlog entry + + if (from_entry.is_dead()) { + // obsolete mutation -- no longer belongs to parent, discard + } else if (child_to) { + assert(!child_to->_is_forwarded()); + + if (tospace_generation_of(child_to) == generation_result::nursery) { + // case + // (b2) loc(P')=T', loc(C')=N1' --> +mlog + // (g) loc(P')=T', loc(C')=N1' --> +mlog + // + to_mlog->push_back(to_entry); + } else { + // case + // (d2) loc(P')=T', loc(C')=T' --> -mlog + } + } + } else { + // impossible - wouldn't have made mlog entry + + + assert(false); + } + } else { + // case + // (a) defer + // (b) defer + // (c) defer + // (d) defer + // (e) defer + // (f) defer + + defer_mlog->push_back(from_entry); + } + + ++i_from; + } + + from_mlog->clear(); + + if (n_rescue == 0) { + // if we didn't rescue any objects + // then we now confirm that otherwise-unreachable parents in defer_mlog + // are garbage + + defer_mlog->clear(); + } + } + + + void + GC::incremental_gc_forward_mlog(ObjectStatistics * per_type_stats) + { + /* control here: + * - incremental gc. + * - gc roots have been copied, along with everything reachable from them. + * + * plan: + * - forward mutations in *from_mutation_log, writing them to + * *to_mutationlog and/or *defer_mutation_log. + * Use defer when mutation P->C encountered, but P was not copied. + * P appears to be garbage, but may turn out to be live if encountered + * in another mutation. + * + */ + + MutationLog * to_mlog = mutation_log_[role2int(role::to_space)].get(); + + for (;;) { + MutationLog * from_mlog = mutation_log_[role2int(role::from_space)].get(); + MutationLog * defer_mlog = defer_mutation_log_.get(); + + this->incremental_gc_forward_mlog_phase(from_mlog, + to_mlog, + defer_mlog, + per_type_stats); + + assert(from_mlog->empty()); + + if (defer_mlog->empty()) { + /* fixpoint reached */ + break; + } + + /* control here: + * 1. at least one mlog triggered a rescue + * 2. at least one mlog was deferred (b/c otherwise-unreachable parent) + * + * it's conceivable deferred parent now reachable thanks to rescues; + * revisit entries in defer_mlog, + * + * using now-empty from_mlog as scratch for any remaining deferred entries + */ + + std::swap(mutation_log_[role2int(role::from_space)], defer_mutation_log_); + } + } + + void + GC::full_gc_forward_mlog(ObjectStatistics * per_type_stats) + { + /* control here: + * - full gc. + * - gc roots have been copied, along with everything reachable + * from them. + * + * plan: + * - forward mutations in *from_mutation_log, writing them to + * *to_mutation_log and/or *defer_mutation_log. + */ + + MutationLog * to_mlog = this->mutation_log(role::to_space); + + for (;;) { + MutationLog * from_mlog = this->mutation_log(role::from_space); + MutationLog * defer_mlog = defer_mutation_log_.get(); + + this->full_gc_forward_mlog_phase(from_mlog, + to_mlog, + defer_mlog, + per_type_stats); + assert(from_mlog->empty()); + + if (defer_mlog->empty()) + break; + + /* control here: + * 1. at least one mlog triggered a rescue + * 2. at least one mlog was deferred (had otherwise-unreachable parent) + * + * possible that deferred parent is now reachable thanks to a rescue; + * to confirm/refute this need to revisit entries in defer_mlog. + */ + std::swap(mutation_log_[role2int(role::from_space)], defer_mutation_log_); + } + } + + void + GC::forward_mutation_log(generation upto) + { + scope log(XO_DEBUG(config_.debug_flag_)); + + if (upto == generation::tenured) { + this->full_gc_forward_mlog + (&object_statistics_sae_[gen2int(generation::tenured)]); + } else { + this->incremental_gc_forward_mlog + (&object_statistics_sae_[gen2int(generation::nursery)]); + } + } + + void + GC::cleanup_phase(generation upto, nanos dt) + { + scope log(XO_DEBUG(config_.stats_flag_)); + + std::size_t N0_before_gc = nursery_from()->after_checkpoint(); + std::size_t N1_before_gc = nursery_from()->before_checkpoint(); + + std::size_t T0_before_gc = tenured_from()->after_checkpoint(); + std::size_t T1_before_gc = tenured_from()->before_checkpoint(); + + std::size_t N_before_gc = nursery_from()->allocated(); + std::size_t T_before_gc = tenured_from()->allocated(); + + std::size_t N_after_gc = nursery_to()->allocated(); + std::size_t T_after_gc = tenured_to()->allocated(); + //std::byte * N_free_ptr = nursery_[role2int(role::to_space)]->free_ptr(); + + std::size_t new_alloc_z = N0_before_gc; + /* survive_z: bytes surviving first collection */ + std::size_t survive_z = N_after_gc; + /* promote_z: bytes surviving 2nd collection */ + std::size_t promote_z = (gc_statistics_.total_promoted_ + - gc_statistics_.total_promoted_sab_); + + /* #of bytes copied by this collection cycle */ + std::size_t effort_z = 0; + if (upto == generation::nursery) { + effort_z = N_after_gc + promote_z; + } else { + effort_z += N_after_gc + T_after_gc; + } + + /* persist_z: bytes surviving 3rd or later collection */ + std::size_t persist_z = 0; + if (upto == generation::tenured) + persist_z = T_after_gc - promote_z; + /* #of bytes found to be garbage on first collection + * (reminder: N_after_gc consists *entirely* of survives from N0_before_gc; + * + all such survivors are in N_after_gc) + */ + std::size_t garbage0_z = (N0_before_gc - N_after_gc); + /* #of bytes found to be garbage on 2nd collection */ + std::size_t garbage1_z = (N1_before_gc - promote_z); + /* #of bytes found to be garbage on 3rd or later collection */ + std::size_t garbageN_z = 0; + if (upto == generation::tenured) + garbageN_z = (T_before_gc - T_after_gc + promote_z); + + /* Don't reset from-space here, it's unnecessary. + * Would be permissible, but interferes with GC object modelling in + * xo-object/utest/GC.test.cpp + */ + //this->nursery_[role2int(role::from_space)]->reset(0); + //this->tenured_[role2int(role::from_space)]->reset(0); + + /* objects currenty in to-space nursery have survived one collection */ + this->nursery_to()->checkpoint(); + if (upto == generation::tenured) + this->tenured_to()->checkpoint(); + + if (log) { + log(xtag("gcseq_before_gc", gc_statistics_.n_gc())); + log(xtag("N0_before_gc", N0_before_gc)); + log(xtag("N1_before_gc", N1_before_gc)); + log(xtag("N_after_gc", N_after_gc)); + + log(xtag("T0_before_gc", T0_before_gc)); + log(xtag("T1_before_gc", T1_before_gc)); + log(xtag("T_after_gc", T_after_gc)); + } + + this->incr_gc_pending_ = false; + this->gc_statistics_.include_gc(generation::nursery, N0_before_gc, N_before_gc, N_after_gc, promote_z); + + if (upto == generation::tenured) { + this->full_gc_pending_ = false; + this->gc_statistics_.include_gc(generation::tenured, T0_before_gc, T_before_gc, T_after_gc, 0); + } else { + // still want to update tenured stats for current alloc size + this->gc_statistics_.update_snapshot(generation::tenured, T_after_gc); + } + + std::size_t sum_effort_z = effort_z; + std::size_t sum_garbage_z = garbage0_z + garbage1_z + garbageN_z; + + if (gc_history_.size() > 0) { + sum_effort_z += gc_history_.back().sum_effort_z_; + sum_garbage_z += gc_history_.back().sum_garbage_z_; + } + + GcStatisticsHistoryItem item(gc_statistics_.n_gc(), + upto, + new_alloc_z, + survive_z, + promote_z, + persist_z, + effort_z, + garbage0_z, + garbage1_z, + garbageN_z, + dt, + sum_effort_z, + sum_garbage_z); + + log && log(xtag("gcseq_after_gc", gc_statistics_.n_gc()), + xtag("item", item)); + + this->gc_history_.push_back(item); + + } /*cleanup_phase*/ + + void + GC::execute_gc(generation upto) + { + scope log(XO_DEBUG(config_.stats_flag_ || config_.debug_flag_)); + + auto t0 = std::chrono::steady_clock::now(); + + bool full_move = (upto == generation::tenured); + + // TODO: RAII version in case of exceptions + this->runstate_ = GCRunstate(true /*in_progress*/, full_move); + + log && log("step 0: snapshot alloc stats"); + + /* new allocation since last GC */ + std::size_t new_alloc = this->after_checkpoint(); + + gc_statistics_.begin_gc(upto, new_alloc); + + log && log(xtag("new_alloc", new_alloc)); + + log && log("step 0: (optional) scan for object statistics"); + + this->capture_object_statistics(upto, capture_phase::sab); + + log && log("step 1 : swap to/from roles"); + + this->swap_spaces(upto); + + log && log("step 2a: copy globals"); + + this->copy_globals(upto); + + log && log("step 2b: TODO: copy pinned"); + + log && log("step 3 : forward mutation log"); + + this->forward_mutation_log(upto); + + log && log("step 4 : TODO: notify destructor log"); + + log && log("step 5 : TODO: keep reachable weak pointers"); + + log && log("step 6 : cleanup"); + + this->capture_object_statistics(upto, capture_phase::sae); + + auto t1 = std::chrono::steady_clock::now(); + auto dt = std::chrono::duration_cast(t1 - t0); + + this->cleanup_phase(upto, xo::qty::qty::nanoseconds(dt.count())); + + log && log("object statistics [nursery]:"); + log && log(refrtag("stats", object_statistics_sab_[gen2int(generation::nursery)])); + log && log("object statistics [tenured]:"); + log && log(refrtag("stats", object_statistics_sab_[gen2int(generation::tenured)])); + + this->runstate_ = GCRunstate(); + + // not this way.. reports cumulative stats + // this->gc_history_.push_back(this->get_gc_statistics()); + + log && log("statistics:"); + log && log(gc_statistics_); + } + + void + GC::request_gc(generation target) + { + /** full collection when >= @ref full_gc_threshold_ bytes added to tenured + * generation, since last full collection + **/ + bool need_full_gc + = ((target == generation::tenured) + || (this->tenured_to()->after_checkpoint() > config_.full_gc_threshold_) + || !config_.allow_incremental_gc_); + + if (need_full_gc) + target = generation::tenured; + + if (!runstate_.in_progress() && (gc_enabled_ == 0)) { + this->execute_gc(target); + } else { + this->incr_gc_pending_ = true; + this->full_gc_pending_ |= need_full_gc; + } + } + + void + GC::disable_gc() { + --gc_enabled_; + } + + bool + GC::enable_gc() { + ++gc_enabled_; + + if (gc_enabled_ == 0) { + /* unblock gc */ + if (incr_gc_pending_) { + this->request_gc(full_gc_pending_ ? generation::tenured : generation::nursery); + return true; + } + } + + return false; + } + + bool + GC::enable_gc_once() { + bool retval = this->enable_gc(); + this->disable_gc(); + return retval; + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end GC.cpp */ diff --git a/xo-alloc/src/alloc/GcStatistics.cpp b/xo-alloc/src/alloc/GcStatistics.cpp new file mode 100644 index 00000000..deb30685 --- /dev/null +++ b/xo-alloc/src/alloc/GcStatistics.cpp @@ -0,0 +1,214 @@ +/* GcStatistics.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "GcStatistics.hpp" +#include "xo/indentlog/print/pretty_vector.hpp" + +namespace xo { + namespace gc { + void + PerGenerationStatistics::include_gc(std::size_t alloc_z, + std::size_t before_z, + std::size_t after_z, + std::size_t promote_z) + { + this->update_snapshot(after_z); + + //++n_gc_; + new_alloc_z_ += alloc_z; + scanned_z_ += before_z; + survive_z_ += after_z; + promote_z_ += promote_z; + } + + void + PerGenerationStatistics::update_snapshot(std::size_t after_z) + { + used_z_ = after_z; + } + + void + PerGenerationStatistics::display(std::ostream & os) const + { + os << ""; + } + + void + GcStatistics::begin_gc(generation upto, + std::size_t new_alloc) + { + ++(this->gen_v_[static_cast(upto)].n_gc_); + this->total_allocated_ += new_alloc; + this->total_promoted_sab_ = total_promoted_; + } + + void + GcStatistics::include_gc(generation upto, + std::size_t alloc_z, + std::size_t before_z, + std::size_t after_z, + std::size_t promote_z) + { + gen_v_[static_cast(upto)].include_gc(alloc_z, before_z, after_z, promote_z); + } + + void + GcStatistics::update_snapshot(generation upto, + std::size_t after_z) + { + gen_v_[static_cast(upto)].update_snapshot(after_z); + } + + void + GcStatistics::display(std::ostream & os) const + { + os << ""; + } + + void + GcStatisticsExt::display(std::ostream & os) const + { + os << ""; + } + + float + GcStatisticsHistoryItem::collection_rate() const { + using namespace xo::qty::qty; + + float gz = this->garbage_z(); + + auto dt_nanos = this->dt_.with_repr(); + auto dt_sec = dt_nanos.rescale_ext(); + auto rate = gz / dt_sec; + float retval = rate.scale(); + + //scope log(XO_DEBUG(true)); + //log && log(xtag("gz", gz), xtag("dt_sec", dt_sec), xtag("rate", rate), xtag("rate/sec", retval)); + + return retval; + } + + void + GcStatisticsHistoryItem::display(std::ostream & os) const + { + os << ""; + } + + } /*namespace gc*/ + + namespace print { + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::PerGenerationStatistics & x) + { + return ppii.pps()->pretty_struct(ppii, + "PerGenerationStatistics", + refrtag("used_z", x.used_z_), + refrtag("n_gc", x.n_gc_), + refrtag("new_alloc_z", x.new_alloc_z_), + refrtag("scanned_z", x.scanned_z_), + refrtag("survive_z", x.survive_z_), + refrtag("promote_z", x.promote_z_) + ); + } + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::GcStatistics & x) + { + return ppii.pps()->pretty_struct(ppii, + "GcStatistics", + refrtag("gen_v", x.gen_v_), + refrtag("total_allocated", x.total_allocated_), + refrtag("total_promoted_sab", x.total_promoted_sab_), + refrtag("total_promoted", x.total_promoted_), + refrtag("n_mutation", x.n_mutation_), + refrtag("n_logged_mutation", x.n_logged_mutation_), + refrtag("n_xgen_mutation", x.n_xgen_mutation_), + refrtag("n_xckp_mutation", x.n_xckp_mutation_) + ); + } + + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::GcStatisticsExt & x) + { + return ppii.pps()->pretty_struct(ppii, + "GcStatisticsExt", + refrtag("gen_v", x.gen_v_), + refrtag("total_allocated", x.total_allocated_), + refrtag("total_promoted_sab", x.total_promoted_sab_), + refrtag("total_promoted", x.total_promoted_), + refrtag("n_mutation", x.n_mutation_), + refrtag("n_logged_mutation", x.n_logged_mutation_), + refrtag("n_xgen_mutation", x.n_xgen_mutation_), + refrtag("n_xckp_mutation", x.n_xckp_mutation_), + refrtag("nursery_z", x.nursery_z_), + refrtag("nursery_before_checkpoint_z", x.nursery_before_checkpoint_z_), + refrtag("nursery_after_checkpoint_z", x.nursery_after_checkpoint_z_), + refrtag("tenured_z", x.tenured_z_)); + } + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::GcStatisticsHistoryItem & x) + { + return ppii.pps()->pretty_struct(ppii, + "GcStatisticsHistoryItem", + refrtag("upto", gen2str(x.upto_)), + refrtag("survive_z", x.survive_z_), + refrtag("promote_z", x.promote_z_), + refrtag("persist_z", x.persist_z_), + refrtag("effort_z", x.effort_z_), + refrtag("garbage0_z", x.garbage0_z_), + refrtag("garbage1_z", x.garbage1_z_), + refrtag("garbageN_z", x.garbageN_z_), + refrtag("dt", x.dt_)); + } + } /*namespace print*/ +} /*namespace xo*/ + +/* end GcStatistics.cpp */ diff --git a/xo-alloc/src/alloc/ListAlloc.cpp b/xo-alloc/src/alloc/ListAlloc.cpp new file mode 100644 index 00000000..9961d59b --- /dev/null +++ b/xo-alloc/src/alloc/ListAlloc.cpp @@ -0,0 +1,400 @@ +/* file ListAlloc.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "ListAlloc.hpp" +#include "ArenaAlloc.hpp" +#include "xo/indentlog/scope.hpp" +#include +#include + +namespace xo { + namespace gc { + ListAlloc::ListAlloc(std::unique_ptr hd, + ArenaAlloc * marked, + std::size_t cz, std::size_t nz, std::size_t tz, + bool debug_flag) + : start_z_{cz}, + hd_{std::move(hd)}, + marked_{marked}, + full_l_{}, + current_z_{cz}, + next_z_{nz}, + total_z_{tz}, + debug_flag_{debug_flag} + {} + + ListAlloc::~ListAlloc() + { + this->clear(); + } + + up + ListAlloc::make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag) + { + std::unique_ptr hd{ArenaAlloc::make(name, + cz, debug_flag)}; + + if (!hd) + return nullptr; + + ArenaAlloc * marked = nullptr; + + up retval{new ListAlloc(std::move(hd), + marked, + cz, nz, cz, + debug_flag)}; + + return retval; + } + + void + ListAlloc::capture_object_statistics(capture_phase phase, + ObjectStatistics * p_dest) const + { + hd_->capture_object_statistics(phase, p_dest); + + for (const auto & arena : full_l_) + arena->capture_object_statistics(phase, p_dest); + + } + + const std::string & + ListAlloc::name() const { + if (hd_) { + return hd_->name(); + } + + static std::string s_default_name = "ListAlloc"; + return s_default_name; + } + + std::size_t + ListAlloc::page_size() const { + return hd_->page_size(); + } + + std::size_t + ListAlloc::hugepage_z() const { + return hd_->hugepage_z(); + } + + std::size_t + ListAlloc::size() const { + return total_z_; + } + + std::size_t + ListAlloc::committed() const { + std::size_t z = 0; + if (hd_) + z += hd_->committed(); + for (const auto & a : full_l_) + z += a->committed(); + + return z; + } + + std::byte * + ListAlloc::free_ptr() const { + return hd_->free_ptr(); + } + + std::size_t + ListAlloc::available() const { + if (hd_) { + /* can only allocate from @ref hd_, + * so even if there were available space in @ref full_l_, + * it's not accessible to ListAlloc. + */ + + return hd_->available(); + } + + return 0; + } + + std::size_t + ListAlloc::allocated() const { + std::size_t total = 0; + + if (hd_) { + total += hd_->allocated(); + } + + for (const auto & alloc : full_l_) + total += alloc->allocated(); + + return total; + } + + bool + ListAlloc::contains(const void * x) const { + if (hd_ && hd_->contains(x)) + return true; + + for (const auto & alloc : full_l_) { + if (alloc->contains(x)) + return true; + } + + return false; + } + + bool + ListAlloc::is_before_checkpoint(const void * x) const { + if (!marked_) + return true; + + if (marked_ && marked_->contains(x)) + return marked_->is_before_checkpoint(x); + + /* + * 1. allocs in full_l_ appear in oldest-to-youngest order + * 2. allocators that appear before marked_ in full_l_ count as 'before checkpoint' + * 3. allocators that appear after marked_ in full_l_ count as 'after checkpoint' + */ + + bool older_than_marked = true; + + for (const auto & alloc : full_l_) { + if (older_than_marked) { + if (alloc.get() == marked_) { + /* nothing else to test on this iteration, + * already checked .marked_ specifically + */ + break; + } else { + /* before checkpoint */ + if (alloc->contains(x)) + return true; + } + } + } + + return false; + } + + std::size_t + ListAlloc::before_checkpoint() const + { + scope log(XO_DEBUG(false && debug_flag_), xtag("marked", marked_ ? marked_->name() : "")); + + if (marked_) { + if (full_l_.empty()) { + assert(marked_ == hd_.get()); + + return marked_->before_checkpoint(); + } else { + std::size_t z = 0; + + /* control here: .marked & .full_l non-empty. */ + if (hd_.get() == marked_) { + z += hd_->before_checkpoint(); + + /* anything in .full_l is older than marked .hd */ + for (const auto & alloc : full_l_) { + z += alloc->allocated(); + } + + return z; + } else { + /* messiest case: .marked is true, + * and not the youngest arena + */ + + /* full_l always in increasing time order: oldest-to-youngest order */ + size_t i_alloc = 0; + for (const auto & alloc : full_l_) { + log && log(xtag("i_alloc", i_alloc), + xtag("alloc", alloc->name()), + xtag("z", z)); + + if (alloc.get() == marked_) { + log && log("marked", xtag("+z", marked_->before_checkpoint())); + z += marked_->before_checkpoint(); + break; + } else { + log && log("older than marked", xtag("+z", alloc->allocated())); + z += alloc->allocated(); + } + ++i_alloc; + } + } + + return z; + } + } else { + /* count *everything* allocated */ + return this->allocated(); + } + } + + std::size_t + ListAlloc::after_checkpoint() const + { + scope log(XO_DEBUG(false && debug_flag_), xtag("marked", marked_ ? marked_->name() : "")); + + if (!marked_) + return 0; + + if (full_l_.empty()) { + assert(marked_ == hd_.get()); + + return marked_->after_checkpoint(); + } + + bool older_than_marked = true; + + std::size_t z = 0; + + std::size_t i_alloc = 0; + for (const auto & alloc : full_l_) { + log && log(xtag("i_alloc", i_alloc), + xtag("alloc", alloc->name()), + xtag("z", z)); + + if (older_than_marked) { + if (alloc.get() == marked_) { + log && log("marked", xtag("+z", marked_->after_checkpoint())); + older_than_marked = false; + z += marked_->after_checkpoint(); + } + } else { + /* younger than marked */ + log && log("younger", xtag("+z", alloc->allocated())); + z += alloc->allocated(); + } + + ++i_alloc; + } + + /** head must be included, since it's always the youngest bucket **/ + z += hd_->after_checkpoint(); + + log && log("z", z); + + return z; + } + + bool + ListAlloc::debug_flag() const { + return debug_flag_; + } + + void + ListAlloc::clear() { + // general hygiene + start_z_ = 0; + hd_.reset(); + marked_ = nullptr; + full_l_.clear(); + current_z_ = 0; + next_z_ = 0; + total_z_ = 0; + } + + bool + ListAlloc::reset(std::size_t z) + { + scope log(XO_DEBUG(debug_flag_), xtag("z", z)); + + bool recycle_head_bucket = hd_ && (z <= hd_->size()); + + this->full_l_.clear(); + this->marked_ = nullptr; + + if (recycle_head_bucket) { + this->hd_->clear(); + this->total_z_ = hd_->size(); + + return true; + } else { + std::string old_name = this->hd_->name(); + + this->hd_.reset(nullptr); + this->total_z_ = 0; + + return this->expand(z, old_name + "+"); + } + } + + bool + ListAlloc::expand(std::size_t z, const std::string & name) + { + scope log(XO_DEBUG(debug_flag_), xtag("name", name)); + + //log && log("before", xtag("before_ckp", this->before_checkpoint())); + + std::size_t cz = current_z_; + std::size_t nz = next_z_; + std::size_t tz; + + do { + tz = cz + nz; + cz = nz; + nz = tz; + } while (cz < z); + + log && log("expand to", xtag("cz", cz)); + + std::unique_ptr new_alloc = ArenaAlloc::make(name, + cz, debug_flag_); + cz = new_alloc->size(); + + if (!new_alloc) + return false; + + this->current_z_ = cz; + this->next_z_ = nz; + this->total_z_ += cz; + + if (hd_) + this->full_l_.push_back(std::move(hd_)); + + this->hd_ = std::move(new_alloc); + + //log && log("after", xtag("before_ckp", this->before_checkpoint())); + + return true; + } + + void + ListAlloc::checkpoint() { + scope log(XO_DEBUG(debug_flag_)); + + hd_->checkpoint(); + + this->marked_ = hd_.get(); + + log && log(xtag("hd", (void*)hd_.get()), xtag("marked", (void*)marked_)); + } + + std::byte * + ListAlloc::alloc(std::size_t z) { + scope log(XO_DEBUG(debug_flag_)); + + /* ArenaAlloc::alloc() may modify its own size */ + + std::size_t z_pre = hd_->size(); + std::byte * retval = hd_->alloc(z); + + if (retval) { + std::size_t z_post = hd_->size(); + this->total_z_ += (z_post - z_pre); + + return retval; + } + + log && log("space exhausted -> expand"); + + if (this->expand(z, hd_->name() + "+")) + return hd_->alloc(z); + + return nullptr; + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* end ListAlloc.cpp */ diff --git a/xo-alloc/src/alloc/Object.cpp b/xo-alloc/src/alloc/Object.cpp new file mode 100644 index 00000000..b0a52f59 --- /dev/null +++ b/xo-alloc/src/alloc/Object.cpp @@ -0,0 +1,230 @@ +/* Object.cpp + * + * author: Roalnd Conybeare, Jul 2025 + */ + +#include "Object.hpp" +#include "GC.hpp" +#include "Forwarding1.hpp" + +using xo::obj::Forwarding1; + +void * +operator new (std::size_t z, const xo::Cpof & cpof) +{ + using xo::gc::GC; + + //GC * gc = reinterpret_cast(cpof.mm_); + + return cpof.mm_->alloc_gc_copy(z, cpof.src_); +} + +namespace xo { + using xo::reflect::TaggedPtr; + + gc::IAlloc * + Object::mm = nullptr; + + TaggedPtr + Object::self_tp() const + { + assert(false); + return TaggedPtr::universal_null(); + } + + void + Object::display(std::ostream & os) const + { + os << ""; + } + + IObject * + Object::_forward(IObject * src, + gc::IAlloc * gc) + { + scope log(XO_DEBUG(gc->debug_flag()), xtag("src", src)); + + if (!src) + return src; + + if (src->_is_forwarded()) { + log && log("already forwarded", xtag("dest", src->_offset_destination(src))); + return src->_offset_destination(src); + } + + if (gc->check_move(src)) { + log && log("needs forwarding"); + Object::_shallow_move(src, gc); + + /* *src is now a forwarding pointer to a copy in to-space */ + return src->_offset_destination(src); + } else { + log && log("already tenured + incr collection"); + /* don't move tenured objects during incremental collection */ + return src; + } + } + + IObject * + Object::_deep_move(IObject * from_src, gc::GC * gc, gc::ObjectStatistics * /*stats*/) + { + scope log(XO_DEBUG(gc->config().debug_flag_)); + + using gc::generation; + + if (!from_src) + return nullptr; + + IObject * retval = from_src->_destination(); + + if (retval) + return retval; + + if (!gc->check_move(from_src)) { + /** incremental collection does not move already-tenured objects **/ + return from_src; + } + + /** + * To-space: + * + * to_lo = start of to-space + * w,W = white objects. An object x is white if x + * + all immediate children of x are in to-space + * (also implies this GC cycle put it there) + * g,G = grey objects. An object x is gray if it's in to-space, + * but possibly has >0 black children + * _ = free to-space memory + * N = nursery space + * T = tenured space + * + * wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________... + * ^ ^ ^ + * to_lo grey_lo(N) free_ptr(N) + * + * After moving children of one object, advancing {nursery_grey_lo, nursery_free_ptr} + * + * wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG______... + * ^ ^ ^ + * to_lo grey_lo(N) free_ptr(N) + * + * Invariant: + * + * objects in [to_lo, gray_lo) are white. + * all gray objects are in [gray_lo, free_ptr) + * memory starting at free_ptr is free. + * + * deep_move terminates when gray_lo catches up to free_ptr + * + * Above is simplified. Complication is that GC (including incremental) may + * promote objects from nursery (N) to tenured (T) + * + * So more accurate before/after picture + * + * N wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________... + * ^ ^ ^ + * to_lo(N) grey_lo(N) free_ptr(N) + * + * T wwwwwwwwwwwwwwgggggggggggg_______________________________... + * ^ ^ ^ + * to_lo(T) grey_lo(T) free_ptr(N) + * + * After moving children of one object, advancing {nursery_grey_lo, nursery_free_ptr} + * + * N wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG_____... + * ^ ^ ^ + * to_lo(N) grey_lo(N) free_ptr(N) + * + * T wwwwwwwwwwwwwwggggggggggggGGGGG_________________________... + * ^ ^ ^ + * to_lo(T) grey_lo(T) free_ptr(T) + * + * deep_move terminates when both: + * - gray_lo(N) catches up with free_ptr(N) + * - gray_lo(T) catches up with free_ptr(T) + * + **/ + + std::array gray_lo_v + = { gc->free_ptr(generation::nursery), gc->free_ptr(generation::tenured) }; + + IObject * to_src = Object::_shallow_move(from_src, gc); + + std::size_t fixup_work = 0; + do { + fixup_work = 0; + + auto fixup_generation = [gc, &log, &gray_lo_v](generation gen) { + std::size_t work = 0; + while(gray_lo_v[gen2int(gen)] < gc->free_ptr(gen)) { + Object * x = reinterpret_cast(gray_lo_v[gen2int(gen)]); + + // update per-class stats here + + log && log("fwd children", xtag("x", x)); + + std::size_t xz = x->_forward_children(gc); + + // must pad xz to multiple of word size, + // to match behavior of LinearAlloc::alloc() + // + xz += gc::IAlloc::alloc_padding(xz); + + gray_lo_v[gen2int(gen)] += xz; + ++work; + } + + return work; + }; + + fixup_work += fixup_generation(generation::nursery); + fixup_work += fixup_generation(generation::tenured); + } while (fixup_work > 0); + + return to_src; + } /*deep_move*/ + + IObject * + Object::_shallow_move(IObject * src, gc::IAlloc * gc) + { + /* filter for source objects that are owned by GC. + * Care required though -- during GC from/to spaces have been swapped already + */ + if (gc->check_owned(src)) + { + IObject * dest = src->_shallow_copy(gc); + + if (dest != src) + src->_forward_to(dest); + + return dest; + } else { + return src; + } + } + + void + Object::_forward_to(IObject * dest) + { + char * mem = reinterpret_cast(this); + + Forwarding1 * fwd = new (mem) Forwarding1(dest); + + (void)fwd; + } + + std::ostream & + operator<< (std::ostream & os, gp x) + { + if (x.ptr()) { + x->display(os); + } else { + os << ""; + } + + return os; + } + +} /*namespace xo*/ + +/* end Object.cpp*/ diff --git a/xo-alloc/src/alloc/ObjectStatistics.cpp b/xo-alloc/src/alloc/ObjectStatistics.cpp new file mode 100644 index 00000000..863a02cb --- /dev/null +++ b/xo-alloc/src/alloc/ObjectStatistics.cpp @@ -0,0 +1,73 @@ +/* file ObjectStatistics.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "ObjectStatistics.hpp" +#include "xo/reflect/TypeDescr.hpp" +#include "xo/indentlog/print/pretty_vector.hpp" + +namespace xo { + namespace gc { + void + PerObjectTypeStatistics::display(std::ostream & os) const + { + os << "short_name()); + else + os << xrtag("td", "nullptr"); + os << xrtag("scanned_n", scanned_n_) + << xrtag("scanned_z", scanned_z_) + << xrtag("survive_n", survive_n_) + << xrtag("survive_z", survive_z_) + << ">"; + } + + void + ObjectStatistics::display(std::ostream & os) const + { + os << ""; + } + } /*namespace gc*/ + + namespace print { + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::PerObjectTypeStatistics & x) + { + static constexpr std::string_view c_nullptr_str = "nullptr"; + + if (x.td_) { + return ppii.pps()->pretty_struct(ppii, + "PerObjectTypeStatistics", + refrtag("td", x.td_ ? x.td_->short_name() : c_nullptr_str), + refrtag("scanned_n", x.scanned_n_), + refrtag("scanned_z", x.scanned_z_), + refrtag("survive_n", x.survive_n_), + refrtag("survive_z", x.survive_z_)); + } else { + /* print nothing -- empty struct */ + return true; + } + } + + bool + ppdetail::print_pretty(const ppindentinfo & ppii, + const xo::gc::ObjectStatistics & x) + { + return ppii.pps()->pretty_struct(ppii, + "ObjectTypeStatistics", + refrtag("per_type_stats_v", x.per_type_stats_v_)); + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* end ObjectStatistics.cpp */ diff --git a/xo-alloc/src/alloc/generation.cpp b/xo-alloc/src/alloc/generation.cpp new file mode 100644 index 00000000..a0ae9e65 --- /dev/null +++ b/xo-alloc/src/alloc/generation.cpp @@ -0,0 +1,31 @@ +/* generation.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "generation.hpp" + +namespace xo { + namespace gc { + const char * gen2str(generation x) { + switch (x) { + case generation::nursery: return "nursery"; + case generation::tenured: return "tenured"; + case generation::N: break; + } + return "?generation"; + } + + const char * genresult2str(generation_result x) { + switch (x) { + case generation_result::nursery: return "nursery"; + case generation_result::tenured: return "tenured"; + case generation_result::not_found: return "not-found"; + } + return "?generation_result"; + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* generation.cpp */ diff --git a/xo-alloc/utest/ArenaAlloc.test.cpp b/xo-alloc/utest/ArenaAlloc.test.cpp new file mode 100644 index 00000000..78055eed --- /dev/null +++ b/xo-alloc/utest/ArenaAlloc.test.cpp @@ -0,0 +1,88 @@ +/* @file ArenaAlloc.test.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "xo/alloc/ArenaAlloc.hpp" +#include + +namespace xo { + using xo::gc::ArenaAlloc; + + namespace ut { + + namespace { + struct testcase_alloc { + explicit testcase_alloc(std::size_t z) + : + arena_z_{z} {} + + std::size_t arena_z_; + }; + + std::vector + s_testcase_v = { + testcase_alloc(4096) + }; + } + + + TEST_CASE("linearalloc", "[alloc]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + const testcase_alloc & tc = s_testcase_v[i_tc]; + + constexpr bool c_debug_flag = false; + + auto alloc = ArenaAlloc::make("linearalloc", + tc.arena_z_, c_debug_flag); + alloc->expand(tc.arena_z_); + + REQUIRE(alloc.get()); + REQUIRE(alloc->name() == "linearalloc"); + REQUIRE(alloc->size() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->available() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->allocated() == 0); + REQUIRE(alloc->is_before_checkpoint(alloc->free_ptr()) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + auto free0 = alloc->free_ptr(); + + auto mem = alloc->alloc(std::max(tc.arena_z_, alloc->hugepage_z())); + + REQUIRE(mem != nullptr); + + REQUIRE(mem == free0); + + REQUIRE(alloc->size() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->available() == 0); + REQUIRE(alloc->allocated() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->is_before_checkpoint(mem) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == std::max(tc.arena_z_, alloc->hugepage_z())); + + alloc->clear(); + + REQUIRE(alloc->free_ptr() == free0); + REQUIRE(alloc->available() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->allocated() == 0); + REQUIRE(alloc->is_before_checkpoint(free0) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + mem = alloc->alloc(1); + + auto used = sizeof(void*); + REQUIRE(alloc->size() == std::max(tc.arena_z_, alloc->hugepage_z())); + REQUIRE(alloc->available() == std::max(tc.arena_z_, alloc->hugepage_z()) - used); + REQUIRE(alloc->allocated() == used); + REQUIRE(alloc->is_before_checkpoint(free0) == false); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == used); + + } + } + + } /*namespace ut */ +} /*namespace xo*/ diff --git a/xo-alloc/utest/CMakeLists.txt b/xo-alloc/utest/CMakeLists.txt new file mode 100644 index 00000000..366cf664 --- /dev/null +++ b/xo-alloc/utest/CMakeLists.txt @@ -0,0 +1,26 @@ +# xo-alloc/utest/CMakeLists.txt +# +# NOTE: more GC tests in xo-object/utest + +set(UTEST_EXE utest.alloc) +set(UTEST_SRCS + alloc_utest_main.cpp + IAlloc.test.cpp + ArenaAlloc.test.cpp + ListAlloc.test.cpp + GC.test.cpp + GcStatistics.test.cpp + ObjectStatistics.test.cpp + Forwarding1.test.cpp + CircularBuffer.test.cpp + generation.test.cpp +) + +if (ENABLE_TESTING) + xo_add_utest_executable(${UTEST_EXE} ${UTEST_SRCS}) + xo_self_dependency(${UTEST_EXE} xo_alloc) + xo_dependency(${UTEST_EXE} reflect) + xo_external_target_dependency(${UTEST_EXE} Catch2 Catch2::Catch2) +endif() + +# end CmakeLists.txt diff --git a/xo-alloc/utest/CircularBuffer.test.cpp b/xo-alloc/utest/CircularBuffer.test.cpp new file mode 100644 index 00000000..f3d63c7e --- /dev/null +++ b/xo-alloc/utest/CircularBuffer.test.cpp @@ -0,0 +1,174 @@ +/* CircularBuffer.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/CircularBuffer.hpp" +#include "xo/indentlog/print/vector.hpp" +#include + +namespace xo { + using xo::gc::CircularBuffer; + + namespace ut { + TEST_CASE("circular_buffer_0", "[circular_buffer]") + { + CircularBuffer q(10, false /*debug_flag*/); + q.push_back("a"); + REQUIRE(q.back() == "a"); + q.push_back("b"); + REQUIRE(q.back() == "b"); + q.push_back("c"); + REQUIRE(q.back() == "c"); + REQUIRE(q.location_of(0) == 0); + REQUIRE(q.location_of(1) == 1); + REQUIRE(q.location_of(2) == 2); + //REQUIRE(q.index_of(0) == 0); + + REQUIRE(q.size() == 3); + REQUIRE(q.front() == "a"); + REQUIRE(q.at(0) == "a"); + REQUIRE(q.at(1) == "b"); + REQUIRE(q.at(2) == "c"); + + CircularBuffer q2; + + q2 = q; + + q.clear(); + + REQUIRE(q2.size() == 3); + REQUIRE(q2.front() == "a"); + REQUIRE(q2.at(0) == "a"); + REQUIRE(q2.at(1) == "b"); + REQUIRE(q2.at(2) == "c"); + } + + TEST_CASE("circular_buffer_1", "[circular_buffer]") + { + CircularBuffer q(2, false /*debug_flag*/); + q.push_back("a"); + REQUIRE(q.back() == "a"); + q.push_back("b"); + REQUIRE(q.back() == "b"); + q.push_back("c"); + REQUIRE(q.back() == "c"); + REQUIRE(q.location_of(0) == 1); + REQUIRE(q.location_of(1) == 0); + //REQUIRE(q.index_of(0) == 0); + + REQUIRE(q.size() == 2); + REQUIRE(q.front() == "b"); + REQUIRE(q.at(0) == "b"); + REQUIRE(q.at(1) == "c"); + + { + std::size_t i = 0; + for (const auto & qi : q) { + REQUIRE(qi == q.at(i)); + ++i; + } + } + + CircularBuffer q2 = q; + + q.clear(); + + REQUIRE(q2.size() == 2); + REQUIRE(q2.front() == "b"); + REQUIRE(q2.at(0) == "b"); + REQUIRE(q2.at(1) == "c"); + + { + std::size_t i = 0; + for (const auto & qi : q) { + REQUIRE(qi == q2.at(i)); + ++i; + } + } + } + } + + namespace { + struct Testcase_CircularBuffer { + explicit Testcase_CircularBuffer(std::size_t capacity, + const std::vector & contents) + : capacity_{capacity}, + contents_{contents} {} + + std::size_t capacity_ = 0; + std::vector contents_; + }; + + std::vector + s_testcase_v = { + Testcase_CircularBuffer(0, {}), + Testcase_CircularBuffer(1, {"a"}), + Testcase_CircularBuffer(2, {"a", "b"}), + Testcase_CircularBuffer(2, {"a", "b", "c", "d"}) + }; + } + + namespace ut { + TEST_CASE("circular_buffer_2", "[circular_buffer]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + const Testcase_CircularBuffer & tc = s_testcase_v[i_tc]; + + INFO(tostr(xtag("i_tc", i_tc), + xtag("capacity", tc.capacity_), + xrtag("contents", tc.contents_))); + + for (std::size_t j_phase = 0; j_phase < 2; ++j_phase) { + constexpr bool c_debug_flag = false; + + CircularBuffer q(tc.capacity_, false /*debug_flag*/); + + REQUIRE(q.empty()); + REQUIRE(q.size() == 0); + REQUIRE(q.begin() == q.end()); + REQUIRE(q.capacity() == tc.capacity_); + + std::size_t n = 0; + for (const auto & s : tc.contents_) { + INFO(tostr(xtag("n0", n), xtag("s", s))); + ++n; + INFO(xtag("n1", n)); + + q.push_back(s); + + REQUIRE(q.back() == s); + REQUIRE(q.capacity() == tc.capacity_); + REQUIRE(q.size() == std::min(n, tc.capacity_)); + + std::size_t i = 0; + for (const auto & qi : q) { + INFO(tostr(xtag("i", i), xtag("qi", qi))); + + if (n <= tc.capacity_) { + REQUIRE(qi == tc.contents_.at(i)); + REQUIRE(qi == tc.contents_[i]); + } else { + REQUIRE(qi == tc.contents_.at(n - tc.capacity_ + i)); + REQUIRE(qi == tc.contents_[n - tc.capacity_ + i]); + } + ++i; + } + + REQUIRE(i == std::min(n, tc.capacity_)); + + if (tc.contents_.size() <= tc.capacity_) + REQUIRE(q.front() == tc.contents_.at(0)); + } + + q.clear(); + + REQUIRE(q.size() == 0); + REQUIRE(q.capacity() == tc.capacity_); + } + } + } + } +} + +/* CircularBuffer.test.cpp */ diff --git a/xo-alloc/utest/Forwarding1.test.cpp b/xo-alloc/utest/Forwarding1.test.cpp new file mode 100644 index 00000000..64c9f3f8 --- /dev/null +++ b/xo-alloc/utest/Forwarding1.test.cpp @@ -0,0 +1,93 @@ +/* Forwarding1.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "Forwarding1.hpp" +#include "ArenaAlloc.hpp" +#include "xo/reflect/Reflect.hpp" +#include +#include +#include + +namespace xo { + using xo::reflect::Reflect; + using xo::obj::Forwarding1; + + namespace gc { + namespace { + class DummyObject : public Object { + public: + explicit DummyObject(const char * data) { + ::strncpy(data_.data(), data, 128); + } + + gp member() const { return member_; } + void assign_member(Object * x) { + Object::mm->assign_member(this, reinterpret_cast(member_.ptr_address()), x); + } + + TaggedPtr self_tp() const final override { + return Reflect::make_tp(const_cast(this)); + } + + void display(std::ostream & os) const final override { os << data_; } + + virtual std::size_t _shallow_size() const final override { return sizeof(*this); } + virtual IObject * _shallow_copy(gc::IAlloc * mm) const final override { return new (Cpof(mm, this)) DummyObject(*this); } + virtual std::size_t _forward_children(gc::IAlloc * gc) final override { return _shallow_size(); } + + private: + std::array data_; + gp member_; + }; + } + + TEST_CASE("Forwarding1", "[gc][alloc]") + { + bool saved = tag_config::tag_color_enabled; + tag_config::tag_color_enabled = false; + + gp obj = new DummyObject("Well, I wasn't expecting that!"); + gp fwd = new Forwarding1(obj); + + REQUIRE(fwd->_destination() == obj.ptr()); + REQUIRE(fwd->_offset_destination(fwd.ptr()) == obj.ptr()); + + REQUIRE(fwd->self_tp().td()->short_name() == "Forwarding1"); + + std::stringstream ss; + ss << fwd; + + // forwarding printer looks like + // "" + // + + std::regex pattern(R"()"); + REQUIRE(std::regex_match(ss.str(), pattern)); + + //REQUIRE(ss.str() == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("IAlloc.assign_member", "[gc][alloc]") + { + /* not giving this nit it's own translation unit. + */ + + gp obj = new DummyObject("This also a surprise.."); + + up arena = ArenaAlloc::make("test", 1024, false); + + Object::mm = arena.get(); + + obj->assign_member(obj.ptr()); + + REQUIRE(obj->member().ptr() == obj.ptr()); + } + + } /*namespace gc*/ +} /*namespace xo*/ + +/* end Forwarding1.test.cpp */ diff --git a/xo-alloc/utest/GC.test.cpp b/xo-alloc/utest/GC.test.cpp new file mode 100644 index 00000000..f178f4a8 --- /dev/null +++ b/xo-alloc/utest/GC.test.cpp @@ -0,0 +1,395 @@ +/* @file GC.test.cpp + * + * author: Roland Conybeare, Jul 2025 + */ + +#include "xo/alloc/GC.hpp" +#include "xo/allocutil/gc_allocator_traits.hpp" +#include + +namespace xo { + using xo::gc::IAlloc; + using xo::gc::GC; + using xo::gc::gc_allocator_traits; + using xo::gc::generation; + using xo::gc::Config; + using xo::reflect::TaggedPtr; + + namespace ut { + + namespace { + struct testcase_gc { + testcase_gc(std::size_t nz, std::size_t tz, std::size_t n_gct, std::size_t t_gct) + : nursery_z_{nz}, tenured_z_{tz}, incr_gc_threshold_{n_gct}, full_gc_threshold_{t_gct} {} + + std::size_t nursery_z_; + std::size_t tenured_z_; + std::size_t incr_gc_threshold_; + std::size_t full_gc_threshold_; + }; + + std::vector + s_testcase_v = { + // n_gct: nursery gc threshold + // t_gct: tenured gc threshold + // + // nz tz n_gct t_gct + testcase_gc(1024, 4096, 1024, 1024) + }; + } + + TEST_CASE("gc", "[alloc][gc]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + try { + const testcase_gc & tc = s_testcase_v[i_tc]; + + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_, + .incr_gc_threshold_ = tc.incr_gc_threshold_, + .full_gc_threshold_ = tc.full_gc_threshold_, + }); + + REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); + REQUIRE(gc->nursery_to_allocated() == 0); + REQUIRE(gc->nursery_to_committed() >= tc.nursery_z_); + REQUIRE(gc->nursery_to_reserved() >= tc.nursery_z_); + REQUIRE(gc->nursery_to_reserved() < tc.nursery_z_ + gc->hugepage_z()); + REQUIRE(gc->size() >= tc.nursery_z_ + tc.tenured_z_); + REQUIRE(gc->size() < tc.nursery_z_ + gc->hugepage_z() + tc.tenured_z_ + gc->hugepage_z()); + REQUIRE(gc->allocated() == 0); + REQUIRE(gc->available() == gc->nursery_to_reserved()); + REQUIRE(gc->before_checkpoint() == 0); + // ListAlloc model is that nothing is before checkpoint + // until it's first established + REQUIRE(gc->after_checkpoint() == 0); + + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->is_gc_enabled() == true); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 0); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + + /* gc with empty state */ + gc->request_gc(generation::nursery); + + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 0); + + /* still empty state */ + gc->request_gc(generation::tenured); + + REQUIRE(gc->gc_in_progress() == false); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::nursery)].n_gc_ == 1); + REQUIRE(gc->native_gc_statistics().gen_v_[gen2int(generation::tenured)].n_gc_ == 1); + } catch (std::exception & ex) { + std::cerr << "caught exception: " << ex.what() << std::endl; + REQUIRE(false); + } + } + } + + /** gc-enabled allocator **/ + namespace { + /** Setup test with custom allocator + * + **/ + template + struct TestClass : public GcObjectInterface { + TestClass() = default; + explicit TestClass(const Nested & member1) : member1_{member1} {} + + // using allocator_type = Allocator; + // using allocator_traits = xo::gc::gc_allocator_traits; + + /** stage1 - just allocates some memory using allocator **/ + template + static TestClass * make_0(Allocator & alloc) { + TestClass * mem = alloc.allocate(sizeof(TestClass)); + + /* but ctor will not have run, so ub to visit object */ + + return mem; + } + + /** stage2 - use allocator_traits construct **/ + template + static TestClass * make_1(Allocator & alloc) { + using traits = gc_allocator_traits; + + TestClass * mem = traits::allocate(alloc, 1); + + /* ctor will not have run here either */ + return mem; + } + + /** stage3 - invoke construct **/ + template + static TestClass * make_2(Allocator & alloc) { + using traits = gc_allocator_traits; + + TestClass * obj = traits::allocate(alloc, 1); + try { + // placement new + traits::construct(alloc, obj); + + return obj; + } catch(...) { + traits::deallocate(alloc, obj, 1); + throw; + } + } + + /** stage4 - init nested type **/ + template + static TestClass * make_3(Allocator & alloc) { + using traits = gc_allocator_traits; + + TestClass * obj = traits::allocate(alloc, 1); + try { + Nested nested; + + // placemenet new + traits::construct(alloc, obj); + + return obj; + } catch(...) { + traits::deallocate(alloc, obj, 1); + throw; + } + } + + // ----- inherited from Object ----- + + virtual TaggedPtr self_tp() const final override { + assert(false); return TaggedPtr::universal_null(); + } + virtual void display(std::ostream & os) const final override { + os << ""; + } + virtual std::size_t _shallow_size() const final override { + assert(false); return sizeof(*this); + } + virtual IObject * _shallow_copy(IAlloc * gc) const final override { + assert(false); return nullptr; + } + virtual std::size_t _forward_children(IAlloc * gc) final override { + assert(false); return _shallow_size(); + } + + Nested member1_; + }; + + //template + struct MemberType { + public: + //using allocator_type = Allocator; + //using vector_allocator_type = typename std::allocator_traits::template rebind_alloc>; + using vector_type = std::vector>; + //using vector_type = std::vector, vector_allocator_type>; + + public: + MemberType() : ctor_ran_{true} {} + //explicit MemberType(const Allocator & alloc) + //: member2_{vector_allocator_type(alloc)}, ctor_ran_{true} {} + + explicit MemberType(const vector_type & mem2) : member2_{mem2}, ctor_ran_{true} {} + //MemberType(const vector_type & mem2, const Allocator & alloc) + //: member2_{mem2, vector_allocator_type(alloc)}, ctor_ran_{true} {} + + vector_type member2_; + bool ctor_ran_ = false; + }; + +#ifdef NOT_YET + struct MemberType2 { + public: + MemberType2() = default; + /** GC hooks rely on copy constructor. But can't write it without allocator state. + * Therefore: need copy-like constructor that takes allocator argument + **/ + + template + explicit MemberType2(Allocator & alloc, uint64 payload) { + using traits = gc_allocator_traits; + + uint64_t * ptr = traits::allocate(alloc, 1); + + this->payload_ = payload; + this->ctor_ran_ = true; + } + + uint64_t * payload_ = nullptr; + bool ctor_ran_ = false; + } +#endif + } + + TEST_CASE("vector_custom_allocator", "[alloc][vector]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + try { + const testcase_gc & tc = s_testcase_v[i_tc]; + + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_, + .incr_gc_threshold_ = tc.incr_gc_threshold_, + .full_gc_threshold_ = tc.full_gc_threshold_, + }); + + REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); + + using NestedElementAllocator = xo::gc::allocator>; + + NestedElementAllocator alloc(gc.get()); + + /** testv will use GC to allocaate element storage + * Attempt to gc will fail, because memory iteration + * won't work. + **/ + std::vector, + NestedElementAllocator> testv(alloc); + + testv.push_back(gp()); + +#ifdef NOPE + using ex_allocator = xo::gc::allocator; + using MyObjectInterface = gc_allocator_traits::template object_interface; + using NestedType = MemberType; + //using NestedType = MemberType; + using MyType = TestClass; + using MyAllocator = xo::gc::allocator; + + MyAllocator alloc(gc.get()); + + { + /* verify that MyType is constructible */ + MyType obj0; + + REQUIRE(obj0.member1_.ctor_ran_ == true); + } + + { + MyType * mem0 = MyType::make_0(alloc); + + REQUIRE(mem0 != nullptr); + REQUIRE(mem0->member1_.ctor_ran_ == false); + } + + { + MyType * mem1 = MyType::make_1(alloc); + + REQUIRE(mem1 != nullptr); + REQUIRE(mem1->member1_.ctor_ran_ == false); + } + + { + MyType * mem2 = MyType::make_2(alloc); + + REQUIRE(mem2 != nullptr); + REQUIRE(mem2->member1_.ctor_ran_ == true); + } + + { + MyType * mem3 = MyType::make_3(alloc); + + REQUIRE(mem3 != nullptr); + REQUIRE(mem3->member1_.ctor_ran_ == true); + } + + gp ptr; + { + REQUIRE(ptr.is_null()); + //ptr = MyType::make_0(); + } +#endif + } catch (std::exception & ex) { + std::cerr << "caught exception: " << ex.what() << std::endl; + REQUIRE(false); + } + } + + } + + TEST_CASE("gc_allocator_traits", "[alloc][gc]") + { + for (std::size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + try { + const testcase_gc & tc = s_testcase_v[i_tc]; + + up gc = GC::make( + {.initial_nursery_z_ = tc.nursery_z_, + .initial_tenured_z_ = tc.tenured_z_, + .incr_gc_threshold_ = tc.incr_gc_threshold_, + .full_gc_threshold_ = tc.full_gc_threshold_, + }); + + REQUIRE(gc.get()); + REQUIRE(gc->name() == "GC"); + + using ex_allocator = xo::gc::allocator; + using MyObjectInterface = gc_allocator_traits::template object_interface; + using NestedElementAllocator = xo::gc::allocator>; + using NestedType = MemberType; + //using NestedType = MemberType; + using MyType = TestClass; + using MyAllocator = xo::gc::allocator; + + MyAllocator alloc(gc.get()); + + { + /* verify that MyType is constructible */ + MyType obj0; + + REQUIRE(obj0.member1_.ctor_ran_ == true); + } + + { + MyType * mem0 = MyType::make_0(alloc); + + REQUIRE(mem0 != nullptr); + REQUIRE(mem0->member1_.ctor_ran_ == false); + } + + { + MyType * mem1 = MyType::make_1(alloc); + + REQUIRE(mem1 != nullptr); + REQUIRE(mem1->member1_.ctor_ran_ == false); + } + + { + MyType * mem2 = MyType::make_2(alloc); + + REQUIRE(mem2 != nullptr); + REQUIRE(mem2->member1_.ctor_ran_ == true); + } + + { + MyType * mem3 = MyType::make_3(alloc); + + REQUIRE(mem3 != nullptr); + REQUIRE(mem3->member1_.ctor_ran_ == true); + } + + gp ptr; + { + REQUIRE(ptr.is_null()); + //ptr = MyType::make_0(); + } + } catch (std::exception & ex) { + std::cerr << "caught exception: " << ex.what() << std::endl; + REQUIRE(false); + } + } + + } + + } /*namespace ut*/ +} /*namespace xo*/ + +/* GC.test.cpp */ diff --git a/xo-alloc/utest/GcStatistics.test.cpp b/xo-alloc/utest/GcStatistics.test.cpp new file mode 100644 index 00000000..f5700f25 --- /dev/null +++ b/xo-alloc/utest/GcStatistics.test.cpp @@ -0,0 +1,216 @@ +/* @file GcStatistics.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/GcStatistics.hpp" +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/tostr.hpp" +#include "xo/indentlog/print/hex.hpp" +#include +#include + +namespace xo { + using xo::gc::GcStatistics; + using xo::gc::GcStatisticsExt; + using xo::gc::PerGenerationStatistics; + using xo::print::ppconfig; + + namespace ut { + TEST_CASE("PerGenerationStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerGenerationStatistics stats; + + std::string s = tostr(stats); + + //std::cerr << hex_view(s.c_str(), s.c_str() + s.length(), true /*as_text*/) << std::endl; + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + GcStatistics stats; + + std::string s = tostr(stats); + + REQUIRE(s == + "" + /***/ " ]" + /**/ " :total_allocated 0" + /**/ " :total_promoted_sab 0" + ">"); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatisticsExt", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + GcStatisticsExt stats; + + std::string s = tostr(stats); + + REQUIRE(s == " ] :total_allocated 0 :total_promoted_sab 0 :nursery_z 0 :nursery_before_ckp_z 0 :nursery_after_ckp_z 0 :tenured_z 0 :n_mutation 0 :n_logged_mutation 0 :n_xgen_mutation 0 :n_xckp_mutation 0>"); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatistics-pretty", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + std::stringstream ss; + ppconfig ppc; + GcStatistics stats; + + std::string actual = toppstr2(ppc, stats); + std::string expected + = (",\n" + " ]\n" + " :total_allocated 0\n" + " :total_promoted_sab 0\n" + " :total_promoted 0\n" + " :n_mutation 0\n" + " :n_logged_mutation 0\n" + " :n_xgen_mutation 0\n" + " :n_xckp_mutation 0>"); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + REQUIRE(actual == expected); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("GcStatisticsExt-pretty", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + std::stringstream ss; + ppconfig ppc; + GcStatisticsExt stats; + + std::string actual = toppstr2(ppc, stats); + std::string expected + = (",\n" + " ]\n" + " :total_allocated 0\n" + " :total_promoted_sab 0\n" + " :total_promoted 0\n" + " :n_mutation 0\n" + " :n_logged_mutation 0\n" + " :n_xgen_mutation 0\n" + " :n_xckp_mutation 0\n" + " :nursery_z 0\n" + " :nursery_before_checkpoint_z 0\n" + " :nursery_after_checkpoint_z 0\n" + " :tenured_z 0>"); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + REQUIRE(actual == expected); + + tag_config::tag_color_enabled = saved; + } + } +} /*namespace xo*/ + +/* GcStatistics.test.cpp */ diff --git a/xo-alloc/utest/IAlloc.test.cpp b/xo-alloc/utest/IAlloc.test.cpp new file mode 100644 index 00000000..7c1fdc56 --- /dev/null +++ b/xo-alloc/utest/IAlloc.test.cpp @@ -0,0 +1,127 @@ +/* @file IAlloc.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +//#include "xo/allocutil/IAlloc.hpp" +#include "xo/alloc/ArenaAlloc.hpp" +#include "xo/indentlog/print/tag.hpp" +#include + +namespace xo { + using xo::gc::IAlloc; + using xo::gc::ArenaAlloc; + + namespace ut { + TEST_CASE("ialloc", "[alloc]") + { + static_assert((sizeof(std::uintptr_t) == 8) && "possibly fine if this fails, but would want to know"); + + REQUIRE(IAlloc::alloc_padding(0) == 0); + + for (std::size_t i = 1; i < sizeof(std::uintptr_t); ++i) { + REQUIRE(IAlloc::alloc_padding(i) + i == IAlloc::c_alloc_alignment); + } + + REQUIRE(IAlloc::alloc_padding(IAlloc::c_alloc_alignment) == 0); + + for (std::size_t i = 1; i < sizeof(std::uintptr_t); ++i) { + REQUIRE(IAlloc::alloc_padding(IAlloc::c_alloc_alignment + i) + i == IAlloc::c_alloc_alignment); + } + } + + /* although xo::gc::allocator is intended for + * IAlloc derivatives (so T is ArenaAlloc | GC), + * + * it only relies on allocate() and deallocate() methods + */ + + namespace { + struct TestCase { + explicit TestCase(size_t arena_z, size_t n, size_t n2) : arena_z_{arena_z}, n_{n}, n2_{n2} {} + + size_t arena_z_ = 0; + size_t n_ = 0; + size_t n2_ = 0; + }; + + std::vector s_testcase_v = { TestCase{1024*1024, 9, 13} }; + } + + TEST_CASE("gc.allocator", "[alloc]") + { + using xo::gc::allocator; + + constexpr bool c_debug_flag = false; + + for (size_t i_tc = 0, n_tc = s_testcase_v.size(); i_tc < n_tc; ++i_tc) { + INFO(xtag("i_tc", i_tc)); + + const TestCase & tc = s_testcase_v[i_tc]; + + up mm1 = ArenaAlloc::make("arena1", + tc.arena_z_, + c_debug_flag); + up mm2 = ArenaAlloc::make("arena2", + tc.arena_z_, + c_debug_flag); + + REQUIRE(mm1.get()); + REQUIRE(mm1->allocated() == 0); + + allocator alloc1(mm1.get()); + allocator alloc1a(mm1.get()); + + REQUIRE(mm2.get()); + REQUIRE(mm2->allocated() == 0); + + allocator alloc2(mm2.get()); + + SECTION("IAlloc identity determines allocator equality") { + REQUIRE(alloc1 == alloc1a); + REQUIRE(alloc1 != alloc2); + } + + int * p1 = nullptr; + size_t z1 = 0; + + SECTION("alloc space for ints") { + p1 = alloc1.allocate(tc.n_); + + REQUIRE(p1 != nullptr); + + // note: allowing for alignment + REQUIRE(mm1->allocated() >= sizeof(int32_t) * tc.n_); + REQUIRE(mm1->allocated() < sizeof(int32_t) * tc.n_ + IAlloc::c_alloc_alignment); + z1 = mm1->allocated(); + + // deallocate exists.. + alloc1.deallocate(p1, tc.n_); + + // ..but is a no-op + REQUIRE(mm1->allocated() == z1); + } + + int * p2 = nullptr; + + SECTION("allocator independence") { + REQUIRE(mm2->allocated() == 0); + + p2 = alloc2.allocate(tc.n2_); + + REQUIRE(p2 != nullptr); + REQUIRE(p1 != p2); + + REQUIRE(mm2->allocated() >= sizeof(int32_t) * tc.n2_); + REQUIRE(mm2->allocated() < sizeof(int32_t) * tc.n2_ + IAlloc::c_alloc_alignment); + + // mm1 unaffected by mm2 allocation + REQUIRE(mm1->allocated() == z1); + } + } + } + + } /*namespace ut*/ +} /*namespace xo*/ + +/* end IAlloc.test.cpp */ diff --git a/xo-alloc/utest/ListAlloc.test.cpp b/xo-alloc/utest/ListAlloc.test.cpp new file mode 100644 index 00000000..5f425568 --- /dev/null +++ b/xo-alloc/utest/ListAlloc.test.cpp @@ -0,0 +1,64 @@ +/* ListAlloc.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/ListAlloc.hpp" +#include + +namespace xo { + using xo::gc::ListAlloc; + + namespace ut { +#ifdef NOT_USING // ListAlloc probably permanently retired. Not maintaining + + TEST_CASE("ListAlloc", "[alloc][gc]") + { + /** teeny weeny allocator. + * but underlying ArenaAlloc works in multiples of VM page size + * (most likely 4k) + **/ + up alloc = ListAlloc::make("test", 16, 32, false); + + REQUIRE(alloc->name() == "test"); + REQUIRE(alloc->size() == 16); + REQUIRE(alloc->before_checkpoint() == 0); + REQUIRE(alloc->after_checkpoint() == 0); + + /* will expand */ + std::byte * mem1 = alloc->alloc(20); + + REQUIRE(mem1); + REQUIRE(alloc->size() == std::max(alloc->page_size(), alloc->hugepage_z())); + /* round up to multiple of 8 */ + REQUIRE(alloc->before_checkpoint() == 24); + REQUIRE(alloc->after_checkpoint() == 0); + + alloc->checkpoint(); + + std::byte * mem2 = alloc->alloc(30); + + REQUIRE(mem2); + REQUIRE(alloc->size() == alloc->page_size()); + REQUIRE(alloc->before_checkpoint() == 24); + /* round up to multiple of 8 */ + REQUIRE(alloc->after_checkpoint() == 32); + + std::byte * mem3 = alloc->alloc(40); + + REQUIRE(mem3); + REQUIRE(alloc->size() == alloc->page_size()); + REQUIRE(alloc->before_checkpoint() == 24); + /* already multiple of 8 */ + REQUIRE(alloc->after_checkpoint() == 32 + 40); + + REQUIRE(alloc->is_before_checkpoint(mem1) == true); + REQUIRE(alloc->is_before_checkpoint(mem2) == false); + REQUIRE(alloc->is_before_checkpoint(mem3) == false); + } +#endif + + } /*namespace ut*/ +} /*namespace xo*/ + +/* ListAlloc.test.cpp */ diff --git a/xo-alloc/utest/ObjectStatistics.test.cpp b/xo-alloc/utest/ObjectStatistics.test.cpp new file mode 100644 index 00000000..7530450d --- /dev/null +++ b/xo-alloc/utest/ObjectStatistics.test.cpp @@ -0,0 +1,185 @@ +/* @file ObjectStatistics.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/ObjectStatistics.hpp" +#include "xo/reflect/Reflect.hpp" +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/ppstr.hpp" +#include "xo/indentlog/print/tostr.hpp" +#include "xo/indentlog/print/hex.hpp" +#include + +namespace xo { + using xo::gc::ObjectStatistics; + using xo::gc::PerObjectTypeStatistics; + using xo::reflect::Reflect; + using xo::print::ppconfig; + + namespace ut { + TEST_CASE("PerObjectTypeStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerObjectTypeStatistics stats; + + std::string s = tostr(stats); + + //std::cerr << hex_view(s.c_str(), s.c_str() + s.length(), true /*as_text*/) << std::endl; + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("PerObjectTypeStatistics-1", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerObjectTypeStatistics stats; + stats.td_ = Reflect::require(); + stats.scanned_n_ = 4; + stats.scanned_z_ = 16; + stats.survive_n_ = 2; + stats.survive_z_ = 8; + + std::string s = tostr(stats); + + //std::cerr << hex_view(s.c_str(), s.c_str() + s.length(), true /*as_text*/) << std::endl; + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + ObjectStatistics stats; + + std::string s = tostr(stats); + + REQUIRE(s == ""); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics-1", "[alloc][gc]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + ObjectStatistics stats; + stats.per_type_stats_v_.push_back(PerObjectTypeStatistics()); + + std::string s = tostr(stats); + + REQUIRE(s == ">"); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics-pretty", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + std::stringstream ss; + ppconfig ppc; + ObjectStatistics stats; + + std::string actual = toppstr2(ppc, stats); + std::string expected + = (""); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + REQUIRE(actual == expected); + + tag_config::tag_color_enabled = saved; + } + + TEST_CASE("ObjectStatistics-pretty-1", "[alloc][gc][pretty]") + { + bool saved = tag_config::tag_color_enabled; + + tag_config::tag_color_enabled = false; + + PerObjectTypeStatistics objstats; + objstats.td_ = Reflect::require(); + objstats.scanned_n_ = 4; + objstats.scanned_z_ = 16; + objstats.survive_n_ = 2; + objstats.survive_z_ = 8; + + std::stringstream ss; + ppconfig ppc; + ObjectStatistics stats; + stats.per_type_stats_v_.push_back(objstats); + + std::string actual = toppstr2(ppc, stats); + + std::string expected + = (" ]>"); + + if (actual != expected) { + CHECK(actual == expected); + CHECK(actual.length() == expected.length()); + + auto a_ix = actual.begin(); + auto e_ix = expected.begin(); + + std::size_t pos = 0; + while (a_ix != actual.end() && e_ix != expected.end()) { + INFO(xtag("pos", pos)); + INFO(xtag("matching_prefix", std::string(actual.c_str(), pos))); + + REQUIRE(*a_ix == *e_ix); + + ++a_ix; + ++e_ix; + ++pos; + } + } + + tag_config::tag_color_enabled = saved; + } + } +} /*namespace xo*/ + +/* ObjectStatistics.test.cpp */ diff --git a/xo-alloc/utest/alloc_utest_main.cpp b/xo-alloc/utest/alloc_utest_main.cpp new file mode 100644 index 00000000..fa384613 --- /dev/null +++ b/xo-alloc/utest/alloc_utest_main.cpp @@ -0,0 +1,6 @@ +/* file alloc_utest_main.cpp */ + +#define CATCH_CONFIG_MAIN +#include "catch2/catch.hpp" + +/* end alloc_utest_main.cpp */ diff --git a/xo-alloc/utest/generation.test.cpp b/xo-alloc/utest/generation.test.cpp new file mode 100644 index 00000000..edbebffa --- /dev/null +++ b/xo-alloc/utest/generation.test.cpp @@ -0,0 +1,39 @@ +/* generation.test.cpp + * + * author: Roland Conybeare, Aug 2025 + */ + +#include "xo/alloc/generation.hpp" +#include +#include +#include + +namespace xo { + namespace gc { + TEST_CASE("generation", "[gc]") { + REQUIRE(::strcmp(gen2str(generation::nursery), "nursery") == 0); + REQUIRE(::strcmp(gen2str(generation::tenured), "tenured") == 0); + REQUIRE(::strcmp(gen2str(generation::N), "?generation") == 0); + + { + std::stringstream ss; + ss << generation::nursery; + REQUIRE(ss.str() == "nursery"); + } + + { + std::stringstream ss; + ss << generation::tenured; + REQUIRE(ss.str() == "tenured"); + } + + { + std::stringstream ss; + ss << generation::N; + REQUIRE(ss.str() == "?generation"); + } + } + } /*namespace gc*/ +} /*namespace xo*/ + +/* generation.test.cpp */