xo-arena: + docs

CMakeLists.txt

@@ -132,6 +132,20 @@ add_subdirectory(xo-imgui)
 # ----------------------------------------------------------------
 # documentation.  must follow add_subdirectory() for satellite projects
 
-xo_umbrella_doxygen_deps(xo_facet xo_alloc2 indentlog xo_flatstring xo_ratio xo_unit xo_tokenizer xo_reader xo_interpreter xo_jit)
+xo_umbrella_doxygen_deps(
+    xo_arena
+    xo_facet
+    xo_alloc2
+    indentlog
+    xo_flatstring
+    xo_ratio
+    xo_unit
+    xo_tokenizer
+    xo_reader
+    xo_interpreter
+    xo_jit)
 xo_umbrella_doxygen_config()
-xo_umbrella_sphinx_config(index.rst docs/install.rst docs/glossary.rst)
+xo_umbrella_sphinx_config(
+    index.rst
+    docs/install.rst
+    docs/glossary.rst)

index.rst

@@ -12,6 +12,7 @@ Some features: kalman filters, stochastic processes, complex event processing, s
    :caption: XO contents
 
    docs/install
+   xo-arena/docs/index
    xo-facet/docs/index
    xo-alloc2/docs/index
    xo-alloc/docs/index

xo-arena/docs/ArenaConfig-reference.rst

@@ -0,0 +1,55 @@
+.. _ArenaConfig-reference:
+
+ArenaConfig Reference
+=====================
+
+Configuration for an arena allocator
+
+Context
+-------
+
+.. ditaa::
+    :--scale: 0.99
+
+    +-----------------------------------------------------+
+    |                       DArena                        |
+    |                   DArenaIterator                    |
+    +-----------------------------------------------------+
+    |                     ArenaConfig                 cBLU|
+    +--------------+------------------------+-------------+
+    |              |        AllocInfo       |             |
+    |              +------------------------+             |
+    | AllocError   |  AllocHeaderConfig     |  cmpresult  |
+    |              +------------------------+             |
+    |              |      AllocHeader       |             |
+    +--------------+------------------------+-------------+
+
+
+.. uml::
+    :caption: example arena config
+    :scale: 99%
+    :align: center
+
+    object cfg<<AreanConfig>>
+    cfg : name = "tmp"
+    cfg : size = 128MB
+    cfg : hugepage_z = 2MB
+    cfg : guard_z = 8
+    cfg : guard_byte = 0xfd
+    cfg : store_header_flag = true
+    cfg : header_size_mask = 0xffffffff
+    cfg : debug_flag = false
+
+.. code-block:: cpp
+
+    #include <xo/arena/ArenaConfig.hpp>
+
+Class
+-----
+
+.. doxygenclass:: xo::mm::ArenaConfig
+
+Instance Variables
+------------------
+
+.. doxygengroup:: mm-arenaconfig-instance-vars

xo-arena/docs/CMakeLists.txt

@@ -0,0 +1,22 @@
+# xo-arena/docs/CMakeLists.txt
+
+xo_doxygen_collect_deps()
+xo_docdir_doxygen_config()
+xo_docdir_sphinx_config(
+    index.rst
+    glossary.rst
+    examples.rst
+    implementation.rst
+    #AAllocator-reference.rst
+    #IAllocator_Xfer-reference.rst
+    #AAllocIterator-reference.rst
+    ArenaConfig-reference.rst
+    DArena-reference.rst
+    AllocInfo-reference.rst
+    cmpresult-reference.rst
+    #install.rst
+    #introduction.rst
+)
+
+# see xo-reader/doc or xo-unit/doc for working examples
+# example.rst install.rst implementation.rst

xo-arena/docs/DArena-reference.rst

@@ -0,0 +1,96 @@
+.. _DArena-reference:
+
+DArena
+======
+
+Native arena allocator
+
+Context
+-------
+
+.. ditaa::
+    :--scale: 0.99
+
+    +-----------------------------------------------------+
+    |                       DArena                    cBLU|
+    |                   DArenaIterator                    |
+    +-----------------------------------------------------+
+    |                     ArenaConfig                     |
+    +--------------+------------------------+-------------+
+    |              |        AllocInfo       |             |
+    |              +------------------------+             |
+    | AllocError   |  AllocHeaderConfig     |  cmpresult  |
+    |              +------------------------+             |
+    |              |      AllocHeader       |             |
+    +--------------+------------------------+-------------+
+
+.. code-block:: cpp
+
+    #include <xo/arena/DArena.hpp>
+
+Arena memory layout
+~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: text
+
+      <------------------------reserved-------------------------->
+      <------------committed-----------><-------uncommitted------>
+      <--allocated--><----available---->
+
+      XXXXXXXXXXXXXXX___________________..........................
+     ^               ^                  ^                         ^
+     lo              free               limit                     hi
+
+      [X] allocated:   in use
+      [_] committed:   physical memory obtained
+      [.] uncommitted: mapped in virtual memory, not backed by memory
+
+
+Representation for a single allocation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: text
+
+                     free_(pre)
+                         v
+
+                         <-------------z1--------------->
+                < guard ><  hz  ><     req_z     >< dz  >< guard >
+
+      used <==  +++++++++0000zzzz@@@@@@@@@@@@@@@@@ppppppp+++++++++ ==> avail
+
+                         ^       ^                                ^
+                         header  mem                              |
+                         ^                                        |
+                         last_header_                   free_(post)
+
+                 [+] guard surrounding each allocation, for simple sanitize checks
+                 [0] unused header bits (available for application metadata)
+                 [z] record allocation size
+                 [@] new allocated memory
+                 [p] padding (to uintptr_t alignment)
+
+Class
+-----
+
+.. doxygenclass:: xo::mm::DArena
+
+Member Variables
+----------------
+
+.. doxygengroup:: mm-arena-instance-vars
+
+Type Traits
+-----------
+
+.. doxygengroup:: mm-arena-traits
+
+Constructors
+------------
+
+.. doxygengroup:: mm-arena-ctors
+
+Methods
+-------
+
+.. doxygengroup:: mm-arena-methods

xo-arena/docs/DArenaIterator-reference.rst

@@ -0,0 +1,50 @@
+.. _DArenaIterator-reference:
+
+DArenaIterator
+==============
+
+Iterator for allocs obtained from a :cpp:class:`xo::mm::DArena`.
+
+Context
+-------
+
+.. ditaa::
+    :--scale: 0.99
+
+    +-----------------------------------------------------+
+    |                       DArena                        |
+    |                   DArenaIterator                cBLU|
+    +-----------------------------------------------------+
+    |                     ArenaConfig                     |
+    +--------------+------------------------+-------------+
+    |              |        AllocInfo       |             |
+    |              +------------------------+             |
+    | AllocError   |  AllocHeaderConfig     |  cmpresult  |
+    |              +------------------------+             |
+    |              |      AllocHeader       |             |
+    +--------------+------------------------+-------------+
+
+
+.. code-block:: cpp
+
+    #include <xo/arena/DArenaIterator.hpp>
+
+Class
+-----
+
+.. doxygenclass:: xo::mm::DArenaIterator
+
+Member Variables
+----------------
+
+.. doxygengroup:: mm-arenaiterator-instance-vars
+
+Constructors
+------------
+
+.. doxygengroup:: mm-arenaiterator-ctors
+
+Methods
+-------
+
+.. doxygengroup:: mm-arenaiterator-methods

xo-arena/docs/README

@@ -0,0 +1,72 @@
+build
+
+        +-----------------------------------------------+
+        | cmake                                         |
+        |   CMakeLists.txt                              |
+        |   $PREFIX/share/cmake/xo_macros/xo_cxx.cmake  |
+        +-----------------------------------------------+
+           |
+           |                                                  +----------------------+
+           +------------------------------------------------->| .build/docs/Doxyfile |
+           |                                                  +----------------------+
+           |                                                     |
+           |                                        /------------/
+           |                                        |
+           |                                        v
+           |     +---------------------------------------+    +-----------------+
+           +---->| doxygen                               |--->| .build/docs/dox |
+           |     |   $PREFIX/share/xo-macros/Doxyfile.in |    |   +- html/      |
+           |     +---------------------------------------+    |   +- xml/       |
+           |                                                  +-----------------+
+           |                                                     |
+           |                                        /------------/
+           |                                        |
+           |                                        v
+           |     +---------------------------------------+    +--------------------+
+           \---->| sphinx                                |--->| .build/docs/sphinx |
+                 |   +- conf.py                          |    |   +- html/         |
+                 |   +- _static/                         |    +--------------------+
+                 |   +- *.rst                            |
+                 +---------------------------------------+
+
+files
+
+   README            this file
+   CMakeLists.txt    build entry point
+   conf.py           sphinx config
+   _static           static files for sphinx
+
+map
+
+   index.rst
+   +- examples.rst
+   +- ArenaConfig-reference.rst
+   +- DArena-reference.rst
+   +- DArenaIterator-reference.rst
+   +- AllocInfo-reference.rst
+   +- cmpresult-reference.rst
+   +- glossary.rst
+   ...
+
+examples
+
+.. doxygenclass:: ${c++ class name}
+   :project:
+   :path:
+   :members:
+   :protected-members:
+   :private-members:
+   :undoc-members:
+   :member-groups:
+   :members-only:
+   :outline:
+   :no-link:
+   :allow-dot-graphs:
+
+.. doxygendefine:: ${c preprocessor define}
+
+.. doxygenconcept:: ${c++ concept definition}
+
+.. doxygenenum:: ${c++ enum definition}
+
+.. doxygenfunction:: ${c++ function name}

xo-arena/docs/_static/README

@@ -0,0 +1 @@
+add any static {.html, .js, ..} files for sphinx to pickup here

xo-arena/docs/cmpresult-reference.rst

@@ -0,0 +1,50 @@
+.. _cmpresult-reference:
+
+cmpresult
+=========
+
+Represent the result of a partially ordered comparison
+
+Context
+-------
+
+.. ditaa::
+    :--scale: 0.99
+
+    +-----------------------------------------------------+
+    |                       DArena                        |
+    |                   DArenaIterator                    |
+    +-----------------------------------------------------+
+    |                     ArenaConfig                     |
+    +--------------+------------------------+-------------+
+    |              |        AllocInfo       |         cBLU|
+    |              +------------------------+             |
+    | AllocError   |  AllocHeaderConfig     |  cmpresult  |
+    |              +------------------------+             |
+    |              |      AllocHeader       |             |
+    +--------------+------------------------+-------------+
+
+
+.. code-block:: cpp
+
+    #include <xo/arena/cmpresult.hpp>
+
+Class
+-----
+
+.. doxygenclass:: xo::mm::cmpresult
+
+Constructors
+------------
+
+.. doxgyengroup:: mm-cmpresult-ctors
+
+Methods
+-------
+
+.. doxygengroup:: mm-cmpresult-methods
+
+Member Variables
+----------------
+
+.. doxygengroup:: mm-cmpresult-instance-vars

xo-arena/docs/conf.py

@@ -0,0 +1,39 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'xo arena documentation'
+copyright = '2025, Roland Conybeare'
+author = 'Roland Conybeare'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+#extensions = []
+extensions = [ "breathe",
+               "sphinx.ext.mathjax",     # inline math
+               "sphinx.ext.autodoc",     # generate info from docstrings
+               "sphinxcontrib.ditaa",    # diagrams-through-ascii-art
+               "sphinxcontrib.plantuml"  # text -> uml diagrams
+              ]
+
+# note: breathe requires doxygen xml output -> must have GENERATE_XML = YES in Doxyfile.in
+#       match project name in Doxyfile.in
+breathe_default_project = "xodoxxml"
+
+templates_path = ['_templates']
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+pygments_style = 'sphinx'
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+#html_theme = 'alabaster'
+html_theme = 'sphinx_rtd_theme'
+html_static_path = ['_static']
+html_favicon = '_static/img/favicon.ico'

xo-arena/docs/examples.rst

@@ -0,0 +1,125 @@
+.. _examples:
+
+.. toctree
+    :maxdepth: 2
+
+Examples
+========
+
+Arena allocation
+-----------------
+
+.. code-block:: cpp
+
+    #include <xo/alloc2/arena/DArena.hpp>
+
+    using namespace xo::mm;
+    using namespace std;
+
+
+Create an arena:
+
+.. code-block:: cpp
+
+    // create arena, size 64k
+    DArena arena = DArena::map(ArenaConfig { .size_ = 64*1024; });
+
+    cout << arena.lo() << ".." << arena.hi();
+
+This determines a VM memory address range.
+Actually address range is rounded up to a whole number of VM pages.
+Size here is a hard maximum. It cannot be changed for this arena instance.
+
+.. code-block:: cpp
+
+    arena.reserved();  // 64k
+    arena.committed(); // 0k
+    arena.allocated(); // ok
+    arena.available(); // 0k
+
+Although we know the address range for arena, it doesn't own any physical
+memory yet.  Two ways to commit memory:
+
+1. Attempt allocation:
+
+   .. code-block:: cpp
+
+       std::byte * mem = arena.alloc(5*1024);
+       if (!mem)
+           throw std::runtime_error("alloc failed");
+
+       arena.reserved();   // 64k
+       arena.committed();  // 8k - 2 pages
+       arena.allocateed(); // 5k
+       arena.available();  // 3k
+
+2. Expand committed memory explicitly:
+
+   .. code-block:: cpp
+
+       bool ok = arena.expand(5*1024);
+       assert(ok);
+
+       arena.reserved();  // 64k
+       arena.committed(); // 8k - 2 pages
+       arena.allocated(); // 0k
+       arena.available(); // 8k
+
+Examining alloc metadata
+------------------------
+
+Given a successful allocation:
+
+.. code-block:: cpp
+
+    std::size_t req_z = 5*1024;
+    std::byte * mem = arena.alloc(req_z);
+    if (!mem)
+        throw std::runtime_error("alloc failed");
+
+    AllocInfo info = arena.alloc_info(mem);
+
+    info.payload();  // [mem, mem + req_z (+ up to 7 bytes padding)]
+    info.is_valid(); // true
+    info.guard_lo(); // guard bytes preceding alloc
+    info.guard_hi(); // guard bytes following alloc
+
+Can alternatively scan all live allocs in arena:
+
+.. code-block:: cpp
+
+    for (AllocInfo info : arena) {
+        info.payload();  // allocated memory range
+        info.is_valid(); // true
+        info.guard_lo(); // guard bytes preceding alloc
+        info.guard_hi(); // guard bytes following alloc
+    }
+
+Recycling memory
+----------------
+
+.. code-block:: cpp
+
+    // arena in non-empty state
+    arena.reserved();   // 64k
+    arena.committed();  // 8k - 2 pages
+    arena.allocateed(); // 5k
+    arena.available();  // 3k
+
+    arena.clear();
+
+    arena.reserved()    // 64k
+    arena.committed();  // 8k - 2 pages
+    arena.allocated();  // 0k
+    arena.available();  // 8k
+
+Memory recycled by :cpp:func:`DArena::clear()`
+is available for reuse by application; it's still owned by arena.
+We're just resetting the free pointer back to the beginning of arena
+memory.
+
+To release memory to the operating system, destroy arena:
+
+.. code-block:: cpp
+
+    arena.~DArena();  // or just let arena go out of scope

xo-arena/docs/glossary.rst

@@ -0,0 +1,24 @@
+.. _glossary:
+
+Glossary
+--------
+
+.. glossary::
+    FOMO
+     | faceted object model
+
+    page
+     | a (4k) page of virtual memory.
+     | O/S manages virtual memory in chunks of this size.
+
+    hugepage
+     | large (2MB) VM page; use to reduce page fault expense and TLB pressure.
+
+    THP
+     | transparent huge pages
+
+    TLB
+     | translation lookaside buffer
+
+    VM
+     | virtual memory

xo-arena/docs/implementation.rst

@@ -0,0 +1,153 @@
+.. _implementation:
+
+Implementation
+==============
+
+Library dependency tower for *xo-arena*
+
+.. ditaa::
+
+    +------------------------------------+
+    |             xo_arena               |
+    +-----------------+------------------+
+    |   xo_indentlog  |  xo_reflectutil  |
+    +-----------------+------------------+
+    |             xo_cmake               |
+    +------------------------------------+
+
+Abstraction tower for *xo-arena* components (simplified)
+
+.. ditaa::
+    :--scale: 0.99
+
+    +-------------------+
+    |   DArena          |
+    |   DArenaIterator  |
+    +-------------------+
+    |   ArenaConfig     |
+    +-------------------+
+    |  auxiliary types  |
+    +-------------------+
+
+
+Abstraction tower for *xo-arena* components (detailed)
+
+.. ditaa::
+    :--scale: 0.99
+
+    +-----------------------------------------------------+
+    |                       DArena                        |
+    |                   DArenaIterator                    |
+    +-----------------------------------------------------+
+    |                     ArenaConfig                     |
+    +--------------+------------------------+-------------+
+    |              |        AllocInfo       |             |
+    |              +------------------------+             |
+    | AllocError   |  AllocHeaderConfig     |  cmpresult  |
+    |              +------------------------+             |
+    |              |      AllocHeader       |             |
+    +--------------+------------------------+-------------+
+
+.. list-table:: Native Arena Allocator
+    :header-rows: 1
+    :widths: 20 90
+
+    * - Class
+      - Description
+    * - ``ArenaConfig``
+      - Configuration for a ``DArena`` instance
+    * - ``DArena``
+      - VM-aware arena allocator
+    * - ``DArenaIterator``
+      - Iterator over ``DArena`` allocations
+
+.. list-table:: Auxiliary/Support Types
+    :header-rows: 1
+    :widths: 20 90
+
+    * - Class
+      - Description
+    * - ``AllocError``
+      - Return type for an alloc request, with error details.
+    * - ``AllocInfo``
+      - An opaque allocation. Value of an alloc-iterator.
+    * - ``AllocHeaderConfig``
+      - Per-allocator configuration of alloc headers
+    * - ``AllocHeader``
+      - Per-allocation header (8 bytes)
+    * - ``cmpresult``
+      - Result of alloc-iterator comparison
+
+Example Object Diagram
+
+.. uml::
+    :caption: representation for an arena allocator
+    :scale: 99%
+    :align: center
+
+    object darena1<<DArena>>
+    darena1 : config
+    darena1 : lo
+    darena1 : hi
+    darena1 : free
+    darena1 : limit
+    darena1 : last_error
+
+    object header1<<ArenaConfig>>
+    header1 : size
+    header1 : header
+
+    object hconfig1<<AllocHeaderConfig>>
+    hconfig1 : guard_z
+    hconfig1 : guard_byte
+    hconfig1 : tseq_bits
+    hconfig1 : age_bits
+    hconfig1 : size_bits
+
+    darena1 o-- header1
+    header1 o-- hconfig1
+
+.. uml::
+    :caption: memory layout
+    :scale: 99%
+    :align: center
+
+    object darena1<<DArena>>
+    darena1 : config
+    darena1 : lo
+    darena1 : hi
+    darena1 : free
+    darena1 : limit
+    darena1 : last_error
+
+    rectangle "allocated" #90EE90 {
+      note as n1
+        lo -> free
+        objects here
+      end note
+    }
+
+    rectangle "available" #FFFFE0 {
+      note as n2
+        free -> limit
+        alloc from here
+      end note
+    }
+
+    rectangle "uncommitted" #D3D3D3 {
+      note as n3
+        limit -> hi
+        not mapped yet
+      end note
+    }
+
+    darena1 -[hidden]down- n1
+    n1 -[hidden]down- n2
+    n2 -[hidden]down- n3
+
+Remarks:
+
+* See xo-alloc2 for abstract allocator trait *AAllocator*
+  along with its application to *DArena*.
+* We split these because in *xo-facet* we rely on *DArena* to implement
+  double-dispatch (two-dimensional vtables, as seen for example in CLOS, Julia, Mathematica).

xo-arena/docs/index.rst

@@ -0,0 +1,31 @@
+# xo-arena documentation master file
+
+xo-arena documentation
+======================
+
+xo-arena provides:
+
+* Fast vm-aware arena allocation.
+* Allocates uncommitted virtual memory, and commits on demand.
+* When available, uses THP (Transparent Huge Pages) to mitigate pagetable pressure.
+* Optional GC support, with per-alloc header.
+
+Diagnostic features:
+
+* with alloc headers: forward iterators over individual allocations
+* configurable guard memory between allocations.
+
+.. toctree::
+    :maxdepth: 2
+    :caption: xo-arena contents
+
+    examples
+    implementation
+    ArenaConfig-reference
+    DArena-reference
+    DArenaIterator-reference
+    AllocInfo-reference
+    cmpresult-reference
+    glossary
+    genindex
+    search