From 289751d3fdaebd2e8b7b3ced8934b7eae35a4f4f Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 23 Dec 2025 21:06:38 -0500 Subject: [PATCH] xo-alloc2: ++ documentation + threshold size for THP feature --- docs/AAllocator-reference.rst | 63 +++++----- docs/AllocInfo-reference.rst | 61 ++++++++++ docs/CMakeLists.txt | 2 + docs/DArena-reference.rst | 48 +++++--- docs/examples.rst | 76 +++++++++++- docs/implementation.rst | 159 +++++++++++++++++++++---- docs/index.rst | 1 + include/xo/alloc2/AllocError.hpp | 6 +- include/xo/alloc2/AllocInfo.hpp | 20 ++++ include/xo/alloc2/alloc/AAllocator.hpp | 57 +++++---- include/xo/alloc2/arena/DArena.hpp | 52 +++++++- include/xo/alloc2/print.hpp | 34 ++++++ src/DArena.cpp | 0 src/alloc2/AllocError.cpp | 45 +++++++ src/alloc2/CMakeLists.txt | 1 + src/alloc2/DArena.cpp | 129 ++++++++++---------- utest/arena.test.cpp | 76 +++++++++--- utest/random_allocs.cpp | 2 +- 18 files changed, 641 insertions(+), 191 deletions(-) create mode 100644 docs/AllocInfo-reference.rst create mode 100644 include/xo/alloc2/print.hpp delete mode 100644 src/DArena.cpp create mode 100644 src/alloc2/AllocError.cpp diff --git a/docs/AAllocator-reference.rst b/docs/AAllocator-reference.rst index 7ae4ea8..1191d6b 100644 --- a/docs/AAllocator-reference.rst +++ b/docs/AAllocator-reference.rst @@ -3,8 +3,15 @@ AAllocator Reference ==================== -Abstract interface facet for arena allocator. -Provides simple arena allocation. +Abstract interface facet for an allocator. + +Base class for runtime polymorphism over allocator implementations, +using faceted object model. + +* runtime size consists of vtable pointer only. + +* per FOMO prinicples, runtime state is stored separately. + Classes that inherit ``AAllocator`` will not add state Context ------- @@ -12,40 +19,24 @@ Context .. ditaa:: :--scale: 0.99 - +-------------------------------+--------------------------------------+ - | | IAllocator_DX1Collector | - | | IAllocIterator_DX1CollectorIterator | - | IAllocator_DArena | | - | IAllocIterator_DArenaIterator +--------------------------------------+ - | | DX1Collector | - | | DX1CollectorIterator | - | | | - +-------------------------------+---------+----------------------------+ - | DArena | | - | DArenaIterator | | - +-----------------------------------------+ CollectorConfig | - | ArenaConfig | | - +-----------------------------------------+----------------------------+ - - +----------------------------------+-----------------------------------+ - | RAllocator | RAllocIterator | - +----------------------------------+-----------------------------------+ - | IAllocator_DX1Collector | IAllocIterator_DX1Collector | - | IAllocator_DArena | IAllocIterator_DArena | - +----------------------------------+-----------------------------------+ - | IAllocator_Xfer | IAllocIterator_Xfer | - | IAllocator_Any | IAllocIterator_Any | - +----------------------------------+-----------------------------------+ - | AAllocator | AAllocIterator | - +----------------------------------+-----------------------------------+ - - +---------------+------------------+----------------------+------------+ - | | | AllocInfo | | - | generation | +----------------------+ | - | object_age | AllocError | AllocHeaderConfig | cmpresult | - | role | +----------------------+ | - | | | AllocHeader | | - +---------------+------------------+----------------------+------------+ + +----------------------+-------------------------+-----------------------------------+ + | RAllocator | RAllocIterator | IAllocator_DArena | + | | | IAllocIterator_DArenaIterator | + +----------------------+-------------------------+-----------------------------------+ + | IAllocator_Xfer | IAllocIterator_Xfer | DArena | + | IAllocator_Any | IAllocIterator_Any | DArenaIterator | + | IAllocator_Impltype | IAllocIterator_Impltype | | + | | | | + +----------------------+-------------------------+-----------------------------------+ + |cBLU AAllocator | AAllocIterator | ArenaConfig | + +----------------------+-------------------------+-----------------------------------+ + +-----------------+----------------------------------------------+-------------------+ + | | AllocInfo | | + | +----------------------------------------------+ | + | AllocError | AllocHeaderConfig | cmpresult | + | +----------------------------------------------+ | + | | AllocHeader | | + +-----------------+----------------------------------------------+-------------------+ .. code-block:: cpp diff --git a/docs/AllocInfo-reference.rst b/docs/AllocInfo-reference.rst new file mode 100644 index 0000000..417a997 --- /dev/null +++ b/docs/AllocInfo-reference.rst @@ -0,0 +1,61 @@ +.. _AllocInfo-reference: + +AllocInfo Reference +=================== + +Information, including alloc metadata, pertaining to a particular allocation. + +Context +------- + +.. ditaa:: + :--scale: 0.99 + + +----------------------+-------------------------+-----------------------------------+ + | RAllocator | RAllocIterator | IAllocator_DArena | + | | | IAllocIterator_DArenaIterator | + +----------------------+-------------------------+-----------------------------------+ + | IAllocator_Xfer | IAllocIterator_Xfer | DArena | + | IAllocator_Any | IAllocIterator_Any | DArenaIterator | + | IAllocator_Impltype | IAllocIterator_Impltype | | + | | | | + +----------------------+-------------------------+-----------------------------------+ + | AAllocator | AAllocIterator | ArenaConfig cBLU | + +----------------------+-------------------------+-----------------------------------+ + +-----------------+----------------------------------------------+-------------------+ + | | AllocInfo | | + | +----------------------------------------------+ | + | AllocError | AllocHeaderConfig | cmpresult | + | +----------------------------------------------+ | + | | AllocHeader | | + +-----------------+----------------------------------------------+-------------------+ + +.. code-block:: cpp + + #include + + +Class +----- + +.. doxygenclass:: xo::mm::AllocInfo + +Member Variables +---------------- + +.. doxygengroup:: mm-allocinfo-instance-vars + +Type Traits +----------- + +.. doxygengroup:: mm-allocinfo-traits + +Constructors +------------ + +.. doxygengroup:: mm-allocinfo-ctors + +Methods +------- + +.. doxygengroup:: mm-allocinfo-methods diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt index aefd71f..d5d24f2 100644 --- a/docs/CMakeLists.txt +++ b/docs/CMakeLists.txt @@ -5,10 +5,12 @@ xo_docdir_doxygen_config() xo_docdir_sphinx_config( index.rst glossary.rst + examples.rst implementation.rst AAllocator-reference.rst ArenaConfig-reference.rst DArena-reference.rst + AllocInfo-reference.rst #install.rst #introduction.rst #implementation.rst diff --git a/docs/DArena-reference.rst b/docs/DArena-reference.rst index 8222f92..35d65fe 100644 --- a/docs/DArena-reference.rst +++ b/docs/DArena-reference.rst @@ -11,17 +11,24 @@ Context .. ditaa:: :--scale: 0.99 - +--------------------------------+ - | IAllocator_DArena | - +--------------------------------+ - | IAllocator_Xfer | - +--------------------------------+ - | IAllocator_ImplType | - +--------------+-----------------+ - | | DArena cBLU| - | AAllocator +-----------------+ - | | ArenaConfig | - +--------------+-----------------+ + +----------------------+-------------------------+-----------------------------------+ + | RAllocator | RAllocIterator | IAllocator_DArena | + | | | IAllocIterator_DArenaIterator | + +----------------------+-------------------------+-----------------------------------+ + | IAllocator_Xfer | IAllocIterator_Xfer | DArena cBLU | + | IAllocator_Any | IAllocIterator_Any +-----------------------------------+ + | IAllocator_Impltype | IAllocIterator_Impltype | DArenaIterator | + | | | | + +----------------------+-------------------------+-----------------------------------+ + | AAllocator | AAllocIterator | ArenaConfig | + +----------------------+-------------------------+-----------------------------------+ + +-----------------+----------------------------------------------+-------------------+ + | | AllocInfo | | + | +----------------------------------------------+ | + | AllocError | AllocHeaderConfig | cmpresult | + | +----------------------------------------------+ | + | | AllocHeader | | + +-----------------+----------------------------------------------+-------------------+ .. code-block:: cpp @@ -29,11 +36,13 @@ Context Arena memory layout +~~~~~~~~~~~~~~~~~~~ + .. code-block:: text - <----------------------------size--------------------------> + <------------------------reserved--------------------------> <------------committed-----------><-------uncommitted------> - <--allocated--> + <--allocated--><----available----> XXXXXXXXXXXXXXX___________________.......................... ^ ^ ^ ^ @@ -44,7 +53,9 @@ Arena memory layout [.] uncommitted: mapped in virtual memory, not backed by memory -Allocation layout +Representation for a single allocation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + .. code-block:: text free_(pre) @@ -60,8 +71,8 @@ Allocation layout ^ | last_header_ free_(post) - [+] guard after each allocation, for simple sanitize checks - [0] unused header bits (avail to application) + [+] guard surrounding each allocation, for simple sanitize checks + [0] unused header bits (available for application metadata) [z] record allocation size [@] new allocated memory [p] padding (to uintptr_t alignment) @@ -85,3 +96,8 @@ Constructors ------------ .. doxygengroup:: mm-arena-ctors + +Methods +------- + +.. doxygengroup:: mm-arena-methods diff --git a/docs/examples.rst b/docs/examples.rst index a60bebd..89c4939 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -32,9 +32,81 @@ Size here is a hard maximum. It cannot be changed for this arena instance. .. code-block:: cpp - arena.reserved(); // 64k + arena.reserved(); // 64k arena.committed(); // 0k + arena.allocated(); // ok arena.available(); // 0k Although we know the address range for arena, it doesn't own any physical -memory yet. +memory yet. Two ways to commit memory: + +1. Attempt allocation: + + .. code-block:: cpp + + std::byte * mem = arena.alloc(5*1024); + if (!mem) + throw std::runtime_error("alloc failed"); + + arena.reserved(); // 64k + arena.committed(); // 8k - 2 pages + arena.allocateed(); // 5k + arena.available(); // 3k + +2. Expand committed memory explicitly: + + .. code-block:: cpp + + bool ok = arena.expand(5*1024); + assert(ok); + + arena.reserved(); // 64k + arena.committed(); // 8k - 2 pages + arena.allocated(); // 0k + arena.available(); // 8k + +Examining alloc metadata +------------------------ + +Given a successful allocation: + +.. code-block:: cpp + + std::size_t req_z = 5*1024; + std::byte * mem = arena.alloc(req_z); + if (!mem) + throw std::runtime_error("alloc failed"); + + AllocInfo info = arena.alloc_info(mem); + + info.payload(); // [mem, mem + req_z (+ up to 7 bytes padding)] + info.is_valid(); // true + info.guard_lo(); // guard bytes preceding alloc + info.guard_hi(); // guard bytes following alloc + +Recycling memory +---------------- + +.. code-block:: cpp + + // arena in non-empty state + arena.reserved(); // 64k + arena.committed(); // 8k - 2 pages + arena.allocateed(); // 5k + arena.available(); // 3k + + arena.clear(); + + arena.reserved() // 64k + arena.committed(); // 8k - 2 pages + arena.allocated(); // 0k + arena.available(); // 8k + +Memory released by @ref Darena::clear is still committed. +It's in use as far as operating system is concerned. + +To release memory to the operating system, destroy arena: + +.. code-block:: cpp + + arena.~DArena(); // or just let arena go out of scope diff --git a/docs/implementation.rst b/docs/implementation.rst index deb79e8..0ef021b 100644 --- a/docs/implementation.rst +++ b/docs/implementation.rst @@ -1,7 +1,7 @@ .. _implementation: -Components -========== +Implementation +============== Library dependency tower for *xo-alloc2* @@ -15,36 +15,147 @@ Library dependency tower for *xo-alloc2* | xo_cmake | +-----------------+ -Abstraction tower for *xo-alloc2* components +Abstraction tower for *xo-alloc2* components (simplified) .. ditaa:: :--scale: 0.99 - +--------------------------------+ - | IAllocator_DArena | - +--------------------------------+ - | IAllocator_Xfer | - | IAllocator_Any | - +--------------+-----------------+ - | | DArena | - | AAllocator +-----------------+ - | | ArenaConfig | - +--------------+-----------------+ + +----------------+-----------------+-------------------+ + | | | DArena | + | Allocator | AllocIterator | DArenaIterator | + | | +-------------------+ + | | | ArenaConfig | + +----------------+-----------------+-------------------+ + | auxiliary types | + +------------------------------------------------------+ -.. list-table:: Descriptions + +Abstraction tower for *xo-alloc2* components (detailed) + +.. ditaa:: + :--scale: 0.99 + + +----------------------+-------------------------+-----------------------------------+ + | RAllocator | RAllocIterator | IAllocator_DArena | + | | | IAllocIterator_DArenaIterator | + +----------------------+-------------------------+-----------------------------------+ + | IAllocator_Xfer | IAllocIterator_Xfer | DArena | + | IAllocator_Any | IAllocIterator_Any | DArenaIterator | + | IAllocator_Impltype | IAllocIterator_Impltype | | + | | | | + +----------------------+-------------------------+-----------------------------------+ + | AAllocator | AAllocIterator | ArenaConfig | + +----------------------+-------------------------+-----------------------------------+ + +-----------------+----------------------------------------------+-------------------+ + | | AllocInfo | | + | +----------------------------------------------+ | + | AllocError | AllocHeaderConfig | cmpresult | + | +----------------------------------------------+ | + | | AllocHeader | | + +-----------------+----------------------------------------------+-------------------+ + +.. list-table:: Polymorphic Allocator :header-rows: 1 :widths: 20 90 - * - Component + * - Class - Description * - ``AAllocator`` - - allocator facet (abstract interface) - * - ``DArena`` - - arena representation - * - ``IAllocator_ImplType`` - - lookup implementation for allocator A - with representation D. + - Abstract allocator interface for runtime polymorphism + * - ``IAllocator_Any`` + - Stub allocator interface for uninitialized variant * - ``IAllocator_Xfer`` - - transfer interface. downcast to native state. - * - ``IAllocator_DArena`` - - allocator implementation for ``DArena`` + - Allocator interface template for representation ``D`` + * - ``IAllocator_Impltype`` + - Lookup allocator interface for representation ``D`` + * - ``RAllocator`` + - Provide allocator methods for FOMO object ``O`` + +.. list-table:: Polymorphich Alloc Iterator + :header-rows: 1 + :widths: 20 90 + + * - Class + - Description + * - ``AAllocIterator`` + - Abstract interface for iteration over allocs + * - ``IAllocIterator_Any`` + - Stub alloc-iterator interface for uninitialized variant + * - ``IAllocIterator_Xfer`` + - Alloc-iterator interface template for representation ``D`` + * - ``IAllocIterator_Impltype`` + - Lookup alloc-iterator interface for representation ``D`` + * - ``RAllocIterator`` + - Provide alloc-iterator methods for FOMO object ``O``. + +.. list-table:: Native Arena Allocator + :header-rows: 1 + :widths: 20 90 + + * - Class + - Description + * - ``ArenaConfig`` + - Configuration for a ``DArena`` instance + * - ``DArena`` + - VM-aware arena allocator + * - ``DArenaIterator`` + - Iterator over ``DArena`` allocations + * - ``IAlllocator_DArena`` + - Adapt a ``DArena`` to facet ``AAllocator`` + * - ``IAllocIterator_DArenaAllocator`` + - Adapt a ``DArenaIterator`` to facet ``AAllocIterator`` + +.. list-table:: Auxiliary/Support Types + :header-rows: 1 + :widths: 20 90 + + * - Class + - Description + * - ``AllocError`` + - Return type for an alloc request, with error details. + * - ``AllocInfo`` + - An opaque allocation. Value of an alloc-iterator. + * - ``AllocHeaderConfig`` + - Per-allocator configuration of alloc headers + * - ``AllocHeader`` + - Per-allocation header (8 bytes) + * - ``cmpresult`` + - Result of alloc-iterator comparison + +Example Object Diagram + +.. uml:: + :caption: representation for an arena allocator + :scale: 99% + :align: center + + object rarena1<> + rarena1 : iface = vtable1 + rarena1 : data = darena1 + + object vtable1<> + vtable1 : alloc() + + object darena1<> + darena1 : config + darena1 : lo + darena1 : hi + darena1 : free + darena1 : limit + darena1 : last_error + + rarena1 o-- vtable1 + rarena1 o-- darena1 + + +Remarks: + +* When we know the allocator representation at compile time (``DArena`` here), + then we also know the interface (``IAllocator_DArena``). + Devirtualization is easy since interface methods are all final. + +* Size of a FOMO object is two pointers; it's natural to create such objects + on the fly and pass them by value. + When storing an allocator in another data structure, we only need to use + the RAllocator stack if we want runtime polymorphism for the stored allocator. + Otherwise can store a ``DArena`` instance. diff --git a/docs/index.rst b/docs/index.rst index ac8a802..325985b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,6 +26,7 @@ Implemented using FOMO (faceted rust-like object model) from xo-facet AAllocator-reference ArenaConfig-reference DArena-reference + AllocInfo-reference glossary genindex search diff --git a/include/xo/alloc2/AllocError.hpp b/include/xo/alloc2/AllocError.hpp index 7d09764..ca98b36 100644 --- a/include/xo/alloc2/AllocError.hpp +++ b/include/xo/alloc2/AllocError.hpp @@ -14,7 +14,7 @@ namespace xo { /** sentinel **/ invalid = -1, /** not an error **/ - none, + ok, /** reserved size exhauged **/ reserve_exhausted, /** unable to commit (i.e. mprotect failure) **/ @@ -57,8 +57,10 @@ namespace xo { committed_z_{com_z}, reserved_z_{rsv_z} {} + static const char * error_description(error x); + /** error code **/ - error error_ = error::none; + error error_ = error::ok; /** sequence# of this error. * Each error event within an allocator gets next sequence number diff --git a/include/xo/alloc2/AllocInfo.hpp b/include/xo/alloc2/AllocInfo.hpp index 5f68396..0f8d0e1 100644 --- a/include/xo/alloc2/AllocInfo.hpp +++ b/include/xo/alloc2/AllocInfo.hpp @@ -18,10 +18,18 @@ namespace xo { * **/ struct AllocInfo { + /** @defgroup mm-allocinfo-traits **/ + ///@{ + using size_type = AllocHeader::size_type; using byte = std::byte; using span_type = std::pair; + ///@} + + /** @defgroup mm-allocinfo-ctors **/ + ///@{ + AllocInfo(const AllocHeaderConfig * p_cfg, const byte * p_guard_lo, const AllocHeader * p_hdr, @@ -39,6 +47,11 @@ namespace xo { return AllocInfo(p_cfg, nullptr, nullptr, nullptr); } + ///@} + + /** @defgroup mm-allocinfo-methods **/ + ///@{ + /** true for non-sentinel AllocInfo instance **/ bool is_valid() const { return (p_config_ != nullptr) && (p_header_ != nullptr); } @@ -59,10 +72,17 @@ namespace xo { /** Value (fixed test pattern) of guard byte **/ char guard_byte() const noexcept { return p_config_->guard_byte_; } + ///@} + + /** @defgroup mm-allocinfo-instance-vars **/ + ///@{ + const AllocHeaderConfig * p_config_ = nullptr; const byte * p_guard_lo_ = nullptr; const AllocHeader * p_header_ = nullptr; const byte * p_guard_hi_ = nullptr; + + ///@} }; } /*namespace mm*/ } /*namespace xo*/ diff --git a/include/xo/alloc2/alloc/AAllocator.hpp b/include/xo/alloc2/alloc/AAllocator.hpp index 809c31b..8e6e5e6 100644 --- a/include/xo/alloc2/alloc/AAllocator.hpp +++ b/include/xo/alloc2/alloc/AAllocator.hpp @@ -25,6 +25,9 @@ namespace xo { /** @class AAllocator * @brief Abstract facet for allocation * + * Methods take a opaque data pointer. + * Implementations of AAllocator will downcast to a + * to some specific representation. **/ struct AAllocator { /** @defgroup mm-allocator-type-traits allocator type traits **/ @@ -56,17 +59,16 @@ namespace xo { /** RTTI: unique id# for actual runtime data representation **/ virtual int32_t _typeseq() const noexcept = 0; - /** optional name for allocator @p d - * Labeling, for diagnostics. + /** optional name for allocator @p d . + * Allows labeling allocators, for diagnostics/instrumentation. **/ virtual std::string_view name(Copaque d) const noexcept = 0; /** reserved size in bytes for allocator @p d. * Includes committed + uncommitted memory. - * Cannot be increased. **/ virtual size_type reserved(Copaque d) const noexcept = 0; /** Synonym for @ref committed. - * Can increase on @ref alloc + * Can increase automatically on @ref alloc **/ virtual size_type size(Copaque d) const noexcept = 0; /** committed size (physical addresses obtained) @@ -74,14 +76,22 @@ namespace xo { * @ref alloc may auto-increase this **/ virtual size_type committed(Copaque d) const noexcept = 0; - /** unallocated (but committed) size in bytes for allocator @p d **/ + /** unallocated (but committed) size in bytes for allocator @p d. + * An alloc request up to this size (including guard / header) + * is guaranteed to succeed. + * An alloc request of more than this size may still succeed, + * if allocator can automatically extend committed memory. + * This is the case for the @ref xo::mm::DArena allocator + **/ virtual size_type available(Copaque d) const noexcept = 0; - /** allocated (i.e. in-use) amount in bytes for allocator @p d **/ + /** allocated (i.e. currently in-use) amount in bytes for allocator @p d. + * Includes alloc headers and guard regions + **/ virtual size_type allocated(Copaque d) const noexcept = 0; /** true iff allocator @p d is responsible for memory at address @p p. **/ virtual bool contains(Copaque d, const void * p) const noexcept = 0; - /** report last error **/ + /** report details of last error for allocator @p d. **/ virtual AllocError last_error(Copaque d) const noexcept = 0; /** fetch alloc info: given memory @p mem previously obtained * from {@ref alloc, @ref super_alloc}, get {tseq, age, size} details @@ -90,28 +100,23 @@ namespace xo { * Non-const @p d because may stash error details **/ virtual AllocInfo alloc_info(Copaque d, value_type mem) const noexcept = 0; - /** Ideally we want to control allocator for iterator here. - * Awkward to supply to compiler since we don't have obj yet. - * OTOH iteration over allocs is a super-niche feature. - * - * Rejected alternatives: - * - put begin/end in separate interface. e.g. extend AAllocator - * - layer of indirection: begin/end return iterator factory. - * Then allocator can be passed to iterator factory separately. - * Helps because factory can be static - * - abandon allocator support in this case. Instead will need to - * reinstate uvt (unique variant), use heap - * - * @p mm is allocator for resulting iterator range + /** + * Create an iterator range for allocator @p d. + * An iterator range has begin and end methods, so supports c++ range iteration. + * Memory for iterator state will be obtained from @p mm. **/ virtual range_type alloc_range(Copaque d, DArena & mm) const noexcept = 0; /** expand committed space in arena @p d - * to size at least @p z - * In practice will round up to a multiple of hugepage size (2MB) + * to size at least @p z. + * In practice will round up to a multiple of page size (4K) or hugepage size (2MB) + * depending on configuration. **/ virtual bool expand(Opaque d, std::size_t z) const noexcept = 0; - /** allocate @p z bytes of memory from allocator @p d. **/ + /** attempt to allocate @p z bytes of memory from allocator @p d. + * If allocation fails returns nullptr. In this case error details may be retrieved + * using last error + **/ virtual value_type alloc(Opaque d, size_type z) const = 0; /** like @ref alloc, but follow with one or more consecutive * @ref sub_alloc() calls. This sequence of allocs will share @@ -125,9 +130,11 @@ namespace xo { * zero @p z **/ virtual value_type sub_alloc(Opaque d, size_type z, bool complete_flag) const = 0; - /** reset allocator @p d to empty state **/ + /** reset allocator @p d to empty state. **/ virtual void clear(Opaque d) const = 0; - /** destruct allocator @p d **/ + /** Destruct allocator @p d. + * Releases allocator memory to operating system. + **/ virtual void destruct_data(Opaque d) const = 0; ///@} diff --git a/include/xo/alloc2/arena/DArena.hpp b/include/xo/alloc2/arena/DArena.hpp index 7c7196e..1b03aaa 100644 --- a/include/xo/alloc2/arena/DArena.hpp +++ b/include/xo/alloc2/arena/DArena.hpp @@ -47,9 +47,15 @@ namespace xo { /** @brief mode argument for @ref _alloc **/ enum class alloc_mode : uint8_t { + /** ordinary alloc. Most common mode **/ standard, + /** begin a sequence of suballocs that share a single alloc header **/ super, + /** make a subsidiary allocation on behalf of a preceding super alloc. + * Will be followed by at least one more suballoc call. + **/ sub_incomplete, + /** make a subsidiary allocation that completes preceding super alloc. **/ sub_complete, }; @@ -64,7 +70,11 @@ namespace xo { /** null ctor **/ DArena() = default; /** ctor from already-mapped (but not committed) address range **/ - DArena(const ArenaConfig & cfg, size_type page_z, value_type lo, value_type hi); + DArena(const ArenaConfig & cfg, + size_type page_z, + size_type arena_align_z, + value_type lo, + value_type hi); /** DArena is not copyable **/ DArena(const DArena & other) = delete; /** move ctor **/ @@ -80,18 +90,49 @@ namespace xo { /** @defgroup mm-arena-methods **/ ///@{ + /** Reserved memory, in bytes. This is the maximum size of this arena. **/ size_type reserved() const noexcept { return hi_ - lo_; } + /** Allocated memory in bytes: memory consumed by allocs from this arena, + * including administrative overhead (alloc headers + guard bytes) + **/ size_type allocated() const noexcept { return free_ - lo_; } + /** Committed memory in bytes: amount of memory actually backed by physical memory **/ size_type committed() const noexcept { return committed_z_; } + /** Available committed memory. + * This is the amount of memory guaranteed to be usable for future allocs from this arena. + **/ size_type available() const noexcept { return limit_ - free_; } + /** True iff address @p addr is owned by this arena, + * i.e. falls within [@ref lo_, @ref hi_) + **/ bool contains(const void * addr) const noexcept { return (lo_ <= addr) && (addr < hi_); } /** obtain uncommitted contiguous memory range comprising - * a whole multiple of @p hugepage_z bytes, of at least size @p req_z, - * aligned on a @p hugepage_z boundary + * a whole multiple of @p align_z bytes, of at least size @p req_z, + * aligned on a @p align_z boundary. Uncommitted memory is not (yet) + * backed by physical memory. + * + * If @p enable_hugepage_flag is true and THP + * (transparent huge pages) are available, use THP for arena memory. + * This relieves TLB and page table memory when @p req_z is a lot larger than + * page size (likely 4KB). Cost is that arena will consum physical memory in unit + * of @p align_z. Arena may waste up to @p align_z bytes of memory as a result. + * + * If @p enable_hugepage_flag is true, @p align_z should be huge page size + * (probably 2MB) for optimal performance. + * + * At present the THP feature is not supported on OSX. + * May be supportable through mach_vm_allocate(). + * + * Note that we reject MAP_HUGETLB|MAP_HUGE_2MB flags to mmap here, + * since requires previously-reserved memory in /proc/sys/vm/nr_hugepages. + * + * @return pair giving reserved memory address range [lo,hi) **/ - static range_type map_aligned_range(size_type req_z, size_type hugepage_z); + static range_type map_aligned_range(size_type req_z, + size_type align_z, + bool enable_hugepage_flag); /** true if arena is mapped i.e. has a reserved address range **/ bool is_mapped() const noexcept { return (lo_ != nullptr) && (hi_ != nullptr); } @@ -177,6 +218,9 @@ namespace xo { /** size of a VM page (obtained automatically via getpagesize()). Likely 4k **/ size_type page_z_ = 0; + /** alignment for this arena. In practice will be either page_z_ or cfg.hugepage_z_ **/ + size_type arena_align_z_ = 0; + /** arena owns memory in range [@ref lo_, @ref hi_) **/ std::byte * lo_ = nullptr; diff --git a/include/xo/alloc2/print.hpp b/include/xo/alloc2/print.hpp new file mode 100644 index 0000000..5c47476 --- /dev/null +++ b/include/xo/alloc2/print.hpp @@ -0,0 +1,34 @@ +/** @file print.hpp +* + * @author Roland Conybeare, Dec 2025 + **/ + +#pragma once + +#include "AllocError.hpp" +#include +#include + +namespace xo { + namespace mm { + inline std::ostream & + operator<<(std::ostream & os, const error & x) { + os << AllocError::error_description(x); + return os; + } + + inline std::ostream & + operator<<(std::ostream & os, const AllocError & x) { + os << ""; + return os; + } + } +} + +/* end print.hpp */ diff --git a/src/DArena.cpp b/src/DArena.cpp deleted file mode 100644 index e69de29..0000000 diff --git a/src/alloc2/AllocError.cpp b/src/alloc2/AllocError.cpp new file mode 100644 index 0000000..43c1678 --- /dev/null +++ b/src/alloc2/AllocError.cpp @@ -0,0 +1,45 @@ +/** @file AllocError.cpp + * + * @author Roland Conybeare, Dec 2025 + **/ + +#include "AllocError.hpp" + +namespace xo { + namespace mm { + + const char * + AllocError::error_description(error x) + { + switch (x) { + case error::invalid: + break; + case error::ok: + return "ok"; + case error::reserve_exhausted: + return "reserve-exhausted"; + case error::commit_failed: + return "commit-failed"; + case error::header_size_mask: + return "header-size-mask"; + case error::orphan_sub_alloc: + return "orphan-sub-alloc"; + case error::alloc_info_disabled: + return "alloc-info-disabled"; + case error::alloc_info_address: + return "alloc-info-address"; + case error::alloc_iterator_not_supported: + return "alloc-iterator-not-supported"; + case error::alloc_iterator_deref: + return "alloc-iterator-deref"; + case error::alloc_iterator_next: + return "alloc-iterator-next"; + } + + return "?error"; + } + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end AllocError.cpp */ diff --git a/src/alloc2/CMakeLists.txt b/src/alloc2/CMakeLists.txt index aabffe8..79aea85 100644 --- a/src/alloc2/CMakeLists.txt +++ b/src/alloc2/CMakeLists.txt @@ -3,6 +3,7 @@ set(SELF_LIB xo_alloc2) set(SELF_SRCS + AllocError.cpp AllocInfo.cpp cmpresult.cpp diff --git a/src/alloc2/DArena.cpp b/src/alloc2/DArena.cpp index a2bcd0e..b353462 100644 --- a/src/alloc2/DArena.cpp +++ b/src/alloc2/DArena.cpp @@ -19,56 +19,30 @@ namespace xo { using std::size_t; namespace mm { - - /** Map a contiguous uncommitted memory range comprising - * a whole multiple of @p hugepage_z, with at least - * @p req_z bytes. - * - * Memory will also be aligned on @p hugepage_z boundary - * (2MB in practice) - * - * - @p req_z is rounded up to a multiple of @p hugepage_z - * - Resulting uncommitted address range not backed by - * physical memory. - * - since hugpage-aligned, can find base of mapped range - * by masking off the bottom log2(align_z) bits. - * May rely on this for GC metadata - * - opt-in to transparent huge pages (THP). - * Reduces page-fault time by a lot, in return for - * lower VM granularity - * - rejecting inferior MAP_HUGETLB|MAP_HUGE_2MB flags on ::mmap here: - * - requires previously-reserved memory in /proc/sys/vm/nr_hugepages - * - reserved pages permenently resident in RAM, never swapped - * - memory cost incurred even if no application is using said pages - * - * TODO: for OSX -> need something else here. - * MAP_ALIGNED_SUPER with mmap() and/or - * use mach_vm_allocate() - * - * @return pair giving mapped address range [lo, hi) - **/ auto - DArena::map_aligned_range(size_t req_z, size_t hugepage_z) -> range_type + DArena::map_aligned_range(size_t req_z, + size_t align_z, + bool enable_hugepage_flag) -> range_type { - // 1. round up to multiple of hugepage_z - size_t target_z = padding::with_padding(req_z, hugepage_z); // 4. + // 1. round up to multiple of align_z + size_t target_z = padding::with_padding(req_z, align_z); // 4. // 2. mmap() will give us page-aligned memory, // but not hugepage-aligned. // - // Over-request by hugepage_z to ensure - // hugepage-aligned subrange of size target_z + // Over-request by align_z to ensure + // aligned subrange of size target_z // byte * base = (byte *)(::mmap(nullptr, - target_z + hugepage_z, + target_z + align_z, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); // on mmap success: upper limit of mapped address range - byte * hi = base + (target_z + hugepage_z); + byte * hi = base + (target_z + align_z); // lowest hugepage-aligned address in [base, hi) - byte * aligned_base = (byte *)(padding::with_padding((size_t)base, hugepage_z)); + byte * aligned_base = (byte *)(padding::with_padding((size_t)base, align_z)); // end of hugeppage-aligned range starting at aligned_base byte * aligned_hi = aligned_base + target_z; @@ -88,9 +62,9 @@ namespace xo { xtag("size", req_z))); } - assert((size_t)aligned_base % hugepage_z == 0); + assert((size_t)aligned_base % align_z == 0); assert(aligned_base >= base); - assert(aligned_base < base + hugepage_z); + assert(aligned_base < base + align_z); } // 4. release unaligned prefix @@ -108,21 +82,25 @@ namespace xo { } #ifdef __linux__ - /** linux: - * opt-in to transparent huge pages (THP) - * provided OS configured to support them. - * otherwise fallback gracefully. - * - * Huge pages -> use fewer TLB entries + faster - * shorter path through page table. - * - * When we commit (i.e. obtain physical memory on page fault), - * typically expect to pay ~1us per superpage. - * Much better than ~500us to commit 512 4k VM pages. - * - * But wasted if we don't use the memory. - **/ - ::madvise(aligned_base, target_z, MADV_HUGEPAGE); // 8. + if (enable_hugepage_flag) { + /** linux: + * opt-in to transparent huge pages (THP) + * provided OS configured to support them. + * otherwise fallback gracefully. + * + * Huge pages -> use fewer TLB entries + faster + * shorter path through page table. + * + * When we commit (i.e. obtain physical memory on page fault), + * typically expect to pay ~1us per superpage. + * Much better than ~500us to commit 512 4k VM pages. + * + * But wasted if we don't use the memory. + * + * Page table has a handful of levels + **/ + ::madvise(aligned_base, target_z, MADV_HUGEPAGE); // 8. + } #endif return std::make_pair(aligned_base, aligned_hi); @@ -133,7 +111,20 @@ namespace xo { { //scope log(XO_DEBUG(debug_flag), xtag("name", name)); - auto [lo, hi] = map_aligned_range(cfg.size_, cfg.hugepage_z_); + /* vm page size. 4KB, probably */ + size_t page_z = getpagesize(); + + bool enable_hugepage_flag = (cfg.size_ >= cfg.hugepage_z_); + + /* Align start of arena memory on this boundary. + * Will use THP (transparent huge pages) if available + * and arena size is at least as large as hugepage size (2MB, probably) + */ + size_t align_z = (enable_hugepage_flag ? cfg.hugepage_z_ : page_z); + + auto [lo, hi] = map_aligned_range(cfg.size_, + align_z, + enable_hugepage_flag); if (!lo) { // control here implies mmap() failed silently @@ -142,8 +133,6 @@ namespace xo { xtag("size", cfg.size_))); } - size_t page_z = getpagesize(); - #ifdef NOPE log && log(xtag("lo", (void*)lo_), @@ -151,14 +140,16 @@ namespace xo { xtag("hugepage_z", hugepage_z_)); #endif - return DArena(cfg, page_z, lo, hi); + return DArena(cfg, page_z, align_z, lo, hi); } /*map*/ DArena::DArena(const ArenaConfig & cfg, size_type page_z, + size_type arena_align_z, byte * lo, byte * hi) : config_{cfg}, page_z_{page_z}, + arena_align_z_{arena_align_z}, lo_{lo}, committed_z_{0}, free_{lo}, @@ -177,6 +168,7 @@ namespace xo { DArena::DArena(DArena && other) { config_ = other.config_; page_z_ = other.page_z_; + arena_align_z_ = other.arena_align_z_; lo_ = other.lo_; committed_z_ = other.committed_z_; free_ = other.free_; @@ -200,6 +192,7 @@ namespace xo { { config_ = other.config_; page_z_ = other.page_z_; + arena_align_z_ = other.arena_align_z_; lo_ = other.lo_; committed_z_ = other.committed_z_; free_ = other.free_; @@ -529,23 +522,25 @@ namespace xo { * */ - std::size_t aligned_target_z = padding::with_padding(target_z, config_.hugepage_z_); + std::size_t aligned_target_z = padding::with_padding(target_z, arena_align_z_); std::byte * commit_start = limit_; // = lo_ + committed_z_; std::size_t add_commit_z = aligned_target_z - committed_z_; assert(limit_ == lo_ + committed_z_); - // log && log(xtag("aligned_offset_z", aligned_offset_z), - // xtag("add_commit_z", add_commit_z)); - // log && log("expand committed range", - // xtag("commit_start", commit_start), - // xtag("add_commit_z", add_commit_z), - // xtag("commit_end", commit_start + add_commit_z)); - if (::mprotect(commit_start, add_commit_z, PROT_READ | PROT_WRITE) != 0) [[unlikely]] { + if (log) { + log("commit failed!"); + log(xtag("aligned_target_z", aligned_target_z), + xtag("commit_start", commit_start), + xtag("add_commit_z", add_commit_z), + xtag("commit_end", commit_start + add_commit_z) + ); + } + capture_error(error::commit_failed, add_commit_z); return false; } @@ -563,8 +558,8 @@ namespace xo { free_ += config_.header_.guard_z_; } - assert(committed_z_ % config_.hugepage_z_ == 0); - assert(reinterpret_cast(limit_) % config_.hugepage_z_ == 0); + assert(committed_z_ % arena_align_z_ == 0); + assert(reinterpret_cast(limit_) % arena_align_z_ == 0); return true; } /*expand*/ diff --git a/utest/arena.test.cpp b/utest/arena.test.cpp index 823e205..09e5e4b 100644 --- a/utest/arena.test.cpp +++ b/utest/arena.test.cpp @@ -8,9 +8,10 @@ //#include "xo/alloc2/DArena.hpp" #include "xo/alloc2/arena/IAllocator_DArena.hpp" //#include "xo/alloc2/alloc/RAllocator.hpp" +#include "xo/alloc2/print.hpp" #include "xo/alloc2/padding.hpp" -#include "xo/indentlog/scope.hpp" -#include "xo/facet/obj.hpp" +#include +#include #include namespace xo { @@ -37,12 +38,55 @@ namespace xo { REQUIRE(IAllocator_Xfer::_valid); } - TEST_CASE("DArena", "[alloc2][DArena]") + TEST_CASE("DArena-tiny", "[alloc2][DArena]") { ArenaConfig cfg { .name_ = "testarena", .size_ = 1 }; DArena arena = DArena::map(cfg); + REQUIRE(arena.config_.name_ == cfg.name_); + REQUIRE(arena.lo_ != nullptr); + REQUIRE(arena.free_ == arena.lo_); + REQUIRE(arena.limit_ == arena.lo_); + REQUIRE(arena.hi_ != nullptr); + REQUIRE(arena.hi_ > arena.lo_); + REQUIRE(((size_t)arena.hi_ - (size_t)arena.lo_) % arena.page_z_ == 0); + REQUIRE(arena.lo_ + cfg.size_ <= arena.hi_); + + /* verify arena.lo_ is aligned on a page boundary */ + REQUIRE(((size_t)(arena.lo_) & (arena.page_z_ - 1)) == 0); + + /* verify arena.hi_ is aligned on a hugepage boundary */ + REQUIRE(((size_t)(arena.hi_) & (arena.page_z_ - 1)) == 0); + + byte * lo = arena.lo_; + byte * free = arena.free_; + byte * limit = arena.limit_; + byte * hi = arena.hi_; + size_t committed_z = arena.committed_z_; + + DArena arena2 = std::move(arena); + + REQUIRE(arena.lo_ == nullptr); + REQUIRE(arena.free_ == nullptr); + REQUIRE(arena.limit_ == nullptr); + REQUIRE(arena.hi_ == nullptr); + REQUIRE(arena.committed_z_ == 0); + + REQUIRE(arena.lo_ == nullptr); + REQUIRE(arena2.lo_ == lo); + REQUIRE(arena2.free_ == free); + REQUIRE(arena2.limit_ == limit); + REQUIRE(arena2.hi_ == hi); + REQUIRE(arena2.committed_z_ == committed_z); + } + + TEST_CASE("DArena-medium", "[alloc2][DArena]") + { + ArenaConfig cfg { .name_ = "testarena", + .size_ = 10*1024*1024 }; + DArena arena = DArena::map(cfg); + REQUIRE(arena.config_.name_ == cfg.name_); REQUIRE(arena.lo_ != nullptr); REQUIRE(arena.free_ == arena.lo_); @@ -52,7 +96,7 @@ namespace xo { REQUIRE(((size_t)arena.hi_ - (size_t)arena.lo_) % cfg.hugepage_z_ == 0); REQUIRE(arena.lo_ + cfg.size_ <= arena.hi_); - /* verify arena.lo_ is aligned on a hugepage boundary */ + /* verify arena.lo_ is aligned on a page boundary */ REQUIRE(((size_t)(arena.lo_) & (cfg.hugepage_z_ - 1)) == 0); /* verify arena.hi_ is aligned on a hugepage boundary */ @@ -103,8 +147,8 @@ namespace xo { == xo::facet::FacetImplType::s_typeseq); REQUIRE(a1o.name() == cfg.name_); REQUIRE(a1o.reserved() >= cfg.size_); - REQUIRE(a1o.reserved() < cfg.size_ + cfg.hugepage_z_); - REQUIRE(a1o.reserved() % cfg.hugepage_z_ == 0); + REQUIRE(a1o.reserved() < cfg.size_ + a1o.data()->page_z_); + REQUIRE(a1o.reserved() % a1o.data()->page_z_ == 0); REQUIRE(a1o.size() == 0); REQUIRE(a1o.committed() == 0); REQUIRE(a1o.allocated() == 0); @@ -123,11 +167,15 @@ namespace xo { REQUIRE(a1o.allocated() == 0); size_t z2 = 512; - REQUIRE(a1o.expand(z2)); + bool ok = a1o.expand(z2); - REQUIRE(a1o.reserved() % cfg.hugepage_z_ == 0); + INFO(xtag("last_error", a1o.last_error())); + + REQUIRE(ok); + + REQUIRE(a1o.reserved() % a1o.data()->page_z_ == 0); REQUIRE(a1o.committed() >= z2); - REQUIRE(a1o.committed() % cfg.hugepage_z_ == 0); + REQUIRE(a1o.committed() % a1o.data()->page_z_ == 0); /* .size() is synonym for .committed() */ REQUIRE(a1o.size() == a1o.committed()); REQUIRE(a1o.available() >= z2); @@ -154,7 +202,7 @@ namespace xo { byte * m0 = a1o.alloc(1); REQUIRE(m0); - REQUIRE(a1o.last_error().error_ == error::none); + REQUIRE(a1o.last_error().error_ == error::ok); REQUIRE(a1o.last_error().error_seq_ == 0); REQUIRE(a1o.allocated() >= z0); REQUIRE(a1o.allocated() < z0 + padding::c_alloc_alignment ); @@ -166,7 +214,7 @@ namespace xo { byte * m1 = a1o.alloc(z1); REQUIRE(m1); - REQUIRE(a1o.last_error().error_ == error::none); + REQUIRE(a1o.last_error().error_ == error::ok); REQUIRE(a1o.last_error().error_seq_ == 0); REQUIRE(a1o.allocated() >= z0 + z1); REQUIRE(a1o.allocated() < z0 + z1 + 2 * padding::c_alloc_alignment ); @@ -209,7 +257,7 @@ namespace xo { REQUIRE(a1o.contains(header)); REQUIRE(cfg.header_.size(*header) == padding::with_padding(z0)); //REQUIRE(((*header) & cfg.header_size_mask_) == padding::with_padding(z0)); - REQUIRE(a1o.last_error().error_ == error::none); + REQUIRE(a1o.last_error().error_ == error::ok); REQUIRE(a1o.last_error().error_seq_ == 0); REQUIRE(a1o.allocated() >= z0); REQUIRE(a1o.allocated() < sizeof(AAllocator::header_type) + z0 + padding::c_alloc_alignment ); @@ -265,7 +313,7 @@ namespace xo { REQUIRE(cfg.header_.size(*header) == padding::with_padding(z0)); //REQUIRE(((*header) & cfg.header_size_mask_) == padding::with_padding(z0)); - REQUIRE(a1o.last_error().error_ == error::none); + REQUIRE(a1o.last_error().error_ == error::ok); REQUIRE(a1o.last_error().error_seq_ == 0); REQUIRE(a1o.allocated() == (cfg.header_.guard_z_ @@ -306,7 +354,7 @@ namespace xo { REQUIRE(err.request_z_ >= z0); REQUIRE(err.request_z_ < z0 + padding::c_alloc_alignment); REQUIRE(err.committed_z_ == 0); - REQUIRE(err.reserved_z_ == cfg.hugepage_z_); + REQUIRE(err.reserved_z_ == arena.reserved()); } } /*namespace ut*/ } /*namespace xo*/ diff --git a/utest/random_allocs.cpp b/utest/random_allocs.cpp index 56d0cb8..4031534 100644 --- a/utest/random_allocs.cpp +++ b/utest/random_allocs.cpp @@ -80,7 +80,7 @@ namespace utest { REQUIRE_ORFAIL(ok_flag, catch_flag, mem != nullptr); REQUIRE_ORFAIL(ok_flag, catch_flag, mm.contains(mem)); REQUIRE_ORFAIL(ok_flag, catch_flag, mm.last_error().error_seq_ == 0); - REQUIRE_ORFAIL(ok_flag, catch_flag, mm.last_error().error_ == xo::mm::error::none); + REQUIRE_ORFAIL(ok_flag, catch_flag, mm.last_error().error_ == xo::mm::error::ok); { auto ix = allocs_by_lo_map.lower_bound(mem);