From 0c7d63bf01854883f00e5829ba13424e5a744c43 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 12 Dec 2025 19:44:38 -0500 Subject: [PATCH] xo-alloc2: streamlining + bugfixes [wip] --- include/xo/alloc2/AAllocator.hpp | 22 ++- include/xo/alloc2/DArena.hpp | 23 ++++ include/xo/alloc2/IAllocator_Any.hpp | 4 +- include/xo/alloc2/IAllocator_DArena.hpp | 35 ++++- include/xo/alloc2/IAllocator_Xfer.hpp | 12 +- include/xo/alloc2/RAllocator.hpp | 7 +- include/xo/alloc2/padding.hpp | 2 +- src/alloc2/DArena.cpp | 3 + src/alloc2/IAllocator_DArena.cpp | 175 +++++++++++++++++++++++- utest/arena.test.cpp | 38 +++++ 10 files changed, 311 insertions(+), 10 deletions(-) diff --git a/include/xo/alloc2/AAllocator.hpp b/include/xo/alloc2/AAllocator.hpp index 38c545f..a0067ee 100644 --- a/include/xo/alloc2/AAllocator.hpp +++ b/include/xo/alloc2/AAllocator.hpp @@ -23,6 +23,10 @@ namespace xo { reserve_exhausted, /** unable to commit (i.e. mprotect failure) **/ commit_failed, + /** allocation size too big (See @ref ArenaConfig::header_size_mask_) **/ + header_size_mask, + /** sub_alloc not preceded by super alloc (or another sub_alloc) **/ + orphan_sub_alloc, }; struct AllocatorError { @@ -67,6 +71,10 @@ namespace xo { ///@{ /** @brief type used for allocation amounts **/ using size_type = std::size_t; + /** @brief type used for allocation responses **/ + using value_type = std::byte *; + /** object header, if configured **/ + using header_type = std::uint64_t; ///@} /* @@ -121,7 +129,19 @@ namespace xo { **/ virtual bool expand(Opaque d, std::size_t z) const noexcept = 0; /** allocate @p z bytes of memory from allocator @p d. **/ - virtual std::byte * alloc(Opaque d, std::size_t z) const = 0; + virtual value_type alloc(Opaque d, size_type z) const = 0; + /** like @ref alloc, but follow with one or more consecutive + * @ref sub_alloc() calls. This sequence of allocs will share + * the initial allocation header. + **/ + virtual value_type super_alloc(Opaque d, size_type z) const = 0; + /** follow a preceding @ref super_alloc call with additional + * subsidiary allocs that share the same object header. + * Must finish sequence with exactly one sub_alloc call + * with @p complete_flag set. This sub_alloc call may have + * zero @p z + **/ + virtual value_type sub_alloc(Opaque d, size_type z, bool complete_flag) const = 0; /** reset allocator @p d to empty state **/ virtual void clear(Opaque d) const = 0; /** destruct allocator @p d **/ diff --git a/include/xo/alloc2/DArena.hpp b/include/xo/alloc2/DArena.hpp index 5919970..a30f5c2 100644 --- a/include/xo/alloc2/DArena.hpp +++ b/include/xo/alloc2/DArena.hpp @@ -26,6 +26,22 @@ namespace xo { * (provided you use their full extent :) **/ std::size_t hugepage_z_ = 2 * 1024 * 1024; + /** true to store header (8 bytes) at the beginning of each allocation. + * necessary and sufficient to allows iterating over allocs + * present in arena + **/ + bool store_header_flag_ = false; + /** if non-zero, allocate extra space between allocs, and fill + * with fixed test-pattern contents. Allows for simple + * runtime arena sanitizing checks. + * Will be rounded up to multiple of @ref padding::c_alloc_alignment + **/ + std::size_t guard_z_ = 0; + /** if store_header_flag_ is true: mask bits for allocation size. + * remaining bits can be stolen for other purposes + * otherwise ignored + **/ + std::uint64_t header_size_mask_ = 0xffffffff; /** true to enable debug logging **/ bool debug_flag_ = false; @@ -59,6 +75,8 @@ namespace xo { using size_type = std::size_t; /** @brief a contiguous memory range **/ using range_type = std::pair; + /** @brief type for allocation header (if enabled) **/ + using header_type = std::uint64_t; ///@} @@ -103,6 +121,11 @@ namespace xo { **/ size_type committed_z_ = 0; + /** if config_.store_header_flag_: + * Pointer to header for last allocation. + **/ + header_type * last_header_ = nullptr; + /** free pointer. * Memory in range [@ref lo_, @ref free_) current in use **/ diff --git a/include/xo/alloc2/IAllocator_Any.hpp b/include/xo/alloc2/IAllocator_Any.hpp index c4cf1b9..0da80ae 100644 --- a/include/xo/alloc2/IAllocator_Any.hpp +++ b/include/xo/alloc2/IAllocator_Any.hpp @@ -41,7 +41,9 @@ namespace xo { [[noreturn]] AllocatorError last_error(Copaque) const noexcept override { _fatal(); } [[noreturn]] bool expand(Opaque, std::size_t) const noexcept override { _fatal(); } - [[noreturn]] std::byte * alloc(Opaque, std::size_t) const override { _fatal(); } + [[noreturn]] value_type alloc(Opaque, std::size_t) const override { _fatal(); } + [[noreturn]] value_type super_alloc(Opaque, std::size_t) const override { _fatal(); } + [[noreturn]] value_type sub_alloc(Opaque, std::size_t, bool) const override { _fatal(); } [[noreturn]] void clear(Opaque) const override { _fatal(); } [[noreturn]] void destruct_data(Opaque) const override { _fatal(); } diff --git a/include/xo/alloc2/IAllocator_DArena.hpp b/include/xo/alloc2/IAllocator_DArena.hpp index 15a9c9f..0cfd762 100644 --- a/include/xo/alloc2/IAllocator_DArena.hpp +++ b/include/xo/alloc2/IAllocator_DArena.hpp @@ -29,6 +29,14 @@ namespace xo { */ struct IAllocator_DArena { using size_type = std::size_t; + using value_type = std::byte *; + + enum class alloc_mode : uint8_t { + standard, + super, + sub, + sub_complete, + }; static std::string_view name(const DArena &) noexcept; static size_type reserved(const DArena &) noexcept; @@ -43,11 +51,34 @@ namespace xo { * to size at least @p z * In practice will round up to a multiple of @ref page_z_. **/ - static bool expand(DArena & d, std::size_t z) noexcept; + static bool expand(DArena & d, size_type z) noexcept; - static std::byte * alloc(DArena &, std::size_t z); + static value_type alloc(DArena &, size_type z); + /** when store_header_flag enabled: + * like alloc(), but combine memory consumed by this alloc + * plus following consecutive sub_alloc()'s into a single header. + * otherwise equivalent to alloc() + **/ + static value_type super_alloc(DArena &, size_type z); + /** when store_header_flag enabled: + * follow preceding super_alloc() by one or more sub_allocs(). + * accumulate total allocated size (including padding) into + * single header. All sub_allocs() except the last must set + * @p complete_flag to false. The last sub_alloc() must set + * @p complete_flag to true. + **/ + static value_type sub_alloc(DArena &, size_type z, bool complete_flag); static void clear(DArena &); static void destruct_data(DArena &); + + private: + /** alloc driver. shared by alloc(), super_alloc(), sub_alloc() **/ + static value_type _alloc(DArena &, + size_type z, + alloc_mode mode, + bool store_header_flag, + bool remember_header_flag); + }; // template <> diff --git a/include/xo/alloc2/IAllocator_Xfer.hpp b/include/xo/alloc2/IAllocator_Xfer.hpp index be5f215..c8ad818 100644 --- a/include/xo/alloc2/IAllocator_Xfer.hpp +++ b/include/xo/alloc2/IAllocator_Xfer.hpp @@ -18,7 +18,8 @@ namespace xo { struct IAllocator_Xfer : public AAllocator { // parallel interface to AAllocator, with specific data type using Impl = IAllocator_DRepr; - using size_type = std::size_t; + using size_type = AAllocator::size_type; + using value_type = AAllocator::value_type; static const DRepr & _dcast(Copaque d) { return *(const DRepr *)d; } static DRepr & _dcast(Opaque d) { return *(DRepr *)d; } @@ -43,8 +44,15 @@ namespace xo { bool expand(Opaque d, std::size_t z) const noexcept override { return I::expand(_dcast(d), z); } - std::byte * alloc(Opaque d, + value_type alloc(Opaque d, std::size_t z) const override { return I::alloc(_dcast(d), z); } + value_type super_alloc(Opaque d, + std::size_t z) const override { return I::super_alloc(_dcast(d), z); } + value_type sub_alloc(Opaque d, + std::size_t z, + bool complete_flag) const override { + return I::sub_alloc(_dcast(d), z, complete_flag); + } void clear(Opaque d) const override { return I::clear(_dcast(d)); } void destruct_data(Opaque d) const override { return I::destruct_data(_dcast(d)); } diff --git a/include/xo/alloc2/RAllocator.hpp b/include/xo/alloc2/RAllocator.hpp index 32e58e6..8f1bc80 100644 --- a/include/xo/alloc2/RAllocator.hpp +++ b/include/xo/alloc2/RAllocator.hpp @@ -18,6 +18,7 @@ namespace xo { public: using ObjectType = Object; using size_type = std::size_t; + using value_type = std::byte *; RAllocator() {} RAllocator(Object::DataPtr data) : Object{std::move(data)} {} @@ -33,7 +34,11 @@ namespace xo { AllocatorError last_error() const { return O::iface()->last_error(O::data()); } bool expand(size_type z) { return O::iface()->expand(O::data(), z); } - std::byte * alloc(size_type z) { return O::iface()->alloc(O::data(), z); } + value_type alloc(size_type z) { return O::iface()->alloc(O::data(), z); } + value_type super_alloc(size_type z) { return O::iface()->super_alloc(O::data(), z); } + value_type sub_alloc(size_type z, + bool complete_flag) { return O::iface()->sub_alloc(O::data(), + z, complete_flag); } static bool _valid; }; diff --git a/include/xo/alloc2/padding.hpp b/include/xo/alloc2/padding.hpp index f1ca0ae..151d558 100644 --- a/include/xo/alloc2/padding.hpp +++ b/include/xo/alloc2/padding.hpp @@ -50,7 +50,7 @@ namespace xo { **/ static inline std::size_t with_padding(std::size_t z, - std::size_t align) + std::size_t align = c_alloc_alignment) { return z + alloc_padding(z, align); } diff --git a/src/alloc2/DArena.cpp b/src/alloc2/DArena.cpp index 09afb1f..4d7bdc0 100644 --- a/src/alloc2/DArena.cpp +++ b/src/alloc2/DArena.cpp @@ -166,6 +166,9 @@ namespace xo { last_error_{} { //retval.checkpoint_ = lo_; + + /** make sure guard size is aligned **/ + config_.guard_z_ = padding::with_padding(config_.guard_z_); } DArena::DArena(DArena && other) { diff --git a/src/alloc2/IAllocator_DArena.cpp b/src/alloc2/IAllocator_DArena.cpp index cd3ec54..412651c 100644 --- a/src/alloc2/IAllocator_DArena.cpp +++ b/src/alloc2/IAllocator_DArena.cpp @@ -145,23 +145,194 @@ namespace xo { std::byte * IAllocator_DArena::alloc(DArena & s, std::size_t req_z) + { + /* - primary allocation path: + * exactly 1 header per alloc() call. + * - store_header_flag follows configuration + */ + + return _alloc(s, req_z, + alloc_mode::standard); + } + + std::byte * + IAllocator_DArena::super_alloc(DArena & s, + std::size_t req_z) + { + /* - (uncommon) pattern for parent alloc immediately followed by + * zero-or-more susidiary allocs, all sharing a single header. + * - collapses into alloc() behavior when + * ArenaConfig.store_header_flag_ disabled + */ + + bool remember_header_flag = s.config_.store_header_flag_; + + return _alloc(s, req_z, + alloc_mode::super); + } + + std::byte * + IAllocator_DArena::sub_alloc(DArena & s, + std::size_t req_z, + bool complete_flag) + { + /* - (uncommon) pattern for subsidiary allocs: + * that piggyback onto preceding super_alloc() + * - collapses into alloc() behavior when + * ArenaConfig.store_header_flag_ disabled + */ + + return _alloc(s, req_z, + (complete_flag + ? alloc_mode::sub_complete + : alloc_mode::sub_incomplete)); + +#ifdef OBSOLETE + if ((req_z == 0) && complete_flag) [[unlikely]] { + /** use zero req_z with complete_flag to clear s.last_header_ **/ + + if (s.config_.store_header_flag_) { + if (!s.last_header_) [[unlikely]] { + ++(s.error_count_); + s.last_error_ = AllocatorError(error::orphan_sub_alloc, + s.error_count_, + 0 /*add_commit_z*/, s.committed_z_, reserved(s)); + } else { + s.last_header_ = nullptr; + } + } + + return nullptr; + } + + byte * free0 = s.free_; + byte * mem = _alloc(s, req_z, + complete_flag ? alloc_mode::sub_complete : alloc_mode::sub, + false /*!store_header_flag*/, + false /*!remember_header_flag*/); + + if (!mem) [[unlikely]] { + /* error already captured */ + return nullptr; + } + + byte * free1 = s.free_; + /* used: accounting for padding applied to req_z */ + size_t z0 = (free1 - free0); + + assert(z0 > 0); + + if (s.config_.store_header_flag_) { + if (!s.last_header_) [[unlikely]] { + ++(s.error_count_); + s.last_error_ = AllocatorError(error::orphan_sub_alloc, + s.error_count_, + 0 /*add_commit_z*/, s.committed_z_, reserved(s)); + return nullptr; + } + + /* s.last_header_ holds aggregate size of preceding super_alloc + * (+ any sub-alloc's). + * + * Accumulate allocation size + */ + uint64_t header = *s.last_header_; + + if ((header & s.config_.header_size_mask_ & z0) != z0) [[unlikely]] { + /* cumulative alloc size doesn't fit in configured header_size_mask bits */ + ++(s.error_count_); + s.last_error_ = AllocatorError(error::header_size_mask, + s.error_count_, + 0 /*add_commit_z*/, s.committed_z_, reserved(s)); + return nullptr; + } + + *s.last_header_ = ((header & ~s.config_.header_size_mask_) | z0); + + if (complete_flag) { + s.last_header_ = nullptr; + } + } + + return mem; +#endif + } + + std::byte * + IAllocator_DArena::_alloc(DArena & s, + std::size_t req_z, + bool store_header_flag, + bool remember_header_flag) { scope log(XO_DEBUG(s.config_.debug_flag_)); assert(padding::is_aligned((size_t)s.free_)); + /* remember_header_flag -implies-> store_header_flag */ + assert(store_header_flag || !remember_header_flag); + /* + * free_(pre) + * v + * + * <-------------z1---------------> + * < guard >< hz >< req_z >< dz >< guard > + * + * used <== +++++++++0000zzzz@@@@@@@@@@@@@@@@@ppppppp+++++++++ ==> unallocated + * + * ^ ^ ^ + * header mem | + * ^ | + * last_header_ free_(post) + * + * [+] guard after each allocation, for simple sanitize checks + * [0] unused header bits (avail to application) + * [z] record allocation size + * [@] new allocated memory + * [p] padding (to uintptr_t alignment) + */ + + /* non-zero if header feature enabled */ + size_t hz = 0; /* dz: pad req_z to alignment size (multiple of 8 bytes, probably) */ size_t dz = padding::alloc_padding(req_z); - size_t z1 = req_z + dz; + size_t z0 = req_z + dz; + /* if non-zero: will store padded alloc size at the beginning of each allocation + * reminder: important to store padded size for correct arena iteration + */ + uint64_t header = req_z + dz; + + if (store_header_flag) { + if ((s.config_.header_size_mask_ & z0) == z0) [[likely]] { + hz = sizeof(header); + } else { + /* req_z doesn't fit in configured header_size_mask bits */ + ++(s.error_count_); + s.last_error_ = AllocatorError(error::header_size_mask, + s.error_count_, + 0 /*add_commit_z*/, s.committed_z_, reserved(s)); + return nullptr; + } + } + + size_t z1 = hz + z0; assert(padding::is_aligned(z1)); if (expand(s, allocated(s) + z1)) [[likely]] { - byte * mem = s.free_; + if (store_header_flag) { + (*(uint64_t *)s.free_) = header; + + if (remember_header_flag) { + s.last_header_ = (uint64_t *)s.free_; + } + } + + byte * mem = s.free_ + hz; s.free_ += z1; log && log(xtag("self", s.config_.name_), + xtag("hz", hz), xtag("z0", req_z), xtag("+pad", dz), xtag("z1", z1), diff --git a/utest/arena.test.cpp b/utest/arena.test.cpp index 1b7643d..9178b34 100644 --- a/utest/arena.test.cpp +++ b/utest/arena.test.cpp @@ -174,6 +174,44 @@ namespace xo { REQUIRE(a1o.committed() <= a1o.reserved()); } + TEST_CASE("allocator-alloc-2", "[alloc2][Allocator]") + { + using header_type = AAllocator::header_type; + + /* typed allocator a1o, with object header */ + ArenaConfig cfg { .name_ = "testarena", + .size_ = 64*1024, + .store_header_flag_ = true, + /* up to 4GB */ + .header_size_mask_ = 0xffffffff, + .debug_flag_ = false, + }; + DArena arena = DArena::map(cfg); + obj a1o{&arena}; + + REQUIRE(a1o.reserved() >= cfg.size_); + REQUIRE(a1o.committed() == 0); + REQUIRE(a1o.available() == 0); + REQUIRE(a1o.allocated() == 0); + + size_t z0 = 1; + byte * m0 = a1o.alloc(1); + + REQUIRE(m0); + + header_type* header = (header_type*)(m0 - sizeof(header_type)); + + REQUIRE(a1o.contains(header)); + REQUIRE(((*header) & cfg.header_size_mask_) == padding::with_padding(z0)); + REQUIRE(a1o.last_error().error_ == error::none); + REQUIRE(a1o.last_error().error_seq_ == 0); + REQUIRE(a1o.allocated() >= z0); + REQUIRE(a1o.allocated() < sizeof(AAllocator::header_type) + z0 + padding::c_alloc_alignment ); + REQUIRE(a1o.allocated() <= a1o.committed()); + REQUIRE(a1o.allocated() + a1o.available() == a1o.committed()); + REQUIRE(a1o.committed() <= a1o.reserved()); + } + TEST_CASE("allocator-fail-1", "[alloc2][AAllocator]") { /* typed allocator a1o */