diff --git a/include/xo/arena/DArena.hpp b/include/xo/arena/DArena.hpp new file mode 100644 index 0000000..f9c2e3c --- /dev/null +++ b/include/xo/arena/DArena.hpp @@ -0,0 +1,297 @@ +/** @file DArena.hpp + * + * @author Roland Conybeare, Dec 2025 + **/ + +#pragma once + +#include "ArenaConfig.hpp" +#include "AllocError.hpp" +#include "AllocInfo.hpp" +#include + +namespace xo { + namespace mm { + struct DArenaIterator; // see DArenaIterator.hpp + + /** @class DArena + * + * @brief represent arena allocator state + * + * Provides minimal RAII functionality around memory mapping. + * For allocation implementation see @ref IAllocator_DArena + **/ + struct DArena { + /* + * <----------------------------size--------------------------> + * <------------committed-----------><-------uncommitted------> + * <--allocated--> + * + * XXXXXXXXXXXXXXX___________________.......................... + * + * [X] allocated: in use + * [_] committed: physical memory obtained + * [.] uncommitted: mapped in virtual memory, not backed by memory + */ + + /** @defgroup mm-arena-traits arena type traits **/ + ///@{ + + /** @brief an amount of memory **/ + using size_type = std::size_t; + /** @brief allocation pointer; use for allocation results **/ + using value_type = std::byte*; + /** @brief a contiguous memory range **/ + using range_type = std::pair; + /** @brief type for allocation header (if enabled) **/ + using header_type = AllocHeader; + /** integer identifying a type (see xo::facet::typeid()) **/ + using typeseq = xo::reflect::typeseq; + + /** @brief mode argument for @ref _alloc **/ + enum class alloc_mode : uint8_t { + /** ordinary alloc. Most common mode **/ + standard, + /** begin a sequence of suballocs that share a single alloc header **/ + super, + /** make a subsidiary allocation on behalf of a preceding super alloc. + * Will be followed by at least one more suballoc call. + **/ + sub_incomplete, + /** make a subsidiary allocation that completes preceding super alloc. **/ + sub_complete, + }; + + ///@} + + /** @defgroup mm-arena-ctors arena constructors and destructors **/ + ///@{ + + /** create arena per configuration @p cfg. **/ + static DArena map(const ArenaConfig & cfg); + + /** null ctor **/ + DArena() = default; + /** ctor from already-mapped (but not committed) address range **/ + DArena(const ArenaConfig & cfg, + size_type page_z, + size_type arena_align_z, + value_type lo, + value_type hi); + /** DArena is not copyable **/ + DArena(const DArena & other) = delete; + /** move ctor **/ + DArena(DArena && other); + /** dtor releases mapped memory **/ + ~DArena(); + + /** move-assignment **/ + DArena & operator=(DArena && other); + + ///@} + + /** @defgroup mm-arena-methods **/ + ///@{ + + /** Reserved memory, in bytes. This is the maximum size of this arena. **/ + size_type reserved() const noexcept { return hi_ - lo_; } + /** Allocated memory in bytes: memory consumed by allocs from this arena, + * including administrative overhead (alloc headers + guard bytes) + **/ + size_type allocated() const noexcept { return free_ - lo_; } + /** Committed memory in bytes: amount of memory actually backed by physical memory **/ + size_type committed() const noexcept { return committed_z_; } + /** Available committed memory. + * This is the amount of memory guaranteed to be usable for future allocs from this arena. + **/ + size_type available() const noexcept { return limit_ - free_; } + + /** True iff address @p addr is owned by this arena, + * i.e. falls within [@ref lo_, @ref hi_) + **/ + bool contains(const void * addr) const noexcept { return (lo_ <= addr) && (addr < hi_); } + + /** obtain uncommitted contiguous memory range comprising + * a whole multiple of @p align_z bytes, of at least size @p req_z, + * aligned on a @p align_z boundary. Uncommitted memory is not (yet) + * backed by physical memory. + * + * If @p enable_hugepage_flag is true and THP + * (transparent huge pages) are available, use THP for arena memory. + * This relieves TLB and page table memory when @p req_z is a lot larger than + * page size (likely 4KB). Cost is that arena will consum physical memory in unit + * of @p align_z. Arena may waste up to @p align_z bytes of memory as a result. + * + * If @p enable_hugepage_flag is true, @p align_z should be huge page size + * (probably 2MB) for optimal performance. + * + * At present the THP feature is not supported on OSX. + * May be supportable through mach_vm_allocate(). + * + * Note that we reject MAP_HUGETLB|MAP_HUGE_2MB flags to mmap here, + * since requires previously-reserved memory in /proc/sys/vm/nr_hugepages. + * + * @return pair giving reserved memory address range [lo,hi) + **/ + static range_type map_aligned_range(size_type req_z, + size_type align_z, + bool enable_hugepage_flag); + + /** true if arena is mapped i.e. has a reserved address range **/ + bool is_mapped() const noexcept { return (lo_ != nullptr) && (hi_ != nullptr); } + + /** @ret iterator pointing to the first allocation in this arena **/ + DArenaIterator begin() const noexcept; + /** @ret iterator pointing to just after the last allocation in this arena **/ + DArenaIterator end() const noexcept; + + /** @ret header for first allocation in this arena **/ + AllocHeader * begin_header() const noexcept; + /** @ret location of header for next (not yet performed!) + * allocation in this arena + **/ + AllocHeader * end_header() const noexcept; + + /** get header from allocated object address **/ + header_type * obj2hdr(void * obj) noexcept; + + /** report alloc book-keeping info for allocation at @p mem + * + * Require: + * 1. @p mem is address returned by allocation on this arena + * i.e. by @ref IAllocator_DArena::alloc() or @ref IAllocator_DArena::alloc_super() + * 2. @p mem has not been invalidated since it was allocated + * i.e. by call to @ref DArena::clear + * + * Note: non-const, may stash error details + **/ + AllocInfo alloc_info(value_type mem) const noexcept; + + /** allocate at least @p z bytes of memory. + * Return nullptr and capture error if unable to satisfy request. + * May expand committed memory, as long as resulting committed size + * is no larger than reserved size + **/ + value_type alloc(typeseq t, size_type z); + + /** when store_header_flag enabled: + * like alloc(), but combine memory consumed by this alloc + * plus following consecutive sub_alloc()'s into a single header. + * otherwise equivalent to alloc() + **/ + value_type super_alloc(typeseq t, size_type z); + + /** when store_header_flag enabled: + * follow preceding super_alloc() by one or more sub_allocs(). + * accumulate total allocated size (including padding) into + * single header. All sub_allocs() except the last must set + * @p complete_flag to false. The last sub_alloc() must set + * @p complete_flag to true. + **/ + value_type sub_alloc(size_type z, bool complete_flag); + + /** alloc copy of @p src **/ + value_type alloc_copy(value_type src); + + /** capture error information: advance error count + set last_error **/ + void capture_error(error err, + size_type target_z = 0) const; + + /** alloc driver. shared by alloc(), super_alloc(), sub_alloc() **/ + value_type _alloc(std::size_t req_z, + alloc_mode mode, + typeseq tseq, + uint32_t age); + + /** expand committed space in arena @p d + * to size at least @p z + * In practice will round up to a multiple of @ref page_z_. + **/ + bool expand(size_type z) noexcept; + + /** create initial guard **/ + void establish_initial_guard() noexcept; + + /** discard all allocated memory, return to empty state + * Promise: + * - committed memory unchanged + * - available memory = committed memory + **/ + void clear() noexcept; + + ///@} + + /** @defgroup mm-arena-instance-vars **/ + ///@{ + + /** arena configuration **/ + ArenaConfig config_; + + /** size of a VM page (obtained automatically via getpagesize()). Likely 4k **/ + size_type page_z_ = 0; + + /** alignment for this arena. In practice will be either page_z_ or cfg.hugepage_z_ **/ + size_type arena_align_z_ = 0; + + /** arena owns memory in range [@ref lo_, @ref hi_) + **/ + std::byte * lo_ = nullptr; + + /** prefix of this size is committed. + * Remainder mapped but uncommitted. + **/ + size_type committed_z_ = 0; + + /** if config_.store_header_flag_: + * Pointer to header for last allocation. + **/ + header_type * last_header_ = nullptr; + + /** free pointer. + * Memory in range [@ref lo_, @ref free_) current in use + **/ + std::byte * free_ = nullptr; + + /** soft limit; end of committed virtual memory + * Memory in range [@ref lo_, @ref limit_) is committed + * (backed by physical memory) + **/ + std::byte * limit_ = nullptr; + + /** hard limit; end of reserved virtual memory + * Memory in range [@ref limit_, @ref hi_) is uncommitted + **/ + std::byte * hi_ = nullptr; + + /** count runtime errors. Each error updates @ref last_error_ **/ + uint32_t error_count_ = 0; + + /** capture some error details if/when error **/ + AllocError last_error_; + + ///@} + }; + + /** construct a @tparam T instance from arguments @p args + * using memory obtained from arena @p ialloc + **/ + template + static T * + construct_with(DArena & ialloc, Args&&... args) + { + using xo::reflect::typeseq; + + typeseq t = typeseq::id(); + std::byte * mem = ialloc.alloc(t, sizeof(T)); + + if (mem) + return new (mem) T(std::forward(args)...); + + return nullptr; + } + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end DArena.hpp */ diff --git a/include/xo/arena/DArenaIterator.hpp b/include/xo/arena/DArenaIterator.hpp new file mode 100644 index 0000000..cb277d6 --- /dev/null +++ b/include/xo/arena/DArenaIterator.hpp @@ -0,0 +1,127 @@ +/** @file DArenaIterator.hpp + * + * @author Roland Conybeare, Dec 2025 + **/ + +#pragma once + +#include "AllocInfo.hpp" +#include "AllocHeader.hpp" +#include "cmpresult.hpp" + +namespace xo { + namespace mm { + struct DArena; + + /** @class DArenaIterator + * @brief Representation for alloc iterator over arena + * + * Map showing an arena allocation: + * + * @verbatim + * + * <-------------z1---------------> + * < guard >< hz >< req_z >< dz >< guard > + * + * +++++++++0000zzzz@@@@@@@@@@@@@@@@@ppppppp+++++++++ + * + * ^ ^ ^ + * header mem header + * ^ (next alloc) + * DArenaIterator::pos_ + * + * guard [+] guard before+after each allocation, for simple sanitize checks + * header [0] alloc header (non-size bits) + * [z] alloc header (size bits) + * mem [@] app-requested memory, including padding [p] + * dz [p] padding (to uintptr_t alignment. req_z+dz recorded in header) + * free_ DArena::free_ just after guard bytes for last allocation + * + * @endverbatim + **/ + struct DArenaIterator { + /** @defgroup mm-arenaiterator-ctors DArenaIterator instance vars **/ + ///@{ + DArenaIterator() = default; + DArenaIterator(const DArena * arena, + AllocHeader * pos) : arena_{arena}, + pos_{pos} {} + + /** Create iterator in invalid state **/ + static DArenaIterator invalid() { return DArenaIterator(); } + + /** Create iterator pointing to the beginning of @p arena + * Iterator cannot modify memory, but can capture + * an iterator error in @p *arena + **/ + static DArenaIterator begin(const DArena * arena); + /** Create iterator pointing to the end of @p arena + * Iterator cannot modify memory, but can capture + * an iterator error in @p *arena + **/ + static DArenaIterator end(const DArena * arena); + ///@} + + /** @defgroup mm-arenaiterator-methods DArenaIterator methods **/ + ///@{ + /** Address of allocation header for beginning of alloc range in @p arena **/ + static AllocHeader * begin_header(const DArena * arena); + /** Address of allocation header for end of alloc range. + * This is the address of header for _next_ allocation in @p arena + * i.e. free pointer + **/ + static AllocHeader * end_header(const DArena * arena); + + /** A valid iterator can be compared, at least with itself + * It can be dereferenced if is also non-empty + **/ + bool is_valid() const noexcept { return (arena_ != nullptr) && (pos_ != nullptr); } + /** An invalid (or sentinel) iterator is incomparable with all + * iterators including itself + **/ + bool is_invalid() const noexcept { return !is_valid(); } + + /** fetch contents at current iterator position **/ + AllocInfo deref() const noexcept; + /** compare two iterators. To be comparable, + * iterators must refer to the same arena + **/ + cmpresult compare(const DArenaIterator & other) const noexcept; + /** advance iterator to next allocation **/ + void next() noexcept; + + /** cast iterator position to byte* */ + std::byte * pos_as_byte() const { return (std::byte *)pos_; } + + /** *ix synonym for ix.deref() **/ + AllocInfo operator*() const noexcept { return this->deref(); } + /** ++ix synonym for ix.next() **/ + DArenaIterator & operator++() noexcept { this->next(); return *this; } + ///@} + + /** @defgroup mm-arenaiterator-instance-vars **/ + ///@{ + /** iterator visits allocations from this arena **/ + const DArena * arena_ = nullptr; + /** current iterator position **/ + AllocHeader * pos_ = nullptr; + ///@} + }; + + inline bool + operator==(const DArenaIterator & x, + const DArenaIterator & y) + { + return x.compare(y).is_equal(); + } + + inline bool + operator!=(const DArenaIterator & x, + const DArenaIterator & y) + { + return !x.compare(y).is_equal(); + } + } /*namespace mm*/ +} /*namespace xo*/ + +/* end DArenaIterator.hpp */ diff --git a/src/arena/CMakeLists.txt b/src/arena/CMakeLists.txt index 806a1fb..111bbe1 100644 --- a/src/arena/CMakeLists.txt +++ b/src/arena/CMakeLists.txt @@ -2,9 +2,11 @@ set(SELF_LIB xo_arena) set(SELF_SRCS + cmpresult.cpp AllocError.cpp AllocInfo.cpp - cmpresult.cpp + DArena.cpp + DArenaIterator.cpp ) xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) diff --git a/src/arena/DArena.cpp b/src/arena/DArena.cpp new file mode 100644 index 0000000..5c0e364 --- /dev/null +++ b/src/arena/DArena.cpp @@ -0,0 +1,622 @@ +/** @file DArena.cpp + * + * @author Roland Conybeare, Dec 2025 + **/ + +//#include "alloc/AAllocator.hpp" +#include "DArena.hpp" +#include "DArenaIterator.hpp" +#include +#include +#include +#include +#include // for ::munmap() +#include // for ::getpagesize() +#include // for ::memset() + +namespace xo { + using xo::reflect::typeseq; + using std::byte; + using std::cerr; + using std::endl; + using std::size_t; + + namespace mm { + auto + DArena::map_aligned_range(size_t req_z, + size_t align_z, + bool enable_hugepage_flag) -> range_type + { + scope log(XO_DEBUG(true), + xtag("req_z", req_z), xtag("align_z", align_z)); + + // 1. round up to multiple of align_z + size_t target_z = padding::with_padding(req_z, align_z); // 4. + + // 2. mmap() will give us page-aligned memory, + // but not hugepage-aligned. + // + // Over-request by align_z to ensure + // aligned subrange of size target_z + // + byte * base = (byte *)(::mmap(nullptr, + target_z + align_z, + PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0)); + + // on mmap success: upper limit of mapped address range + byte * hi = base + (target_z + align_z); + // lowest hugepage-aligned address in [base, hi) + byte * aligned_base = (byte *)(padding::with_padding((size_t)base, align_z)); + // end of hugeppage-aligned range starting at aligned_base + byte * aligned_hi = aligned_base + target_z; + + log && log("acquired memory [lo,hi) using mmap", + xtag("lo", base), + xtag("aligned_lo", aligned_base), + xtag("req_z", req_z), + xtag("target_z", target_z), + xtag("aligned_hi", aligned_hi), + xtag("hi", hi)); + + // 3. assess mmap success + { + if (base == MAP_FAILED) { + throw std::runtime_error(tostr("ArenaAlloc: uncommitted allocation failed", + xtag("size", req_z))); + } + + assert((size_t)aligned_base % align_z == 0); + assert(aligned_base >= base); + assert(aligned_base < base + align_z); + } + + // 4. release unaligned prefix + if (base < aligned_base) { + size_t ua_prefix = aligned_base - base; + + ::munmap(base, ua_prefix); + } + + // 5. release unaligned suffix + if (aligned_hi < hi) { + size_t suffix = hi - aligned_hi; + + ::munmap(aligned_hi, suffix); + } + + if (enable_hugepage_flag) { +#ifdef __linux__ + /** linux: + * opt-in to transparent huge pages (THP) + * provided OS configured to support them. + * otherwise fallback gracefully. + * + * Huge pages -> use fewer TLB entries + faster + * shorter path through page table. + * + * When we commit (i.e. obtain physical memory on page fault), + * typically expect to pay ~1us per superpage. + * Much better than ~500us to commit 512 4k VM pages. + * + * But wasted if we don't use the memory. + * + * Page table has a handful of levels + **/ + ::madvise(aligned_base, target_z, MADV_HUGEPAGE); // 8. +#endif + } + + return std::make_pair(aligned_base, aligned_hi); + } + + DArena + DArena::map(const ArenaConfig & cfg) + { + scope log(XO_DEBUG(true)); + + /* vm page size. 4KB, probably */ + size_t page_z = getpagesize(); + + bool enable_hugepage_flag = (cfg.size_ >= cfg.hugepage_z_); + + /* Align start of arena memory on this boundary. + * Will use THP (transparent huge pages) if available + * and arena size is at least as large as hugepage size (2MB, probably) + */ + size_t align_z = (enable_hugepage_flag ? cfg.hugepage_z_ : page_z); + + log && log(xtag("page_z", page_z), + xtag("align_z", align_z)); + + auto [lo, hi] = map_aligned_range(cfg.size_, + align_z, + enable_hugepage_flag); + + if (!lo) { + // control here implies mmap() failed silently + + throw std::runtime_error(tostr("ArenaAlloc: allocation failed", + xtag("size", cfg.size_))); + } + + +#ifdef NOPE + log && log(xtag("lo", (void*)lo_), + xtag("page_z", page_z_), + xtag("hugepage_z", hugepage_z_)); +#endif + + return DArena(cfg, page_z, align_z, lo, hi); + } /*map*/ + + DArena::DArena(const ArenaConfig & cfg, + size_type page_z, + size_type arena_align_z, + byte * lo, + byte * hi) : config_{cfg}, + page_z_{page_z}, + arena_align_z_{arena_align_z}, + lo_{lo}, + committed_z_{0}, + free_{lo}, + limit_{lo}, + hi_{hi}, + error_count_{0}, + last_error_{} + { + //retval.checkpoint_ = lo_; + + /** make sure guard size is aligned **/ + config_.header_.guard_z_ + = padding::with_padding(config_.header_.guard_z_); + } + + DArena::DArena(DArena && other) { + config_ = other.config_; + page_z_ = other.page_z_; + arena_align_z_ = other.arena_align_z_; + lo_ = other.lo_; + committed_z_ = other.committed_z_; + free_ = other.free_; + limit_ = other.limit_; + hi_ = other.hi_; + error_count_ = other.error_count_; + last_error_ = other.last_error_; + + other.config_ = ArenaConfig(); + other.lo_ = nullptr; + other.committed_z_ = 0; + other.free_ = nullptr; + other.limit_ = nullptr; + other.hi_ = nullptr; + other.error_count_ = 0; + other.last_error_ = AllocError(); + } + + DArena & + DArena::operator=(DArena && other) + { + config_ = other.config_; + page_z_ = other.page_z_; + arena_align_z_ = other.arena_align_z_; + lo_ = other.lo_; + committed_z_ = other.committed_z_; + free_ = other.free_; + limit_ = other.limit_; + hi_ = other.hi_; + error_count_ = other.error_count_; + last_error_ = other.last_error_; + + other.config_ = ArenaConfig(); + other.lo_ = nullptr; + other.committed_z_ = 0; + other.free_ = nullptr; + other.limit_ = nullptr; + other.hi_ = nullptr; + other.error_count_ = 0; + other.last_error_ = AllocError(); + + return *this; + } + + DArena::~DArena() + { + if (lo_) { + //log && log("unmap [lo,hi)", + // xtag("lo", lo_), + // xtag("z", hi_ - lo_), + // xtag("hi", hi_)); + + ::munmap(lo_, hi_ - lo_); + } + + // hygiene + lo_ = nullptr; + committed_z_ = 0; + // checkpoint_ = nullptr; + free_ = nullptr; + limit_ = nullptr; + hi_ = nullptr; + error_count_ = 0; + last_error_ = AllocError(); + } + + auto + DArena::obj2hdr(void * obj) noexcept -> header_type * + { + assert(config_.store_header_flag_); + + return (header_type *)((byte *)obj - sizeof(header_type)); + } + + AllocInfo + DArena::alloc_info(value_type mem) const noexcept + { + if (!config_.store_header_flag_) [[unlikely]] { + this->capture_error(error::alloc_info_disabled); + + return AllocInfo::error_not_configured(&config_.header_); + } + + byte * header_mem = mem - sizeof(AllocHeader); + +#ifdef OBSOLETE // relying on cross-alloc header shenanigans in DX1Collector + if (!this->contains(header_mem)) { + this->capture_error(error::alloc_info_address); + } +#endif + + AllocHeader * header = (AllocHeader *)header_mem; + + const byte * guard_lo + = header_mem - config_.header_.guard_z_; + const byte * guard_hi + = mem + config_.header_.size(*header); + + return AllocInfo(&config_.header_, + guard_lo, + (AllocHeader *)header_mem, + guard_hi); + } + + DArenaIterator + DArena::begin() const noexcept + { + return DArenaIterator::begin(this); + } + + DArenaIterator + DArena::end() const noexcept + { + return DArenaIterator::end(this); + } + + AllocHeader * + DArena::begin_header() const noexcept + { + if (config_.store_header_flag_ == false) { + this->capture_error(error::alloc_iterator_not_supported); + + return nullptr; + } + + return (AllocHeader *)(lo_ + config_.header_.guard_z_); + } + + AllocHeader * + DArena::end_header() const noexcept + { + if (config_.store_header_flag_ == false) { + this->capture_error(error::alloc_iterator_not_supported); + + return nullptr; + } + + return (AllocHeader *)free_; + } + + std::byte * + DArena::alloc(typeseq t, std::size_t req_z) + { + /* - primary allocation path: + * exactly 1 header per alloc() call. + * - store_header_flag follows configuration + */ + return _alloc(req_z, alloc_mode::standard, t, 0 /*age*/); + } + + std::byte * + DArena::super_alloc(typeseq t, std::size_t req_z) + { + /* - (uncommon) pattern for parent alloc immediately followed by + * zero-or-more susidiary allocs, all sharing a single header. + * - collapses into alloc() behavior when + * ArenaConfig.store_header_flag_ disabled + */ + + (void)t; + + return _alloc(req_z, + alloc_mode::super, + t, + 0 /*age*/); + } + + std::byte * + DArena::sub_alloc(std::size_t req_z, + bool complete_flag) + { + /* - (uncommon) pattern for subsidiary allocs: + * that piggyback onto preceding super_alloc() + * - collapses into alloc() behavior when + * ArenaConfig.store_header_flag_ disabled + */ + + return _alloc(req_z, + (complete_flag + ? alloc_mode::sub_complete + : alloc_mode::sub_incomplete), + typeseq::anon() /*typeseq: ignored*/, + 0 /*age - ignored */); + } + + std::byte * + DArena::alloc_copy(std::byte * src) + { + /* NOTE: allocator that owns src must have the same header configuration */ + + assert(config_.store_header_flag_); + + /* src will come from an allocator other than this one; + * we rely on header layout from destination + * allocator -> assumes compatible header config + */ + AllocInfo src_info = alloc_info(src); + + size_t req_z = src_info.size(); + typeseq tseq = typeseq(src_info.tseq()); + uint32_t age = src_info.age(); + + return _alloc(req_z, alloc_mode::standard, tseq, age + 1); + } + + void + DArena::capture_error(error err, + size_type target_z) const + { + DArena * self = const_cast(this); + + ++(self->error_count_); + self->last_error_ = AllocError(err, + error_count_, + target_z, + committed_z_, + reserved()); + } + + byte * + DArena::_alloc(std::size_t req_z, + alloc_mode mode, + typeseq tseq, + uint32_t age) + { + scope log(XO_DEBUG(config_.debug_flag_)); + + /* + * sub_complete + * sub_incomplete | + * standard super | | + * v v v v + */ + std::array store_header_v = {{ true, true, false, false }}; + std::array retain_header_v = {{ false, true, false, false }}; + std::array store_guard_v = {{ true, false, false, true }}; + + /* -> write header at free_ */ + bool store_header_flag = false; + /* -> stash last_header_*/ + bool retain_header_flag = false; + /* -> write guard bytes */ + bool store_guard = false; + + if (config_.store_header_flag_) { + store_header_flag = store_header_v[(int)mode]; + retain_header_flag = retain_header_v[(int)mode]; + store_guard = store_guard_v[(int)mode]; + } + + assert(padding::is_aligned((size_t)free_)); + + /* + * free_(pre) + * v + * + * <-------------z1---------------> + * < guard >< hz >< req_z >< dz >< guard > + * + * used <== +++++++++0000zzzz@@@@@@@@@@@@@@@@@ppppppp+++++++++ ==> avail + * + * ^ ^ ^ + * header mem | + * ^ | + * last_header_ free_(post) + * + * [+] guard after each allocation, for simple sanitize checks + * [0] unused header bits (avail to application) + * [z] record allocation size + * [@] new allocated memory + * [p] padding (to uintptr_t alignment) + */ + + /* non-zero if header feature enabled */ + size_t hz = 0; + /* dz: pad req_z to alignment size (multiple of 8 bytes, probably) */ + size_t dz = padding::alloc_padding(req_z); + size_t z0 = req_z + dz; + /* if non-zero: + * will store padded alloc size at the beginning of each allocation + * reminder: + * important to store padded size for correct arena iteration + */ + uint64_t header = (req_z + dz); + + if (store_header_flag) + { + if (config_.header_.is_size_enabled()) [[likely]] { + header = this->config_.header_.mkheader(tseq.seqno(), age, req_z + dz); + hz = sizeof(header); + } else { + /* req_z doesn't fit in configured header_size_mask bits */ + capture_error(error::header_size_mask); + return nullptr; + } + } + + size_t z1 = hz + z0; + + assert(padding::is_aligned(z1)); + + if (!this->expand(this->allocated() + z1)) [[unlikely]] { + /* (error state already captured) */ + return nullptr; + } + + if (store_header_flag) { + /* capturing header */ + *(uint64_t *)free_ = header; + + if (retain_header_flag) { + /* and rembering for subsequent + * sub_alloc() + */ + last_header_ = (AllocHeader *)free_; + } + } + + byte * mem = free_ + hz; + + this->free_ += z1; + + if (store_guard) { + /* write guard bytes for overrun detection */ + ::memset(free_, + config_.header_.guard_byte_, + config_.header_.guard_z_); + + this->free_ += config_.header_.guard_z_; + } + + log && log(xtag("self", config_.name_), + xtag("hz", hz), + xtag("z0", req_z), + xtag("+pad", dz), + xtag("z1", z1), + xtag("size", this->committed()), + xtag("avail", this->available())); + log && log(xtag("mem", mem), + xtag("free", free_)); + + return mem; + } + + void + DArena::establish_initial_guard() noexcept + { + assert(free_ == lo_); + + ::memset(this->free_, + config_.header_.guard_byte_, + config_.header_.guard_z_); + + this->free_ += config_.header_.guard_z_; + } + + bool + DArena::expand(size_t target_z) noexcept + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("target_z", target_z), + xtag("committed_z", committed_z_)); + + if (target_z <= committed_z_) [[likely]] { + log && log("trivial success, offset within committed range", + xtag("target_z", target_z), + xtag("committed_z", committed_z_)); + return true; + } + + if (lo_ + target_z > hi_) [[unlikely]] { + this->capture_error(error::reserve_exhausted, target_z); + return false; + } + + /* + * pre: + * + * _______________................................... + * ^ ^ ^ + * lo limit hi + * + * < committed_z > + * <----------target_z-----------> + * > <- z: 0 <= z < hugepage_z + * <---------aligned_target_z---------> + * <--- add_commit_z --> + * + * post: + * ____________________________________.............. + * ^ ^ ^ + * lo limit hi + * + */ + + std::size_t aligned_target_z = padding::with_padding(target_z, arena_align_z_); + std::byte * commit_start = limit_; // = lo_ + committed_z_; + std::size_t add_commit_z = aligned_target_z - committed_z_; + + assert(limit_ == lo_ + committed_z_); + + if (::mprotect(commit_start, + add_commit_z, + PROT_READ | PROT_WRITE) != 0) [[unlikely]] + { + if (log) { + log("commit failed!"); + log(xtag("aligned_target_z", aligned_target_z), + xtag("commit_start", commit_start), + xtag("add_commit_z", add_commit_z), + xtag("commit_end", commit_start + add_commit_z) + ); + } + + capture_error(error::commit_failed, add_commit_z); + return false; + } + + committed_z_ = aligned_target_z; + limit_ = lo_ + committed_z_; + + if (commit_start == lo_) [[unlikely]] { + /* first expand() for this allocator - start with guard_z_ bytes */ + + this->establish_initial_guard(); + } + + assert(committed_z_ % arena_align_z_ == 0); + assert(reinterpret_cast(limit_) % arena_align_z_ == 0); + + return true; + } /*expand*/ + + void + DArena::clear() noexcept + { + this->free_ = lo_; + this->establish_initial_guard(); + } + } +} /*namespace xo*/ + +/* end DArena.cpp */ diff --git a/src/arena/DArenaIterator.cpp b/src/arena/DArenaIterator.cpp new file mode 100644 index 0000000..931108b --- /dev/null +++ b/src/arena/DArenaIterator.cpp @@ -0,0 +1,144 @@ +/** @file DArenaIterator.cpp +* + * @author Roland Conybeare, Dec 2025 + **/ + +#include "DArenaIterator.hpp" +#include "DArena.hpp" +#include +#include +#include + +namespace xo { + using std::byte; + + namespace mm { + DArenaIterator + DArenaIterator::begin(const DArena * arena) + { + constexpr bool c_debug_flag = false; + scope log(XO_DEBUG(c_debug_flag)); + + AllocHeader * begin_hdr = begin_header(arena); + + if (!begin_hdr) + return DArenaIterator::invalid(); + + log && log(xtag("begin_hdr", begin_hdr)); + + return DArenaIterator(arena, begin_hdr); + } + + DArenaIterator + DArenaIterator::end(const DArena * arena) + { + constexpr bool c_debug_flag = false; + scope log(XO_DEBUG(c_debug_flag)); + + AllocHeader * end_hdr = end_header(arena); + + if (!end_hdr) + return DArenaIterator::invalid(); + + log && log(xtag("end_hdr", end_hdr)); + + return DArenaIterator(arena, end_hdr); + } + + AllocHeader * + DArenaIterator::begin_header(const DArena * arena) + { + assert(arena); + + return arena->begin_header(); + } + + AllocHeader * + DArenaIterator::end_header(const DArena * arena) + { + assert(arena); + + return arena->end_header(); + } + + AllocInfo + DArenaIterator::deref() const noexcept + { + constexpr bool c_debug_flag = false; + scope log(XO_DEBUG(c_debug_flag)); + + bool contains_flag = arena_->contains(this->pos_as_byte()); + bool bounds_flag = (this->pos_as_byte() < arena_->free_); + + log && log(xtag("contains_flag", contains_flag), + xtag("bounds_flag", bounds_flag)); + + if (!contains_flag || !bounds_flag) { + arena_->capture_error(error::alloc_iterator_deref); + + return AllocInfo::error_invalid_iterator(&(arena_->config_.header_)); + } + + /* iterator points to beginning of header. + * memory given to application start immediately followed header + */ + + byte * mem = (byte *)(pos_ + 1); + + return arena_->alloc_info(mem); + } + + cmpresult + DArenaIterator::compare(const DArenaIterator & other_ix) const noexcept + { + scope log(XO_DEBUG(false), + xtag("arena", arena_), + xtag("pos", pos_), + xtag("other.arena", other_ix.arena_), + xtag("other.pos", other_ix.pos_)); + + if (is_invalid() || (arena_ != other_ix.arena_)) + return cmpresult::incomparable(); + + if (pos_ < other_ix.pos_) { + return cmpresult::lesser(); + } else if(pos_ == other_ix.pos_) { + return cmpresult::equal(); + } else { + return cmpresult::greater(); + } + } + + void + DArenaIterator::next() noexcept + { + constexpr bool c_debug_flag = false; + scope log(XO_DEBUG(c_debug_flag)); + + bool contains_flag = arena_->contains(this->pos_as_byte()); + bool bounds_flag = (this->pos_as_byte() < arena_->free_); + + log && log(xtag("contains_flag", contains_flag), + xtag("bounds_flag", bounds_flag)); + + if (!contains_flag || !bounds_flag) { + arena_->capture_error(error::alloc_iterator_next); + return; + } + + size_t mem_z = arena_->config_.header_.size(*pos_); + size_t guard_z = arena_->config_.header_.guard_z_; + + byte * next_as_byte = ((byte *)pos_ + sizeof(AllocHeader) + mem_z + guard_z); + /* next == ix.arena_free_ --> iterator is at end of allocator */ + assert(next_as_byte <= arena_->free_); + + AllocHeader * next = (AllocHeader *)next_as_byte; + + this->pos_ = next; + } + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end DArenaIterator.cpp */