diff --git a/xo-arena/include/xo/arena/DArena.hpp b/xo-arena/include/xo/arena/DArena.hpp index a3253220..8eb6125c 100644 --- a/xo-arena/include/xo/arena/DArena.hpp +++ b/xo-arena/include/xo/arena/DArena.hpp @@ -115,6 +115,7 @@ namespace xo { **/ bool contains(const void * addr) const noexcept { return (lo_ <= addr) && (addr < hi_); } +#ifdef OBSOLETE /** obtain uncommitted contiguous memory range comprising * a whole multiple of @p align_z bytes, of at least size @p req_z, * aligned on a @p align_z boundary. Uncommitted memory is not (yet) @@ -140,6 +141,7 @@ namespace xo { static range_type map_aligned_range(size_type req_z, size_type align_z, bool enable_hugepage_flag); +#endif /** true if arena is mapped i.e. has a reserved address range **/ bool is_mapped() const noexcept { return (lo_ != nullptr) && (hi_ != nullptr); } diff --git a/xo-arena/include/xo/arena/DArenaHashMap.hpp b/xo-arena/include/xo/arena/DArenaHashMap.hpp index 2a51eaa8..aef9ed3a 100644 --- a/xo-arena/include/xo/arena/DArenaHashMap.hpp +++ b/xo-arena/include/xo/arena/DArenaHashMap.hpp @@ -65,6 +65,10 @@ namespace xo { size_type capacity() const noexcept { return store_.capacity(); } float load_factor() const noexcept { return store_.load_factor(); } + /** verify DArenaHashMap invariants + * Act on failure according to policy @p + * (combination of throw|log bits) + **/ bool verify_ok(verify_policy p = verify_policy::throw_only()) const; iterator begin() { diff --git a/xo-arena/include/xo/arena/mmap_util.hpp b/xo-arena/include/xo/arena/mmap_util.hpp new file mode 100644 index 00000000..64cd4d25 --- /dev/null +++ b/xo-arena/include/xo/arena/mmap_util.hpp @@ -0,0 +1,49 @@ +/** @file mmap_util.hpp +* + * @author Roland Conybeare, Jan 2026 + **/ + +#pragma once + +#include "span.hpp" + +namespace xo { + namespace mm { + struct mmap_util { + using byte = std::byte; + using span_type = span; + using size_type = std::size_t; + + /** obtain uncommitted contiguous memory range comprising + * a whole multiple of @p align_z bytes, of at least size @p req_z, + * aligned on a @p align_z boundary. Uncommitted memory is not (yet) + * backed by physical memory. + * + * If @p enable_hugepage_flag is true and THP + * (transparent huge pages) are available, use THP for arena memory. + * This relieves TLB and page table memory when @p req_z is a lot larger than + * page size (likely 4KB). Cost is that arena will consum physical memory in unit + * of @p align_z. Arena may waste up to @p align_z bytes of memory as a result. + * + * If @p enable_hugepage_flag is true, @p align_z should be huge page size + * (probably 2MB) for optimal performance. + * + * At present the THP feature is not supported on OSX. + * May be supportable through mach_vm_allocate(). + * + * Note that we reject MAP_HUGETLB|MAP_HUGE_2MB flags to mmap here, + * since requires previously-reserved memory in /proc/sys/vm/nr_hugepages. + * + * Write log messages iff @p debug_flag is true. + * + * @return spqn giving reserved memory address range [lo,hi) + **/ + static span_type map_aligned_range(size_type req_z, + size_type align_z, + bool enable_hugepage_flag, + bool debug_flag); + }; + } /*namespace mm*/ +} /*namespace xo*/ + +/* end mmap_util.hpp */ diff --git a/xo-arena/include/xo/arena/span.hpp b/xo-arena/include/xo/arena/span.hpp new file mode 100644 index 00000000..e902193f --- /dev/null +++ b/xo-arena/include/xo/arena/span.hpp @@ -0,0 +1,304 @@ +/** @file span.hpp + * + * @author Roland Conybeare, Jul 2024 + **/ + +#pragma once + +#include "xo/indentlog/scope.hpp" +#include "xo/indentlog/print/ppdetail_atomic.hpp" +#include +#include +#include + +namespace xo { + namespace mm { + /** @class span compression/span.hpp + * + * @brief A contiguous range of characters, without ownership. + * + * @tparam CharT type for elements referred to by this span. + **/ + template + class span { + public: + /** @defgroup span-type-traits span type traits **/ + ///@{ + + /** typealias for span size (in units of CharT) **/ + using size_type = std::uint64_t; + + ///@} + + public: + /** @defgroup span-ctors span constructors **/ + ///@{ + + /** null span **/ + span() : lo_{nullptr}, hi_{nullptr} {} + + /** Create span for the contiguous memory range [@p lo, @p hi) **/ + span(CharT * lo, CharT * hi) : lo_{lo}, hi_{hi} {} + + /** Create span for the contiguous memory range [@p lo, @p lo + z) **/ + span(CharT * lo, size_t z) : lo_{lo}, hi_{lo + z} {} + + /** explicit conversion from span **/ + template + span(const span & other, + std::enable_if_t + && !std::is_same_v> * = nullptr) + : lo_{other.lo()}, hi_{other.hi()} {} + + /** copy ctor (explicit to avoid ambiguity with template ctor) **/ + span(const span & other) = default; + span & operator=(const span & other) = default; + + /** Create a null span (i.e. with null @p lo, @p hi pointers) + * A null span can be concatenated with any other span + * without triggering matching-endpoint asserts. + **/ + static span make_null() { return span(static_cast(nullptr), static_cast(nullptr)); } + + /** @brief create span for C-style string @p cstr **/ + static span from_cstr(const CharT * cstr) { + CharT * lo = cstr; + CharT * hi = cstr ? cstr + strlen(cstr) : nullptr; + + return span(lo, hi); + } + + /** @brief create span from std::string @p str **/ + static span from_string(const std::string& str) { + CharT * lo = &(*str.begin()); + CharT * hi = &(*str.end()); + + return span(lo, hi); + } + + /** @brief concatenate two contiguous spans */ + static span concat(const span & span1, const span & span2) { + if (span1.is_null()) + return span2; + if (span2.is_null()) + return span1; + + if (span1.hi() != span2.lo()) { + scope log(XO_DEBUG(true)); + + log && log(xtag("span1.hi", (void*)span1.hi()), xtag("span2.lo", (void*)span2.lo())); + } + + assert(span1.hi() == span2.lo()); + + CharT * lo = span1.lo(); + CharT * hi = span2.hi(); + + return span(lo, hi); + } + + ///@} + + /** @defgroup span-access-methods **/ + ///@{ + + CharT * lo() const { return lo_; } /* get member span::lo_ */ + CharT * hi() const { return hi_; } /* get member span::hi_ */ + + /** true iff this span is null. distinct from empty. **/ + bool is_null() const { return lo_ == nullptr && hi_ == nullptr; } + /** true iff this span is empty (comprises 0 elements). **/ + bool empty() const { return lo_ == hi_; } + /** report the number of elements (of type CharT) in this span. **/ + size_type size() const { return hi_ - lo_; } + + /** true iff this span is a subspan of @p other. + * i.e. other.lo() <= this->lo() && this->hi() <= other.hi() + **/ + bool is_subspan_of(const span & other) const noexcept { + return (other.lo() <= lo_) && (hi_ <= other.hi()); + } + + ///@} + + /** @defgroup span-general-methods **/ + ///@{ + + /** @brief strip prefix until first occurence of '\n', including the newline **/ + void discard_until_newline() { + for (const CharT * p = lo_; p < hi_; ++p) { + if (*p == '\n') { + lo_ = p + 1; + return; + } + } + + lo_ = hi_; + } + + /** Create new span over supplied type, + * with identical (possibly misaligned) endpoints. + * + * @warning + * 1. New span uses exactly the same memory addresses. + * Endpoint pointers may not be aligned. + * 2. Implementation assumes code compiled with + * @code -fno-strict-aliasing @endcode enabled. + * + * @tparam OtherT element type for new span + **/ + template + span + cast() const { return span(reinterpret_cast(lo_), + reinterpret_cast(hi_)); } + + /** @brief create span including the first @p z members of this span. **/ + span prefix(size_type z) const { return span(lo_, lo_ + z); } + + /** @brief create span representing prefix up to (but not including) @p *p + **/ + span prefix_upto(CharT * p) const { + if (p <= hi_) + return span(lo_, p); + else + return span(lo_, hi_); + } + + /** @brief create span with first @p z members of this span removed **/ + span after_prefix(size_type z) const { + if (lo_ + z > hi_) + z = hi_ - lo_; + + return span(lo_ + z, hi_); + } + + /** @brief create span with @p prefix of this span removed **/ + span after_prefix(const span & prefix) const { + if (!prefix.is_null() && (prefix.lo() != lo_)) { + throw std::runtime_error + ("after_prefix: expected prefix of this span"); + } + + return after_prefix(prefix.size()); + } + + /** Create span starting with position @p p. + * Does boundary checking; will return empty span if @p p is outside @c [lo_,hi) + **/ + span suffix_from(CharT * p) const { + if ((lo_ <= p) && (p <= hi_)) + return span(p, hi_); + else + return span(hi_, hi_); + } + + /** increase extent of this spans to include @p x. + * Requires @c hi() == @c x.lo() + **/ + span & operator+=(const span & x) { + if (hi_ == x.lo_) { + hi_ = x.hi_; + } else if (!x.is_null()) { + assert(false); + } + + return *this; + } + + /** print representation for this span on stream @p os **/ + void print(std::ostream & os) const { + os << ""; + } + ///@} + + private: + /** @defgroup span-instance-vars **/ + ///@{ + + /** start of span. + Span comprises memory address between @p lo (inclusive) and @p hi (exclusive) + **/ + CharT * lo_ = nullptr; + + /** @brief end of span. + Span comprises memory address between @p lo (inclusive) and @p hi (exclusive) + **/ + CharT * hi_ = nullptr; + + ///@} + }; /*span*/ + + /** @defgroup span-operators **/ + ///@{ + + /** compare spans for equality. + * Two spans are equal iff both endpoints match exactly. + **/ + template + inline bool + operator==(const span & lhs, const span & rhs) { + return ((lhs.lo() == rhs.lo()) + && (lhs.hi() == rhs.hi())); + } + + /** compare spans for inequality. + * Two spans are unequal if either paired endpoint differs. + **/ + template + inline bool + operator!=(const span & lhs, const span & rhs) { + return ((lhs.lo() != rhs.lo()) + || (lhs.hi() != rhs.hi())); + } + + /** print a summary of @p x on stream @p os. Intended for diagnostics **/ + template + inline std::ostream & + operator<<(std::ostream & os, + const span & x) { + x.print(os); + return os; + } + + ///@} + } /*namespace scm*/ + + namespace print { + template + class printspan_impl { + public: + printspan_impl(xo::mm::span x) : span_{x} {} + + xo::mm::span span_; + }; + + template + printspan_impl printspan(const xo::mm::span& span) { + return printspan_impl(span); + } + + template + inline std::ostream & + operator<< (std::ostream & os, + const printspan_impl & x) + { + for (const CharT * p = x.span_.lo(); p < x.span_.hi(); ++p) + os << *p; + + return os; + } + +#ifndef ppdetail_atomic + template \ + PPDETAIL_ATOMIC_BODY(printspan_impl); + + template \ + PPDETAIL_ATOMIC_BODY(xo::scm::span); +#endif + + } /*namespace mm*/ +} /*namespace xo*/ diff --git a/xo-arena/src/arena/CMakeLists.txt b/xo-arena/src/arena/CMakeLists.txt index 111bbe15..5288c393 100644 --- a/xo-arena/src/arena/CMakeLists.txt +++ b/xo-arena/src/arena/CMakeLists.txt @@ -3,10 +3,12 @@ set(SELF_LIB xo_arena) set(SELF_SRCS cmpresult.cpp + mmap_util.cpp AllocError.cpp AllocInfo.cpp DArena.cpp DArenaIterator.cpp + DCircularBuffer.cpp ) xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) diff --git a/xo-arena/src/arena/DArena.cpp b/xo-arena/src/arena/DArena.cpp index ee03ddea..670666f5 100644 --- a/xo-arena/src/arena/DArena.cpp +++ b/xo-arena/src/arena/DArena.cpp @@ -6,6 +6,7 @@ //#include "alloc/AAllocator.hpp" #include "DArena.hpp" #include "DArenaIterator.hpp" +#include "mmap_util.hpp" #include #include #include @@ -22,94 +23,6 @@ namespace xo { using std::size_t; namespace mm { - auto - DArena::map_aligned_range(size_t req_z, - size_t align_z, - bool enable_hugepage_flag) -> range_type - { - scope log(XO_DEBUG(false), - xtag("req_z", req_z), xtag("align_z", align_z)); - - // 1. round up to multiple of align_z - size_t target_z = padding::with_padding(req_z, align_z); // 4. - - // 2. mmap() will give us page-aligned memory, - // but not hugepage-aligned. - // - // Over-request by align_z to ensure - // aligned subrange of size target_z - // - byte * base = (byte *)(::mmap(nullptr, - target_z + align_z, - PROT_NONE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0)); - - // on mmap success: upper limit of mapped address range - byte * hi = base + (target_z + align_z); - // lowest hugepage-aligned address in [base, hi) - byte * aligned_base = (byte *)(padding::with_padding((size_t)base, align_z)); - // end of hugeppage-aligned range starting at aligned_base - byte * aligned_hi = aligned_base + target_z; - - log && log("acquired memory [lo,hi) using mmap", - xtag("lo", base), - xtag("aligned_lo", aligned_base), - xtag("req_z", req_z), - xtag("target_z", target_z), - xtag("aligned_hi", aligned_hi), - xtag("hi", hi)); - - // 3. assess mmap success - { - if (base == MAP_FAILED) { - throw std::runtime_error(tostr("ArenaAlloc: uncommitted allocation failed", - xtag("size", req_z))); - } - - assert((size_t)aligned_base % align_z == 0); - assert(aligned_base >= base); - assert(aligned_base < base + align_z); - } - - // 4. release unaligned prefix - if (base < aligned_base) { - size_t ua_prefix = aligned_base - base; - - ::munmap(base, ua_prefix); - } - - // 5. release unaligned suffix - if (aligned_hi < hi) { - size_t suffix = hi - aligned_hi; - - ::munmap(aligned_hi, suffix); - } - - if (enable_hugepage_flag) { -#ifdef __linux__ - /** linux: - * opt-in to transparent huge pages (THP) - * provided OS configured to support them. - * otherwise fallback gracefully. - * - * Huge pages -> use fewer TLB entries + faster - * shorter path through page table. - * - * When we commit (i.e. obtain physical memory on page fault), - * typically expect to pay ~1us per superpage. - * Much better than ~500us to commit 512 4k VM pages. - * - * But wasted if we don't use the memory. - * - * Page table has a handful of levels - **/ - ::madvise(aligned_base, target_z, MADV_HUGEPAGE); // 8. -#endif - } - - return std::make_pair(aligned_base, aligned_hi); - } DArena DArena::map(const ArenaConfig & cfg) @@ -130,14 +43,15 @@ namespace xo { log && log(xtag("page_z", page_z), xtag("align_z", align_z)); - auto [lo, hi] = map_aligned_range(cfg.size_, - align_z, - enable_hugepage_flag); + auto span = mmap_util::map_aligned_range(cfg.size_, + align_z, + enable_hugepage_flag, + cfg.debug_flag_); - if (!lo) { + if (!span.lo()) { // control here implies mmap() failed silently - throw std::runtime_error(tostr("ArenaAlloc: allocation failed", + throw std::runtime_error(tostr("ArenaAlloc: reserve address range failed", xtag("size", cfg.size_))); } @@ -148,7 +62,7 @@ namespace xo { xtag("hugepage_z", hugepage_z_)); #endif - return DArena(cfg, page_z, align_z, lo, hi); + return DArena(cfg, page_z, align_z, span.lo(), span.hi()); } /*map*/ DArena::DArena(const ArenaConfig & cfg,