diff --git a/include/xo/arena/ArenaConfig.hpp b/include/xo/arena/ArenaConfig.hpp index db5a440..4d72b91 100644 --- a/include/xo/arena/ArenaConfig.hpp +++ b/include/xo/arena/ArenaConfig.hpp @@ -19,6 +19,12 @@ namespace xo { struct ArenaConfig { /** @defgroup mm-arenaconfig-ctors **/ + ArenaConfig with_name(std::string name) { + ArenaConfig copy(*this); + copy.name_ = name; + return copy; + } + ArenaConfig with_size(std::size_t z) { ArenaConfig copy(*this); copy.size_ = z; diff --git a/include/xo/arena/DArena.hpp b/include/xo/arena/DArena.hpp index ffd710f..8c7203d 100644 --- a/include/xo/arena/DArena.hpp +++ b/include/xo/arena/DArena.hpp @@ -7,6 +7,7 @@ #include "ArenaConfig.hpp" #include "AllocError.hpp" +#include "MemorySizeInfo.hpp" #include "AllocInfo.hpp" #include @@ -140,6 +141,9 @@ namespace xo { /** get header from allocated object address **/ header_type * obj2hdr(void * obj) noexcept; + /** resource ocnsumption in normal form **/ + MemorySizeInfo _store_info() const noexcept; + /** report alloc book-keeping info for allocation at @p mem * * Require: @@ -206,6 +210,7 @@ namespace xo { /** restore arena state to previously-established checkpoint **/ void restore(Checkpoint ckp) noexcept { free_ = ckp.free_; } + /** discard all allocated memory, return to empty state * Promise: * - committed memory unchanged diff --git a/include/xo/arena/DArenaHashMap.hpp b/include/xo/arena/DArenaHashMap.hpp index 50b7b4a..042dc90 100644 --- a/include/xo/arena/DArenaHashMap.hpp +++ b/include/xo/arena/DArenaHashMap.hpp @@ -45,6 +45,7 @@ namespace xo { using value_type = std::pair; using key_hash = Hash; using key_equal = Equal; + using MemorySizeInfo = xo::mm::MemorySizeInfo; using byte = std::byte; using group_type = detail::ControlGroup; using store_type = detail::HashMapStore; @@ -76,6 +77,11 @@ namespace xo { iterator begin() { return _promote_iterator(_begin_aux()); } iterator end() { return _promote_iterator(_end_aux()); } + std::size_t _n_store() const noexcept { return store_._n_store(); } + MemorySizeInfo _store_info(std::size_t i) const noexcept { + return store_._store_info(i); + } + /** insert @p kv_pair into hash map. * Replaces any previous value stored under the same key. * diff --git a/include/xo/arena/DArenaVector.hpp b/include/xo/arena/DArenaVector.hpp index f8bd982..b7a0b2a 100644 --- a/include/xo/arena/DArenaVector.hpp +++ b/include/xo/arena/DArenaVector.hpp @@ -79,6 +79,11 @@ namespace xo { constexpr T * data() { return reinterpret_cast(store_.lo_); } constexpr const T * data() const { return reinterpret_cast(store_.lo_); } + /** arena used for element storage + * (Might prefer obj here; refrain to avoid leveling violation) + **/ + MemorySizeInfo _store_info() const { return store_._store_info(); } + /** reserve space, if possible, for at least @p z elements. * Always limited by ArenaConfig.size_ **/ diff --git a/include/xo/arena/DCircularBuffer.hpp b/include/xo/arena/DCircularBuffer.hpp index 30a3896..538d272 100644 --- a/include/xo/arena/DCircularBuffer.hpp +++ b/include/xo/arena/DCircularBuffer.hpp @@ -83,6 +83,9 @@ namespace xo { const_span_type occupied_range() const noexcept { return occupied_range_; } const_span_type input_range() const noexcept { return input_range_; } + std::size_t _n_store() const noexcept; + MemorySizeInfo _store_info(std::size_t i) const noexcept; + /** verify DCircularBuffer invariants. * Act on failure according to policy @p p * (combination of throw|log bits) diff --git a/include/xo/arena/ErrorArena.hpp b/include/xo/arena/ErrorArena.hpp new file mode 100644 index 0000000..0236c32 --- /dev/null +++ b/include/xo/arena/ErrorArena.hpp @@ -0,0 +1,54 @@ +/** @file ErrorArena.hpp +* + * @author Roland Conybeare, Feb 2026 + **/ + +#pragma once + +#include "DArena.hpp" + +namespace xo { + namespace mm { + + /** @brief Dedicated arena for error reporting + * + * Reserving memory for error messaages. + * Motivation + * 1. so we have room to report an out-of-memory condition + * 2. so we have place to allocate for an error that + * doesn't interfere with other allocator state + * + * Expect to reset arena between errors, so only need + * enough room to report one error. + * + * To initialize explicitly: + * @code + * // before any other ErrorArena method calls: + * ErrorArena::init_once(cfg...); + * + * // do stuff with ErrorArena.. + * ErrorArena::instance() + * @endcode + * + * Reminder: can't use obj here, + * would be leveling violation. + **/ + class ErrorArena { + public: + /** default configuration for error arena **/ + static ArenaConfig default_config(); + + /** idempotent initialization **/ + static void init_once(const ArenaConfig & cfg = default_config()); + + /** get initialized instnace **/ + static DArena * instance(); + + private: + static DArena s_instance; + }; + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end ErrorArena.hpp */ diff --git a/include/xo/arena/MemorySizeInfo.hpp b/include/xo/arena/MemorySizeInfo.hpp new file mode 100644 index 0000000..7cba082 --- /dev/null +++ b/include/xo/arena/MemorySizeInfo.hpp @@ -0,0 +1,37 @@ +/** @file MemorySizeInfo.hpp +* + * @author Roland Conybeare, Feb 2026 + **/ + +#pragma once + +#include + +namespace xo { + namespace mm { + + struct MemorySizeInfo { + using size_type = std::size_t; + + MemorySizeInfo(std::string_view name, std::size_t a, std::size_t c, std::size_t r) + : resource_name_{name}, allocated_{a}, committed_{c}, reserved_{r} + {} + + static MemorySizeInfo sentinel() { return MemorySizeInfo("", 0, 0, 0); } + + /** resource name **/ + std::string_view resource_name_; + /** memory in-use **/ + std::size_t allocated_ = 0; + /** memory committed (backed by physical memory) **/ + std::size_t committed_ = 0; + /** memory reserved: + * virtual memory addresses range obtained, whether or not committed + **/ + std::size_t reserved_ = 0; + }; + + } +} + +/* end MemorySizeInfo.hpp */ diff --git a/include/xo/arena/arena_streambuf.hpp b/include/xo/arena/arena_streambuf.hpp new file mode 100644 index 0000000..2b36462 --- /dev/null +++ b/include/xo/arena/arena_streambuf.hpp @@ -0,0 +1,236 @@ +/** @file arena_streambuf.hpp +* + * @author Roland Conybeare, Feb 2026 + **/ + +#pragma once + +#include "DArena.hpp" +//#include "print/quoted_char.hpp" +#include +#include +#include +#include // e.g. for std::memcpy() +#include +#include + +namespace xo { + namespace mm { + /** @brief Arena-based buffer for logging and pretty-printing + * + * Arena-based using mmap + * Write to self-extending storage array + * Track position relative to start of line + **/ + class arena_streambuf : public std::streambuf { + public: + struct rewind_state { + explicit rewind_state(std::size_t solpos, std::size_t color_esc, std::uint32_t p) + : solpos{solpos}, color_escape_chars{color_esc}, pos{p} {} + + std::size_t solpos = 0; + std::size_t color_escape_chars = 0; + std::uint32_t pos = 0; + }; + + public: + /** arena should be ready-to-allocate i.e. have committed > 0 **/ + arena_streambuf(DArena * arena, bool debug_flag = false) : arena_{arena}, debug_flag_{debug_flag} { + this->reset_stream(); + } /*ctor*/ + + std::streamsize capacity() const { return arena_->committed(); } + const char * lo() const { return this->pbase(); } + const char * hi() const { return this->lo() + this->capacity(); } + std::uint32_t pos() const { return this->pptr() - this->pbase(); } + + /** output position (relative to pbase) when local state last computed. Exposed here for unit tests **/ + std::size_t _local_ppos() const { return local_ppos_; } + /** position (relative to pbase) one character after last \n or \r. For unit tests **/ + std::uint32_t _solpos() const { return solpos_; } + /** start of incomplete color-escape sequence **/ + const char * _color_escape_start() const { return color_escape_start_; } + /** number of non-printing chars after @ref solpos_ from completed color-escape sequences **/ + std::uint32_t _color_escape_chars() const { return color_escape_chars_; } + + /** number of visible characters since start of line (last \n or \r) **/ + std::uint32_t lpos() const; + + rewind_state checkpoint() const; + + bool debug_flag() const { return debug_flag_; } + + operator std::string_view () const { return std::string_view(this->pbase(), this->pptr()); } + + void reset_stream(); + + void rewind_to(rewind_state s); + + protected: + /** expand buffer storage (by 2x), preserve current contents **/ + void expand_to(std::size_t new_z); + + virtual std::streamsize xsputn(const char * s, std::streamsize n) override; + + virtual int_type overflow(int_type new_ch) override; + + /* off. offset, relative to starting point dir. + * dir. + * which. in|out|both + * + * Note that off=0,dir=cur,which=out reads offset + */ + virtual pos_type seekoff(off_type off, + std::ios_base::seekdir dir, + std::ios_base::openmode which) override; + + private: + void _update_local_state_char(const char * p_lo, const char * p) + { + if ((*p == '\n') || (*p == '\r')) { + this->solpos_ = (p+1 - this->pbase()); + /* reset, since these chars relevant as correction to solpos */ + this->color_escape_chars_ = 0; + /* -> incomplete color escape, broken by newline */ + this->color_escape_start_ = nullptr; + } else if (*p == '\033') { + if (debug_flag_) [[unlikely]] { + std::cout << "xsputn: \\033 at p-p_lo=" << (p - p_lo) << std::endl; + } + this->color_escape_start_ = p; + } else if (this->color_escape_start_ != nullptr) { + if (*p == 'm') { + /* escape seq non-printing including both endpoints */ + std::int64_t esc_chars = (p+1 - color_escape_start_); + + this->color_escape_chars_ += esc_chars; + + if (debug_flag_) [[unlikely]] { + std::cout << "xsputn: m at p-p_lo" << (p - p_lo) << " +" << esc_chars + << " -> color_escape_chars=" << color_escape_chars_ << std::endl; + } + this->color_escape_start_ = nullptr; + } else if (!isdigit(*p) && (*p != '[') && (*p != ';')) { + /* not color escape after all */ + this->color_escape_start_ = nullptr; + } + } + } + + /** recognize stale local state vars: + * @ref solpos_, @ref color_escape_chars_, @ref color_escape_start_. + * + * Require: + * - {pbase, pptr} in consistent state + * Promise: + * - @c local_ppos_ + @c pbase = @c pptr + * - @c solpos_, @c color_escape_chars_, @c color_escape_start_ all up-to-date + **/ + void _check_update_local_state() { + const char * p0 = this->pbase(); + const char * pn = this->pptr(); + + if (debug_flag_) { + std::cerr << "_check_update_local_state:" << std::endl; + std::cerr << " buf: (p0=" << (void*)p0 << ", pn=" << (void*)pn << ")" << std::endl; + std::cerr << " solpos_=" << solpos_ << ", color_escape_chars_=" << color_escape_chars_ << std::endl; + } + + if (p0 + local_ppos_ == pn) [[likely]] { + // solpos_, color_escape_chars_, color_escape_start_ all up-to-date + } else { + // [pnew, pn): input that hasn't been incorporated into + // {solpos_, color_escape_chars_, color_escape_start_) + + const char * pnew = this->pbase() + this->local_ppos_; + + if (debug_flag_) { + std::cerr << "_check_update_local_state: range: (pnew=" << (void*)pnew << ", pn=" << (void*)pn << ")" << std::endl; + } + + for(const char * p = pnew; p < pn; ++p) { + this->_update_local_state_char(p0, p); + } + } + + // solpos_, color_escape_chars_, color_escape_start_ all up-to-date + // for current buffered contents + + this->local_ppos_ = pn - p0; + + if (debug_flag_) { + std::cerr << "_check_update_local_state: pos=" << pos(); + std::cerr << ", solpos=" << solpos_; + std::cerr << ", color_escape_chars=" << color_escape_chars_ << std::endl; + } + + assert(pos() >= solpos_ + color_escape_chars_); + } + + private: + /* + * pbase: start of buffered text. Thils will be arena_->lo_ + * + * + * pbase pptr epptr + * v >e1< >e2< v v + * |xx\xxEEExxx\xxxxxxxEExxxxEExxxxxxxEExxx\xEExxxxxx..................| + * ^ ^<------new-------> + * solpos local_ppos + * + * solpos : first character after newline (stale) + * color_escape_pos : e1+e2+.. (stale) + * new : new characters not reflected + * in local_ppos_, color_escape_chars_ etc. + * + * Legend: + * [\] newline + * [x] visible character + * [E] color escape chars + * + * + * after _check_update_local_state(): + * + * + * pbase pptr epptr + * v >e1< v v + * |xx\xxEEExxx\xxxxxxxEExxxxEExxxxxxxEExxx\xEExxxxxx..................| + * ^ ^ + * solpos local_ppos + * + */ + + /** @defgroup logstreambuf-instance-vars **/ + ///@{ + + /** value of pptr (relative to pbase) when _check_update_local_state() last ran **/ + std::size_t local_ppos_ = 0; + /** position (relative to pbase) one character after last \n or \r. + * Use to drive @ref lpos. This _has_ to be lazy, since + * xsputn() isn'g guaranteed to be called when there's room in + * in buffer. + **/ + std::size_t solpos_ = 0; + /** number of non-printing chars after @ref solpos_, from + * completed color escape sequences. + * (ansi color escapes = text between '\033' and 'm') + **/ + std::size_t color_escape_chars_ = 0; + /** non-null: start of incomplete color escape sequence **/ + const char * color_escape_start_ = nullptr; + + /** buffered output stored here. + * We don't use arena's allocation api, just treat as a block of available memory + **/ + DArena * arena_ = nullptr;; + /** true to debug log_streambuf itself **/ + bool debug_flag_ = false; + + ///@} + }; /*log_streambuf*/ + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end arena_streambuf.hpp */ + diff --git a/include/xo/arena/hashmap/HashMapStore.hpp b/include/xo/arena/hashmap/HashMapStore.hpp index fd7c87f..74f2f4e 100644 --- a/include/xo/arena/hashmap/HashMapStore.hpp +++ b/include/xo/arena/hashmap/HashMapStore.hpp @@ -19,6 +19,7 @@ namespace xo { using group_type = detail::ControlGroup; using control_vector_type = xo::mm::DArenaVector; using slot_vector_type = xo::mm::DArenaVector; + using MemorySizeInfo = xo::mm::MemorySizeInfo; public: /** group_exp2: number of groups {x, 2^x} **/ @@ -47,6 +48,18 @@ namespace xo { size_type capacity() const noexcept { return n_group_ * c_group_size; } float load_factor() const noexcept { return size_ / static_cast(n_slot_); } + std::size_t _n_store() const noexcept { return 2; } + MemorySizeInfo _store_info(std::size_t i) const noexcept { + switch (i) { + case 0: + return control_._store_info(); + case 1: + return slots_._store_info(); + } + + return MemorySizeInfo::sentinel(); + } + void resize_from_empty(const std::pair & group_exp2) { diff --git a/src/arena/CMakeLists.txt b/src/arena/CMakeLists.txt index 5288c39..b3bb35a 100644 --- a/src/arena/CMakeLists.txt +++ b/src/arena/CMakeLists.txt @@ -2,6 +2,8 @@ set(SELF_LIB xo_arena) set(SELF_SRCS + arena_streambuf.cpp + ErrorArena.cpp cmpresult.cpp mmap_util.cpp AllocError.cpp diff --git a/src/arena/DArena.cpp b/src/arena/DArena.cpp index 670666f..3156fc1 100644 --- a/src/arena/DArena.cpp +++ b/src/arena/DArena.cpp @@ -165,6 +165,15 @@ namespace xo { return (header_type *)((byte *)obj - sizeof(header_type)); } + MemorySizeInfo + DArena::_store_info() const noexcept + { + return MemorySizeInfo(config_.name_, + this->allocated(), + this->committed(), + this->reserved()); + } + AllocInfo DArena::alloc_info(value_type mem) const noexcept { diff --git a/src/arena/DCircularBuffer.cpp b/src/arena/DCircularBuffer.cpp index 2a415c4..2a5fb48 100644 --- a/src/arena/DCircularBuffer.cpp +++ b/src/arena/DCircularBuffer.cpp @@ -79,6 +79,31 @@ namespace xo { { } + std::size_t + DCircularBuffer::_n_store() const noexcept + { + return 2; + } + + MemorySizeInfo + DCircularBuffer::_store_info(std::size_t i) const noexcept + { + switch (i) { + case 0: + return MemorySizeInfo(config_.name_, + occupied_range_.size(), + mapped_range_.size(), + reserved_range_.size()); + case 1: + return pinned_spans_._store_info(); + default: + break; + } + + return MemorySizeInfo::sentinel(); + } + + bool DCircularBuffer::verify_ok(verify_policy policy) const { diff --git a/src/arena/ErrorArena.cpp b/src/arena/ErrorArena.cpp new file mode 100644 index 0000000..d1d3d62 --- /dev/null +++ b/src/arena/ErrorArena.cpp @@ -0,0 +1,43 @@ +/** @file ErrorArena.cpp +* + * @author Roland Conybeare, Feb 2026 + **/ + +#include "ErrorArena.hpp" + +namespace xo { + namespace mm { + DArena + ErrorArena::s_instance; + + ArenaConfig + ErrorArena::default_config() + { + return ArenaConfig().with_name("error-arena").with_size(16 * 1024); + } + + namespace { + bool s_init_done = false; + } + + void + ErrorArena::init_once(const ArenaConfig & cfg) + { + if (!s_init_done) { + s_init_done = true; + s_instance = DArena::map(cfg); + } + } + + DArena * + ErrorArena::instance() + { + init_once(default_config()); + + return &s_instance; + } + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end ErrorArena.cpp */ diff --git a/src/arena/arena_streambuf.cpp b/src/arena/arena_streambuf.cpp new file mode 100644 index 0000000..18ea132 --- /dev/null +++ b/src/arena/arena_streambuf.cpp @@ -0,0 +1,214 @@ +/** @file arena_streambuf.cpp + * + * @author Roland Conybeare, Feb 2026 + **/ + +#include "arena_streambuf.hpp" + +namespace xo { + namespace mm { + + std::uint32_t + arena_streambuf::lpos() const + { + if (debug_flag_) { + std::cerr << "log_streambuf::lpos: enter" << std::endl; + } + + // logically-const. lazy implementation + arena_streambuf * self = const_cast(this); + + self->_check_update_local_state(); + + return pos() - solpos_ - color_escape_chars_; + } + + auto + arena_streambuf::checkpoint() const -> rewind_state + { + // logically-const. lazy implementation + arena_streambuf * self = const_cast(this); + + self->_check_update_local_state(); + + return rewind_state(solpos_, color_escape_chars_, pos()); + } + + void + arena_streambuf::reset_stream() + { + assert(arena_); + assert(arena_->committed() > 0); + + char * p_lo = (char *)(arena_->lo_); + char * p_hi = (char *)(arena_->limit_); + + /* tells parent our buffer extent */ + this->setp(p_lo, p_hi); + + this->local_ppos_ = 0; + this->solpos_ = 0; + this->color_escape_chars_ = 0; + this->color_escape_start_ = nullptr; + } + + void + arena_streambuf::rewind_to(rewind_state s) + { + if (debug_flag_) { + std::cout << "rewind_to: pos " << pos() << "->" << s.pos + << " solpos " << solpos_ << "->" << s.solpos + << " color_esc " << color_escape_chars_ << "->" << s.color_escape_chars + << std::endl; + } + + /* .setp(): using just for side effect: sets .pptr to .pbase */ + this->setp(this->pbase(), this->epptr()); + /* advance pptr to saved position */ + this->pbump(s.pos); + + this->local_ppos_ = this->pptr() - this->pbase(); + this->solpos_ = s.solpos; + this->color_escape_chars_ = s.color_escape_chars; + /* assuming we never try to capture rewind state with incomplete color escape */ + this->color_escape_start_ = nullptr; + } + + void + arena_streambuf::expand_to(std::size_t new_z) + { + char * old_pptr = pptr(); + std::streamsize old_n = old_pptr - pbase(); + + assert(old_n <= static_cast(arena_->allocated())); + assert(new_z > arena_->committed()); + + /* note: local_ppos_ invariant across expand_to() */ + + arena_->expand(new_z); + + char * p_base = (char *)(arena_->lo_); + char * p_hi = (char *)(arena_->limit_); + + this->setp(p_base, p_hi); + this->pbump(old_n); + } + + std::streamsize + arena_streambuf::xsputn(const char * s, std::streamsize n) + { + /* s must be an address in [this->lo() .. this->lo() + capacity()] */ + + assert(hi() >= pptr()); + + if (pptr() + n > hi()) { + std::size_t new_z = std::max(2 * arena_->committed(), std::size_t(this->pos() + n + 1)); + + if (new_z > arena_->reserved()) + new_z = arena_->reserved(); + + this->expand_to(new_z); + } + + if (debug_flag_) { + std::cout << "xsputn: pbase=" << (void *)(this->pbase()) + << ", pptr=" << (void*)(this->pptr()) + << "(+" << (this->pptr() - this->lo()) << ")" + << ", n=" << n << " -> (+" << (this->pptr() + n - this->lo()) << ")" + << ", arena.size=" << this->arena_->committed() + << std::endl; + } + + std::streamsize ncopied = 0; + + if (this->pptr() + n > this->hi()) { + ncopied = this->hi() - this->pptr(); + } else { + ncopied = n; + } + + if (false /*debug_flag_*/) { + std::cout << "xsputn: copying ncopied=" << ncopied << " (/n=" << n << ") bytes into range [lo,hi)" + << ", lo=" << (void*)this->pptr() + << ", hi=" << (void*)(this->pptr() + n) + << std::endl; + } + + std::memcpy(this->pptr(), s, ncopied); + + this->pbump(ncopied); + + /* now {pbase, pptr} consistent with new input */ + + this->_check_update_local_state(); + + return ncopied; + } + + auto + arena_streambuf::overflow(int_type new_ch) -> int_type + { + char * old_base = this->pbase(); + char * old_pptr = this->pptr(); + /* #of chars buffered */ + std::streamsize old_n = old_pptr - old_base; + + assert(old_n <= static_cast(arena_->committed())); + + // if (debug_flag_) { + // std::cout << "overflow: new_ch=" << quoted_char(new_ch) << std::endl; + // } + + /* increase buffer size */ + this->expand_to(2 * arena_->committed()); + + arena_->lo_[old_n] = static_cast(new_ch); + this->pbump(1); + + if ((new_ch == static_cast('\n')) || (new_ch == static_cast('\r'))) { + this->solpos_ = this->pos(); + + // what if new_ch starts color escape ? + } + + if (new_ch == std::char_traits::eof()) { + /* reminder: returning eof sets badbit on ostream */ + return std::char_traits::not_eof(new_ch); + } else { + return new_ch; + } + } + + auto + arena_streambuf::seekoff(off_type off, + std::ios_base::seekdir dir, + std::ios_base::openmode which) -> pos_type + { + //std::cout << "seekoff: off=" << off << ", dir=" << dir << ", which=" << which << std::endl; + if (debug_flag_) { + std::cout << "seekoff(off,dir,which)" << std::endl; + } + + // Only output stream is supported + if (which != std::ios_base::out) + throw std::runtime_error("log_streambuf: only output mode supported"); + + if (dir == std::ios_base::cur) { + this->pbump(off); + } else if (dir == std::ios_base::end) { + /* .setp(): using for side effect: sets .pptr to .pbase */ + this->setp(this->pbase(), this->epptr()); + this->pbump(off); + } else if (dir == std::ios_base::beg) { + /* .setp(): using for side effect: sets .pptr to .pbase */ + this->setp(this->pbase(), this->epptr()); + this->pbump(this->capacity() + off); + } + + return this->pptr() - this->pbase(); + } /*seekoff*/ + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end arena_streambuf.cpp */