roland/xo-arena
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-interpreter2: scaffold repl + alloc measurement frameowkr

Browse source
This commit is contained in:
Roland Conybeare 2026-02-02 21:55:34 -05:00
commit c89e28367c
14 changed files with 658 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
include/xo/arena/ArenaConfig.hpp
View file

@ -19,6 +19,12 @@ namespace xo {
struct ArenaConfig {
/** @defgroup mm-arenaconfig-ctors **/
ArenaConfig with_name(std::string name) {
ArenaConfig copy(*this);
copy.name_ = name;
return copy;
}
ArenaConfig with_size(std::size_t z) {
ArenaConfig copy(*this);
copy.size_ = z;

5
include/xo/arena/DArena.hpp
View file

@ -7,6 +7,7 @@
#include "ArenaConfig.hpp"
#include "AllocError.hpp"
#include "MemorySizeInfo.hpp"
#include "AllocInfo.hpp"
#include <xo/reflectutil/typeseq.hpp>
@ -140,6 +141,9 @@ namespace xo {
/** get header from allocated object address **/
header_type * obj2hdr(void * obj) noexcept;
/** resource ocnsumption in normal form **/
MemorySizeInfo _store_info() const noexcept;
/** report alloc book-keeping info for allocation at @p mem
*
* Require:
@ -206,6 +210,7 @@ namespace xo {
/** restore arena state to previously-established checkpoint **/
void restore(Checkpoint ckp) noexcept { free_ = ckp.free_; }
/** discard all allocated memory, return to empty state
* Promise:
* - committed memory unchanged

6
include/xo/arena/DArenaHashMap.hpp
View file

@ -45,6 +45,7 @@ namespace xo {
using value_type = std::pair<const Key, Value>;
using key_hash = Hash;
using key_equal = Equal;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using byte = std::byte;
using group_type = detail::ControlGroup;
using store_type = detail::HashMapStore<Key, Value>;
@ -76,6 +77,11 @@ namespace xo {
iterator begin() { return _promote_iterator(_begin_aux()); }
iterator end() { return _promote_iterator(_end_aux()); }
std::size_t _n_store() const noexcept { return store_._n_store(); }
MemorySizeInfo _store_info(std::size_t i) const noexcept {
return store_._store_info(i);
}
/** insert @p kv_pair into hash map.
* Replaces any previous value stored under the same key.
*

5
include/xo/arena/DArenaVector.hpp
View file

@ -79,6 +79,11 @@ namespace xo {
constexpr T * data() { return reinterpret_cast<T*>(store_.lo_); }
constexpr const T * data() const { return reinterpret_cast<const T*>(store_.lo_); }
/** arena used for element storage
* (Might prefer obj<AResourceVisitor> here; refrain to avoid leveling violation)
**/
MemorySizeInfo _store_info() const { return store_._store_info(); }
/** reserve space, if possible, for at least @p z elements.
* Always limited by ArenaConfig.size_
**/

3
include/xo/arena/DCircularBuffer.hpp
View file

@ -83,6 +83,9 @@ namespace xo {
const_span_type occupied_range() const noexcept { return occupied_range_; }
const_span_type input_range() const noexcept { return input_range_; }
std::size_t _n_store() const noexcept;
MemorySizeInfo _store_info(std::size_t i) const noexcept;
/** verify DCircularBuffer invariants.
* Act on failure according to policy @p p
* (combination of throw|log bits)

54
include/xo/arena/ErrorArena.hpp Normal file
View file

@ -0,0 +1,54 @@
/** @file ErrorArena.hpp
*
* @author Roland Conybeare, Feb 2026
**/
#pragma once
#include "DArena.hpp"
namespace xo {
namespace mm {
/** @brief Dedicated arena for error reporting
*
* Reserving memory for error messaages.
* Motivation
* 1. so we have room to report an out-of-memory condition
* 2. so we have place to allocate for an error that
* doesn't interfere with other allocator state
*
* Expect to reset arena between errors, so only need
* enough room to report one error.
*
* To initialize explicitly:
* @code
* // before any other ErrorArena method calls:
* ErrorArena::init_once(cfg...);
*
* // do stuff with ErrorArena..
* ErrorArena::instance()
* @endcode
*
* Reminder: can't use obj<AAllocator> here,
* would be leveling violation.
**/
class ErrorArena {
public:
/** default configuration for error arena **/
static ArenaConfig default_config();
/** idempotent initialization **/
static void init_once(const ArenaConfig & cfg = default_config());
/** get initialized instnace **/
static DArena * instance();
private:
static DArena s_instance;
};
} /*namespace mm*/
} /*namespace xo*/
/* end ErrorArena.hpp */

37
include/xo/arena/MemorySizeInfo.hpp Normal file
View file

@ -0,0 +1,37 @@
/** @file MemorySizeInfo.hpp
*
* @author Roland Conybeare, Feb 2026
**/
#pragma once
#include <cstdint>
namespace xo {
namespace mm {
struct MemorySizeInfo {
using size_type = std::size_t;
MemorySizeInfo(std::string_view name, std::size_t a, std::size_t c, std::size_t r)
: resource_name_{name}, allocated_{a}, committed_{c}, reserved_{r}
{}
static MemorySizeInfo sentinel() { return MemorySizeInfo("", 0, 0, 0); }
/** resource name **/
std::string_view resource_name_;
/** memory in-use **/
std::size_t allocated_ = 0;
/** memory committed (backed by physical memory) **/
std::size_t committed_ = 0;
/** memory reserved:
* virtual memory addresses range obtained, whether or not committed
**/
std::size_t reserved_ = 0;
};
}
}
/* end MemorySizeInfo.hpp */

236
include/xo/arena/arena_streambuf.hpp Normal file
View file

@ -0,0 +1,236 @@
/** @file arena_streambuf.hpp
*
* @author Roland Conybeare, Feb 2026
**/
#pragma once
#include "DArena.hpp"
//#include "print/quoted_char.hpp"
#include <iostream>
#include <string_view>
#include <vector>
#include <cstring> // e.g. for std::memcpy()
#include <cstdint>
#include <cassert>
namespace xo {
namespace mm {
/** @brief Arena-based buffer for logging and pretty-printing
*
* Arena-based using mmap
* Write to self-extending storage array
* Track position relative to start of line
**/
class arena_streambuf : public std::streambuf {
public:
struct rewind_state {
explicit rewind_state(std::size_t solpos, std::size_t color_esc, std::uint32_t p)
: solpos{solpos}, color_escape_chars{color_esc}, pos{p} {}
std::size_t solpos = 0;
std::size_t color_escape_chars = 0;
std::uint32_t pos = 0;
};
public:
/** arena should be ready-to-allocate i.e. have committed > 0 **/
arena_streambuf(DArena * arena, bool debug_flag = false) : arena_{arena}, debug_flag_{debug_flag} {
this->reset_stream();
} /*ctor*/
std::streamsize capacity() const { return arena_->committed(); }
const char * lo() const { return this->pbase(); }
const char * hi() const { return this->lo() + this->capacity(); }
std::uint32_t pos() const { return this->pptr() - this->pbase(); }
/** output position (relative to pbase) when local state last computed. Exposed here for unit tests **/
std::size_t _local_ppos() const { return local_ppos_; }
/** position (relative to pbase) one character after last \n or \r. For unit tests **/
std::uint32_t _solpos() const { return solpos_; }
/** start of incomplete color-escape sequence **/
const char * _color_escape_start() const { return color_escape_start_; }
/** number of non-printing chars after @ref solpos_ from completed color-escape sequences **/
std::uint32_t _color_escape_chars() const { return color_escape_chars_; }
/** number of visible characters since start of line (last \n or \r) **/
std::uint32_t lpos() const;
rewind_state checkpoint() const;
bool debug_flag() const { return debug_flag_; }
operator std::string_view () const { return std::string_view(this->pbase(), this->pptr()); }
void reset_stream();
void rewind_to(rewind_state s);
protected:
/** expand buffer storage (by 2x), preserve current contents **/
void expand_to(std::size_t new_z);
virtual std::streamsize xsputn(const char * s, std::streamsize n) override;
virtual int_type overflow(int_type new_ch) override;
/* off. offset, relative to starting point dir.
* dir.
* which. in|out|both
*
* Note that off=0,dir=cur,which=out reads offset
*/
virtual pos_type seekoff(off_type off,
std::ios_base::seekdir dir,
std::ios_base::openmode which) override;
private:
void _update_local_state_char(const char * p_lo, const char * p)
{
if ((*p == '\n') || (*p == '\r')) {
this->solpos_ = (p+1 - this->pbase());
/* reset, since these chars relevant as correction to solpos */
this->color_escape_chars_ = 0;
/* -> incomplete color escape, broken by newline */
this->color_escape_start_ = nullptr;
} else if (*p == '\033') {
if (debug_flag_) [[unlikely]] {
std::cout << "xsputn: \\033 at p-p_lo=" << (p - p_lo) << std::endl;
}
this->color_escape_start_ = p;
} else if (this->color_escape_start_ != nullptr) {
if (*p == 'm') {
/* escape seq non-printing including both endpoints */
std::int64_t esc_chars = (p+1 - color_escape_start_);
this->color_escape_chars_ += esc_chars;
if (debug_flag_) [[unlikely]] {
std::cout << "xsputn: m at p-p_lo" << (p - p_lo) << " +" << esc_chars
<< " -> color_escape_chars=" << color_escape_chars_ << std::endl;
}
this->color_escape_start_ = nullptr;
} else if (!isdigit(*p) && (*p != '[') && (*p != ';')) {
/* not color escape after all */
this->color_escape_start_ = nullptr;
}
}
}
/** recognize stale local state vars:
* @ref solpos_, @ref color_escape_chars_, @ref color_escape_start_.
*
* Require:
* - {pbase, pptr} in consistent state
* Promise:
* - @c local_ppos_ + @c pbase = @c pptr
* - @c solpos_, @c color_escape_chars_, @c color_escape_start_ all up-to-date
**/
void _check_update_local_state() {
const char * p0 = this->pbase();
const char * pn = this->pptr();
if (debug_flag_) {
std::cerr << "_check_update_local_state:" << std::endl;
std::cerr << " buf: (p0=" << (void*)p0 << ", pn=" << (void*)pn << ")" << std::endl;
std::cerr << " solpos_=" << solpos_ << ", color_escape_chars_=" << color_escape_chars_ << std::endl;
}
if (p0 + local_ppos_ == pn) [[likely]] {
// solpos_, color_escape_chars_, color_escape_start_ all up-to-date
} else {
// [pnew, pn): input that hasn't been incorporated into
// {solpos_, color_escape_chars_, color_escape_start_)
const char * pnew = this->pbase() + this->local_ppos_;
if (debug_flag_) {
std::cerr << "_check_update_local_state: range: (pnew=" << (void*)pnew << ", pn=" << (void*)pn << ")" << std::endl;
}
for(const char * p = pnew; p < pn; ++p) {
this->_update_local_state_char(p0, p);
}
}
// solpos_, color_escape_chars_, color_escape_start_ all up-to-date
// for current buffered contents
this->local_ppos_ = pn - p0;
if (debug_flag_) {
std::cerr << "_check_update_local_state: pos=" << pos();
std::cerr << ", solpos=" << solpos_;
std::cerr << ", color_escape_chars=" << color_escape_chars_ << std::endl;
}
assert(pos() >= solpos_ + color_escape_chars_);
}
private:
/*
* pbase: start of buffered text. Thils will be arena_->lo_
*
*
* pbase pptr epptr
* v >e1< >e2< v v
* |xx\xxEEExxx\xxxxxxxEExxxxEExxxxxxxEExxx\xEExxxxxx..................|
* ^ ^<------new------->
* solpos local_ppos
*
* solpos : first character after newline (stale)
* color_escape_pos : e1+e2+.. (stale)
* new : new characters not reflected
* in local_ppos_, color_escape_chars_ etc.
*
* Legend:
* [\] newline
* [x] visible character
* [E] color escape chars
*
*
* after _check_update_local_state():
*
*
* pbase pptr epptr
* v >e1< v v
* |xx\xxEEExxx\xxxxxxxEExxxxEExxxxxxxEExxx\xEExxxxxx..................|
* ^ ^
* solpos local_ppos
*
*/
/** @defgroup logstreambuf-instance-vars **/
///@{
/** value of pptr (relative to pbase) when _check_update_local_state() last ran **/
std::size_t local_ppos_ = 0;
/** position (relative to pbase) one character after last \n or \r.
* Use to drive @ref lpos. This _has_ to be lazy, since
* xsputn() isn'g guaranteed to be called when there's room in
* in buffer.
**/
std::size_t solpos_ = 0;
/** number of non-printing chars after @ref solpos_, from
* completed color escape sequences.
* (ansi color escapes = text between '\033' and 'm')
**/
std::size_t color_escape_chars_ = 0;
/** non-null: start of incomplete color escape sequence **/
const char * color_escape_start_ = nullptr;
/** buffered output stored here.
* We don't use arena's allocation api, just treat as a block of available memory
**/
DArena * arena_ = nullptr;;
/** true to debug log_streambuf itself **/
bool debug_flag_ = false;
///@}
}; /*log_streambuf*/
} /*namespace mm*/
} /*namespace xo*/
/* end arena_streambuf.hpp */

13
include/xo/arena/hashmap/HashMapStore.hpp
View file

@ -19,6 +19,7 @@ namespace xo {
using group_type = detail::ControlGroup;
using control_vector_type = xo::mm::DArenaVector<uint8_t>;
using slot_vector_type = xo::mm::DArenaVector<value_type>;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
public:
/** group_exp2: number of groups {x, 2^x} **/
@ -47,6 +48,18 @@ namespace xo {
size_type capacity() const noexcept { return n_group_ * c_group_size; }
float load_factor() const noexcept { return size_ / static_cast<float>(n_slot_); }
std::size_t _n_store() const noexcept { return 2; }
MemorySizeInfo _store_info(std::size_t i) const noexcept {
switch (i) {
case 0:
return control_._store_info();
case 1:
return slots_._store_info();
}
return MemorySizeInfo::sentinel();
}
void resize_from_empty(const std::pair<size_type,
size_type> & group_exp2)
{

2
src/arena/CMakeLists.txt
View file

@ -2,6 +2,8 @@
set(SELF_LIB xo_arena)
set(SELF_SRCS
arena_streambuf.cpp
ErrorArena.cpp
cmpresult.cpp
mmap_util.cpp
AllocError.cpp

9
src/arena/DArena.cpp
View file

@ -165,6 +165,15 @@ namespace xo {
return (header_type *)((byte *)obj - sizeof(header_type));
}
MemorySizeInfo
DArena::_store_info() const noexcept
{
return MemorySizeInfo(config_.name_,
this->allocated(),
this->committed(),
this->reserved());
}
AllocInfo
DArena::alloc_info(value_type mem) const noexcept
{

25
src/arena/DCircularBuffer.cpp
View file

@ -79,6 +79,31 @@ namespace xo {
{
}
std::size_t
DCircularBuffer::_n_store() const noexcept
{
return 2;
}
MemorySizeInfo
DCircularBuffer::_store_info(std::size_t i) const noexcept
{
switch (i) {
case 0:
return MemorySizeInfo(config_.name_,
occupied_range_.size(),
mapped_range_.size(),
reserved_range_.size());
case 1:
return pinned_spans_._store_info();
default:
break;
}
return MemorySizeInfo::sentinel();
}
bool
DCircularBuffer::verify_ok(verify_policy policy) const
{

43
src/arena/ErrorArena.cpp Normal file
View file

@ -0,0 +1,43 @@
/** @file ErrorArena.cpp
*
* @author Roland Conybeare, Feb 2026
**/
#include "ErrorArena.hpp"
namespace xo {
namespace mm {
DArena
ErrorArena::s_instance;
ArenaConfig
ErrorArena::default_config()
{
return ArenaConfig().with_name("error-arena").with_size(16 * 1024);
}
namespace {
bool s_init_done = false;
}
void
ErrorArena::init_once(const ArenaConfig & cfg)
{
if (!s_init_done) {
s_init_done = true;
s_instance = DArena::map(cfg);
}
}
DArena *
ErrorArena::instance()
{
init_once(default_config());
return &s_instance;
}
} /*namespace mm*/
} /*namespace xo*/
/* end ErrorArena.cpp */

214
src/arena/arena_streambuf.cpp Normal file
View file

@ -0,0 +1,214 @@
/** @file arena_streambuf.cpp
*
* @author Roland Conybeare, Feb 2026
**/
#include "arena_streambuf.hpp"
namespace xo {
namespace mm {
std::uint32_t
arena_streambuf::lpos() const
{
if (debug_flag_) {
std::cerr << "log_streambuf::lpos: enter" << std::endl;
}
// logically-const. lazy implementation
arena_streambuf * self = const_cast<arena_streambuf *>(this);
self->_check_update_local_state();
return pos() - solpos_ - color_escape_chars_;
}
auto
arena_streambuf::checkpoint() const -> rewind_state
{
// logically-const. lazy implementation
arena_streambuf * self = const_cast<arena_streambuf *>(this);
self->_check_update_local_state();
return rewind_state(solpos_, color_escape_chars_, pos());
}
void
arena_streambuf::reset_stream()
{
assert(arena_);
assert(arena_->committed() > 0);
char * p_lo = (char *)(arena_->lo_);
char * p_hi = (char *)(arena_->limit_);
/* tells parent our buffer extent */
this->setp(p_lo, p_hi);
this->local_ppos_ = 0;
this->solpos_ = 0;
this->color_escape_chars_ = 0;
this->color_escape_start_ = nullptr;
}
void
arena_streambuf::rewind_to(rewind_state s)
{
if (debug_flag_) {
std::cout << "rewind_to: pos " << pos() << "->" << s.pos
<< " solpos " << solpos_ << "->" << s.solpos
<< " color_esc " << color_escape_chars_ << "->" << s.color_escape_chars
<< std::endl;
}
/* .setp(): using just for side effect: sets .pptr to .pbase */
this->setp(this->pbase(), this->epptr());
/* advance pptr to saved position */
this->pbump(s.pos);
this->local_ppos_ = this->pptr() - this->pbase();
this->solpos_ = s.solpos;
this->color_escape_chars_ = s.color_escape_chars;
/* assuming we never try to capture rewind state with incomplete color escape */
this->color_escape_start_ = nullptr;
}
void
arena_streambuf::expand_to(std::size_t new_z)
{
char * old_pptr = pptr();
std::streamsize old_n = old_pptr - pbase();
assert(old_n <= static_cast<std::streamsize>(arena_->allocated()));
assert(new_z > arena_->committed());
/* note: local_ppos_ invariant across expand_to() */
arena_->expand(new_z);
char * p_base = (char *)(arena_->lo_);
char * p_hi = (char *)(arena_->limit_);
this->setp(p_base, p_hi);
this->pbump(old_n);
}
std::streamsize
arena_streambuf::xsputn(const char * s, std::streamsize n)
{
/* s must be an address in [this->lo() .. this->lo() + capacity()] */
assert(hi() >= pptr());
if (pptr() + n > hi()) {
std::size_t new_z = std::max(2 * arena_->committed(), std::size_t(this->pos() + n + 1));
if (new_z > arena_->reserved())
new_z = arena_->reserved();
this->expand_to(new_z);
}
if (debug_flag_) {
std::cout << "xsputn: pbase=" << (void *)(this->pbase())
<< ", pptr=" << (void*)(this->pptr())
<< "(+" << (this->pptr() - this->lo()) << ")"
<< ", n=" << n << " -> (+" << (this->pptr() + n - this->lo()) << ")"
<< ", arena.size=" << this->arena_->committed()
<< std::endl;
}
std::streamsize ncopied = 0;
if (this->pptr() + n > this->hi()) {
ncopied = this->hi() - this->pptr();
} else {
ncopied = n;
}
if (false /*debug_flag_*/) {
std::cout << "xsputn: copying ncopied=" << ncopied << " (/n=" << n << ") bytes into range [lo,hi)"
<< ", lo=" << (void*)this->pptr()
<< ", hi=" << (void*)(this->pptr() + n)
<< std::endl;
}
std::memcpy(this->pptr(), s, ncopied);
this->pbump(ncopied);
/* now {pbase, pptr} consistent with new input */
this->_check_update_local_state();
return ncopied;
}
auto
arena_streambuf::overflow(int_type new_ch) -> int_type
{
char * old_base = this->pbase();
char * old_pptr = this->pptr();
/* #of chars buffered */
std::streamsize old_n = old_pptr - old_base;
assert(old_n <= static_cast<std::streamsize>(arena_->committed()));
// if (debug_flag_) {
// std::cout << "overflow: new_ch=" << quoted_char(new_ch) << std::endl;
// }
/* increase buffer size */
this->expand_to(2 * arena_->committed());
arena_->lo_[old_n] = static_cast<std::byte>(new_ch);
this->pbump(1);
if ((new_ch == static_cast<int_type>('\n')) || (new_ch == static_cast<int_type>('\r'))) {
this->solpos_ = this->pos();
// what if new_ch starts color escape ?
}
if (new_ch == std::char_traits<char>::eof()) {
/* reminder: returning eof sets badbit on ostream */
return std::char_traits<char>::not_eof(new_ch);
} else {
return new_ch;
}
}
auto
arena_streambuf::seekoff(off_type off,
std::ios_base::seekdir dir,
std::ios_base::openmode which) -> pos_type
{
//std::cout << "seekoff: off=" << off << ", dir=" << dir << ", which=" << which << std::endl;
if (debug_flag_) {
std::cout << "seekoff(off,dir,which)" << std::endl;
}
// Only output stream is supported
if (which != std::ios_base::out)
throw std::runtime_error("log_streambuf: only output mode supported");
if (dir == std::ios_base::cur) {
this->pbump(off);
} else if (dir == std::ios_base::end) {
/* .setp(): using for side effect: sets .pptr to .pbase */
this->setp(this->pbase(), this->epptr());
this->pbump(off);
} else if (dir == std::ios_base::beg) {
/* .setp(): using for side effect: sets .pptr to .pbase */
this->setp(this->pbase(), this->epptr());
this->pbump(this->capacity() + off);
}
return this->pptr() - this->pbase();
} /*seekoff*/
} /*namespace mm*/
} /*namespace xo*/
/* end arena_streambuf.cpp */