roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-interpreter2: scaffold repl + alloc measurement frameowkr

Browse source
This commit is contained in:
Roland Conybeare 2026-02-02 21:55:34 -05:00
commit f6aae4190e
42 changed files with 1398 additions and 198 deletions
Download patch file Download diff file Expand all files Collapse all files

17
xo-alloc2/CMakeLists.txt
View file

@ -18,6 +18,23 @@ add_definitions(${PROJECT_CXX_FLAGS})
# ----------------------------------------------------------------
# note: manual target; generated code committed to git
xo_add_genfacet(
TARGET xo-alloc2-facet-resourcevisitor
FACET ResourceVisitor
INPUT idl/ResourceVisitor.json5
OUTPUT_HPP_DIR include/xo/alloc2
OUTPUT_IMPL_SUBDIR visitor
OUTPUT_CPP_DIR src/alloc2
)
# ----------------------------------------------------------------
xo_add_genfacet_all(xo-alloc2-genfacet-all)
# ----------------------------------------------------------------
# must complete definition of expression lib before configuring examples
add_subdirectory(src/alloc2)
add_subdirectory(utest)

49
xo-alloc2/idl/ResourceVisitor.json5 Normal file
View file

@ -0,0 +1,49 @@
{
mode: "facet",
includes: [
"\"Allocator.hpp\""
],
// extra includes in ResourceVisitor.hpp, if any
user_hpp_includes: [],
namespace1: "xo",
namespace2: "mm",
// text after includes, before AResourceVisitor
pretext: [ "// {pretext} here" ],
facet: "ResourceVisitor",
detail_subdir: "visitor",
brief: "visitor to inspect resource consumption",
using_doxygen: true,
doc: [
"Visitor to receive measured resource consumption"
],
types: [
// using size_type = std::size_t
{
name: "size_type",
doc: ["type for length of a sequence"],
definition: "std::size_t",
},
// // using AGCObject = xo::mm::AGCObject
// {
// name: "AGCObject",
// doc: ["facet for types with GC support"],
// definition: "xo::mm::AGCObject",
// }
],
const_methods: [
// bool on_memory(name, allocated, committed, reserved) const noexcept
{
name: "on_allocator",
doc: ["report memory consumption"],
return_type: "void",
args: [
{type: "obj<AAllocator>", name: "mm"},
],
const: true,
noexcept: true,
attributes: [],
},
],
nonconst_methods: [
],
}

22
xo-alloc2/include/xo/alloc2/ResourceVisitor.hpp Normal file
View file

@ -0,0 +1,22 @@
/** @file ResourceVisitor.hpp
*
* Generated automagically from ingredients:
* 1. code generator:
* [xo-facet/codegen/genfacet]
* arguments:
* --input [idl/ResourceVisitor.json5]
* 2. jinja2 template for facet .hpp file:
* [facet.hpp.j2]
* 3. idl for facet methods
* [idl/ResourceVisitor.json5]
**/
#pragma once
#include "visitor/AResourceVisitor.hpp"
#include "visitor/IResourceVisitor_Any.hpp"
#include "visitor/IResourceVisitor_Xfer.hpp"
#include "visitor/RResourceVisitor.hpp"
/* end ResourceVisitor.hpp */

74
xo-alloc2/include/xo/alloc2/visitor/AResourceVisitor.hpp Normal file
View file

@ -0,0 +1,74 @@
/** @file AResourceVisitor.hpp
*
* Generated automagically from ingredients:
* 1. code generator:
* [xo-facet/codegen/genfacet]
* arguments:
* --input [idl/ResourceVisitor.json5]
* 2. jinja2 template for abstract facet .hpp file:
* [abstract_facet.hpp.j2]
* 3. idl for facet methods
* [idl/ResourceVisitor.json5]
**/
#pragma once
// includes (via {facet_includes})
#include "Allocator.hpp"
#include <xo/facet/obj.hpp>
#include <xo/facet/facet_implementation.hpp>
#include <xo/facet/typeseq.hpp>
// {pretext} here
namespace xo {
namespace mm {
using Copaque = const void *;
using Opaque = void *;
/**
Visitor to receive measured resource consumption
**/
class AResourceVisitor {
public:
/** @defgroup mm-resourcevisitor-type-traits **/
///@{
// types
/** integer identifying a type **/
using typeseq = xo::facet::typeseq;
using Copaque = const void *;
using Opaque = void *;
/** type for length of a sequence **/
using size_type = std::size_t;
///@}
/** @defgroup mm-resourcevisitor-methods **/
///@{
// const methods
/** RTTI: unique id# for actual runtime data representation **/
virtual typeseq _typeseq() const noexcept = 0;
/** report memory consumption **/
virtual void on_allocator(Copaque data, obj<AAllocator> mm) const noexcept = 0;
// nonconst methods
///@}
}; /*AResourceVisitor*/
/** Implementation IResourceVisitor_DRepr of AResourceVisitor for state DRepr
* should provide a specialization:
*
* template <>
* struct xo::facet::FacetImplementation<AResourceVisitor, DRepr> {
* using Impltype = IResourceVisitor_DRepr;
* };
*
* then IResourceVisitor_ImplType<DRepr> --> IResourceVisitor_DRepr
**/
template <typename DRepr>
using IResourceVisitor_ImplType = xo::facet::FacetImplType<AResourceVisitor, DRepr>;
} /*namespace mm*/
} /*namespace xo*/
/* AResourceVisitor.hpp */

86
xo-alloc2/include/xo/alloc2/visitor/IResourceVisitor_Any.hpp Normal file
View file

@ -0,0 +1,86 @@
/** @file IResourceVisitor_Any.hpp
*
* Generated automagically from ingredients:
* 1. code generator:
* [xo-facet/codegen/genfacet]
* arguments:
* --input [idl/ResourceVisitor.json5]
* 2. jinja2 template for abstract facet .hpp file:
* [iface_facet_any.hpp.j2]
* 3. idl for facet methods
* [idl/ResourceVisitor.json5]
**/
#pragma once
#include "AResourceVisitor.hpp"
#include <xo/facet/obj.hpp>
namespace xo { namespace mm { class IResourceVisitor_Any; } }
namespace xo {
namespace facet {
template <>
struct FacetImplementation<xo::mm::AResourceVisitor,
DVariantPlaceholder>
{
using ImplType = xo::mm::IResourceVisitor_Any;
};
}
}
namespace xo {
namespace mm {
/** @class IResourceVisitor_Any
* @brief AResourceVisitor implementation for empty variant instance
**/
class IResourceVisitor_Any : public AResourceVisitor {
public:
/** @defgroup mm-resourcevisitor-any-type-traits **/
///@{
/** integer identifying a type **/
using typeseq = xo::facet::typeseq;
using size_type = AResourceVisitor::size_type;
///@}
/** @defgroup mm-resourcevisitor-any-methods **/
///@{
const AResourceVisitor * iface() const { return std::launder(this); }
// from AResourceVisitor
// const methods
typeseq _typeseq() const noexcept override { return s_typeseq; }
[[noreturn]] void on_allocator(Copaque, obj<AAllocator>) const noexcept override { _fatal(); }
// nonconst methods
///@}
private:
/** @defgraoup mm-resourcevisitor-any-private-methods **/
///@{
[[noreturn]] static void _fatal();
///@}
public:
/** @defgroup mm-resourcevisitor-any-member-vars **/
///@{
static typeseq s_typeseq;
static bool _valid;
///@}
};
} /*namespace mm */
} /*namespace xo */
/* IResourceVisitor_Any.hpp */

81
xo-alloc2/include/xo/alloc2/visitor/IResourceVisitor_Xfer.hpp Normal file
View file

@ -0,0 +1,81 @@
/** @file IResourceVisitor_Xfer.hpp
*
* Generated automagically from ingredients:
* 1. code generator:
* [xo-facet/codegen/genfacet]
* arguments:
* --input [idl/ResourceVisitor.json5]
* 2. jinja2 template for abstract facet .hpp file:
* [iface_facet_any.hpp.j2]
* 3. idl for facet methods
* [idl/ResourceVisitor.json5]
**/
#pragma once
#include "Allocator.hpp"
namespace xo {
namespace mm {
/** @class IResourceVisitor_Xfer
**/
template <typename DRepr, typename IResourceVisitor_DRepr>
class IResourceVisitor_Xfer : public AResourceVisitor {
public:
/** @defgroup mm-resourcevisitor-xfer-type-traits **/
///@{
/** actual implementation (not generated; often delegates to DRepr) **/
using Impl = IResourceVisitor_DRepr;
/** integer identifying a type **/
using typeseq = AResourceVisitor::typeseq;
using size_type = AResourceVisitor::size_type;
///@}
/** @defgroup mm-resourcevisitor-xfer-methods **/
///@{
static const DRepr & _dcast(Copaque d) { return *(const DRepr *)d; }
static DRepr & _dcast(Opaque d) { return *(DRepr *)d; }
// from AResourceVisitor
// const methods
typeseq _typeseq() const noexcept override { return s_typeseq; }
void on_allocator(Copaque data, obj<AAllocator> mm) const noexcept override {
return I::on_allocator(_dcast(data), mm);
}
// non-const methods
///@}
private:
using I = Impl;
public:
/** @defgroup mm-resourcevisitor-xfer-member-vars **/
///@{
/** typeseq for template parameter DRepr **/
static typeseq s_typeseq;
/** true iff satisfies facet implementation **/
static bool _valid;
///@}
};
template <typename DRepr, typename IResourceVisitor_DRepr>
xo::facet::typeseq
IResourceVisitor_Xfer<DRepr, IResourceVisitor_DRepr>::s_typeseq
= xo::facet::typeseq::id<DRepr>();
template <typename DRepr, typename IResourceVisitor_DRepr>
bool
IResourceVisitor_Xfer<DRepr, IResourceVisitor_DRepr>::_valid
= xo::facet::valid_facet_implementation<AResourceVisitor,
IResourceVisitor_Xfer>();
} /*namespace mm */
} /*namespace xo*/
/* end IResourceVisitor_Xfer.hpp */

80
xo-alloc2/include/xo/alloc2/visitor/RResourceVisitor.hpp Normal file
View file

@ -0,0 +1,80 @@
/** @file RResourceVisitor.hpp
*
* Generated automagically from ingredients:
* 1. code generator:
* [xo-facet/codegen/genfacet]
* arguments:
* --input [idl/ResourceVisitor.json5]
* 2. jinja2 template for abstract facet .hpp file:
* [iface_facet_any.hpp.j2]
* 3. idl for facet methods
* [idl/ResourceVisitor.json5]
**/
#pragma once
#include "AResourceVisitor.hpp"
namespace xo {
namespace mm {
/** @class RResourceVisitor
**/
template <typename Object>
class RResourceVisitor : public Object {
private:
using O = Object;
public:
/** @defgroup mm-resourcevisitor-router-type-traits **/
///@{
using ObjectType = Object;
using DataPtr = Object::DataPtr;
using typeseq = xo::reflect::typeseq;
using size_type = AResourceVisitor::size_type;
///@}
/** @defgroup mm-resourcevisitor-router-ctors **/
///@{
RResourceVisitor() {}
RResourceVisitor(Object::DataPtr data) : Object{std::move(data)} {}
RResourceVisitor(const AResourceVisitor * iface, void * data)
requires std::is_same_v<typename Object::DataType, xo::facet::DVariantPlaceholder>
: Object(iface, data) {}
///@}
/** @defgroup mm-resourcevisitor-router-methods **/
///@{
// const methods
typeseq _typeseq() const noexcept { return O::iface()->_typeseq(); }
void on_allocator(obj<AAllocator> mm) const noexcept {
return O::iface()->on_allocator(O::data(), mm);
}
// non-const methods (still const in router!)
///@}
/** @defgroup mm-resourcevisitor-member-vars **/
///@{
static bool _valid;
///@}
};
template <typename Object>
bool
RResourceVisitor<Object>::_valid = xo::facet::valid_object_router<Object>();
} /*namespace mm*/
} /*namespace xo*/
namespace xo { namespace facet {
template <typename Object>
struct RoutingFor<xo::mm::AResourceVisitor, Object> {
using RoutingType = xo::mm::RResourceVisitor<Object>;
};
} }
/* end RResourceVisitor.hpp */

2
xo-alloc2/src/alloc2/CMakeLists.txt
View file

@ -17,6 +17,8 @@ set(SELF_SRCS
# DArenaIterator.cpp
IAllocIterator_DArenaIterator.cpp
IResourceVisitor_Any.cpp
)
xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS})

41
xo-alloc2/src/alloc2/IResourceVisitor_Any.cpp Normal file
View file

@ -0,0 +1,41 @@
/** @file IResourceVisitor_Any.cpp
*
**/
#include "visitor/IResourceVisitor_Any.hpp"
#include <iostream>
namespace xo {
namespace mm {
using xo::facet::DVariantPlaceholder;
using xo::facet::typeseq;
using xo::facet::valid_facet_implementation;
void
IResourceVisitor_Any::_fatal()
{
/* control here on uninitialized IAllocator_Any.
* Initialized instance will have specific implementation type
*/
std::cerr << "fatal"
<< ": attempt to call uninitialized"
<< " IResourceVisitor_Any method"
<< std::endl;
std::terminate();
}
typeseq
IResourceVisitor_Any::s_typeseq = typeseq::id<DVariantPlaceholder>();
bool
IResourceVisitor_Any::_valid
= valid_facet_implementation<AResourceVisitor, IResourceVisitor_Any>();
// nonconst methods
} /*namespace mm*/
} /*namespace xo*/
/* end IResourceVisitor_Any.cpp */

6
xo-arena/include/xo/arena/ArenaConfig.hpp
View file

@ -19,6 +19,12 @@ namespace xo {
struct ArenaConfig {
/** @defgroup mm-arenaconfig-ctors **/
ArenaConfig with_name(std::string name) {
ArenaConfig copy(*this);
copy.name_ = name;
return copy;
}
ArenaConfig with_size(std::size_t z) {
ArenaConfig copy(*this);
copy.size_ = z;

5
xo-arena/include/xo/arena/DArena.hpp
View file

@ -7,6 +7,7 @@
#include "ArenaConfig.hpp"
#include "AllocError.hpp"
#include "MemorySizeInfo.hpp"
#include "AllocInfo.hpp"
#include <xo/reflectutil/typeseq.hpp>
@ -140,6 +141,9 @@ namespace xo {
/** get header from allocated object address **/
header_type * obj2hdr(void * obj) noexcept;
/** resource ocnsumption in normal form **/
MemorySizeInfo _store_info() const noexcept;
/** report alloc book-keeping info for allocation at @p mem
*
* Require:
@ -206,6 +210,7 @@ namespace xo {
/** restore arena state to previously-established checkpoint **/
void restore(Checkpoint ckp) noexcept { free_ = ckp.free_; }
/** discard all allocated memory, return to empty state
* Promise:
* - committed memory unchanged

6
xo-arena/include/xo/arena/DArenaHashMap.hpp
View file

@ -45,6 +45,7 @@ namespace xo {
using value_type = std::pair<const Key, Value>;
using key_hash = Hash;
using key_equal = Equal;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using byte = std::byte;
using group_type = detail::ControlGroup;
using store_type = detail::HashMapStore<Key, Value>;
@ -76,6 +77,11 @@ namespace xo {
iterator begin() { return _promote_iterator(_begin_aux()); }
iterator end() { return _promote_iterator(_end_aux()); }
std::size_t _n_store() const noexcept { return store_._n_store(); }
MemorySizeInfo _store_info(std::size_t i) const noexcept {
return store_._store_info(i);
}
/** insert @p kv_pair into hash map.
* Replaces any previous value stored under the same key.
*

5
xo-arena/include/xo/arena/DArenaVector.hpp
View file

@ -79,6 +79,11 @@ namespace xo {
constexpr T * data() { return reinterpret_cast<T*>(store_.lo_); }
constexpr const T * data() const { return reinterpret_cast<const T*>(store_.lo_); }
/** arena used for element storage
* (Might prefer obj<AResourceVisitor> here; refrain to avoid leveling violation)
**/
MemorySizeInfo _store_info() const { return store_._store_info(); }
/** reserve space, if possible, for at least @p z elements.
* Always limited by ArenaConfig.size_
**/

3
xo-arena/include/xo/arena/DCircularBuffer.hpp
View file

@ -83,6 +83,9 @@ namespace xo {
const_span_type occupied_range() const noexcept { return occupied_range_; }
const_span_type input_range() const noexcept { return input_range_; }
std::size_t _n_store() const noexcept;
MemorySizeInfo _store_info(std::size_t i) const noexcept;
/** verify DCircularBuffer invariants.
* Act on failure according to policy @p p
* (combination of throw|log bits)

54
xo-arena/include/xo/arena/ErrorArena.hpp Normal file
View file

@ -0,0 +1,54 @@
/** @file ErrorArena.hpp
*
* @author Roland Conybeare, Feb 2026
**/
#pragma once
#include "DArena.hpp"
namespace xo {
namespace mm {
/** @brief Dedicated arena for error reporting
*
* Reserving memory for error messaages.
* Motivation
* 1. so we have room to report an out-of-memory condition
* 2. so we have place to allocate for an error that
* doesn't interfere with other allocator state
*
* Expect to reset arena between errors, so only need
* enough room to report one error.
*
* To initialize explicitly:
* @code
* // before any other ErrorArena method calls:
* ErrorArena::init_once(cfg...);
*
* // do stuff with ErrorArena..
* ErrorArena::instance()
* @endcode
*
* Reminder: can't use obj<AAllocator> here,
* would be leveling violation.
**/
class ErrorArena {
public:
/** default configuration for error arena **/
static ArenaConfig default_config();
/** idempotent initialization **/
static void init_once(const ArenaConfig & cfg = default_config());
/** get initialized instnace **/
static DArena * instance();
private:
static DArena s_instance;
};
} /*namespace mm*/
} /*namespace xo*/
/* end ErrorArena.hpp */

37
xo-arena/include/xo/arena/MemorySizeInfo.hpp Normal file
View file

@ -0,0 +1,37 @@
/** @file MemorySizeInfo.hpp
*
* @author Roland Conybeare, Feb 2026
**/
#pragma once
#include <cstdint>
namespace xo {
namespace mm {
struct MemorySizeInfo {
using size_type = std::size_t;
MemorySizeInfo(std::string_view name, std::size_t a, std::size_t c, std::size_t r)
: resource_name_{name}, allocated_{a}, committed_{c}, reserved_{r}
{}
static MemorySizeInfo sentinel() { return MemorySizeInfo("", 0, 0, 0); }
/** resource name **/
std::string_view resource_name_;
/** memory in-use **/
std::size_t allocated_ = 0;
/** memory committed (backed by physical memory) **/
std::size_t committed_ = 0;
/** memory reserved:
* virtual memory addresses range obtained, whether or not committed
**/
std::size_t reserved_ = 0;
};
}
}
/* end MemorySizeInfo.hpp */

236
xo-arena/include/xo/arena/arena_streambuf.hpp Normal file
View file

@ -0,0 +1,236 @@
/** @file arena_streambuf.hpp
*
* @author Roland Conybeare, Feb 2026
**/
#pragma once
#include "DArena.hpp"
//#include "print/quoted_char.hpp"
#include <iostream>
#include <string_view>
#include <vector>
#include <cstring> // e.g. for std::memcpy()
#include <cstdint>
#include <cassert>
namespace xo {
namespace mm {
/** @brief Arena-based buffer for logging and pretty-printing
*
* Arena-based using mmap
* Write to self-extending storage array
* Track position relative to start of line
**/
class arena_streambuf : public std::streambuf {
public:
struct rewind_state {
explicit rewind_state(std::size_t solpos, std::size_t color_esc, std::uint32_t p)
: solpos{solpos}, color_escape_chars{color_esc}, pos{p} {}
std::size_t solpos = 0;
std::size_t color_escape_chars = 0;
std::uint32_t pos = 0;
};
public:
/** arena should be ready-to-allocate i.e. have committed > 0 **/
arena_streambuf(DArena * arena, bool debug_flag = false) : arena_{arena}, debug_flag_{debug_flag} {
this->reset_stream();
} /*ctor*/
std::streamsize capacity() const { return arena_->committed(); }
const char * lo() const { return this->pbase(); }
const char * hi() const { return this->lo() + this->capacity(); }
std::uint32_t pos() const { return this->pptr() - this->pbase(); }
/** output position (relative to pbase) when local state last computed. Exposed here for unit tests **/
std::size_t _local_ppos() const { return local_ppos_; }
/** position (relative to pbase) one character after last \n or \r. For unit tests **/
std::uint32_t _solpos() const { return solpos_; }
/** start of incomplete color-escape sequence **/
const char * _color_escape_start() const { return color_escape_start_; }
/** number of non-printing chars after @ref solpos_ from completed color-escape sequences **/
std::uint32_t _color_escape_chars() const { return color_escape_chars_; }
/** number of visible characters since start of line (last \n or \r) **/
std::uint32_t lpos() const;
rewind_state checkpoint() const;
bool debug_flag() const { return debug_flag_; }
operator std::string_view () const { return std::string_view(this->pbase(), this->pptr()); }
void reset_stream();
void rewind_to(rewind_state s);
protected:
/** expand buffer storage (by 2x), preserve current contents **/
void expand_to(std::size_t new_z);
virtual std::streamsize xsputn(const char * s, std::streamsize n) override;
virtual int_type overflow(int_type new_ch) override;
/* off. offset, relative to starting point dir.
* dir.
* which. in|out|both
*
* Note that off=0,dir=cur,which=out reads offset
*/
virtual pos_type seekoff(off_type off,
std::ios_base::seekdir dir,
std::ios_base::openmode which) override;
private:
void _update_local_state_char(const char * p_lo, const char * p)
{
if ((*p == '\n') || (*p == '\r')) {
this->solpos_ = (p+1 - this->pbase());
/* reset, since these chars relevant as correction to solpos */
this->color_escape_chars_ = 0;
/* -> incomplete color escape, broken by newline */
this->color_escape_start_ = nullptr;
} else if (*p == '\033') {
if (debug_flag_) [[unlikely]] {
std::cout << "xsputn: \\033 at p-p_lo=" << (p - p_lo) << std::endl;
}
this->color_escape_start_ = p;
} else if (this->color_escape_start_ != nullptr) {
if (*p == 'm') {
/* escape seq non-printing including both endpoints */
std::int64_t esc_chars = (p+1 - color_escape_start_);
this->color_escape_chars_ += esc_chars;
if (debug_flag_) [[unlikely]] {
std::cout << "xsputn: m at p-p_lo" << (p - p_lo) << " +" << esc_chars
<< " -> color_escape_chars=" << color_escape_chars_ << std::endl;
}
this->color_escape_start_ = nullptr;
} else if (!isdigit(*p) && (*p != '[') && (*p != ';')) {
/* not color escape after all */
this->color_escape_start_ = nullptr;
}
}
}
/** recognize stale local state vars:
* @ref solpos_, @ref color_escape_chars_, @ref color_escape_start_.
*
* Require:
* - {pbase, pptr} in consistent state
* Promise:
* - @c local_ppos_ + @c pbase = @c pptr
* - @c solpos_, @c color_escape_chars_, @c color_escape_start_ all up-to-date
**/
void _check_update_local_state() {
const char * p0 = this->pbase();
const char * pn = this->pptr();
if (debug_flag_) {
std::cerr << "_check_update_local_state:" << std::endl;
std::cerr << " buf: (p0=" << (void*)p0 << ", pn=" << (void*)pn << ")" << std::endl;
std::cerr << " solpos_=" << solpos_ << ", color_escape_chars_=" << color_escape_chars_ << std::endl;
}
if (p0 + local_ppos_ == pn) [[likely]] {
// solpos_, color_escape_chars_, color_escape_start_ all up-to-date
} else {
// [pnew, pn): input that hasn't been incorporated into
// {solpos_, color_escape_chars_, color_escape_start_)
const char * pnew = this->pbase() + this->local_ppos_;
if (debug_flag_) {
std::cerr << "_check_update_local_state: range: (pnew=" << (void*)pnew << ", pn=" << (void*)pn << ")" << std::endl;
}
for(const char * p = pnew; p < pn; ++p) {
this->_update_local_state_char(p0, p);
}
}
// solpos_, color_escape_chars_, color_escape_start_ all up-to-date
// for current buffered contents
this->local_ppos_ = pn - p0;
if (debug_flag_) {
std::cerr << "_check_update_local_state: pos=" << pos();
std::cerr << ", solpos=" << solpos_;
std::cerr << ", color_escape_chars=" << color_escape_chars_ << std::endl;
}
assert(pos() >= solpos_ + color_escape_chars_);
}
private:
/*
* pbase: start of buffered text. Thils will be arena_->lo_
*
*
* pbase pptr epptr
* v >e1< >e2< v v
* |xx\xxEEExxx\xxxxxxxEExxxxEExxxxxxxEExxx\xEExxxxxx..................|
* ^ ^<------new------->
* solpos local_ppos
*
* solpos : first character after newline (stale)
* color_escape_pos : e1+e2+.. (stale)
* new : new characters not reflected
* in local_ppos_, color_escape_chars_ etc.
*
* Legend:
* [\] newline
* [x] visible character
* [E] color escape chars
*
*
* after _check_update_local_state():
*
*
* pbase pptr epptr
* v >e1< v v
* |xx\xxEEExxx\xxxxxxxEExxxxEExxxxxxxEExxx\xEExxxxxx..................|
* ^ ^
* solpos local_ppos
*
*/
/** @defgroup logstreambuf-instance-vars **/
///@{
/** value of pptr (relative to pbase) when _check_update_local_state() last ran **/
std::size_t local_ppos_ = 0;
/** position (relative to pbase) one character after last \n or \r.
* Use to drive @ref lpos. This _has_ to be lazy, since
* xsputn() isn'g guaranteed to be called when there's room in
* in buffer.
**/
std::size_t solpos_ = 0;
/** number of non-printing chars after @ref solpos_, from
* completed color escape sequences.
* (ansi color escapes = text between '\033' and 'm')
**/
std::size_t color_escape_chars_ = 0;
/** non-null: start of incomplete color escape sequence **/
const char * color_escape_start_ = nullptr;
/** buffered output stored here.
* We don't use arena's allocation api, just treat as a block of available memory
**/
DArena * arena_ = nullptr;;
/** true to debug log_streambuf itself **/
bool debug_flag_ = false;
///@}
}; /*log_streambuf*/
} /*namespace mm*/
} /*namespace xo*/
/* end arena_streambuf.hpp */

13
xo-arena/include/xo/arena/hashmap/HashMapStore.hpp
View file

@ -19,6 +19,7 @@ namespace xo {
using group_type = detail::ControlGroup;
using control_vector_type = xo::mm::DArenaVector<uint8_t>;
using slot_vector_type = xo::mm::DArenaVector<value_type>;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
public:
/** group_exp2: number of groups {x, 2^x} **/
@ -47,6 +48,18 @@ namespace xo {
size_type capacity() const noexcept { return n_group_ * c_group_size; }
float load_factor() const noexcept { return size_ / static_cast<float>(n_slot_); }
std::size_t _n_store() const noexcept { return 2; }
MemorySizeInfo _store_info(std::size_t i) const noexcept {
switch (i) {
case 0:
return control_._store_info();
case 1:
return slots_._store_info();
}
return MemorySizeInfo::sentinel();
}
void resize_from_empty(const std::pair<size_type,
size_type> & group_exp2)
{

2
xo-arena/src/arena/CMakeLists.txt
View file

@ -2,6 +2,8 @@
set(SELF_LIB xo_arena)
set(SELF_SRCS
arena_streambuf.cpp
ErrorArena.cpp
cmpresult.cpp
mmap_util.cpp
AllocError.cpp

9
xo-arena/src/arena/DArena.cpp
View file

@ -165,6 +165,15 @@ namespace xo {
return (header_type *)((byte *)obj - sizeof(header_type));
}
MemorySizeInfo
DArena::_store_info() const noexcept
{
return MemorySizeInfo(config_.name_,
this->allocated(),
this->committed(),
this->reserved());
}
AllocInfo
DArena::alloc_info(value_type mem) const noexcept
{

25
xo-arena/src/arena/DCircularBuffer.cpp
View file

@ -79,6 +79,31 @@ namespace xo {
{
}
std::size_t
DCircularBuffer::_n_store() const noexcept
{
return 2;
}
MemorySizeInfo
DCircularBuffer::_store_info(std::size_t i) const noexcept
{
switch (i) {
case 0:
return MemorySizeInfo(config_.name_,
occupied_range_.size(),
mapped_range_.size(),
reserved_range_.size());
case 1:
return pinned_spans_._store_info();
default:
break;
}
return MemorySizeInfo::sentinel();
}
bool
DCircularBuffer::verify_ok(verify_policy policy) const
{

43
xo-arena/src/arena/ErrorArena.cpp Normal file
View file

@ -0,0 +1,43 @@
/** @file ErrorArena.cpp
*
* @author Roland Conybeare, Feb 2026
**/
#include "ErrorArena.hpp"
namespace xo {
namespace mm {
DArena
ErrorArena::s_instance;
ArenaConfig
ErrorArena::default_config()
{
return ArenaConfig().with_name("error-arena").with_size(16 * 1024);
}
namespace {
bool s_init_done = false;
}
void
ErrorArena::init_once(const ArenaConfig & cfg)
{
if (!s_init_done) {
s_init_done = true;
s_instance = DArena::map(cfg);
}
}
DArena *
ErrorArena::instance()
{
init_once(default_config());
return &s_instance;
}
} /*namespace mm*/
} /*namespace xo*/
/* end ErrorArena.cpp */

214
xo-arena/src/arena/arena_streambuf.cpp Normal file
View file

@ -0,0 +1,214 @@
/** @file arena_streambuf.cpp
*
* @author Roland Conybeare, Feb 2026
**/
#include "arena_streambuf.hpp"
namespace xo {
namespace mm {
std::uint32_t
arena_streambuf::lpos() const
{
if (debug_flag_) {
std::cerr << "log_streambuf::lpos: enter" << std::endl;
}
// logically-const. lazy implementation
arena_streambuf * self = const_cast<arena_streambuf *>(this);
self->_check_update_local_state();
return pos() - solpos_ - color_escape_chars_;
}
auto
arena_streambuf::checkpoint() const -> rewind_state
{
// logically-const. lazy implementation
arena_streambuf * self = const_cast<arena_streambuf *>(this);
self->_check_update_local_state();
return rewind_state(solpos_, color_escape_chars_, pos());
}
void
arena_streambuf::reset_stream()
{
assert(arena_);
assert(arena_->committed() > 0);
char * p_lo = (char *)(arena_->lo_);
char * p_hi = (char *)(arena_->limit_);
/* tells parent our buffer extent */
this->setp(p_lo, p_hi);
this->local_ppos_ = 0;
this->solpos_ = 0;
this->color_escape_chars_ = 0;
this->color_escape_start_ = nullptr;
}
void
arena_streambuf::rewind_to(rewind_state s)
{
if (debug_flag_) {
std::cout << "rewind_to: pos " << pos() << "->" << s.pos
<< " solpos " << solpos_ << "->" << s.solpos
<< " color_esc " << color_escape_chars_ << "->" << s.color_escape_chars
<< std::endl;
}
/* .setp(): using just for side effect: sets .pptr to .pbase */
this->setp(this->pbase(), this->epptr());
/* advance pptr to saved position */
this->pbump(s.pos);
this->local_ppos_ = this->pptr() - this->pbase();
this->solpos_ = s.solpos;
this->color_escape_chars_ = s.color_escape_chars;
/* assuming we never try to capture rewind state with incomplete color escape */
this->color_escape_start_ = nullptr;
}
void
arena_streambuf::expand_to(std::size_t new_z)
{
char * old_pptr = pptr();
std::streamsize old_n = old_pptr - pbase();
assert(old_n <= static_cast<std::streamsize>(arena_->allocated()));
assert(new_z > arena_->committed());
/* note: local_ppos_ invariant across expand_to() */
arena_->expand(new_z);
char * p_base = (char *)(arena_->lo_);
char * p_hi = (char *)(arena_->limit_);
this->setp(p_base, p_hi);
this->pbump(old_n);
}
std::streamsize
arena_streambuf::xsputn(const char * s, std::streamsize n)
{
/* s must be an address in [this->lo() .. this->lo() + capacity()] */
assert(hi() >= pptr());
if (pptr() + n > hi()) {
std::size_t new_z = std::max(2 * arena_->committed(), std::size_t(this->pos() + n + 1));
if (new_z > arena_->reserved())
new_z = arena_->reserved();
this->expand_to(new_z);
}
if (debug_flag_) {
std::cout << "xsputn: pbase=" << (void *)(this->pbase())
<< ", pptr=" << (void*)(this->pptr())
<< "(+" << (this->pptr() - this->lo()) << ")"
<< ", n=" << n << " -> (+" << (this->pptr() + n - this->lo()) << ")"
<< ", arena.size=" << this->arena_->committed()
<< std::endl;
}
std::streamsize ncopied = 0;
if (this->pptr() + n > this->hi()) {
ncopied = this->hi() - this->pptr();
} else {
ncopied = n;
}
if (false /*debug_flag_*/) {
std::cout << "xsputn: copying ncopied=" << ncopied << " (/n=" << n << ") bytes into range [lo,hi)"
<< ", lo=" << (void*)this->pptr()
<< ", hi=" << (void*)(this->pptr() + n)
<< std::endl;
}
std::memcpy(this->pptr(), s, ncopied);
this->pbump(ncopied);
/* now {pbase, pptr} consistent with new input */
this->_check_update_local_state();
return ncopied;
}
auto
arena_streambuf::overflow(int_type new_ch) -> int_type
{
char * old_base = this->pbase();
char * old_pptr = this->pptr();
/* #of chars buffered */
std::streamsize old_n = old_pptr - old_base;
assert(old_n <= static_cast<std::streamsize>(arena_->committed()));
// if (debug_flag_) {
// std::cout << "overflow: new_ch=" << quoted_char(new_ch) << std::endl;
// }
/* increase buffer size */
this->expand_to(2 * arena_->committed());
arena_->lo_[old_n] = static_cast<std::byte>(new_ch);
this->pbump(1);
if ((new_ch == static_cast<int_type>('\n')) || (new_ch == static_cast<int_type>('\r'))) {
this->solpos_ = this->pos();
// what if new_ch starts color escape ?
}
if (new_ch == std::char_traits<char>::eof()) {
/* reminder: returning eof sets badbit on ostream */
return std::char_traits<char>::not_eof(new_ch);
} else {
return new_ch;
}
}
auto
arena_streambuf::seekoff(off_type off,
std::ios_base::seekdir dir,
std::ios_base::openmode which) -> pos_type
{
//std::cout << "seekoff: off=" << off << ", dir=" << dir << ", which=" << which << std::endl;
if (debug_flag_) {
std::cout << "seekoff(off,dir,which)" << std::endl;
}
// Only output stream is supported
if (which != std::ios_base::out)
throw std::runtime_error("log_streambuf: only output mode supported");
if (dir == std::ios_base::cur) {
this->pbump(off);
} else if (dir == std::ios_base::end) {
/* .setp(): using for side effect: sets .pptr to .pbase */
this->setp(this->pbase(), this->epptr());
this->pbump(off);
} else if (dir == std::ios_base::beg) {
/* .setp(): using for side effect: sets .pptr to .pbase */
this->setp(this->pbase(), this->epptr());
this->pbump(this->capacity() + off);
}
return this->pptr() - this->pbase();
} /*seekoff*/
} /*namespace mm*/
} /*namespace xo*/
/* end arena_streambuf.cpp */

4
xo-expression2/include/xo/expression2/StringTable.hpp
View file

@ -21,6 +21,7 @@ namespace xo {
class StringTable {
public:
using DArena = xo::mm::DArena;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using StringMap = xo::map::DArenaHashMap<std::string_view,
DUniqueString*>;
using size_type = StringMap::size_type;
@ -45,6 +46,9 @@ namespace xo {
**/
bool verify_ok(verify_policy p = verify_policy::throw_only()) const;
std::size_t _n_store() const noexcept;
MemorySizeInfo _store_info(std::size_t i) const noexcept;
private:
/** allocate string storage in this arena; use DString to represent each string.
* Can't use DArenaVector b/c DString has variable size

19
xo-expression2/src/expression2/StringTable.cpp
View file

@ -10,6 +10,7 @@
namespace xo {
using xo::mm::ArenaConfig;
using xo::mm::AAllocator;
using xo::mm::MemorySizeInfo;
using xo::facet::with_facet;
using xo::facet::obj;
@ -159,6 +160,24 @@ namespace xo {
return true;
}
std::size_t
StringTable::_n_store() const noexcept
{
return 1 + map_._n_store();
}
MemorySizeInfo
StringTable::_store_info(std::size_t i) const noexcept
{
if (i == 0)
return strings_._store_info();
if (i+1 < map_._n_store())
return map_._store_info(i-1);
return MemorySizeInfo::sentinel();
}
} /*namespace scm*/
} /*namespace xo*/

7
xo-facet/include/xo/facet/box.hpp
View file

@ -38,6 +38,7 @@ namespace xo {
{}
/** (copy ctor not supported -- ownership is unique) **/
box(const box & other) = delete;
// --------------------------------
@ -73,8 +74,10 @@ namespace xo {
~box() {
auto p = this->data();
this->_drop();
::operator delete(p);
if (p) {
this->_drop();
::operator delete(p);
}
}
};
} /*namespace facet*/

151
xo-indentlog/include/xo/indentlog/log_state.hpp
View file

@ -255,157 +255,6 @@ namespace xo {
// instead of post-processing, rely on newline-aware log_streambuf
// to indent in advance
#ifdef OBSOLETE
log_streambuf_type * sbuf2 = this->p_sbuf_phase2_.get();
/* often sbuf contains one line of output.
* if it contains multiple newlines, need to indent
* after each one.
*
* will scan output in *sbuf1, post-process to *sbuf2,
* then write *sbuf2 to output stream
*
* note: we inherit .lpos from prec call to .flush2sbuf(),
* in the unlikely event that it's non-zero
*/
char const * s = sbuf1->lo();
char const * e = s + sbuf1->pos();
char const * p = s;
/* point to first space following a non-space character.
* will indent to just after this space
*/
char const * space_after_nonspace = nullptr;
/* true on VT100 color escape (\033); in which case false on terminating char (m)
* don't advance lpos during escape
*/
bool in_color_escape = false;
while(true) {
bool have_nonspace = false;
/* invariant: s<=p<=e */
/* for indenting, looking for first 'space following non-space, on first line', if any */
#ifdef OBSOLETE
// ..multiline input should have already been indented by custom log_streambuf.
// may need to extend to recognize terminal control sequences like below
std::size_t lpos_on_newline = 0;
#endif
#ifdef OBSOLETE
while(p < e) {
if(space_after_nonspace) {
;
} else {
if(*p != ' ')
have_nonspace = true;
if(have_nonspace && (*p == ' ')) {
space_after_nonspace = p;
}
}
if (in_color_escape && (*p != '\n')) {
/* in color escape -> don't advance .lpos */
if (*p == 'm')
in_color_escape = false;
++p;
} else if (*p == '\033') {
/* begin color escape sequence */
in_color_escape = true;
++p;
} else if (*p == '\n') {
/* reset .pos on newline; also drop any (incomplete + ill-formed) color escape */
in_color_escape = false;
#ifdef OBSOLETE
lpos_on_newline = this->lpos_;
this->lpos_ = 0;
#endif
++p;
break;
} else {
/* increment .lpos on non-newline */
++(this->lpos_);
++p;
}
}
#endif
/* p=e or *p=\n */
/* charseq [s,p) does not contain any newlines, print it */
if (lpos_on_newline > 0) {
/* charseq [s,p) does not contain any newlines, print it */
sbuf2->sputn(s, p - s - 1);
if (this->location_flag_) {
/* 'tab' to position lpos for [file:line] */
sbuf2->sputc(' ');
for (std::uint32_t i = lpos_on_newline + 1; i < log_config::location_tab; ++i)
sbuf2->sputc(' ');
std::stringstream ss;
ss << code_location(this->file_, this->line_,
log_config::code_location_color);
std::string ss_str = ss.str(); /*hoping for copy elision here*/
sbuf2->sputn(ss_str.c_str(), ss_str.size());
this->location_flag_ = false;
this->file_ = "";
this->line_ = 0;
}
sbuf2->sputc('\n');
} else {
/* control here if .flush2sbuf() called without trailing newline in .p_sbuf_phase1 */
sbuf2->sputn(s, p - s);
}
if (p == e)
break;
// {
// char buf[80];
// snprintf(buf, sizeof(buf), "*** indent=[%d] next=[%c]", this->nesting_level_, *(p+1));
//
// std::clog.rdbuf()->sputn(buf, strlen(buf));
//}
/* control here only for continuation lines (application logging code embedding its own newlines)
* - minimum indent = nesting level;
* - however if space_after_nonspace defined, also indent for that
*/
std::uint32_t n_indent = 0;
n_indent += this->calc_time_indent();
n_indent += std::min(this->nesting_level_ * log_config::indent_width,
log_config::max_indent_width);
#ifdef OBSOLETE // nice try, broken for multiline input + written before log_streambuf calculated lpos
/* this is just to indent for per-line entry/exit label */
if(space_after_nonspace)
n_indent += (space_after_nonspace - s);
#endif
for(std::uint32_t i = 0; i < n_indent; ++i)
sbuf2->sputc(' ');
s = p;
}
/* now write entire contents of *sbuf2 to clog */
p_sbuf->sputn(sbuf2->lo(), sbuf2->pos());
#endif
p_sbuf->sputn(sbuf1->lo(), sbuf1->pos());
/* reset streams for next message */

63
xo-interpreter2/include/xo/interpreter2/VirtualSchematikaMachine.hpp
View file

@ -14,21 +14,42 @@
namespace xo {
namespace scm {
struct EvaluationError {
/** source location (in vsm implementation) at which error identified **/
std::string_view src_function_;
/** error description (allocated from ErrorArena) **/
std::string_view error_description_;
// TODO: info about location in schematika source
};
/** similar to @ref xo::scm::ReaderResult **/
struct VsmResult {
using AGCObject = xo::mm::AGCObject;
using span_type = xo::mm::span<const char>;
bool is_tk_error() const { return tk_error_.is_error(); }
VsmResult() = default;
VsmResult(obj<AGCObject> value) : result_{value} {}
VsmResult(TokenizerError err) : result_{err} {}
bool is_value() const { return std::holds_alternative<obj<AGCObject>>(result_); }
bool is_tk_error() const { return std::holds_alternative<TokenizerError>(result_); }
bool is_eval_error() const { return std::holds_alternative<EvaluationError>(result_); }
const obj<AGCObject> * value() const { return std::get_if<obj<AGCObject>>(&result_); }
/** result of evaluating first expression encountered in input **/
obj<AGCObject> value_;
std::variant<obj<AGCObject>, TokenizerError, EvaluationError> result_;
};
/** unconsumed portion of input span **/
span_type remaining_input_;
/** vsm result + reamining span **/
struct VsmResultExt : public VsmResult {
using span_type = VsmResult::span_type;
/** {src_function, error_description, input_state, error_pos} **/
TokenizerError tk_error_;
VsmResultExt() = default;
VsmResultExt(const VsmResult & result, span_type rem) : VsmResult{result}, remaining_{rem} {}
/** unconsumed portion of input **/
VsmResult::span_type remaining_;
};
/** @class VirtualSchematikaMachine
@ -40,16 +61,29 @@ namespace xo {
using Stack = void *;
using AAllocator = xo::mm::AAllocator;
using AGCObject = xo::mm::AGCObject;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using span_type = xo::mm::span<const char>;
public:
VirtualSchematikaMachine(const VsmConfig & config);
/** consume input @p input_cstr **/
VsmResult read_eval_print(span_type input_span, bool eof);
size_t _n_store() const noexcept;
MemorySizeInfo _store_info(std::size_t i) const noexcept;
/** evaluate expression @p expr **/
std::pair<obj<AGCObject>, TokenizerError> eval(obj<AExpression> expr);
/** begin interactive session. **/
void begin_interactive_session();
/** begin batch session **/
void begin_batch_session();
/** consume input @p input_cstr.
* Require: must first start interactive/batch session
**/
VsmResultExt read_eval_print(span_type input_span, bool eof);
/** evaluate expression @p expr
* Require: must first start interactive/batch session
**/
VsmResult start_eval(obj<AExpression> expr);
/** borrow calling thread to run indefinitely,
* until halt instruction
@ -124,13 +158,16 @@ namespace xo {
/** configuration **/
VsmConfig config_;
/** allocator (likely collector) for
* expressions and values
**/
box<AAllocator> mm_;
/** reader: text -> expression **/
SchematikaReader reader_;
/** program counter **/
VsmInstr pc_ = VsmInstr::halt();
VsmInstr pc_ = VsmInstr::c_halt;
#ifdef NOT_YET
/** stack pointer **/
@ -141,10 +178,10 @@ namespace xo {
obj<AExpression> expr_;
/** result register **/
obj<AGCObject> value_;
VsmResult value_;
/** continuation register **/
VsmInstr cont_ = VsmInstr::halt();
VsmInstr cont_ = VsmInstr::c_halt;
};
} /*namespace scm*/
} /*namespace xo*/

3
xo-interpreter2/include/xo/interpreter2/VsmConfig.hpp
View file

@ -17,6 +17,9 @@ namespace xo {
VsmConfig() = default;
/** true for interactive parser session; false for batch session **/
bool interactive_flag_ = true;
/** reader configuration **/
ReaderConfig rdr_config_;
/** Configuration for allocator/collector.

4
xo-interpreter2/include/xo/interpreter2/VsmInstr.hpp
View file

@ -13,8 +13,8 @@ namespace xo {
public:
explicit VsmInstr(vsm_opcode oc) : opcode_{oc} {}
static VsmInstr halt() { return VsmInstr{vsm_opcode::halt}; }
static VsmInstr eval() { return VsmInstr{vsm_opcode::eval}; }
static VsmInstr c_halt;
static VsmInstr c_eval;
vsm_opcode opcode() const noexcept { return opcode_; }

1
xo-interpreter2/src/interpreter2/CMakeLists.txt
View file

@ -4,6 +4,7 @@ set(SELF_LIB xo_interpreter2)
set(SELF_SRCS
init_interpreter2.cpp
VirtualSchematikaMachine.cpp
VsmInstr.cpp
#IExpression_Any.cpp
#interpreter2_register_facets.cpp
)

73
xo-interpreter2/src/interpreter2/VirtualSchematikaMachine.cpp
View file

@ -17,6 +17,7 @@ namespace xo {
using xo::print::ppconfig;
using xo::print::ppstate_standalone;
using xo::mm::AGCObject;
using xo::mm::MemorySizeInfo;
using xo::mm::DX1Collector;
using xo::facet::FacetRegistry;
using std::cout;
@ -29,50 +30,84 @@ namespace xo {
reader_{config.rdr_config_, mm_.to_op()}
{}
VsmResult
std::size_t
VirtualSchematikaMachine::_n_store() const noexcept
{
// oops. need something that goes through AAllocator api
return reader_._n_store();
}
MemorySizeInfo
VirtualSchematikaMachine::_store_info(std::size_t i) const noexcept
{
// oops. need something poly that goes through AAllocator api
return reader_._store_info(i);
}
void
VirtualSchematikaMachine::begin_interactive_session()
{
reader_.begin_interactive_session();
}
void
VirtualSchematikaMachine::begin_batch_session()
{
reader_.begin_batch_session();
}
VsmResultExt
VirtualSchematikaMachine::read_eval_print(span_type input, bool eof)
{
if (input.empty()) {
return VsmResult();
return VsmResultExt();
}
auto [expr, remaining, error1]
= reader_.read_expr(input, eof);
if (!expr) {
return {
.remaining_input_ = remaining,
.tk_error_ = error1
};
/* tokenizer error */
return VsmResultExt(VsmResult(error1), remaining);
}
auto [value, error2] = this->eval(expr);
VsmResult evalresult = this->start_eval(expr);
if (!value) {
return {
.remaining_input_ = remaining,
.tk_error_ = error2
};
if (evalresult.is_eval_error() || evalresult.is_tk_error()) {
return VsmResultExt(evalresult, remaining);
}
assert(evalresult.is_value());
obj<AGCObject> * p_value = std::get_if<obj<AGCObject>>(&(evalresult.result_));
assert(p_value);
obj<APrintable> value_pr
= FacetRegistry::instance().variant<APrintable,AGCObject>(value);
= FacetRegistry::instance().variant<APrintable,AGCObject>(*p_value);
// pretty_toplevel(value_pr, &cout, ppconfig());
ppconfig ppc;
ppstate_standalone pps(&cout, 0, &ppc);
pps.prettyn(value_pr);
return { .remaining_input_ = remaining };
return VsmResultExt(VsmResult(*p_value), remaining);
}
std::pair<obj<AGCObject>, TokenizerError>
VirtualSchematikaMachine::eval(obj<AExpression> expr)
VsmResult
VirtualSchematikaMachine::start_eval(obj<AExpression> expr)
{
(void)expr;
this->pc_ = VsmInstr::c_eval;
this->expr_ = expr;
this->value_ = obj<AGCObject>();
this->cont_ = VsmInstr::c_halt;
assert(false);
return std::make_pair(obj<AGCObject>(), TokenizerError());
this->run();
return value_;
}
void

18
xo-interpreter2/src/interpreter2/VsmInstr.cpp Normal file
View file

@ -0,0 +1,18 @@
/** @file VsmInstr.cpp
*
* @author Roland Conybeare, Feb 2026
**/
#include "VsmInstr.hpp"
namespace xo {
namespace scm {
VsmInstr
VsmInstr::c_halt = VsmInstr(vsm_opcode::halt);
VsmInstr
VsmInstr::c_eval = VsmInstr(vsm_opcode::eval);
} /*namespace scm*/
} /*namespace xo*/
/* end VsmInstr.cpp */

35
xo-interpreter2/utest/VirtualSchematikaMachine.test.cpp
View file

@ -4,6 +4,8 @@
**/
#include <xo/interpreter2/VirtualSchematikaMachine.hpp>
#include <xo/object2/DFloat.hpp>
#include <xo/object2/number/IGCObject_DFloat.hpp>
#ifdef NOT_YET
#include <xo/reader2/SchematikaParser.hpp>
@ -16,12 +18,18 @@
#ifdef NOT_YET
#include <xo/alloc2/arena/IAllocator_DArena.hpp>
#endif
#include <xo/indentlog/print/hex.hpp>
#include <catch2/catch.hpp>
namespace xo {
using xo::scm::VirtualSchematikaMachine;
using xo::scm::VsmConfig;
using xo::scm::VsmResultExt;
using xo::scm::DFloat;
using xo::mm::AGCObject;
using span_type = xo::scm::VirtualSchematikaMachine::span_type;
using Catch::Matchers::WithinAbs;
#ifdef NOT_YET
using xo::scm::SchematikaParser;
@ -39,27 +47,32 @@ namespace xo {
using xo::mm::DArena;
using xo::facet::with_facet;
#endif
using std::cout;
using std::endl;
static InitEvidence s_init = (InitSubsys<S_interpreter2_tag>::require());
namespace ut {
TEST_CASE("VirtualSchematikaMachine-ctor", "[interpreter2][VSM]")
{
VirtualSchematikaMachine vsm(VsmConfig);
VsmConfig cfg;
VirtualSchematikaMachine vsm(cfg);
#ifdef NOT_YET
ArenaConfig config;
config.name_ = "test-arena";
config.size_ = 16 * 1024;
bool eof_flag = false;
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
vsm.begin_interactive_session();
VsmResultExt res = vsm.read_eval_print(span_type::from_cstr("3.141592635;"), eof_flag);
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
REQUIRE(res.is_value());
REQUIRE(res.value());
REQUIRE(parser.debug_flag() == false);
REQUIRE(parser.is_at_toplevel() == true);
#endif
auto x = obj<AGCObject,DFloat>::from(*res.value());
REQUIRE(x);
REQUIRE_THAT(x.data()->value(), WithinAbs(3.141592635, 1e-6));
REQUIRE(res.remaining_.size() == 1);
REQUIRE(*res.remaining_.lo() == '\n');
}
} /*namespace ut*/

6
xo-reader2/include/xo/reader2/ParserStateMachine.hpp
View file

@ -36,6 +36,7 @@ namespace xo {
using AAllocator = xo::mm::AAllocator;
using ArenaConfig = xo::mm::ArenaConfig;
using DArena = xo::mm::DArena;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using size_type = std::size_t;
public:
@ -61,6 +62,11 @@ namespace xo {
/** top of parser stack **/
obj<ASyntaxStateMachine> top_ssm() const;
/** number of distinct memory pools owned by PS **/
std::size_t _n_store() const noexcept;
/** memory consumption for i'th memory pool **/
MemorySizeInfo _store_info(std::size_t i) const noexcept;
///@}
/** @defgroup scm-parserstatemachine-bookkeeping bookkeeping methods **/

6
xo-reader2/include/xo/reader2/SchematikaParser.hpp
View file

@ -156,6 +156,7 @@ namespace xo {
using token_type = Token;
using ArenaConfig = xo::mm::ArenaConfig;
using AAllocator = xo::mm::AAllocator;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using ppindentinfo = xo::print::ppindentinfo;
using size_type = std::size_t;
@ -192,6 +193,11 @@ namespace xo {
/** top of parser stack **/
obj<ASyntaxStateMachine> top_ssm() const;
/** number of distinct memory pools owned by PS **/
std::size_t _n_store() const noexcept;
/** memory consumption for i'th memory pool **/
MemorySizeInfo _store_info(std::size_t i) const noexcept;
///@}
/** scm-schematikaparser-general-methods **/
///@{

9
xo-reader2/include/xo/reader2/SchematikaReader.hpp
View file

@ -36,6 +36,7 @@ namespace xo {
class SchematikaReader {
public:
using AAllocator = xo::mm::AAllocator;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using span_type = xo::mm::span<const char>;
using size_type = std::size_t;
@ -43,6 +44,9 @@ namespace xo {
SchematikaReader(const ReaderConfig & config,
obj<AAllocator> expr_alloc);
std::size_t _n_store() const noexcept;
MemorySizeInfo _store_info(std::size_t i) const noexcept;
/** true iff parser is at top-level.
* false iff parser is working on incomplete expression
**/
@ -53,6 +57,11 @@ namespace xo {
**/
void begin_interactive_session();
/** prepare batch session
* (limits expression types at toplevel)
**/
void begin_batch_session();
/** consume input @p input_cstr **/
const ReaderResult & read_expr(span_type input_span, bool eof);

24
xo-reader2/src/reader2/ParserStateMachine.cpp
View file

@ -16,6 +16,7 @@
#include <stdexcept>
namespace xo {
using xo::mm::MemorySizeInfo;
using xo::print::APrintable;
using xo::facet::FacetRegistry;
using xo::facet::with_facet;
@ -53,6 +54,29 @@ namespace xo {
return this->stack_->top();
}
std::size_t
ParserStateMachine::_n_store() const noexcept
{
return stringtable_._n_store() + 1;
}
MemorySizeInfo
ParserStateMachine::_store_info(std::size_t i) const noexcept
{
size_t n0 = stringtable_._n_store();
if (i < n0)
return stringtable_._store_info(i);
if (i == n0)
return parser_alloc_._store_info();
// not counting expr_alloc_. We don't consider
// that to be owned by ParserStateMachine
return MemorySizeInfo::sentinel();
}
void
ParserStateMachine::establish_toplevel_ssm(obj<ASyntaxStateMachine> ssm)
{

13
xo-reader2/src/reader2/SchematikaParser.cpp
View file

@ -13,6 +13,7 @@
namespace xo {
using xo::mm::AAllocator;
using xo::mm::MemorySizeInfo;
using xo::tostr;
using xo::xtag;
@ -46,6 +47,18 @@ namespace xo {
return psm_.top_ssm();
}
std::size_t
SchematikaParser::_n_store() const noexcept
{
return psm_._n_store();
}
MemorySizeInfo
SchematikaParser::_store_info(std::size_t i) const noexcept
{
return psm_._store_info(i);
}
void
SchematikaParser::begin_interactive_session()
{

31
xo-reader2/src/reader2/SchematikaReader.cpp
View file

@ -6,6 +6,8 @@
#include "SchematikaReader.hpp"
namespace xo {
using xo::mm::MemorySizeInfo;
namespace scm {
SchematikaReader::SchematikaReader(const ReaderConfig & config,
obj<AAllocator> expr_alloc)
@ -19,6 +21,29 @@ namespace xo {
{
}
std::size_t
SchematikaReader::_n_store() const noexcept
{
return tokenizer_._n_store() + parser_._n_store();
}
MemorySizeInfo
SchematikaReader::_store_info(std::size_t i) const noexcept
{
size_t n_tk = tokenizer_._n_store();
if (i < n_tk) {
return tokenizer_._store_info(i);
}
size_t n_pr = parser_._n_store();
if (i < n_tk + n_pr)
return parser_._store_info(i - n_tk);
return MemorySizeInfo::sentinel();
}
bool
SchematikaReader::is_at_toplevel() const noexcept
{
@ -31,6 +56,12 @@ namespace xo {
parser_.begin_interactive_session();
}
void
SchematikaReader::begin_batch_session()
{
parser_.begin_batch_session();
}
// TODO:
// Schematika::end_interactive_session()

6
xo-tokenizer2/include/xo/tokenizer2/Tokenizer.hpp
View file

@ -61,6 +61,7 @@ namespace xo {
using error_type = TokenizerError;
using DCircularBuffer = xo::mm::DCircularBuffer;
using CircularBufferConfig = xo::mm::CircularBufferConfig;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using span_type = xo::mm::span<const CharT>;
//using input_state_type = TkInputState;
using result_type = scan_result;
@ -90,6 +91,11 @@ namespace xo {
const TkInputState & input_state() const { return input_state_; }
#pragma GCC diagnostic pop
/** number of distinct memory pools owned by tokenizer **/
std::size_t _n_store() const noexcept;
/** memory consumption for i'th memory pool **/
MemorySizeInfo _store_info(std::size_t i) const noexcept;
///@}
/** @defgroup tokenizer-general-methods tokenizer methods **/

13
xo-tokenizer2/src/tokenizer2/Tokenizer.cpp
View file

@ -6,6 +6,7 @@
#include "Tokenizer.hpp"
namespace xo {
using xo::mm::MemorySizeInfo;
using std::byte;
namespace scm {
@ -21,6 +22,18 @@ namespace xo {
this->input_state_.discard_current_line();
}
std::size_t
Tokenizer::_n_store() const noexcept
{
return input_buffer_._n_store();
}
MemorySizeInfo
Tokenizer::_store_info(std::size_t i) const noexcept
{
return input_buffer_._store_info(i);
}
bool
Tokenizer::is_1char_punctuation(CharT ch)
{