roland/xo-reader2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-interpreter2 stack: plumbing for aux_mm and use opportunistically

Browse source
This commit is contained in:
Roland Conybeare 2026-02-15 14:13:38 -05:00
commit 78f1b8a0b6
8 changed files with 110 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

22
example/readerreplxx/readerreplxx.cpp
View file

@ -2,8 +2,9 @@
#include <xo/reader2/init_reader2.hpp>
#include <xo/reader2/SchematikaReader.hpp>
#include <xo/gc/DX1Collector.hpp>
#include <xo/gc/X1Collector.hpp>
#include <xo/gc/detail/IAllocator_DX1Collector.hpp>
#include <xo/alloc2/Arena.hpp>
#include <xo/alloc2/Allocator.hpp>
#include <xo/printable2/Printable.hpp>
#include <xo/facet/FacetRegistry.hpp>
@ -137,6 +138,7 @@ namespace {
struct AppConfig {
using ReaderConfig = xo::scm::ReaderConfig;
using X1CollectorConfig = xo::mm::X1CollectorConfig;
using ArenaConfig = xo::mm::ArenaConfig;
AppConfig() {
rdr_config_.reader_debug_flag_ = true;
@ -147,17 +149,24 @@ struct AppConfig {
std::size_t max_history_size_ = 1000;
std::string repl_history_fname_ = "repl_history.txt";;
ReaderConfig rdr_config_;
X1CollectorConfig x1_config_ = (X1CollectorConfig().with_size(4*1024*1024));
X1CollectorConfig x1_config_ = (X1CollectorConfig().with_name("gc").with_size(4*1024*1024));
ArenaConfig fixed_config_ = (ArenaConfig().with_name("fixed").with_size(4*1024));
};
struct AppContext {
using AAllocator = xo::mm::AAllocator;
using DX1Collector = xo::mm::DX1Collector;
using X1CollectorConfig = xo::mm::X1CollectorConfig;
using DArena = xo::mm::DArena;
using ArenaConfig = xo::mm::ArenaConfig;
using Replxx = replxx::Replxx;
AppContext(const AppConfig & cfg = AppConfig()) : config_{cfg},
x1_{X1CollectorConfig().with_size(4*1024*1024)},
rdr_{config_.rdr_config_, x1_.ref()}
x1_{cfg.x1_config_},
fixed_{cfg.fixed_config_},
rdr_{config_.rdr_config_,
x1_.ref(),
obj<AAllocator,DArena>(&fixed_)}
{
rx_.set_max_history_size(config_.max_history_size_);
rx_.history_load(config_.repl_history_fname_);
@ -167,9 +176,12 @@ struct AppContext {
AppConfig config_;
Replxx rx_;
/** collector/allocator for schematika expressions **/
DX1Collector x1_;
/** e.g. for DArenaHashMap within global symtab **/
DArena fixed_;
SchematikaReader rdr_;
};
};
int
main()

27
include/xo/reader2/ParserStateMachine.hpp
View file

@ -41,9 +41,21 @@ namespace xo {
using size_type = std::size_t;
public:
/**
* @p config arena configuration for parser state
* @p max_stringtable_capacity
* hard max size for unique stringtable
* @p expr_alloc allocator for schematika expressions.
* Probably shared with execution.
* @p aux_alloc auxiliary allocator for non-copyable memory
* (e.g. DArenaHashMap for global symtable).
* If not using X1Collector, this can be the
* same as @p expr_alloc.
**/
ParserStateMachine(const ArenaConfig & config,
size_type max_stringtable_capacity,
obj<AAllocator> expr_alloc);
obj<AAllocator> expr_alloc,
obj<AAllocator> aux_alloc);
/** @defgroup scm-parserstatemachine-accessors accessor methods **/
///@{
@ -280,6 +292,19 @@ namespace xo {
**/
obj<AAllocator> expr_alloc_;
/** Allocator for data with lifetime bounded by this ParserStateMachine
*
* Cannot be DX1Collector; for example DArenaHashMap will
* for global symtab will be allocated from here,
* and does not support gc.
*
* If @ref expr_alloc_ is an ordinary arena (e.g. DArenaAlloc)
* can have aux_alloc_ = expr_alloc_.
* When expr_alloc_ is a garbage collector (e.g. DX1Collector)
* this needs to be distinct.
**/
obj<AAllocator> aux_alloc_;
/** symbol table with local bindings.
* non-null during parsing of lambda expressions.
* Always allocated from @p expr_alloc_.

10
include/xo/reader2/SchematikaParser.hpp
View file

@ -164,14 +164,18 @@ namespace xo {
/** create parser in initial state;
* parser is ready to receive tokens via @ref include_token
*
* @p config arena configuration for parser stack
* @p expr_alloc allocator for schematika expressions.
* Probably shared with execution.
* @p config arena configuration for parser stack
* @p expr_alloc allocator for schematika expressions.
* Probably shared with execution.
* @p aux_alloc aux allocator for non-copyable memory
* with lifetime bounded by this
* SchematikeParser itself
* @p debug_flag true to enable debug logging
**/
SchematikaParser(const ArenaConfig & config,
size_t max_stringtable_capacity,
obj<AAllocator> expr_alloc,
obj<AAllocator> aux_alloc,
bool debug_flag);
/** scm-schematikaparser-access-methods **/

9
include/xo/reader2/SchematikaReader.hpp
View file

@ -41,8 +41,15 @@ namespace xo {
using size_type = std::size_t;
public:
/**
* @p expr_alloc. allocator for Schematika expressions
* @p aux_alloc. allocator for miscellaneous objects
* (e.g. DArenaHashMap for global symtab)
* that have lifetime bounded by Schematika reader itself.
**/
SchematikaReader(const ReaderConfig & config,
obj<AAllocator> expr_alloc);
obj<AAllocator> expr_alloc,
obj<AAllocator> fixed_alloc);
/** visit reader-owned memory pools; call visitor(info) for each.
* Specifically exclude expr_alloc, since we don't consider

9
src/reader2/ParserStateMachine.cpp
View file

@ -24,13 +24,14 @@ namespace xo {
namespace scm {
ParserStateMachine::ParserStateMachine(const ArenaConfig & config,
size_type max_stringtable_capacity,
obj<AAllocator> expr_alloc)
obj<AAllocator> expr_alloc,
obj<AAllocator> aux_alloc)
: stringtable_{max_stringtable_capacity},
parser_alloc_{DArena::map(config)},
expr_alloc_{expr_alloc},
aux_alloc_{aux_alloc},
debug_flag_{config.debug_flag_}
{
}
bool
@ -60,8 +61,8 @@ namespace xo {
stringtable_.visit_pools(visitor);
parser_alloc_.visit_pools(visitor);
// not counting expr_alloc_. We don't consider
// that to be owned by ParserStateMachine
// not counting {expr_alloc_, fixed_alloc_}. We don't consider
// either to be owned by ParserStateMachine
}
void

5
src/reader2/SchematikaParser.cpp
View file

@ -23,9 +23,10 @@ namespace xo {
SchematikaParser::SchematikaParser(const ArenaConfig & config,
size_t max_stringtable_capacity,
obj<AAllocator> expr_alloc,
obj<AAllocator> fixed_alloc,
bool debug_flag)
: psm_{config, max_stringtable_capacity, expr_alloc},
debug_flag_{debug_flag}
: psm_{config, max_stringtable_capacity, expr_alloc, fixed_alloc},
debug_flag_{debug_flag}
{
}

4
src/reader2/SchematikaReader.cpp
View file

@ -10,12 +10,14 @@ namespace xo {
namespace scm {
SchematikaReader::SchematikaReader(const ReaderConfig & config,
obj<AAllocator> expr_alloc)
obj<AAllocator> expr_alloc,
obj<AAllocator> fixed_alloc)
: tokenizer_{config.tk_buffer_config_,
config.tk_debug_flag_},
parser_{config.parser_arena_config_,
config.max_stringtable_cap_,
expr_alloc,
fixed_alloc,
config.parser_debug_flag_},
debug_flag_{config.reader_debug_flag_}
{

66
utest/SchematikaParser.test.cpp
View file

@ -88,8 +88,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
REQUIRE(parser.debug_flag() == false);
REQUIRE(parser.is_at_toplevel() == true);
@ -103,8 +104,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -121,8 +123,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_batch_session();
@ -133,17 +136,20 @@ namespace xo {
TEST_CASE("SchematikaParser-batch-def", "[reader2][SchematikaParser]")
{
const auto & testname = Catch::getResultCapture().getCurrentTestName();
constexpr bool c_debug_flag = false;
scope log(XO_DEBUG(c_debug_flag));
scope log(XO_DEBUG(c_debug_flag), xtag("test", testname));
ArenaConfig config;
config.name_ = "test-arena";
config.name_ = testname;
config.size_ = 16 * 1024;
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_batch_session();
@ -185,8 +191,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -250,8 +257,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -315,8 +323,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -380,8 +389,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -435,16 +445,17 @@ namespace xo {
const auto & testname = Catch::getResultCapture().getCurrentTestName();
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag), xtag("test", testname));
scope log(XO_DEBUG(c_debug_flag),
xtag("test", testname));
ArenaConfig config;
config.name_ = "test-arena";
config.size_ = 16 * 1024;
ArenaConfig config
= (ArenaConfig().with_name(testname).with_size(16 * 1024));
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto expr_alloc = obj<AAllocator,DArena>(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -507,8 +518,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -517,7 +529,7 @@ namespace xo {
* lambda (n : i64, r : i64) -> i64 { 123 }
* ^ ^^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
* 0 1| 3 4 5 6 7 8 9 a b c d e
* 2
* 2
**/
std::vector<Token> tk_v{
@ -549,13 +561,14 @@ namespace xo {
scope log(XO_DEBUG(c_debug_flag), xtag("test", testname));
ArenaConfig config;
config.name_ = "test-arena";
config.name_ = testname;
config.size_ = 16 * 1024;
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -592,8 +605,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -632,13 +646,14 @@ namespace xo {
scope log(XO_DEBUG(c_debug_flag), xtag("test", testname));
ArenaConfig config;
config.name_ = "test-arena";
config.name_ = testname;
config.size_ = 16 * 1024;
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -689,8 +704,9 @@ namespace xo {
DArena expr_arena = DArena::map(config);
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&expr_arena);
auto aux_alloc = expr_alloc;
SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/);
SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/);
parser.begin_interactive_session();
@ -699,7 +715,7 @@ namespace xo {
* (lambda (x : i64, y : i64) { x * y })(13, 15) ;
* ^^ ^^ ^ ^ ^ ^^ ^ ^ ^ ^ ^ ^ ^^^^ ^ ^ ^ ^
* 0| 2| 4 5 6 7| 9 a b c d e f|h| j k l m
* 1 3 8 g i
* 1 3 8 g i
**/
std::vector<Token> tk_v{