roland/xo-reader2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2: readerreplxx works + streamline debugging

Browse source
This commit is contained in:
Roland Conybeare 2026-01-23 14:57:43 -05:00
commit 6894055548
7 changed files with 99 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

80
example/readerreplxx/readerreplxx.cpp
View file

@ -1,11 +1,14 @@
/** @file readerreplxx.cpp **/
#include <xo/reader2/init_reader2.hpp>
#include <xo/reader2/SchematikaReader.hpp>
#include <xo/gc/DX1Collector.hpp>
#include <xo/gc/detail/IAllocator_DX1Collector.hpp>
#include <xo/alloc2/Allocator.hpp>
//#include <xo/facet/facet.hpp>
#include <xo/printable2/Printable.hpp>
#include <xo/facet/FacetRegistry.hpp>
#include <xo/facet/obj.hpp>
#include <xo/subsys/Subsystem.hpp>
#include <replxx.hxx>
#include <iostream>
#include <unistd.h> // for isatty
@ -32,7 +35,8 @@ bool replxx_getline(bool interactive,
if (retval)
*p_input = input_cstr;
rx.history_add(input_cstr);
if (input_cstr)
rx.history_add(input_cstr);
return retval;
}
@ -56,8 +60,14 @@ welcome(std::ostream & os)
namespace {
using xo::scm::SchematikaReader;
using xo::scm::AExpression;
using xo::print::APrintable;
using xo::print::ppstate_standalone;
using xo::print::ppconfig;
using xo::facet::FacetRegistry;
using xo::facet::obj;
using xo::xtag;
using xo::scope;
using std::cout;
using std::endl;
@ -69,15 +79,36 @@ namespace {
bool
reader_seq(SchematikaReader * p_reader,
SchematikaReader::span_type * p_input,
bool eof)
bool eof,
bool debug_flag)
{
scope log(XO_DEBUG(debug_flag));
if (!p_input || p_input->empty())
return true;
auto [expr, remaining, error] = p_reader->read_expr(*p_input, eof);
obj<APrintable> expr_pr;
if (expr) {
expr_pr = FacetRegistry::instance().variant<APrintable,AExpression>(expr);
assert(expr_pr);
}
if (log) {
if (expr_pr) {
log(xtag("expr", expr_pr));
}
log(xtag("remaining", remaining));
log(xtag("error", error));
}
if (expr) {
ppconfig ppc;
ppstate_standalone pps(&cout, 0, &ppc);
pps.prettyn(expr);
pps.prettyn(expr_pr);
*p_input = remaining;
@ -93,6 +124,8 @@ namespace {
return false;
} else {
*p_input = remaining;
/* partial expression or whitespace input, no error */
return true;
}
@ -110,15 +143,19 @@ main()
using xo::mm::DX1Collector;
using xo::mm::CollectorConfig;
using xo::mm::DArena;
//using xo::print::ppconfig;
//using xo::print::ppstate_standalone;
using xo::facet::with_facet;
using xo::facet::obj;
using xo::S_reader2_tag;
using xo::InitSubsys;
using xo::Subsystem;
using xo::scope;
using namespace std;
bool interactive = isatty(STDIN_FILENO);
InitSubsys<S_reader2_tag>::require();
Subsystem::initialize_all();
Replxx rx;
rx.set_max_history_size(1000);
rx.history_load("repl_history.txt");
@ -134,7 +171,12 @@ main()
obj<AAllocator> expr_alloc = with_facet<AAllocator>::mkobj(&x1);
// accepting defaults too
ReaderConfig rdr_config = ReaderConfig();
ReaderConfig rdr_config;
{
//rdr_config.reader_debug_flag_ = true;
//rdr_config.parser_debug_flag_ = true;
//rdr_config.tk_debug_flag_ = true;
}
SchematikaReader rdr(rdr_config, expr_alloc);
using span_type = SchematikaReader::span_type;
@ -144,24 +186,28 @@ main()
rdr.begin_interactive_session();
bool eof = false;
const char * input_str;
const char * input_str = nullptr;
span_type input;
while (replxx_getline(interactive, rdr.is_at_toplevel(), rx, &input_str)) {
input = span_type::from_cstr(input_str);
if (input_str && *input_str) {
input = span_type::from_cstr(input_str);
while (!input.empty() && reader_seq(&rdr, &input, false /*eof*/)) {
;
while (!input.empty()
&& reader_seq(&rdr, &input, false /*eof*/, c_debug_flag))
{
;
}
/* here: either:
* 1. input.empty() or
* 2. error encountered
*/
}
/* here: either:
* 1. input.empty() or
* 2. error encountered
*/
}
/* reminder: eof can complete at most one token */
reader_seq(&rdr, &input, true /*eof*/);
reader_seq(&rdr, &input, true /*eof*/, c_debug_flag);
rx.history_save("repl_history.txt");
}

20
include/xo/reader2/ReaderConfig.hpp
View file

@ -11,7 +11,7 @@
namespace xo {
namespace scm {
/** @brief Configuration for SchemtikaReader
/** @brief Configuration for SchematikaReader
**/
struct ReaderConfig {
using CircularBufferConfig = xo::mm::CircularBufferConfig;
@ -26,6 +26,7 @@ namespace xo {
.max_captured_span_ = 128 };
/** debug flag for schematika tokenizer **/
bool tk_debug_flag_ = false;
/** arena configuration for parser stack **/
ArenaConfig parser_arena_config_ { .name_ = "parer-arena",
.size_ = 2*1024*1024,
@ -33,19 +34,14 @@ namespace xo {
.store_header_flag_ = false,
.header_{},
.debug_flag_ = false };
/** max size (in bytes) of stringtable **/
size_t max_stringtable_cap_ = 64*1024;
/** debug flag for schematika parser **/
bool parser_debug_flag_ = false;
#ifdef NOT_YET
/** arena configuration for output expressions **/
ArenaConfig expr_arena_config_ { .name_ = "expr-arena",
.size_ = 2*1024*1024,
.hugepage_z_ = 2*1024*1024,
.store_header_flag_ = false,
.header_{},
.debug_flag_ = false };
#endif
/** max size (in bytes) of stringtable **/
size_t max_stringtable_cap_ = 64*1024;
/** debug flag for schematika_reader **/
bool reader_debug_flag_ = false;;
};
} /*namespace scm*/

2
include/xo/reader2/SchematikaReader.hpp
View file

@ -77,6 +77,8 @@ namespace xo {
/** current output from reader **/
ReaderResult result_;
/** true to enable reader debug logging **/
bool debug_flag_ = false;
};
} /*namespace scm*/
} /*namespace xo*/

2
src/reader2/DDefineSsm.cpp
View file

@ -550,7 +550,7 @@ namespace xo {
DDefineSsm::on_singleassign_token(const Token & tk,
ParserStateMachine * p_psm)
{
scope log(XO_DEBUG(true), xtag("defstate", defstate_));
scope log(XO_DEBUG(p_psm->debug_flag()), xtag("defstate", defstate_));
if ((defstate_ == defexprstatetype::def_2)
|| (defstate_ == defexprstatetype::def_4))

2
src/reader2/DExpectTypeSsm.cpp
View file

@ -114,7 +114,7 @@ namespace xo {
DExpectTypeSsm::on_symbol_token(const Token & tk,
ParserStateMachine * p_psm)
{
scope log(XO_DEBUG(true));
scope log(XO_DEBUG(p_psm->debug_flag()));
TypeDescr td = nullptr;

3
src/reader2/DProgressSsm.cpp
View file

@ -216,8 +216,7 @@ namespace xo {
DProgressSsm::on_semicolon_token(const Token & tk,
ParserStateMachine * p_psm)
{
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
scope log(XO_DEBUG(p_psm->debug_flag()));
/* note: implementation should parallel .on_rightparen_token() */

26
src/reader2/SchematikaReader.cpp
View file

@ -9,11 +9,13 @@ namespace xo {
namespace scm {
SchematikaReader::SchematikaReader(const ReaderConfig & config,
obj<AAllocator> expr_alloc)
: tokenizer_{config.tk_buffer_config_, config.tk_debug_flag_},
: tokenizer_{config.tk_buffer_config_,
config.tk_debug_flag_},
parser_{config.parser_arena_config_,
config.max_stringtable_cap_,
expr_alloc,
config.parser_debug_flag_}
config.parser_debug_flag_},
debug_flag_{config.reader_debug_flag_}
{
}
@ -35,18 +37,34 @@ namespace xo {
const ReaderResult &
SchematikaReader::read_expr(span_type input_ext, bool eof)
{
scope log(XO_DEBUG(debug_flag_));
if (log) {
log(xtag("input_ext", input_ext));
log(xtag("eof", eof));
}
if (!input_ext.empty()) {
auto [error, input]
= tokenizer_.buffer_input_line(input_ext, eof);
// log && log(xtag("msg", "buffered input line"));
// log && log(xtag("input", input));
if (log) {
log(xtag("msg", "before loop: buffered input line"));
log(xtag("input", input));
}
while (!input.empty()) {
log && log(xtag("msg", "loop"),
xtag("input", input));
auto [tk, consumed, error] = tokenizer_.scan(input);
log && log(xtag("tk", tk), xtag("consumed", consumed));
auto rem_input = input.after_prefix(consumed);
log && log(xtag("rem_input", rem_input));
if (!tk.is_valid() && error.is_error()) {
this->result_
= ReaderResult