roland/xo-reader2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2 stack: top-level lambda w/ apply parses

Browse source
This commit is contained in:
Roland Conybeare 2026-02-05 15:45:40 -05:00
commit ca1370570b
46 changed files with 329 additions and 191 deletions
Download patch file Download diff file Expand all files Collapse all files

27
src/reader2/DDefineSsm.cpp
View file

@ -626,7 +626,7 @@ namespace xo {
{
if (defstate_ == defexprstatetype::def_6) {
p_psm->pop_ssm();
p_psm->on_parsed_expression_with_semicolon(def_expr_);
p_psm->on_parsed_expression_with_token(def_expr_, tk);
return;
}
@ -649,11 +649,28 @@ namespace xo {
}
void
DDefineSsm::on_parsed_expression_with_semicolon(obj<AExpression> expr,
ParserStateMachine * p_psm)
DDefineSsm::on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk,
ParserStateMachine * p_psm)
{
this->on_parsed_expression(expr, p_psm);
this->on_semicolon_token(Token::semicolon_token(), p_psm);
/* must end with semicolon */
if (tk.tk_type() == tokentype::tk_semicolon) {
if (defstate_ == defexprstatetype::def_5)
{
this->defstate_ = defexprstatetype::def_6;
def_expr_.data()->assign_rhs(expr);
// completes this definition syntax
this->on_semicolon_token(tk, p_psm);
return;
}
}
// error in all other cases
Super::on_parsed_expression_with_token(expr, tk, p_psm);
}
bool

11
src/reader2/DExpectExprSsm.cpp
View file

@ -195,7 +195,7 @@ namespace xo {
log && log(xtag("tk", tk));
const DVariable * var = p_psm->lookup_variable(tk.text());
DVarRef * var = p_psm->lookup_varref(tk.text());
if (!var) {
p_psm->error_unbound_variable(ssm_classname(),
@ -210,7 +210,7 @@ namespace xo {
//
DProgressSsm::start(p_psm->parser_alloc(),
obj<AExpression,DVariable>(const_cast<DVariable *>(var)),
obj<AExpression,DVarRef>(var),
p_psm);
}
@ -388,14 +388,15 @@ namespace xo {
}
void
DExpectExprSsm::on_parsed_expression_with_semicolon(obj<AExpression> expr,
ParserStateMachine * p_psm)
DExpectExprSsm::on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk,
ParserStateMachine * p_psm)
{
// expression (reported by nested ProgressSsm)
// completes this DExpectExprSsm's assignment
p_psm->pop_ssm();
p_psm->on_parsed_expression_with_semicolon(expr);
p_psm->on_parsed_expression_with_token(expr, tk);
}
bool

15
src/reader2/DExprSeqState.cpp
View file

@ -167,7 +167,10 @@ namespace xo {
case tokentype::tk_dot:
case tokentype::tk_comma:
case tokentype::tk_colon:
break;
case tokentype::tk_semicolon:
assert(false);
break;
case tokentype::tk_doublecolon:
case tokentype::tk_singleassign:
case tokentype::tk_assign:
@ -396,10 +399,16 @@ namespace xo {
}
void
DExprSeqState::on_parsed_expression_with_semicolon(obj<AExpression> expr,
ParserStateMachine * p_psm)
DExprSeqState::on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk,
ParserStateMachine * p_psm)
{
p_psm->capture_result("DExprSeqState::on_parsed_expression_with_semicolon", expr);
if (tk.tk_type() == tokentype::tk_semicolon) {
p_psm->capture_result("DExprSeqState::on_parsed_expression_with_token", expr);
return;
}
Super::on_parsed_expression_with_token(expr, tk, p_psm);
}
bool

24
src/reader2/DIfElseSsm.cpp
View file

@ -150,13 +150,15 @@ namespace xo {
case tokentype::tk_else:
this->on_else_token(tk, p_psm);
return;
case tokentype::tk_semicolon:
this->on_semicolon_token(tk, p_psm);
return;
case tokentype::tk_colon:
case tokentype::tk_singleassign:
case tokentype::tk_string:
case tokentype::tk_f64:
case tokentype::tk_i64:
case tokentype::tk_bool:
case tokentype::tk_semicolon:
case tokentype::tk_invalid:
case tokentype::tk_leftparen:
case tokentype::tk_rightparen:
@ -405,13 +407,25 @@ namespace xo {
}
void
DIfElseSsm::on_parsed_expression_with_semicolon(obj<AExpression> expr,
ParserStateMachine * p_psm)
DIfElseSsm::on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk,
ParserStateMachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
this->on_parsed_expression(expr, p_psm);
this->on_semicolon_token(Token::semicolon_token(), p_psm);
// TODO: may consider allowing if-else to terminate on other particular tokens
// e.g. ')'
if ((tk.tk_type() == tokentype::tk_then)
|| (tk.tk_type() == tokentype::tk_else)
|| (tk.tk_type() == tokentype::tk_semicolon))
{
this->on_parsed_expression(expr, p_psm);
this->on_token(tk, p_psm);
return;
}
Super::on_parsed_expression_with_token(expr, tk, p_psm);
}
bool

7
src/reader2/DLambdaSsm.cpp
View file

@ -364,10 +364,11 @@ namespace xo {
}
void
DLambdaSsm::on_parsed_expression_with_semicolon(obj<AExpression> expr,
ParserStateMachine * p_psm)
DLambdaSsm::on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk,
ParserStateMachine * p_psm)
{
Super::on_parsed_expression_with_semicolon(expr, p_psm);
Super::on_parsed_expression_with_token(expr, tk, p_psm);
}
#ifdef NOT_YET

45
src/reader2/DProgressSsm.cpp
View file

@ -319,10 +319,7 @@ namespace xo {
obj<AExpression> expr = this->assemble_expr(p_psm);
p_psm->pop_ssm(); // completes self
// TODO: perhaps need to generalize on_parsed_expression_with_semicolon() ..?
p_psm->on_parsed_expression(expr);
p_psm->on_token(tk);
p_psm->on_parsed_expression_with_token(expr, tk);
}
void
@ -458,25 +455,7 @@ namespace xo {
}
p_psm->pop_ssm();
p_psm->on_parsed_expression_with_semicolon(expr);
/* control here on input like:
* (1.234;
*
* a. '(' sets up stack [lparen_0:expect_rhs_expression]
* (see exprstate::on_leftparen())
* b. 1.234 pushes (in case operators) [lparen_0:expect_rhs_expression:expr_progress]
* (see exprstate::on_f64())
* c. semicolon completes expr_progress [lparen_0:expect_rhs_expression]
* deliver expresssion to expect_rhs_expression.on_expr_with_semicolon()
* (see exprstate::on_expr_with_semicolon())
* d. expr_rhs_expression forwards expression to [lparen_0]
* e. lparen_0 would advance to [lparen_1], but rejects semicolon
*/
#ifdef OBSOLETE
Super::on_token(tk, p_psm);
#endif
p_psm->on_parsed_expression_with_token(expr, tk);
}
void
@ -489,11 +468,15 @@ namespace xo {
obj<AExpression> expr = this->assemble_expr(p_psm);
{
if (expr) {
obj<APrintable> expr_pr
= FacetRegistry::instance().variant<APrintable,AExpression>(expr);
= FacetRegistry::instance().try_variant<APrintable,AExpression>(expr);
assert(expr_pr);
log && log(xtag("expr", expr_pr));
} else {
// illegal token if assemble failed
Super::on_token(tk, p_psm);
return;
}
p_psm->pop_ssm();
@ -501,15 +484,19 @@ namespace xo {
}
void
DProgressSsm::on_parsed_expression_with_semicolon(obj<AExpression> expr,
ParserStateMachine * p_psm)
DProgressSsm::on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk,
ParserStateMachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()),
xtag("expr", expr));
if (op_type_ == optype::invalid) {
// e.g. control here on input like
// x : = 4 4
p_psm->illegal_parsed_expression
("DProgressSsm::on_parsed_expression_with_semicolon",
("DProgressSsm::on_parsed_expression_with_token",
expr,
this->get_expect_str());
return;
@ -521,7 +508,7 @@ namespace xo {
if (expr2) {
p_psm->pop_ssm();
p_psm->on_parsed_expression_with_semicolon(expr2);
p_psm->on_parsed_expression_with_token(expr2, tk);
}
}

33
src/reader2/DSequenceSsm.cpp
View file

@ -147,10 +147,8 @@ namespace xo {
{
scope log(XO_DEBUG(p_psm->debug_flag()));
// TODO: stream inserter that sets up pretty-printing.
// Or integrate with indentlog.
// Maybe trouble is that indentlog doesn't #include Printable ?
//
// TODO: switch to printable facet
log && log(xtag("expr", expr));
#ifdef NOT_YET
@ -206,8 +204,31 @@ namespace xo {
}
#endif
this->seq_expr_->push_back(p_psm->expr_alloc(),
expr);
this->seq_expr_->push_back(p_psm->expr_alloc(), expr);
}
void
DSequenceSsm::on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk,
ParserStateMachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
if (tk.tk_type() == tokentype::tk_semicolon) {
// keep sequence on stack, consuming semicolon
this->seq_expr_->push_back(p_psm->expr_alloc(),
expr);
return;
} else if (tk.tk_type() == tokentype::tk_rightbrace) {
// rightbrace ends sequence
this->seq_expr_->push_back(p_psm->expr_alloc(), expr);
this->on_rightbrace_token(tk, p_psm);
return;
}
Super::on_parsed_expression_with_token(expr, tk, p_psm);
}
#ifdef NOT_YET

2
src/reader2/ISyntaxStateMachine_Any.cpp
View file

@ -71,7 +71,7 @@ ISyntaxStateMachine_Any::on_parsed_expression(Opaque, obj<AExpression>, ParserSt
}
auto
ISyntaxStateMachine_Any::on_parsed_expression_with_semicolon(Opaque, obj<AExpression>, ParserStateMachine *) -> void
ISyntaxStateMachine_Any::on_parsed_expression_with_token(Opaque, obj<AExpression>, const Token &, ParserStateMachine *) -> void
{
_fatal();
}

4
src/reader2/ISyntaxStateMachine_DDefineSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DDefineSsm::on_parsed_expression_with_semicolon(DDefineSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DDefineSsm::on_parsed_expression_with_token(DDefineSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DExpectExprSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DExpectExprSsm::on_parsed_expression_with_semicolon(DExpectExprSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DExpectExprSsm::on_parsed_expression_with_token(DExpectExprSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DExpectFormalArgSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DExpectFormalArgSsm::on_parsed_expression_with_semicolon(DExpectFormalArgSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DExpectFormalArgSsm::on_parsed_expression_with_token(DExpectFormalArgSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DExpectFormalArglistSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DExpectFormalArglistSsm::on_parsed_expression_with_semicolon(DExpectFormalArglistSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DExpectFormalArglistSsm::on_parsed_expression_with_token(DExpectFormalArglistSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DExpectSymbolSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DExpectSymbolSsm::on_parsed_expression_with_semicolon(DExpectSymbolSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DExpectSymbolSsm::on_parsed_expression_with_token(DExpectSymbolSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DExpectTypeSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DExpectTypeSsm::on_parsed_expression_with_semicolon(DExpectTypeSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DExpectTypeSsm::on_parsed_expression_with_token(DExpectTypeSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DExprSeqState.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DExprSeqState::on_parsed_expression_with_semicolon(DExprSeqState & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DExprSeqState::on_parsed_expression_with_token(DExprSeqState & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DIfElseSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DIfElseSsm::on_parsed_expression_with_semicolon(DIfElseSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DIfElseSsm::on_parsed_expression_with_token(DIfElseSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DLambdaSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DLambdaSsm::on_parsed_expression_with_semicolon(DLambdaSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DLambdaSsm::on_parsed_expression_with_token(DLambdaSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DProgressSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DProgressSsm::on_parsed_expression_with_semicolon(DProgressSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DProgressSsm::on_parsed_expression_with_token(DProgressSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

4
src/reader2/ISyntaxStateMachine_DSequenceSsm.cpp
View file

@ -58,9 +58,9 @@ namespace xo {
self.on_parsed_expression(expr, p_psm);
}
auto
ISyntaxStateMachine_DSequenceSsm::on_parsed_expression_with_semicolon(DSequenceSsm & self, obj<AExpression> expr, ParserStateMachine * p_psm) -> void
ISyntaxStateMachine_DSequenceSsm::on_parsed_expression_with_token(DSequenceSsm & self, obj<AExpression> expr, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_parsed_expression_with_semicolon(expr, p_psm);
self.on_parsed_expression_with_token(expr, tk, p_psm);
}
} /*namespace scm*/

118
src/reader2/ParserStateMachine.cpp
View file

@ -107,38 +107,57 @@ namespace xo {
return stringtable_.gensym(str);
}
Binding
ParserStateMachine::lookup_binding(std::string_view symbolname)
DVarRef *
ParserStateMachine::lookup_varref(std::string_view symbolname)
{
scope log(XO_DEBUG(debug_flag_));
if (!local_symtab_)
return Binding::null();
const DUniqueString * ustr = stringtable_.lookup(symbolname);
if (!ustr) {
// if not in string table, then can't be a variable either
return Binding::null();
}
DLocalSymtab * symtab = local_symtab_;
// count #of nested scopes to cross, to reach symbol
// TODO:
// 1. check global symtab
// 2. combine local+global symtab into indept struct
// 3. move lookup_varref implementation there.
//
int32_t link_count = 0;
while (symtab) {
Binding b = symtab->lookup_binding(ustr);
if (local_symtab_) {
const DUniqueString * ustr = stringtable_.lookup(symbolname);
if (b.is_local()) {
assert(b.i_link() == 0);
if (ustr) {
DLocalSymtab * symtab = local_symtab_;
return Binding(link_count, b.j_slot());
// count #of nested scopes to cross, to reach symbol
//
int32_t link_count = 0;
while (symtab) {
Binding b = symtab->lookup_binding(ustr);
if (b.is_local()) {
assert(b.i_link() == 0);
DVariable * vardef = symtab->lookup_var(b);
assert(vardef);
/** ascii diagram here
**/
return DVarRef::make(expr_alloc_,
vardef,
link_count);
} else {
assert(b.is_null());
}
++link_count;
symtab = symtab->parent();
}
} else {
// if we don't already know the symbol,
// -> can't be a valid variable reference
// (whether global or local)
return nullptr;
}
++link_count;
symtab = symtab->parent();
}
// TODO: check global symtab also
@ -146,7 +165,7 @@ namespace xo {
log.retroactively_enable();
log("STUB: check global symtab");
return Binding::null();
return nullptr;
}
void
@ -239,16 +258,6 @@ namespace xo {
this->top_ssm().on_parsed_expression(expr, this);
}
void
ParserStateMachine::on_parsed_expression_with_semicolon(obj<AExpression> expr)
{
scope log(XO_DEBUG(debug_flag_), xtag("expr", expr));
assert(stack_);
this->top_ssm().on_parsed_expression_with_semicolon(expr, this);
}
void
ParserStateMachine::on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk)
@ -257,11 +266,7 @@ namespace xo {
assert(stack_);
this->top_ssm().on_parsed_expression(expr, this);
assert(stack_);
this->top_ssm().on_token(tk, this);
this->top_ssm().on_parsed_expression_with_token(expr, tk, this);
}
void
@ -443,6 +448,39 @@ namespace xo {
this->capture_error(ssm_name, errmsg);
}
void
ParserStateMachine::illegal_parsed_expression_with_token(std::string_view ssm_name,
obj<AExpression> expr,
const Token & tk,
std::string_view expect_str)
{
// TODO:
// - want to write error message using DArena
// - need something like log_streambuf and/or tostr() that's arena-aware
obj<APrintable> expr_pr
= FacetRegistry::instance().variant<APrintable,AExpression>(expr);
assert(expr_pr);
/** TODO
* problem here: we have pretty() support for obj<AExpression>,
* but not "ordinary printing" support. So expression doesn't get printed
**/
auto errmsg_string = tostr("Unexpected expression",
xtag("expr", expr_pr),
xtag("tk", tk),
xtag("expecting", expect_str),
xtag("ssm", ssm_name),
xtag("via", "ParserStateMachine::illegal_parsed_expression"));
assert(expr_alloc_);
auto errmsg = DString::from_view(expr_alloc_,
std::string_view(errmsg_string));
this->capture_error(ssm_name, errmsg);
}
void
ParserStateMachine::error_unbound_variable(std::string_view ssm_name,
std::string_view sym)