roland/xo-expression2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2: DLambdaSsm work towards producing DLambdaExpr [WIP]

Browse source
This commit is contained in:
Roland Conybeare 2026-01-31 21:33:39 -05:00
commit e0eeeb12c2
9 changed files with 219 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

17
xo-reader2/include/xo/reader2/DProgressSsm.hpp
View file

@ -166,6 +166,8 @@ namespace xo {
ParserStateMachine * p_psm);
void on_semicolon_token(const Token & tk,
ParserStateMachine * p_psm);
void on_rightbrace_token(const Token & tk,
ParserStateMachine * p_psm);
void on_parsed_expression_with_semicolon(obj<AExpression> expr,
ParserStateMachine * p_psm);
@ -183,27 +185,12 @@ namespace xo {
void on_typedescr(TypeDescr td,
parserstatemachine * p_psm) override;
void on_semicolon_token(const token_type & tk,
parserstatemachine * p_psm) override;
void on_assign_token(const token_type & tk,
parserstatemachine * p_psm) final override;
void on_leftparen_token(const token_type & tk,
parserstatemachine * p_psm) override;
void on_rightparen_token(const token_type & tk,
parserstatemachine * p_psm) override;
void on_rightbrace_token(const token_type & tk,
parserstatemachine * p_psm) override;
/* entry point for an infix operator token */
void on_operator_token(const token_type & tk,
parserstatemachine * p_psm) final override;
void on_bool_token(const token_type & tk,
parserstatemachine * p_psm) override;
void on_i64_token(const token_type & tk,
parserstatemachine * p_psm) override;
void print(std::ostream & os) const override;
#endif

32
xo-reader2/include/xo/reader2/DSequenceSsm.hpp
View file

@ -7,12 +7,25 @@
#include "DSyntaxStateMachine.hpp"
#include <xo/expression2/DSequenceExpr.hpp>
#include "syntaxstatetype.hpp"
#include <xo/expression2/detail/IExpression_DSequenceExpr.hpp>
#include <xo/expression2/DSequenceExpr.hpp>
#include <xo/facet/obj.hpp>
namespace xo {
namespace scm { class Sequence; }
namespace scm { class Lambda; }
namespace scm {
// TODO: need switching between 1a,1b states.
// Allow
// { }
// { 1 }
// { 1; }
// Reject
// { 1 2 }
//
class DSequenceSsm : public DSyntaxStateMachine<DSequenceSsm> {
public:
//using Sequence = xo::scm::Sequence;
@ -38,15 +51,14 @@ namespace xo {
obj<AAllocator> expr_mm);
#ifdef NOT_YET
virtual void on_expr(bp<Expression> expr,
parserstatemachine * p_psm) override;
virtual void on_expr_with_semicolon(bp<Expression> expr,
parserstatemachine * p_psm) override;
virtual void on_rightbrace_token(const token_type & tk,
parserstatemachine * p_psm) override;
#endif
/** update ssm for incoming rightbrace token '}' **/
void on_rightbrace_token(const Token & tk,
ParserStateMachine * p_psm);
/** @defgroup scm-sequencessm-syntaxstatemachine-facet ssm facet **/
///@{
@ -56,6 +68,16 @@ namespace xo {
/** mnemonic for syntax sequence ssm expects given current state **/
std::string_view get_expect_str() const noexcept;
/** operate state machine for this syntax on incoming token @p tk
* with overall parser state in @p p_psm
**/
void on_token(const Token & tk,
ParserStateMachine * p_psm);
/** consume expression @p expr produced by nested ssm; overall parser state in @p p_psm **/
void on_parsed_expression(obj<AExpression> expr,
ParserStateMachine * p_psm);
///@}
/** @defgroup scm-sequencessm-printable-facet printable facet **/
///@{

12
xo-reader2/include/xo/reader2/ParserStateMachine.hpp
View file

@ -142,6 +142,18 @@ namespace xo {
**/
void on_parsed_expression_with_semicolon(obj<AExpression> expr);
/** update state to respond to parsed expression @p expr
* (from nested parsing state), with trailing token @p tk.
*
* Need to distinguish cases like:
* 6 // ) ? ; allowed } ?
* f(6 // ) allowed ; forbidden } forbidden
* 6 + // ) forbidden ; forbidden } forbidden
*
**/
void on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk);
/** update state to respond to input token @p tk.
* record output (if any) in @ref result_
**/

17
xo-reader2/src/reader2/DLambdaSsm.cpp
View file

@ -336,6 +336,23 @@ namespace xo {
DLambdaSsm::on_parsed_expression(obj<AExpression> expr,
ParserStateMachine * p_psm)
{
if (lm_state_ == lambdastatetype::lm_4) {
this->lmstate_ = lambdastatetype::lm_5;
this->body_ = expr;
// assemble lambda
obj<AExpression,DLambda> lm_expr = DLambda::make(p_psm->expr_alloc(),
xxx typeref,
xxx name,
local_symtab_,
body_);
p_psm->pop_ssm(); // this lambda
p_psm->on_parsed_expression(lm_expr);
return;
}
Super::on_parsed_expression(expr, p_psm);
}

35
xo-reader2/src/reader2/DProgressSsm.cpp
View file

@ -205,7 +205,7 @@ namespace xo {
std::string_view
DProgressSsm::get_expect_str() const noexcept {
if (op_type_ == optype::invalid) {
return "oper|semicolon|rightparen";
return "oper|semicolon|rightparen|righbrace";
} else {
return "expr|leftparen";
}
@ -255,6 +255,10 @@ namespace xo {
this->on_semicolon_token(tk, p_psm);
return;
case tokentype::tk_rightbrace:
this->on_rightbrace_token(tk, p_psm);
return;
// all the not-yet handled cases
case tokentype::tk_invalid:
case tokentype::tk_def:
@ -264,7 +268,6 @@ namespace xo {
case tokentype::tk_leftbracket:
case tokentype::tk_rightbracket:
case tokentype::tk_leftbrace:
case tokentype::tk_rightbrace:
case tokentype::tk_leftangle:
case tokentype::tk_rightangle:
case tokentype::tk_lessequal:
@ -448,7 +451,8 @@ namespace xo {
obj<AExpression> expr = this->assemble_expr(p_psm);
{
obj<APrintable> expr_pr = FacetRegistry::instance().variant<APrintable,AExpression>(expr);
obj<APrintable> expr_pr
= FacetRegistry::instance().variant<APrintable,AExpression>(expr);
assert(expr_pr);
log && log(xtag("expr", expr_pr));
}
@ -475,6 +479,27 @@ namespace xo {
#endif
}
void
DProgressSsm::on_rightbrace_token(const Token & tk,
ParserStateMachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
(void)tk;
obj<AExpression> expr = this->assemble_expr(p_psm);
{
obj<APrintable> expr_pr
= FacetRegistry::instance().variant<APrintable,AExpression>(expr);
assert(expr_pr);
log && log(xtag("expr", expr_pr));
}
p_psm->pop_ssm();
p_psm->on_parsed_expression_with_token(expr, tk);
}
void
DProgressSsm::on_parsed_expression_with_semicolon(obj<AExpression> expr,
ParserStateMachine * p_psm)
@ -1024,7 +1049,8 @@ namespace xo {
"DProgressSsm",
refrtag("lhs", lhs),
refrtag("op", op_type_),
cond(rhs, refrtag("rhs", rhs), "nullptr")
cond(rhs, refrtag("rhs", rhs), "nullptr"),
refrtag("expect", this->get_expect_str())
);
}
@ -1048,6 +1074,7 @@ namespace xo {
std::string_view(errmsg_string));
p_psm->capture_error(c_self_name, errmsg);
return obj<AExpression>();
}
/* consecutive expressions not legal, e.g:

122
xo-reader2/src/reader2/DSequenceSsm.cpp
View file

@ -55,12 +55,6 @@ namespace xo {
DSequenceSsm::DSequenceSsm(DSequenceExpr * seq_expr) : seq_expr_{seq_expr}
{}
#ifdef NOT_YET
sequence_xs::sequence_xs()
: exprstate(exprstatetype::sequenceexpr)
{}
#endif
syntaxstatetype
DSequenceSsm::ssm_type() const noexcept
{
@ -73,15 +67,93 @@ namespace xo {
return "expr|semicolon|rightbrace";
}
#ifdef NOT_YET
void
sequence_xs::on_expr(bp<Expression> expr,
parserstatemachine * p_psm)
DSequenceSsm::on_token(const Token & tk,
ParserStateMachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()), xtag("tk", tk));
switch (tk.tk_type()) {
case tokentype::tk_rightbrace:
this->on_rightbrace_token(tk, p_psm);
return;
case tokentype::tk_symbol:
case tokentype::tk_def:
case tokentype::tk_if:
case tokentype::tk_then:
case tokentype::tk_else:
case tokentype::tk_colon:
case tokentype::tk_singleassign:
case tokentype::tk_string:
case tokentype::tk_f64:
case tokentype::tk_i64:
case tokentype::tk_bool:
case tokentype::tk_semicolon:
case tokentype::tk_invalid:
case tokentype::tk_leftparen:
case tokentype::tk_rightparen:
case tokentype::tk_leftbracket:
case tokentype::tk_rightbracket:
case tokentype::tk_leftbrace:
case tokentype::tk_leftangle:
case tokentype::tk_rightangle:
case tokentype::tk_lessequal:
case tokentype::tk_greatequal:
case tokentype::tk_dot:
case tokentype::tk_comma:
case tokentype::tk_doublecolon:
case tokentype::tk_assign:
case tokentype::tk_yields:
case tokentype::tk_plus:
case tokentype::tk_minus:
case tokentype::tk_star:
case tokentype::tk_slash:
case tokentype::tk_cmpeq:
case tokentype::tk_cmpne:
case tokentype::tk_type:
case tokentype::tk_lambda:
case tokentype::tk_let:
case tokentype::tk_in:
case tokentype::tk_end:
case tokentype::N:
break;
}
// default = illegal token error
DSyntaxStateMachine<DSequenceSsm>::on_token(tk, p_psm);
}
void
DSequenceSsm::on_rightbrace_token(const Token & tk,
ParserStateMachine * p_psm)
{
(void)tk;
/** rightbrace ends DSequenceSsm **/
obj<AExpression,DSequenceExpr> expr(seq_expr_);
p_psm->pop_ssm();
/* make sequence from expressions seen at this level,
* and report it to parent
*/
p_psm->top_ssm().on_parsed_expression(expr, p_psm);
}
void
DSequenceSsm::on_parsed_expression(obj<AExpression> expr,
ParserStateMachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
log && log(xtag("expr", expr.promote()));
// TODO: stream inserter that sets up pretty-printing.
// Or integrate with indentlog.
// Maybe trouble is that indentlog doesn't #include Printable ?
//
log && log(xtag("expr", expr));
#ifdef NOT_YET
/* TODO: if expr is a DefineExpr,
* then need to rewrite...
*
@ -132,8 +204,13 @@ namespace xo {
true /*cxl_on_rightbrace*/,
p_psm);
}
#endif
this->seq_expr_->push_back(p_psm->expr_alloc(),
expr);
}
#ifdef NOT_YET
void
sequence_xs::on_expr_with_semicolon(bp<Expression> expr,
parserstatemachine * p_psm)
@ -141,25 +218,6 @@ namespace xo {
/* sequence continues until right brace */
this->on_expr(expr, p_psm);
}
void
sequence_xs::on_rightbrace_token(const token_type & /*tk*/,
parserstatemachine * p_psm)
{
auto self = p_psm->pop_exprstate();
/* make sequence from expressions seen at this level,
* and report it to parent
*/
auto expr = Sequence::make(this->expr_v_);
p_psm->top_exprstate().on_expr(expr, p_psm);
}
void
sequence_xs::print(std::ostream & os) const {
os << "<sequence_xs" << xtag("expr_v.size", expr_v_.size()) << ">";
}
#endif
bool
@ -167,9 +225,9 @@ namespace xo {
{
return ppii.pps()->pretty_struct
(ppii,
"SequenceSsm",
xrefrtag("seq_expr.size", seq_expr_->size()),
xrefrtag("expect", this->get_expect_str()));
"DSequenceSsm",
refrtag("seq_expr.size", seq_expr_->size()),
refrtag("expect", this->get_expect_str()));
}
} /*namespace scm*/

35
xo-reader2/src/reader2/ParserStateMachine.cpp
View file

@ -9,6 +9,7 @@
#include <xo/object2/array/IPrintable_DArray.hpp>
#include <xo/printable2/Printable.hpp>
#include <xo/alloc2/arena/IAllocator_DArena.hpp>
#include <xo/facet/FacetRegistry.hpp>
#include <xo/indentlog/scope.hpp>
#include <xo/indentlog/print/tostr.hpp>
#include <xo/indentlog/print/tag.hpp>
@ -16,6 +17,7 @@
namespace xo {
using xo::print::APrintable;
using xo::facet::FacetRegistry;
using xo::facet::with_facet;
namespace scm {
@ -188,6 +190,21 @@ namespace xo {
this->top_ssm().on_parsed_expression_with_semicolon(expr, this);
}
void
ParserStateMachine::on_parsed_expression_with_token(obj<AExpression> expr,
const Token & tk)
{
scope log(XO_DEBUG(debug_flag_), xtag("expr", expr), xtag("tk", tk));
assert(stack_);
this->top_ssm().on_parsed_expression(expr, this);
assert(stack_);
this->top_ssm().on_token(tk, this);
}
void
ParserStateMachine::on_token(const Token & tk)
{
@ -215,7 +232,11 @@ namespace xo {
ParserStateMachine::capture_error(std::string_view ssm_name,
const DString * errmsg)
{
this->result_ = ParserResult::error(ssm_name, errmsg);
if (result_.is_error()) {
/* in case one error triggers another, remmber just the first one */
} else {
this->result_ = ParserResult::error(ssm_name, errmsg);
}
}
void
@ -302,7 +323,7 @@ namespace xo {
xtag("param_type", param_type),
xtag("expecting", expect_str),
xtag("ssm", ssm_name),
xtag("via", "ParserStateMachine::illegal_parsed_expression"));
xtag("via", "ParserStateMachine::illegal_parsed_formal"));
assert(expr_alloc_);
@ -341,8 +362,16 @@ namespace xo {
// - want to write error message using DArena
// - need something like log_streambuf and/or tostr() that's arena-aware
obj<APrintable> expr_pr
= FacetRegistry::instance().variant<APrintable,AExpression>(expr);
assert(expr_pr);
/** TODO
* problem here: we have pretty() support for obj<AExpression>,
* but not "ordinary printing" support. So expression doesn't get printed
**/
auto errmsg_string = tostr("Unexpected expression",
xtag("expr", expr),
xtag("expr", expr_pr),
xtag("expecting", expect_str),
xtag("ssm", ssm_name),
xtag("via", "ParserStateMachine::illegal_parsed_expression"));

16
xo-reader2/utest/SchematikaParser.test.cpp
View file

@ -215,7 +215,7 @@ namespace xo {
/** Walkthrough parsing input equivalent to:
*
* lambda (n : i64,
* lambda (n : i64, r : i64) -> i64 { 123 }
*
**/
@ -375,11 +375,10 @@ namespace xo {
REQUIRE(result.is_incomplete());
}
#ifdef NOT_YET
{
auto & result = parser.on_token(Token::string_token("fooey"));
auto & result = parser.on_token(Token::i64_token("123"));
log && log("after string token:");
log && log("after f64(123) token:");
log && log(xtag("parser", &parser));
log && log(xtag("result", result));
@ -389,17 +388,16 @@ namespace xo {
}
{
auto & result = parser.on_token(Token::semicolon_token());
auto & result = parser.on_token(Token::rightbrace_token());
log && log("after semicolon token:");
log && log("after rightbrace token:");
log && log(xtag("parser", &parser));
log && log(xtag("result", result));
REQUIRE(parser.has_incomplete_expr() == false);
REQUIRE(parser.has_incomplete_expr() == true);
REQUIRE(!result.is_error());
REQUIRE(!result.is_incomplete());
REQUIRE(result.is_incomplete());
}
#endif
//REQUIRE(result.is_error());
//// illegal input on token

2
xo-tokenizer2/include/xo/tokenizer2/Token.hpp
View file

@ -102,7 +102,7 @@ namespace xo {
/** token representing left brace @c "{" **/
static Token leftbrace_token() { return Token(tokentype::tk_leftbrace); }
/** token representing right brace @c "}" **/
static Token rightbrace() { return Token(tokentype::tk_rightbrace); }
static Token rightbrace_token() { return Token(tokentype::tk_rightbrace); }
/** token representing period @c "." **/
static Token dot() { return Token(tokentype::tk_dot); }
/** token representing comma @c "," **/