roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2: + on_bool_token scaffold in parser etc.

Browse source
This commit is contained in:
Roland Conybeare 2026-01-23 15:25:30 -05:00
commit 132b8d231a
32 changed files with 166 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

9
xo-reader2/idl/SyntaxStateMachine.json5
View file

@ -99,6 +99,15 @@
{type: "ParserStateMachine *", name: "p_psm"},
],
},
{
name: "on_bool_token",
doc: ["update state machine for incoming bool-token @p tk"],
return_type: "void",
args: [
{type: "const Token &", name: "tk"},
{type: "ParserStateMachine *", name: "p_psm"},
],
},
{
name: "on_semicolon_token",
doc: ["update state machine for incoming semicolon-token @p tk"],

6
xo-reader2/include/xo/reader2/DDefineSsm.hpp
View file

@ -153,6 +153,12 @@ namespace xo {
void on_f64_token(const Token & tk,
ParserStateMachine * p_psm);
/** update state for this syntax on incoming bool token @p tk,
* overall parser state in @p p_psm
**/
void on_bool_token(const Token & tk,
ParserStateMachine * p_psm);
/** update state for this syntax on incoming semicolon token @p tk,
* overall parser state in @p p_psm
**/

6
xo-reader2/include/xo/reader2/DExpectExprSsm.hpp
View file

@ -89,6 +89,12 @@ namespace xo {
void on_f64_token(const Token & tk,
ParserStateMachine * p_psm);
/** update state for this syntax on incoming bool token @p tk,
* overall parser state in @p p_psm
**/
void on_bool_token(const Token & tk,
ParserStateMachine * p_psm);
/** update state for this syntax on incoming semicolon token @p tk,
* overall parser state in @p p_psm
**/

6
xo-reader2/include/xo/reader2/DExpectSymbolSsm.hpp
View file

@ -123,6 +123,12 @@ namespace xo {
void on_f64_token(const Token & tk,
ParserStateMachine * p_psm);
/** update state for this syntax on incoming bool token @p tk,
* overall parser state in @p p_psm
**/
void on_bool_token(const Token & tk,
ParserStateMachine * p_psm);
/** update state for this syntax on incoming semicolon token @p tk,
* overall parser state in @p p_psm
**/

6
xo-reader2/include/xo/reader2/DExpectTypeSsm.hpp
View file

@ -83,6 +83,12 @@ namespace xo {
void on_f64_token(const Token & tk,
ParserStateMachine * p_psm);
/** update state for this syntax on incoming bool token @p tk,
* overall parser state in @p p_psm
**/
void on_bool_token(const Token & tk,
ParserStateMachine * p_psm);
/** update state for this syntax on incoming semicolon token @p tk,
* overall parser state in @p p_psm
**/

5
xo-reader2/include/xo/reader2/DExprSeqState.hpp
View file

@ -95,6 +95,11 @@ namespace xo {
**/
void on_f64_token(const Token & tk, ParserStateMachine * p_psm);
/** update state for this syntax on incoming bool token @p tk,
* overall parser state in @p p_psm
**/
void on_bool_token(const Token & tk, ParserStateMachine * p_psm);
/** update state for this syntax on incoming semicolon token @p tk,
* overall parser state in @p p_psm
**/

2
xo-reader2/include/xo/reader2/DProgressSsm.hpp
View file

@ -143,6 +143,8 @@ namespace xo {
ParserStateMachine * p_psm);
void on_f64_token(const Token & tk,
ParserStateMachine * p_psm);
void on_bool_token(const Token & tk,
ParserStateMachine * p_psm);
void on_semicolon_token(const Token & tk,
ParserStateMachine * p_psm);
void on_parsed_symbol(std::string_view sym,

3
xo-reader2/include/xo/reader2/ParserStateMachine.hpp
View file

@ -141,6 +141,9 @@ namespace xo {
/** operate state machine for incoming f64-token @p tk **/
void on_f64_token(const Token & tk);
/** operate state machine for incoming bool-token @p tk **/
void on_bool_token(const Token & tk);
/** operate state machine for incoming semicolon-token @p tk **/
void on_semicolon_token(const Token & tk);

2
xo-reader2/include/xo/reader2/ssm/ASyntaxStateMachine.hpp
View file

@ -69,6 +69,8 @@ public:
virtual void on_singleassign_token(Opaque data, const Token & tk, ParserStateMachine * p_psm) = 0;
/** update state machine for incoming f64-token @p tk **/
virtual void on_f64_token(Opaque data, const Token & tk, ParserStateMachine * p_psm) = 0;
/** update state machine for incoming bool-token @p tk **/
virtual void on_bool_token(Opaque data, const Token & tk, ParserStateMachine * p_psm) = 0;
/** update state machine for incoming semicolon-token @p tk **/
virtual void on_semicolon_token(Opaque data, const Token & tk, ParserStateMachine * p_psm) = 0;
/** update stat machine for incoming parsed symbol @p sym **/

1
xo-reader2/include/xo/reader2/ssm/ISyntaxStateMachine_Any.hpp
View file

@ -66,6 +66,7 @@ namespace scm {
[[noreturn]] void on_colon_token(Opaque, const Token &, ParserStateMachine *) override;
[[noreturn]] void on_singleassign_token(Opaque, const Token &, ParserStateMachine *) override;
[[noreturn]] void on_f64_token(Opaque, const Token &, ParserStateMachine *) override;
[[noreturn]] void on_bool_token(Opaque, const Token &, ParserStateMachine *) override;
[[noreturn]] void on_semicolon_token(Opaque, const Token &, ParserStateMachine *) override;
[[noreturn]] void on_parsed_symbol(Opaque, std::string_view, ParserStateMachine *) override;
[[noreturn]] void on_parsed_typedescr(Opaque, TypeDescr, ParserStateMachine *) override;

2
xo-reader2/include/xo/reader2/ssm/ISyntaxStateMachine_DDefineSsm.hpp
View file

@ -66,6 +66,8 @@ namespace xo {
static void on_singleassign_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming f64-token @p tk **/
static void on_f64_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming bool-token @p tk **/
static void on_bool_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming semicolon-token @p tk **/
static void on_semicolon_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update stat machine for incoming parsed symbol @p sym **/

2
xo-reader2/include/xo/reader2/ssm/ISyntaxStateMachine_DExpectExprSsm.hpp
View file

@ -66,6 +66,8 @@ namespace xo {
static void on_singleassign_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming f64-token @p tk **/
static void on_f64_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming bool-token @p tk **/
static void on_bool_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming semicolon-token @p tk **/
static void on_semicolon_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update stat machine for incoming parsed symbol @p sym **/

2
xo-reader2/include/xo/reader2/ssm/ISyntaxStateMachine_DExpectSymbolSsm.hpp
View file

@ -66,6 +66,8 @@ namespace xo {
static void on_singleassign_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming f64-token @p tk **/
static void on_f64_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming bool-token @p tk **/
static void on_bool_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming semicolon-token @p tk **/
static void on_semicolon_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update stat machine for incoming parsed symbol @p sym **/

2
xo-reader2/include/xo/reader2/ssm/ISyntaxStateMachine_DExpectTypeSsm.hpp
View file

@ -66,6 +66,8 @@ namespace xo {
static void on_singleassign_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming f64-token @p tk **/
static void on_f64_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming bool-token @p tk **/
static void on_bool_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming semicolon-token @p tk **/
static void on_semicolon_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update stat machine for incoming parsed symbol @p sym **/

2
xo-reader2/include/xo/reader2/ssm/ISyntaxStateMachine_DExprSeqState.hpp
View file

@ -66,6 +66,8 @@ namespace xo {
static void on_singleassign_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming f64-token @p tk **/
static void on_f64_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming bool-token @p tk **/
static void on_bool_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming semicolon-token @p tk **/
static void on_semicolon_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm);
/** update stat machine for incoming parsed symbol @p sym **/

2
xo-reader2/include/xo/reader2/ssm/ISyntaxStateMachine_DProgressSsm.hpp
View file

@ -66,6 +66,8 @@ namespace xo {
static void on_singleassign_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming f64-token @p tk **/
static void on_f64_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming bool-token @p tk **/
static void on_bool_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update state machine for incoming semicolon-token @p tk **/
static void on_semicolon_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm);
/** update stat machine for incoming parsed symbol @p sym **/

3
xo-reader2/include/xo/reader2/ssm/ISyntaxStateMachine_Xfer.hpp
View file

@ -70,6 +70,9 @@ namespace scm {
void on_f64_token(Opaque data, const Token & tk, ParserStateMachine * p_psm) override {
return I::on_f64_token(_dcast(data), tk, p_psm);
}
void on_bool_token(Opaque data, const Token & tk, ParserStateMachine * p_psm) override {
return I::on_bool_token(_dcast(data), tk, p_psm);
}
void on_semicolon_token(Opaque data, const Token & tk, ParserStateMachine * p_psm) override {
return I::on_semicolon_token(_dcast(data), tk, p_psm);
}

3
xo-reader2/include/xo/reader2/ssm/RSyntaxStateMachine.hpp
View file

@ -74,6 +74,9 @@ public:
void on_f64_token(const Token & tk, ParserStateMachine * p_psm) {
return O::iface()->on_f64_token(O::data(), tk, p_psm);
}
void on_bool_token(const Token & tk, ParserStateMachine * p_psm) {
return O::iface()->on_bool_token(O::data(), tk, p_psm);
}
void on_semicolon_token(const Token & tk, ParserStateMachine * p_psm) {
return O::iface()->on_semicolon_token(O::data(), tk, p_psm);
}

9
xo-reader2/src/reader2/DDefineSsm.cpp
View file

@ -576,6 +576,15 @@ namespace xo {
this->get_expect_str());
}
void
DDefineSsm::on_bool_token(const Token & tk,
ParserStateMachine * p_psm)
{
p_psm->illegal_input_on_token("DDefineSsm::on_bool_token",
tk,
this->get_expect_str());
}
void
DDefineSsm::on_semicolon_token(const Token & tk,
ParserStateMachine * p_psm)

9
xo-reader2/src/reader2/DExpectExprSsm.cpp
View file

@ -170,6 +170,15 @@ namespace xo {
p_psm);
}
void
DExpectExprSsm::on_bool_token(const Token & tk,
ParserStateMachine * p_psm)
{
p_psm->illegal_input_on_token("DExpectExprSsm::on_bool_token",
tk,
this->get_expect_str());
}
void
DExpectExprSsm::on_semicolon_token(const Token & tk,
ParserStateMachine * p_psm)

9
xo-reader2/src/reader2/DExpectSymbolSsm.cpp
View file

@ -158,6 +158,15 @@ namespace xo {
this->get_expect_str());
}
void
DExpectSymbolSsm::on_bool_token(const Token & tk,
ParserStateMachine * p_psm)
{
p_psm->illegal_input_on_token("DExpectSymbolSsm::on_bool_token",
tk,
this->get_expect_str());
}
void
DExpectSymbolSsm::on_semicolon_token(const Token & tk,
ParserStateMachine * p_psm)

9
xo-reader2/src/reader2/DExpectTypeSsm.cpp
View file

@ -101,6 +101,15 @@ namespace xo {
this->get_expect_str());
}
void
DExpectTypeSsm::on_bool_token(const Token & tk,
ParserStateMachine * p_psm)
{
p_psm->illegal_input_on_token("DExpectTypeSsm::on_bool_token",
tk,
this->get_expect_str());
}
void
DExpectTypeSsm::on_semicolon_token(const Token & tk,
ParserStateMachine * p_psm)

9
xo-reader2/src/reader2/DExprSeqState.cpp
View file

@ -217,6 +217,15 @@ namespace xo {
this->get_expect_str());
}
void
DExprSeqState::on_bool_token(const Token & tk,
ParserStateMachine * p_psm)
{
p_psm->illegal_input_on_token("DExprSeqState::on_bool_token",
tk,
this->get_expect_str());
}
void
DExprSeqState::on_semicolon_token(const Token & tk,
ParserStateMachine * p_psm)

9
xo-reader2/src/reader2/DProgressSsm.cpp
View file

@ -212,6 +212,15 @@ namespace xo {
this->get_expect_str());
}
void
DProgressSsm::on_bool_token(const Token & tk,
ParserStateMachine * p_psm)
{
p_psm->illegal_input_on_token("DProgressSsm::on_bool_token",
tk,
this->get_expect_str());
}
void
DProgressSsm::on_semicolon_token(const Token & tk,
ParserStateMachine * p_psm)

6
xo-reader2/src/reader2/ISyntaxStateMachine_Any.cpp
View file

@ -70,6 +70,12 @@ ISyntaxStateMachine_Any::on_f64_token(Opaque, const Token &, ParserStateMachine
_fatal();
}
auto
ISyntaxStateMachine_Any::on_bool_token(Opaque, const Token &, ParserStateMachine *) -> void
{
_fatal();
}
auto
ISyntaxStateMachine_Any::on_semicolon_token(Opaque, const Token &, ParserStateMachine *) -> void
{

5
xo-reader2/src/reader2/ISyntaxStateMachine_DDefineSsm.cpp
View file

@ -58,6 +58,11 @@ namespace xo {
self.on_f64_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DDefineSsm::on_bool_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_bool_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DDefineSsm::on_semicolon_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_semicolon_token(tk, p_psm);

5
xo-reader2/src/reader2/ISyntaxStateMachine_DExpectExprSsm.cpp
View file

@ -58,6 +58,11 @@ namespace xo {
self.on_f64_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DExpectExprSsm::on_bool_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_bool_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DExpectExprSsm::on_semicolon_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_semicolon_token(tk, p_psm);

5
xo-reader2/src/reader2/ISyntaxStateMachine_DExpectSymbolSsm.cpp
View file

@ -58,6 +58,11 @@ namespace xo {
self.on_f64_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DExpectSymbolSsm::on_bool_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_bool_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DExpectSymbolSsm::on_semicolon_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_semicolon_token(tk, p_psm);

5
xo-reader2/src/reader2/ISyntaxStateMachine_DExpectTypeSsm.cpp
View file

@ -58,6 +58,11 @@ namespace xo {
self.on_f64_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DExpectTypeSsm::on_bool_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_bool_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DExpectTypeSsm::on_semicolon_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_semicolon_token(tk, p_psm);

5
xo-reader2/src/reader2/ISyntaxStateMachine_DExprSeqState.cpp
View file

@ -58,6 +58,11 @@ namespace xo {
self.on_f64_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DExprSeqState::on_bool_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_bool_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DExprSeqState::on_semicolon_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_semicolon_token(tk, p_psm);

5
xo-reader2/src/reader2/ISyntaxStateMachine_DProgressSsm.cpp
View file

@ -58,6 +58,11 @@ namespace xo {
self.on_f64_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DProgressSsm::on_bool_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_bool_token(tk, p_psm);
}
auto
ISyntaxStateMachine_DProgressSsm::on_semicolon_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
{
self.on_semicolon_token(tk, p_psm);

13
xo-reader2/src/reader2/ParserStateMachine.cpp
View file

@ -187,13 +187,16 @@ namespace xo {
this->on_f64_token(tk);
break;
case tokentype::tk_bool:
this->on_bool_token(tk);
break;
case tokentype::tk_semicolon:
this->on_semicolon_token(tk);
break;
// all the not-yet handled cases
case tokentype::tk_invalid:
case tokentype::tk_bool:
case tokentype::tk_i64:
case tokentype::tk_string:
case tokentype::tk_leftparen:
@ -280,6 +283,14 @@ namespace xo {
stack_->top().on_f64_token(tk, this);
}
void
ParserStateMachine::on_bool_token(const Token & tk)
{
scope log(XO_DEBUG(debug_flag_), xtag("tk", tk));
stack_->top().on_bool_token(tk, this);
}
void
ParserStateMachine::on_semicolon_token(const Token & tk)
{