From ce760bd5cf75e829d71003d091af0d6bdc25868c Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 22 Jul 2025 07:20:56 -0500 Subject: [PATCH] xo-tokenizer xo-reader: + bool literals + if-expr parsing --- .../xo/expression/GeneralizedExpression.hpp | 1 - .../include/xo/expression/IfExpr.hpp | 32 ++- xo-expression/src/expression/IfExpr.cpp | 53 +++- xo-reader/examples/exprrepl/exprrepl.cpp | 4 +- xo-reader/examples/exprreplxx/exprreplxx.cpp | 4 +- .../include/xo/reader/expect_expr_xs.hpp | 17 +- xo-reader/include/xo/reader/exprseq_xs.hpp | 20 +- xo-reader/include/xo/reader/exprstate.hpp | 21 ++ xo-reader/include/xo/reader/if_else_xs.hpp | 97 +++++++ xo-reader/include/xo/reader/parser.hpp | 10 +- .../include/xo/reader/parserstatemachine.hpp | 16 +- xo-reader/include/xo/reader/progress_xs.hpp | 6 +- xo-reader/include/xo/reader/reader.hpp | 2 +- xo-reader/src/reader/CMakeLists.txt | 1 + xo-reader/src/reader/define_xs.cpp | 4 +- xo-reader/src/reader/expect_expr_xs.cpp | 22 +- xo-reader/src/reader/exprseq_xs.cpp | 80 +++++- xo-reader/src/reader/exprstate.cpp | 69 +++++ xo-reader/src/reader/exprstatestack.cpp | 4 +- xo-reader/src/reader/if_else_xs.cpp | 269 ++++++++++++++++++ xo-reader/src/reader/parser.cpp | 12 +- xo-reader/src/reader/parserstatemachine.cpp | 27 +- xo-reader/src/reader/progress_xs.cpp | 69 ++++- xo-reader/src/reader/reader.cpp | 4 + xo-reader/utest/parser.test.cpp | 5 +- xo-reader/utest/reader.test.cpp | 2 +- xo-tokenizer/include/xo/tokenizer/token.hpp | 38 ++- .../include/xo/tokenizer/tokenizer.hpp | 21 +- .../include/xo/tokenizer/tokentype.hpp | 9 + xo-tokenizer/src/tokenizer/tokentype.cpp | 3 + 30 files changed, 848 insertions(+), 74 deletions(-) create mode 100644 xo-reader/include/xo/reader/if_else_xs.hpp create mode 100644 xo-reader/src/reader/if_else_xs.cpp diff --git a/xo-expression/include/xo/expression/GeneralizedExpression.hpp b/xo-expression/include/xo/expression/GeneralizedExpression.hpp index 585b22fb..1d513a2d 100644 --- a/xo-expression/include/xo/expression/GeneralizedExpression.hpp +++ b/xo-expression/include/xo/expression/GeneralizedExpression.hpp @@ -38,7 +38,6 @@ namespace xo { /** pretty printing support. See [xo-indentlog/xo/indentlog/pretty.hpp] **/ virtual std::uint32_t pretty_print(const ppindentinfo & ppii) const = 0; - protected: /** useful when scaffolding expressions in a parser **/ void assign_valuetype(TypeDescr x) { valuetype_ = x; } diff --git a/xo-expression/include/xo/expression/IfExpr.hpp b/xo-expression/include/xo/expression/IfExpr.hpp index 946fda9d..462a049f 100644 --- a/xo-expression/include/xo/expression/IfExpr.hpp +++ b/xo-expression/include/xo/expression/IfExpr.hpp @@ -104,7 +104,7 @@ namespace xo { virtual void display(std::ostream & os) const override; virtual std::uint32_t pretty_print(const ppindentinfo & ppi) const override; - private: + protected: /** * @p ifexpr_type type for value produced by if-expression. * same as both when_true->valuetype() and @@ -122,7 +122,13 @@ namespace xo { when_true_{std::move(when_true)}, when_false_{std::move(when_false)} {} - private: + static TypeDescr check_consistent_valuetype(const rp & when_true, + const rp & when_false); + + /** determine if-expr valuetype **/ + void establish_valuetype(); + + protected: /** if: * (if x y z) * @@ -140,6 +146,28 @@ namespace xo { { return IfExpr::make(test, when_true, when_false); } + + class IfExprAccess : public IfExpr { + public: + static rp make(rp test, + rp when_true, + rp when_false); + static rp make_empty(); + + void assign_test(rp x) { test_ = std::move(x); } + void assign_when_true(rp x); + void assign_when_false(rp x); + + private: + IfExprAccess(TypeDescr ifexpr_type, + rp test, + rp when_true, + rp when_false) + : IfExpr(ifexpr_type, + std::move(test), + std::move(when_true), + std::move(when_false)) {} + }; } /*namespace ast*/ } /*namespace xo*/ diff --git a/xo-expression/src/expression/IfExpr.cpp b/xo-expression/src/expression/IfExpr.cpp index f6377ce3..7027f433 100644 --- a/xo-expression/src/expression/IfExpr.cpp +++ b/xo-expression/src/expression/IfExpr.cpp @@ -7,6 +7,21 @@ namespace xo { namespace ast { + auto IfExpr::check_consistent_valuetype(const rp & when_true, + const rp & when_false) -> TypeDescr + { + if (when_true->valuetype() != when_false->valuetype()) + return nullptr; + + return when_true->valuetype(); + } + + void IfExpr::establish_valuetype() + { + if (this->when_true_.get() && this->when_false_.get()) + this->assign_valuetype(check_consistent_valuetype(this->when_true_, this->when_false_)); + } + rp IfExpr::make(const rp & test, const rp & when_true, @@ -36,9 +51,10 @@ namespace xo { IfExpr::display(std::ostream & os) const { os << ""; + << xtag("when_true", when_true_); + if (when_false_) + os << xtag("when_false", when_false_); + os << ">"; } /*display*/ std::uint32_t @@ -48,6 +64,37 @@ namespace xo { refrtag("when_true", when_true_), refrtag("when_false", when_false_)); } + + rp + IfExprAccess::make(rp test, + rp when_true, + rp when_false) + { + auto ifexpr_type = check_consistent_valuetype(when_true, when_false); + + return new IfExprAccess(ifexpr_type, std::move(test), std::move(when_true), std::move(when_false)); + } + + rp + IfExprAccess::make_empty() + { + return new IfExprAccess(nullptr /*ifexpr_valuetype*/, + nullptr /*test*/, + nullptr /*when_true*/, + nullptr /*when_false*/); + } + + void + IfExprAccess::assign_when_true(rp x) + { + this->when_true_ = std::move(x); + } + + void + IfExprAccess::assign_when_false(rp x) + { + this->when_false_ = std::move(x); + } } /*namespace ast*/ } /*namespace xo*/ diff --git a/xo-reader/examples/exprrepl/exprrepl.cpp b/xo-reader/examples/exprrepl/exprrepl.cpp index bf771d02..6555e847 100644 --- a/xo-reader/examples/exprrepl/exprrepl.cpp +++ b/xo-reader/examples/exprrepl/exprrepl.cpp @@ -47,7 +47,9 @@ main() { bool interactive = isatty(STDIN_FILENO); - reader rdr; + bool c_debug_flag = false; + + reader rdr(c_debug_flag); rdr.begin_interactive_session(); string input_str; diff --git a/xo-reader/examples/exprreplxx/exprreplxx.cpp b/xo-reader/examples/exprreplxx/exprreplxx.cpp index 17abcbe6..03410625 100644 --- a/xo-reader/examples/exprreplxx/exprreplxx.cpp +++ b/xo-reader/examples/exprreplxx/exprreplxx.cpp @@ -77,7 +77,9 @@ main() { // rx.bind_key_internal(Replxx::KEY::control('p'), "history_previous"); // rx.bind_key_internal(Replxx::KEY::control('n'), "history_next"); - reader rdr; + constexpr bool c_debug_flag = true; + + reader rdr(c_debug_flag); rdr.begin_interactive_session(); string input_str; diff --git a/xo-reader/include/xo/reader/expect_expr_xs.hpp b/xo-reader/include/xo/reader/expect_expr_xs.hpp index b416c7eb..148d728f 100644 --- a/xo-reader/include/xo/reader/expect_expr_xs.hpp +++ b/xo-reader/include/xo/reader/expect_expr_xs.hpp @@ -12,6 +12,18 @@ namespace xo { /** @class expect_expr_xs * @brief state machine to expect + capture an expression + * + * Examples: + * @text + * def x : i64 = 5 ; // with allow_defs + * lambda (f : x64) { ... } ; + * if (prime(x)) { ... } ; + * 5 + x ; + * @endtext + * + * top exprstate when expect_expr_xs::start() invoked + * will receive parsed expression via + * exprstate::on_expr() or exprstate::on_expr_with_semicolon(). **/ class expect_expr_xs : public exprstate { public: @@ -41,6 +53,9 @@ namespace xo { virtual void on_symbol_token(const token_type & tk, parserstatemachine * p_psm) override; + virtual void on_bool_token(const token_type & tk, + parserstatemachine * p_psm) override; + virtual void on_i64_token(const token_type & tk, parserstatemachine * p_psm) override; @@ -50,7 +65,7 @@ namespace xo { /** update exprstate in response to a successfully-parsed subexpression **/ virtual void on_expr(bp expr, parserstatemachine * p_psm) override; - /** update exprstate in response to a successfully-parsed subexpression, + /** update exprstate in response to a successfully-parsed subexpression * that's terminated by semicolon ';' **/ virtual void on_expr_with_semicolon(bp expr, diff --git a/xo-reader/include/xo/reader/exprseq_xs.hpp b/xo-reader/include/xo/reader/exprseq_xs.hpp index bfd82fe2..2dc58e13 100644 --- a/xo-reader/include/xo/reader/exprseq_xs.hpp +++ b/xo-reader/include/xo/reader/exprseq_xs.hpp @@ -17,8 +17,6 @@ namespace xo { * This used for top-level expressions in a translation unit. **/ toplevel_batch, - /** nested sequence, for example in function body **/ - nested, }; /** @class exprseq_xs @@ -27,14 +25,14 @@ namespace xo { * expression sequences come in several types: * 1. top-level interactive * 2. top-level batch - * 3. nested * * @text - * 1 2 3 - * +-------- - * def | y y y - * symbol | y n n 1: evaluate as variable - * i64 | y n n 1: evaluate as constant + * 1 2 + * +----- + * def | y y + * if | y n 1: eval as expression + * symbol | y n 1: evaluate as variable + * i64 | y n 1: evaluate as constant * * @endtext **/ @@ -45,12 +43,18 @@ namespace xo { static void start(exprseqtype seqtype, parserstatemachine * p_psm); public: + virtual const char * get_expect_str() const override; + // ----- token input methods ----- virtual void on_def_token(const token_type & tk, parserstatemachine * p_psm) override; + virtual void on_if_token(const token_type & tk, + parserstatemachine * p_psm) override; virtual void on_symbol_token(const token_type & tk, parserstatemachine * p_psm) override; + virtual void on_bool_token(const token_type & tk, + parserstatemachine * p_psm) override; virtual void on_i64_token(const token_type & tk, parserstatemachine * p_psm) override; diff --git a/xo-reader/include/xo/reader/exprstate.hpp b/xo-reader/include/xo/reader/exprstate.hpp index 529b6a06..781026a1 100644 --- a/xo-reader/include/xo/reader/exprstate.hpp +++ b/xo-reader/include/xo/reader/exprstate.hpp @@ -45,6 +45,11 @@ namespace xo { **/ let1expr, + /** handle if-else expression + * see @ref if_else_xs + **/ + ifexpr, + expect_rhs_expression, expect_symbol, expect_type, @@ -177,6 +182,22 @@ namespace xo { virtual void on_operator_token(const token_type & tk, parserstatemachine * p_psm); + /** handle incoming if-keyword token **/ + virtual void on_if_token(const token_type & tk, + parserstatemachine * p_psm); + + /** handle incoming then-keyword token **/ + virtual void on_then_token(const token_type & tk, + parserstatemachine * p_psm); + + /** handle incoming then-keyword token **/ + virtual void on_else_token(const token_type & tk, + parserstatemachine * p_psm); + + /** handle incoming bool-literal token **/ + virtual void on_bool_token(const token_type & tk, + parserstatemachine * p_psm); + /** handle incoming integer-literal token **/ virtual void on_i64_token(const token_type & tk, parserstatemachine * p_psm); diff --git a/xo-reader/include/xo/reader/if_else_xs.hpp b/xo-reader/include/xo/reader/if_else_xs.hpp new file mode 100644 index 00000000..ce7947d9 --- /dev/null +++ b/xo-reader/include/xo/reader/if_else_xs.hpp @@ -0,0 +1,97 @@ +/** @file if_else_xs.hpp + * + * author: Roland Conybeare, Jul 2025 + **/ + +#pragma once + +#include "xo/expression/IfExpr.hpp" +#include "exprstate.hpp" +#include "xo/indentlog/print/ppdetail_atomic.hpp" + +namespace xo { + namespace scm { + /** + * if test-expr then then-expr else else-expr ; + * ^ ^ ^ ^ ^ ^ ^ + * | | | | | | | + * | if_1 if_2 if_3 if_4 if_5 if_6 + * if_0 + * + * if_0 --on_if_token()--> if_1 + * if_1 --on_expr()--> if_2 + * if_2 --on_then_token()--> if_3 + * if_3 --on_expr()--> if_4 + * if_4 --on_else_token()--> if_5 + * --on_semicolon_token()--> (done) + * if_5 --on_expr()-->if_6 + * if_6 --on_semicolon_token()--> (done) + **/ + enum class ifexprstatetype { + invalid = -1, + + if_0, + if_1, + if_2, + if_3, + if_4, + if_5, + if_6, + + n_ifexprstatetype, + }; + + extern const char * ifexprstatetype_descr(ifexprstatetype x); + + std::ostream & + operator<<(std::ostream & os, ifexprstatetype x); + + /** @class if_else_xs + * @brief state to provide parsing of a conditional expression + **/ + class if_else_xs : public exprstate { + public: + using IfExprAccess = xo::ast::IfExprAccess; + + public: + if_else_xs(rp if_expr); + virtual ~if_else_xs() = default; + + static const if_else_xs * from(const exprstate * x) { return dynamic_cast(x); } + + static void start(parserstatemachine * p_psm); + + // ----- inherited from exprstate ----- + + virtual const char * get_expect_str() const override; + + virtual void on_if_token(const token_type & tk, + parserstatemachine * p_psm) override; + virtual void on_then_token(const token_type & tk, + parserstatemachine * p_psm) override; + virtual void on_else_token(const token_type & tk, + parserstatemachine * p_psm) override; + virtual void on_semicolon_token(const token_type & tk, + parserstatemachine * p_psm) override; + + virtual void on_expr(bp expr, + parserstatemachine * p_psm) override; + virtual void on_expr_with_semicolon(bp expr, + parserstatemachine * p_psm) override; + + virtual void print(std::ostream & os) const override; + + private: + static std::unique_ptr make(); + + private: + ifexprstatetype ifxs_type_; + /** scaffold output expression here **/ + rp if_expr_; + + }; + } /*namespace scm*/ +} /*namespace xo*/ + + +/** end if_else_xs.hpp **/ diff --git a/xo-reader/include/xo/reader/parser.hpp b/xo-reader/include/xo/reader/parser.hpp index 323e9ff6..5ac3cec9 100644 --- a/xo-reader/include/xo/reader/parser.hpp +++ b/xo-reader/include/xo/reader/parser.hpp @@ -112,6 +112,9 @@ namespace xo { * | symbol-literal * | struct-literal * + * + * boolean-literal = true | false + * * variable-expr = $varname * apply-expr = fn-expr(arg-expr(1), .., arg-expr(n)) * fn-expr = expression @@ -155,8 +158,10 @@ namespace xo { public: /** create parser in initial state; * parser is ready to receive tokens via @ref include_token + * + * @p debug_flag true to enable debug logging **/ - parser(); + explicit parser(bool debug_flag); /** true if parser is at top-level, i.e. ready for next top-level expression **/ bool is_at_toplevel() const { return stack_size() == 0; } @@ -243,6 +248,9 @@ namespace xo { /** parser result state **/ parser_result result_; + + /** enable/disable debug logging **/ + bool debug_flag_; }; /*parser*/ inline std::ostream & diff --git a/xo-reader/include/xo/reader/parserstatemachine.hpp b/xo-reader/include/xo/reader/parserstatemachine.hpp index 49ec9cef..1bf0905c 100644 --- a/xo-reader/include/xo/reader/parserstatemachine.hpp +++ b/xo-reader/include/xo/reader/parserstatemachine.hpp @@ -28,10 +28,12 @@ namespace xo { public: parserstatemachine(exprstatestack * p_stack, envframestack * p_env_stack, - parser_result * p_result) + parser_result * p_result, + bool debug_flag) : p_stack_{p_stack}, p_env_stack_{p_env_stack}, - p_result_{p_result} + p_result_{p_result}, + debug_flag_{debug_flag} {} //const parser_result & result() const { return result_; } @@ -44,6 +46,8 @@ namespace xo { exprstate & top_exprstate(); void push_exprstate(std::unique_ptr x); + bool debug_flag() const { return debug_flag_; } + /** lookup variable name in lexical context represented by * this psm. nullptr if not found **/ @@ -54,9 +58,13 @@ namespace xo { **/ void upsert_var(bp x); + /** @return available variable bindings in current parsing state **/ bp top_envframe() const; + /** push frame @p x (with new variable bindings) onto environment stack **/ void push_envframe(const rp & x); + /** @return pop innermost environment frame and return it **/ rp pop_envframe(); + /** @return number of stacked environment frames **/ size_t env_stack_size() const { return p_env_stack_->size(); } // ----- parsing outputs ----- @@ -71,6 +79,8 @@ namespace xo { void on_operator_token(const token_type & tk); void on_leftbrace_token(const token_type & tk); void on_rightbrace_token(const token_type & tk); + void on_then_token(const token_type & tk); + void on_else_token(const token_type & tk); // ----- parsing error ----- @@ -91,6 +101,8 @@ namespace xo { envframestack * p_env_stack_ = nullptr; /** parser result object **/ parser_result * p_result_ = nullptr; + /** enable debug logging **/ + bool debug_flag_ = false; }; inline std::ostream & diff --git a/xo-reader/include/xo/reader/progress_xs.hpp b/xo-reader/include/xo/reader/progress_xs.hpp index 9f304d98..808c964b 100644 --- a/xo-reader/include/xo/reader/progress_xs.hpp +++ b/xo-reader/include/xo/reader/progress_xs.hpp @@ -95,6 +95,10 @@ namespace xo { parserstatemachine * p_psm) override; virtual void on_rightparen_token(const token_type & tk, parserstatemachine * p_psm) override; + virtual void on_then_token(const token_type & tk, + parserstatemachine * p_psm) override; + virtual void on_else_token(const token_type & tk, + parserstatemachine * p_psm) override; /* entry point for an infix operator token */ virtual void on_operator_token(const token_type & tk, @@ -121,7 +125,7 @@ namespace xo { * * where f determined by @ref op_type_ **/ - rp assemble_expr(); + rp assemble_expr(parserstatemachine * p_psm); private: /** populate an expression here, may be followed by an operator **/ diff --git a/xo-reader/include/xo/reader/reader.hpp b/xo-reader/include/xo/reader/reader.hpp index ffbdee56..3283065f 100644 --- a/xo-reader/include/xo/reader/reader.hpp +++ b/xo-reader/include/xo/reader/reader.hpp @@ -78,7 +78,7 @@ namespace xo { using span_type = tokenizer_type::span_type; public: - reader() = default; + explicit reader(bool debug_flag); /** call once before calling .read_expr() * for a new interactive session diff --git a/xo-reader/src/reader/CMakeLists.txt b/xo-reader/src/reader/CMakeLists.txt index 7314e974..89e2cf98 100644 --- a/xo-reader/src/reader/CMakeLists.txt +++ b/xo-reader/src/reader/CMakeLists.txt @@ -9,6 +9,7 @@ set(SELF_SRCS exprstate.cpp exprstatestack.cpp define_xs.cpp + if_else_xs.cpp progress_xs.cpp paren_xs.cpp sequence_xs.cpp diff --git a/xo-reader/src/reader/define_xs.cpp b/xo-reader/src/reader/define_xs.cpp index 30dcffa2..8d91ea9d 100644 --- a/xo-reader/src/reader/define_xs.cpp +++ b/xo-reader/src/reader/define_xs.cpp @@ -14,7 +14,7 @@ namespace xo { const char * defexprstatetype_descr(defexprstatetype x) { - switch(x) { + switch (x) { case defexprstatetype::invalid: return "invalid"; case defexprstatetype::def_0: return "def_0"; case defexprstatetype::def_1: return "def_1"; @@ -208,7 +208,7 @@ namespace xo { return; } - constexpr const char * c_self_name = "define_xs::on_symbol"; + constexpr const char * c_self_name = "define_xs::on_def_token"; const char * exp = this->get_expect_str(); this->illegal_input_on_token(c_self_name, tk, exp, p_psm); diff --git a/xo-reader/src/reader/expect_expr_xs.cpp b/xo-reader/src/reader/expect_expr_xs.cpp index b318966e..68a489b0 100644 --- a/xo-reader/src/reader/expect_expr_xs.cpp +++ b/xo-reader/src/reader/expect_expr_xs.cpp @@ -129,6 +129,8 @@ namespace xo { log && log(xtag("tk", tk)); + constexpr const char * c_self_name = "expect_expr_xs::on_symbol_token"; + /* various possibilities when looking for rhs expression: * * x := y // (1) @@ -143,10 +145,8 @@ namespace xo { bp var = p_psm->lookup_var(tk.text()); if (!var) { - throw std::runtime_error - (tostr("expect_expr_xs::on_symbol_token", - ": unknown symbol", - xtag("name", tk.text()))); + this->unknown_variable_error(c_self_name, tk, p_psm); + return; } /* e.g. @@ -173,12 +173,22 @@ namespace xo { return; } + void + expect_expr_xs::on_bool_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + progress_xs::start + (Constant::make(tk.bool_value()), + p_psm); + } + void expect_expr_xs::on_i64_token(const token_type & tk, parserstatemachine * p_psm) { - constexpr bool c_debug_flag = true; - scope log(XO_DEBUG(c_debug_flag)); + scope log(XO_DEBUG(p_psm->debug_flag())); progress_xs::start (Constant::make(tk.i64_value()), diff --git a/xo-reader/src/reader/exprseq_xs.cpp b/xo-reader/src/reader/exprseq_xs.cpp index 72d06e5e..03399cca 100644 --- a/xo-reader/src/reader/exprseq_xs.cpp +++ b/xo-reader/src/reader/exprseq_xs.cpp @@ -7,6 +7,7 @@ //#include "expect_expr_xs.hpp" #include "progress_xs.hpp" #include "define_xs.hpp" +#include "if_else_xs.hpp" #include "expect_symbol_xs.hpp" #include "xo/expression/Constant.hpp" @@ -30,6 +31,24 @@ namespace xo { { } + const char * + exprseq_xs::get_expect_str() const + { + /* + * def... + * ^ + * exprseq_xs + */ + switch (this->xseqtype_) { + case exprseqtype::toplevel_interactive: + return "def|expression"; + case exprseqtype::toplevel_batch: + return "def"; + } + + return "?expect"; + } + void exprseq_xs::on_def_token(const token_type & /*tk*/, parserstatemachine * p_psm) @@ -45,6 +64,25 @@ namespace xo { */ } + void + exprseq_xs::on_if_token(const token_type & tk, + parserstatemachine * p_psm) + { + if (xseqtype_ == exprseqtype::toplevel_interactive) + { + /* in interactive session, allow top-level if-expressions. + * Could be: + * if sometest() do_something() do_otherthing(); + */ + if_else_xs::start(p_psm); + } else { + constexpr const char * c_self_name = "exprseq_xs::on_if_token"; + const char * exp = get_expect_str(); + + this->illegal_input_on_token(c_self_name, tk, exp, p_psm); + } + } + void exprseq_xs::on_symbol_token(const token_type & tk, parserstatemachine * p_psm) @@ -72,7 +110,39 @@ namespace xo { /* policy: don't allow variable references as toplevel expressions * unless interactive session */ - this->illegal_input_error(c_self_name, tk); + const char * exp = get_expect_str(); + + this->illegal_input_on_token(c_self_name, + tk, + exp, + p_psm); + } + } + + void + exprseq_xs::on_bool_token(const token_type & tk, + parserstatemachine * p_psm) + { + using xo::ast::Constant; + + constexpr bool c_debug_flag = true; + scope log(XO_DEBUG(c_debug_flag)); + + constexpr const char * c_self_name = "exprseq_xs::on_bool_token"; + + if (xseqtype_ == exprseqtype::toplevel_interactive) + { + progress_xs::start(Constant::make(tk.bool_value()), p_psm); + } else { + /* policy: don't allow literals as toplevel expressions + * unless interactive session + */ + const char * exp = get_expect_str(); + + this->illegal_input_on_token(c_self_name, + tk, + exp, + p_psm); } } @@ -94,7 +164,12 @@ namespace xo { /* policy: don't allow literals as toplevel expressions * unless interactive session. */ - this->illegal_input_error(c_self_name, tk); + const char * exp = get_expect_str(); + + this->illegal_input_on_token(c_self_name, + tk, + exp, + p_psm); } } @@ -115,7 +190,6 @@ namespace xo { * arbitrary number of expressions. */ - *(p_psm->p_result_) = parser_result::expression(expr.promote()); } diff --git a/xo-reader/src/reader/exprstate.cpp b/xo-reader/src/reader/exprstate.cpp index 3d447311..57d0c22f 100644 --- a/xo-reader/src/reader/exprstate.cpp +++ b/xo-reader/src/reader/exprstate.cpp @@ -40,6 +40,8 @@ namespace xo { return "sequenceexpr"; case exprstatetype::let1expr: return "let1expr"; + case exprstatetype::ifexpr: + return "ifexpr"; case exprstatetype::expect_rhs_expression: return "expect_rhs_expression"; case exprstatetype::expect_symbol: @@ -279,6 +281,55 @@ namespace xo { this->illegal_input_on_token(c_self_name, tk, exp, p_psm); } + void + exprstate::on_if_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + constexpr const char * c_self_name = "exprstate::on_if_token"; + const char * exp = get_expect_str(); + + this->illegal_input_on_token(c_self_name, tk, exp, p_psm); + } + + void + exprstate::on_then_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + constexpr const char * c_self_name = "exprstate::on_then_token"; + const char * exp = get_expect_str(); + + this->illegal_input_on_token(c_self_name, tk, exp, p_psm); + } + + void + exprstate::on_else_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + constexpr const char * c_self_name = "exprstate::on_else_token"; + const char * exp = get_expect_str(); + + this->illegal_input_on_token(c_self_name, tk, exp, p_psm); + } + + void + exprstate::on_bool_token(const token_type & tk, + parserstatemachine * p_psm) + { + constexpr bool c_debug_flag = true; + scope log(XO_DEBUG(c_debug_flag)); + + constexpr const char * c_self_name = "exprstate::on_bool"; + const char * exp = get_expect_str(); + + this->illegal_input_on_token(c_self_name, tk, exp, p_psm); + } + void exprstate::on_i64_token(const token_type & tk, parserstatemachine * p_psm) @@ -326,6 +377,10 @@ namespace xo { this->on_lambda_token(tk, p_psm); return; + case tokentype::tk_bool: + this->on_bool_token(tk, p_psm); + return; + case tokentype::tk_i64: this->on_i64_token(tk, p_psm); return; @@ -402,7 +457,21 @@ namespace xo { return; case tokentype::tk_type: + assert(false); + return; + case tokentype::tk_if: + this->on_if_token(tk, p_psm); + return; + + case tokentype::tk_then: + this->on_then_token(tk, p_psm); + return; + + case tokentype::tk_else: + this->on_else_token(tk, p_psm); + return; + case tokentype::tk_let: case tokentype::tk_in: diff --git a/xo-reader/src/reader/exprstatestack.cpp b/xo-reader/src/reader/exprstatestack.cpp index 354586d7..effff3a9 100644 --- a/xo-reader/src/reader/exprstatestack.cpp +++ b/xo-reader/src/reader/exprstatestack.cpp @@ -89,7 +89,7 @@ namespace xo { /** always multiple lines if more than one element in stack **/ if ((stack_.size() > 0) - && !pps->print_upto_tag("[0]", *stack_[0].get())) + && !pps->print_upto_tag("[0]", stack_[0].get())) { return false; } @@ -105,7 +105,7 @@ namespace xo { for (std::size_t i = 0, z = stack_.size(); i < z; ++i) { std::string i_str = tostr("[", z-i-1, "]"); - pps->newline_pretty_tag(ppii.ci1(), i_str, *stack_[i].get()); + pps->newline_pretty_tag(ppii.ci1(), i_str, stack_[i].get()); } pps->write(">"); diff --git a/xo-reader/src/reader/if_else_xs.cpp b/xo-reader/src/reader/if_else_xs.cpp new file mode 100644 index 00000000..f94e6e9d --- /dev/null +++ b/xo-reader/src/reader/if_else_xs.cpp @@ -0,0 +1,269 @@ +/** @file if_else_xs.cpp + * + * author: Roland Conybeare, Jul 2025 + **/ + +#include "if_else_xs.hpp" +//#include "exprstatestack.hpp" +#include "parserstatemachine.hpp" +#include "expect_expr_xs.hpp" + +namespace xo { + namespace scm { + // ----- ifexprstatetype ----- + + const char * + ifexprstatetype_descr(ifexprstatetype x) { + switch (x) { + case ifexprstatetype::invalid: return "invalid"; + case ifexprstatetype::if_0: return "if_0"; + case ifexprstatetype::if_1: return "if_1"; + case ifexprstatetype::if_2: return "if_2"; + case ifexprstatetype::if_3: return "if_3"; + case ifexprstatetype::if_4: return "if_4"; + case ifexprstatetype::if_5: return "if_5"; + case ifexprstatetype::if_6: return "if_6"; + case ifexprstatetype::n_ifexprstatetype: break; + } + + return "???ifexprstatetype"; + } + + std::ostream & + operator<<(std::ostream & os, ifexprstatetype x) { + os << ifexprstatetype_descr(x); + return os; + } + + // ----- if_else_xs ----- + + std::unique_ptr + if_else_xs::make() { + return std::make_unique(if_else_xs(IfExprAccess::make_empty())); + } + + void + if_else_xs::start(parserstatemachine * p_psm) + { + constexpr bool c_debug_flag = true; + scope log(XO_DEBUG(c_debug_flag)); + + p_psm->push_exprstate(if_else_xs::make()); + p_psm->top_exprstate().on_if_token(token_type::if_token(), p_psm); + } + + if_else_xs::if_else_xs(rp if_expr) + : exprstate(exprstatetype::ifexpr), + ifxs_type_{ifexprstatetype::if_0}, + if_expr_{std::move(if_expr)} + {} + + const char * + if_else_xs::get_expect_str() const + { + /** + * if test-expr then then-expr else else-expr ; + * ^ ^ ^ ^ ^ ^ ^ + * | | | | | | | + * | if_1 if_2 if_3 if_4 if_5 if_6 + * if_0 + * + * if_0 --on_if_token()--> if_1 + * if_1 --on_expr()--> if_2 + * if_2 --on_then_token()--> if_3 + * if_3 --on_expr()--> if_4 + * if_4 --on_else_token()--> if_5 + * --on_semicolon_token()--> (done) + * if_5 --on_expr()-->if_6 + * if_6 --on_semicolon_token()--> (done) + **/ + switch (this->ifxs_type_) { + case ifexprstatetype::invalid: + case ifexprstatetype::if_0: + case ifexprstatetype::n_ifexprstatetype: + assert(false); // unreachable + return nullptr; + case ifexprstatetype::if_1: + return "expression"; + case ifexprstatetype::if_2: + return "then"; + case ifexprstatetype::if_3: + return "expression"; + case ifexprstatetype::if_4: + return "else|semicolon"; + case ifexprstatetype::if_5: + return "expression"; + case ifexprstatetype::if_6: + return "semicolon"; + } + + return "?expect"; + } + + void + if_else_xs::on_if_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + log && log("ifxs_type", ifxs_type_); + + if (this->ifxs_type_ == ifexprstatetype::if_0) { + this->ifxs_type_ = ifexprstatetype::if_1; + + expect_expr_xs::start(p_psm); + return; + } + + constexpr const char * c_self_name = "if_else_xs::on_if_token"; + const char * exp = this->get_expect_str(); + + this->illegal_input_on_token(c_self_name, tk, exp, p_psm); + } + + void + if_else_xs::on_then_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + log && log("ifxs_type", ifxs_type_); + + if (this->ifxs_type_ == ifexprstatetype::if_2) { + this->ifxs_type_ = ifexprstatetype::if_3; + + expect_expr_xs::start(p_psm); + return; + } + + constexpr const char * c_self_name = "if_else_xs::on_then_token"; + const char * exp = this->get_expect_str(); + + this->illegal_input_on_token(c_self_name, tk, exp, p_psm); + } + + void + if_else_xs::on_else_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + log && log("ifxs_type", ifxs_type_); + + if (this->ifxs_type_ == ifexprstatetype::if_4) { + this->ifxs_type_ = ifexprstatetype::if_5; + + expect_expr_xs::start(p_psm); + return; + } + + constexpr const char * c_self_name = "if_else_xs::on_else_token"; + const char * exp = this->get_expect_str(); + + this->illegal_input_on_token(c_self_name, tk, exp, p_psm); + } + + void + if_else_xs::on_semicolon_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + log && log("ifxs_type", ifxs_type_); + + const char * c_self_name = "if_else_xs::on_semicolon_token"; + + switch (this->ifxs_type_) { + case ifexprstatetype::invalid: + case ifexprstatetype::if_0: + case ifexprstatetype::n_ifexprstatetype: + // unreachable + assert(false); + return; + + case ifexprstatetype::if_1: + case ifexprstatetype::if_2: + case ifexprstatetype::if_3: + case ifexprstatetype::if_5: + this->illegal_input_on_token(c_self_name, tk, get_expect_str(), p_psm); + return; + case ifexprstatetype::if_4: + case ifexprstatetype::if_6: { + rp if_expr = this->if_expr_; + + std::unique_ptr self = p_psm->pop_exprstate(); + TypeDescr td = nullptr; + + if_expr->assign_valuetype(td); + p_psm->top_exprstate().on_expr(if_expr, p_psm); + return; + } + } + } + + void + if_else_xs::on_expr(bp expr, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + log && log(xtag("ifxs_type", ifxs_type_)); + + switch (this->ifxs_type_) { + case ifexprstatetype::invalid: + case ifexprstatetype::if_0: + case ifexprstatetype::n_ifexprstatetype: + assert(false); // unreachable + return; + case ifexprstatetype::if_1: + if_expr_->assign_test(expr.promote()); + ifxs_type_ = ifexprstatetype::if_2; + return; + case ifexprstatetype::if_2: + /** error: expecting 'then' **/ + break; + case ifexprstatetype::if_3: + if_expr_->assign_when_true(expr.promote()); + ifxs_type_ = ifexprstatetype::if_4; + return; + case ifexprstatetype::if_4: + /** error: expecting 'else' or ';' **/ + break; + case ifexprstatetype::if_5: + if_expr_->assign_when_false(expr.promote()); + ifxs_type_ = ifexprstatetype::if_6; + return; + case ifexprstatetype::if_6: + /** error: expecting ';' **/ + break; + } + + constexpr const char* c_self_name = "if_else_xs::on_expr"; + const char * exp = get_expect_str(); + + this->illegal_input_on_expr(c_self_name, expr, exp, p_psm); + } + + void + if_else_xs::on_expr_with_semicolon(bp expr, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + + log && log(xtag("ifxs_type", ifxs_type_)); + + this->on_expr(expr, p_psm); + this->on_semicolon_token(token_type::semicolon(), p_psm); + } + + void + if_else_xs::print(std::ostream & os) const { + os << ""; + } + + } /*namespace scm*/ +} /*namespace xo*/ diff --git a/xo-reader/src/reader/parser.cpp b/xo-reader/src/reader/parser.cpp index 85ce0231..5e9aed09 100644 --- a/xo-reader/src/reader/parser.cpp +++ b/xo-reader/src/reader/parser.cpp @@ -29,8 +29,8 @@ namespace xo { namespace scm { // ----- parser ----- - parser::parser() - : xs_stack_{}, env_stack_{}, result_{} + parser::parser(bool debug_flag) + : xs_stack_{}, env_stack_{}, result_{}, debug_flag_{debug_flag} { /* top-level environment. initially empty */ rp toplevel_env = LocalEnv::make_empty(); @@ -48,7 +48,8 @@ namespace xo { parser::begin_interactive_session() { parserstatemachine psm(&xs_stack_, &env_stack_, - &result_); + &result_, + debug_flag_); exprseq_xs::start(exprseqtype::toplevel_interactive, &psm); } @@ -57,7 +58,8 @@ namespace xo { parser::begin_translation_unit() { parserstatemachine psm(&xs_stack_, &env_stack_, - &result_); + &result_, + debug_flag_); exprseq_xs::start(exprseqtype::toplevel_batch, &psm); } @@ -79,7 +81,7 @@ namespace xo { log && log(xtag("top", xs_stack_.top_exprstate())); - parserstatemachine psm(&xs_stack_, &env_stack_, &result_); + parserstatemachine psm(&xs_stack_, &env_stack_, &result_, debug_flag_); xs_stack_.top_exprstate().on_input(tk, &psm); diff --git a/xo-reader/src/reader/parserstatemachine.cpp b/xo-reader/src/reader/parserstatemachine.cpp index 660c8ede..bb53cde4 100644 --- a/xo-reader/src/reader/parserstatemachine.cpp +++ b/xo-reader/src/reader/parserstatemachine.cpp @@ -146,8 +146,7 @@ namespace xo { void parserstatemachine::on_rightbrace_token(const token_type & tk) { - constexpr bool c_debug_flag = true; - scope log(XO_DEBUG(c_debug_flag)); + scope log(XO_DEBUG(debug_flag_)); log && log(xtag("tk", tk), xtag("psm", *this)); @@ -156,6 +155,30 @@ namespace xo { ->top_exprstate().on_rightbrace_token(tk, this); } + void + parserstatemachine::on_then_token(const token_type & tk) + { + scope log(XO_DEBUG(debug_flag_)); + + log && log(xtag("tk", tk), + xtag("psm", *this)); + + this->p_stack_ + ->top_exprstate().on_then_token(tk, this); + } + + void + parserstatemachine::on_else_token(const token_type & tk) + { + scope log(XO_DEBUG(debug_flag_)); + + log && log(xtag("tk", tk), + xtag("psm", *this)); + + this->p_stack_ + ->top_exprstate().on_else_token(tk, this); + } + void parserstatemachine::on_error(const char * self_name, std::string errmsg) { diff --git a/xo-reader/src/reader/progress_xs.cpp b/xo-reader/src/reader/progress_xs.cpp index 43fcc373..ca137fdd 100644 --- a/xo-reader/src/reader/progress_xs.cpp +++ b/xo-reader/src/reader/progress_xs.cpp @@ -104,7 +104,7 @@ namespace xo { } rp - progress_xs::assemble_expr() { + progress_xs::assemble_expr(parserstatemachine * p_psm) { /* need to defer building Apply incase expr followed by higher-precedence operator: * consider input like * 3.14 + 2.0 * ... @@ -113,10 +113,11 @@ namespace xo { constexpr const char * c_self_name = "progress_xs::assemble_expr"; if ((op_type_ != optype::invalid) && (rhs_.get() == nullptr)) { - throw std::runtime_error(tostr(c_self_name, - ": expected expr on rhs of operator", - xtag("lhs", lhs_), - xtag("op", op_type_))); + std::string errmsg = tostr("expected expression on rhs of operator op", + xtag("lhs", lhs_), + xtag("op", op_type_)); + + p_psm->on_error(c_self_name, errmsg); } /* consecutive expressions not legal, e.g: @@ -220,7 +221,7 @@ namespace xo { // this->on_semicolon_token(token_type::semicolon(), p_psm); // INSTEAD, spell out the body - rp expr2 = this->assemble_expr(); + rp expr2 = this->assemble_expr(p_psm); std::unique_ptr self = p_psm->pop_exprstate(); @@ -261,10 +262,9 @@ namespace xo { { /* note: implementation parallels .on_rightparen_token() */ - constexpr bool c_debug_flag = true; - scope log(XO_DEBUG(c_debug_flag)); + scope log(XO_DEBUG(p_psm->debug_flag())); - rp expr = this->assemble_expr(); + rp expr = this->assemble_expr(p_psm); log && log(xtag("assembled-expr", expr)); @@ -316,9 +316,7 @@ namespace xo { { /* note: implementation parallels .on_semicolon_token() */ - - constexpr bool c_debug_flag = true; - scope log(XO_DEBUG(c_debug_flag)); + scope log(XO_DEBUG(p_psm->debug_flag())); constexpr const char * self_name = "progress_xs::on_rightparen"; @@ -336,7 +334,7 @@ namespace xo { */ /* right paren confirms stack expression */ - rp expr = this->assemble_expr(); + rp expr = this->assemble_expr(p_psm); std::unique_ptr self = p_psm->pop_exprstate(); @@ -353,6 +351,49 @@ namespace xo { p_psm->top_exprstate().on_rightparen_token(tk, p_psm); } + void + progress_xs::on_then_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + rp expr = this->assemble_expr(p_psm); + + log && log(xtag("assembled-expr", expr)); + + std::unique_ptr self = p_psm->pop_exprstate(); + + p_psm->on_expr(expr); + p_psm->on_then_token(tk); + + /* control here on input like: + * + * if a > b then.. + * + */ + } + + void + progress_xs::on_else_token(const token_type & tk, + parserstatemachine * p_psm) + { + scope log(XO_DEBUG(p_psm->debug_flag())); + + rp expr = this->assemble_expr(p_psm); + + log && log(xtag("assembled-expr", expr)); + + std::unique_ptr self = p_psm->pop_exprstate(); + + p_psm->on_expr(expr); + p_psm->on_else_token(tk); + + /* control here on input like: + * + * if a > b then c else.. + */ + } + namespace { optype tk2op(const tokentype & tktype) { @@ -406,7 +447,7 @@ namespace xo { */ /* 1. instantiate expression for *this */ - auto expr = this->assemble_expr(); + auto expr = this->assemble_expr(p_psm); /* 2. remove from stack */ std::unique_ptr self = p_psm->pop_exprstate(); diff --git a/xo-reader/src/reader/reader.cpp b/xo-reader/src/reader/reader.cpp index 6e73400e..7f1080f3 100644 --- a/xo-reader/src/reader/reader.cpp +++ b/xo-reader/src/reader/reader.cpp @@ -4,6 +4,10 @@ namespace xo { namespace scm { + reader::reader(bool debug_flag) + : parser_{debug_flag} + {} + void reader::begin_interactive_session() { parser_.begin_interactive_session(); diff --git a/xo-reader/utest/parser.test.cpp b/xo-reader/utest/parser.test.cpp index e29d030e..18b52723 100644 --- a/xo-reader/utest/parser.test.cpp +++ b/xo-reader/utest/parser.test.cpp @@ -21,9 +21,10 @@ namespace xo { namespace ut { TEST_CASE("parser", "[parser]") { for (std::size_t i_tc = 0; i_tc < 2; ++i_tc) { - parser_type parser; - constexpr bool c_debug_flag = true; + + parser_type parser(c_debug_flag); + scope log(XO_DEBUG(c_debug_flag), xtag("i_tc", i_tc)); parser.begin_translation_unit(); diff --git a/xo-reader/utest/reader.test.cpp b/xo-reader/utest/reader.test.cpp index 16951874..d9a55dd5 100644 --- a/xo-reader/utest/reader.test.cpp +++ b/xo-reader/utest/reader.test.cpp @@ -30,7 +30,7 @@ namespace xo { for (std::size_t i_tc = 0; i_tc < s_testcase_v.size(); ++i_tc) { const test_case & tc = s_testcase_v[i_tc]; - reader rdr; + reader rdr(c_debug_flag); scope log(XO_ENTER2(always, c_debug_flag, "reader.testcase"), xtag("i_tc", i_tc)); diff --git a/xo-tokenizer/include/xo/tokenizer/token.hpp b/xo-tokenizer/include/xo/tokenizer/token.hpp index 5a053d11..689a4512 100644 --- a/xo-tokenizer/include/xo/tokenizer/token.hpp +++ b/xo-tokenizer/include/xo/tokenizer/token.hpp @@ -60,6 +60,12 @@ namespace xo { /** create invalid token (same as null ctor, but explicit) **/ static token invalid() { return token(); } + /** Create token representing a boolean literal from text @p txt + * @p txt must be @c true or @c false + **/ + static token bool_token(const std::string & txt) { + return token(tokentype::tk_bool, txt); + } /** Create token representing 64-bit signed integer literal parsed from decimal @p txt. * The string @p txt must be a decimal integer literal, since @ref i64_value re-parses @p txt. **/ @@ -132,6 +138,8 @@ namespace xo { static token lambda() { return token(tokentype::tk_lambda); } /** token representing keyword @c if **/ static token if_token() { return token(tokentype::tk_if); } + /** token representing keyword @c else **/ + static token else_token() { return token(tokentype::tk_else); } /** token representing keyword @c let **/ static token let() { return token(tokentype::tk_let); } /** token representing keyword @c in **/ @@ -165,10 +173,13 @@ namespace xo { || tk_type_ == tokentype::tk_string || tk_type_ == tokentype::tk_symbol); } - /** expect input matching @c "[+|-][0-9][0-9]*" **/ + /** expect input matching @c true or @c false **/ + bool bool_value() const; + + /** expect input matching @c [+|-][0-9][0-9]* **/ std::int64_t i64_value() const; - /** expect input matching @c "[+|-][0-9]*[.][0-9]*[e|E][+|-][0-9]*" **/ + /** expect input matching @c [+|-][0-9]*[.][0-9]*[e|E][+|-][0-9]* **/ double f64_value() const; /** print human-readable token representation on stream @p os **/ @@ -196,6 +207,29 @@ namespace xo { ///@} }; + template + bool + token::bool_value() const { + if (tk_type_ != tokentype::tk_bool) { + throw (std::runtime_error + (tostr("token::bool_value", + ": token with type tk found where tk_bool expected", + xtag("tk", tk_type_)))); + } + + if (text_ == "true") + return true; + if (text_ == "false") + return false; + + throw (std::runtime_error + (tostr("token::bool_value", + ": unexpected input string tk_bool token", + xtag("text", text_)))); + + return false; + } + template std::int64_t token::i64_value() const { diff --git a/xo-tokenizer/include/xo/tokenizer/tokenizer.hpp b/xo-tokenizer/include/xo/tokenizer/tokenizer.hpp index 9b017486..8c1b131a 100644 --- a/xo-tokenizer/include/xo/tokenizer/tokenizer.hpp +++ b/xo-tokenizer/include/xo/tokenizer/tokenizer.hpp @@ -479,9 +479,6 @@ namespace xo { (error_type(__FUNCTION__ /*src_function*/, "expecting key following escape character \\", input_state_, - //current_line_, - //current_pos_, - //initial_whitespace, (ix - tk_start))); } @@ -511,9 +508,6 @@ namespace xo { (error_type(__FUNCTION__ /*src_function*/, "expecting one of n|r|\"|\\ following escape \\", input_state_, - //current_line_, - //current_pos_, - //initial_whitespace, (ix - tk_start))); } break; @@ -531,9 +525,6 @@ namespace xo { (error_type(__FUNCTION__ /*src_function*/, "missing terminating '\"' to complete literal string", input_state_, - //current_line_, - //current_pos_, - //initial_whitespace, (ix - tk_start))); } @@ -683,7 +674,10 @@ namespace xo { bool keep_text = false; - if (tk_text == "type") { + if ((tk_text == "true") || (tk_text == "false")) { + tk_type = tokentype::tk_bool; + keep_text = true; + } else if (tk_text == "type") { tk_type = tokentype::tk_type; } else if (tk_text == "def") { tk_type = tokentype::tk_def; @@ -691,6 +685,10 @@ namespace xo { tk_type = tokentype::tk_lambda; } else if (tk_text == "if") { tk_type = tokentype::tk_if; + } else if (tk_text == "then") { + tk_type = tokentype::tk_then; + } else if (tk_text == "else") { + tk_type = tokentype::tk_else; } else if (tk_text == "let") { tk_type = tokentype::tk_let; } else if (tk_text == "in") { @@ -842,9 +840,6 @@ namespace xo { (error_type(__FUNCTION__ /*src_function*/, "must use \\n or \\r to encode newline/cr in string literal", input_state_, - //current_line_, - //current_pos_, - //whitespace.size(), (ix - tk_start))); } diff --git a/xo-tokenizer/include/xo/tokenizer/tokentype.hpp b/xo-tokenizer/include/xo/tokenizer/tokentype.hpp index bb1e8000..81a852fa 100644 --- a/xo-tokenizer/include/xo/tokenizer/tokentype.hpp +++ b/xo-tokenizer/include/xo/tokenizer/tokentype.hpp @@ -49,6 +49,9 @@ namespace xo { /** sentinel value **/ tk_invalid = -1, + /** a boolean constant **/ + tk_bool, + /** an integer constant (signed 64-bit integer) **/ tk_i64, @@ -135,6 +138,12 @@ namespace xo { /** keyword @c 'if' **/ tk_if, + /** keyworkd @c 'then' **/ + tk_then, + + /** keyword @c 'else' **/ + tk_else, + /** keyword @c 'let' **/ tk_let, diff --git a/xo-tokenizer/src/tokenizer/tokentype.cpp b/xo-tokenizer/src/tokenizer/tokentype.cpp index 54d86540..c19d023b 100644 --- a/xo-tokenizer/src/tokenizer/tokentype.cpp +++ b/xo-tokenizer/src/tokenizer/tokentype.cpp @@ -13,6 +13,7 @@ namespace xo { #define CASE(x) case tokentype::x: return STRINGIFY(x) switch(tk_type) { + CASE(tk_bool); CASE(tk_i64); CASE(tk_f64); CASE(tk_string); @@ -46,6 +47,8 @@ namespace xo { CASE(tk_def); CASE(tk_lambda); CASE(tk_if); + CASE(tk_then); + CASE(tk_else); CASE(tk_let); CASE(tk_in);