roland/xo-alloc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer xo-reader: + bool literals + if-expr parsing

Browse source
This commit is contained in:
Roland Conybeare 2025-07-22 07:20:56 -05:00
commit ce760bd5cf
30 changed files with 848 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

1
xo-expression/include/xo/expression/GeneralizedExpression.hpp
View file

@ -38,7 +38,6 @@ namespace xo {
/** pretty printing support. See [xo-indentlog/xo/indentlog/pretty.hpp] **/ /** pretty printing support. See [xo-indentlog/xo/indentlog/pretty.hpp] **/
virtual std::uint32_t pretty_print(const ppindentinfo & ppii) const = 0; virtual std::uint32_t pretty_print(const ppindentinfo & ppii) const = 0;
protected:
/** useful when scaffolding expressions in a parser **/ /** useful when scaffolding expressions in a parser **/
void assign_valuetype(TypeDescr x) { valuetype_ = x; } void assign_valuetype(TypeDescr x) { valuetype_ = x; }

32
xo-expression/include/xo/expression/IfExpr.hpp
View file

@ -104,7 +104,7 @@ namespace xo {
virtual void display(std::ostream & os) const override; virtual void display(std::ostream & os) const override;
virtual std::uint32_t pretty_print(const ppindentinfo & ppi) const override; virtual std::uint32_t pretty_print(const ppindentinfo & ppi) const override;
private: protected:
/** /**
* @p ifexpr_type type for value produced by if-expression. * @p ifexpr_type type for value produced by if-expression.
* same as both when_true->valuetype() and * same as both when_true->valuetype() and
@ -122,7 +122,13 @@ namespace xo {
when_true_{std::move(when_true)}, when_true_{std::move(when_true)},
when_false_{std::move(when_false)} {} when_false_{std::move(when_false)} {}
private: static TypeDescr check_consistent_valuetype(const rp<Expression> & when_true,
const rp<Expression> & when_false);
/** determine if-expr valuetype **/
void establish_valuetype();
protected:
/** if: /** if:
* (if x y z) * (if x y z)
* *
@ -140,6 +146,28 @@ namespace xo {
{ {
return IfExpr::make(test, when_true, when_false); return IfExpr::make(test, when_true, when_false);
} }
class IfExprAccess : public IfExpr {
public:
static rp<IfExprAccess> make(rp<Expression> test,
rp<Expression> when_true,
rp<Expression> when_false);
static rp<IfExprAccess> make_empty();
void assign_test(rp<Expression> x) { test_ = std::move(x); }
void assign_when_true(rp<Expression> x);
void assign_when_false(rp<Expression> x);
private:
IfExprAccess(TypeDescr ifexpr_type,
rp<Expression> test,
rp<Expression> when_true,
rp<Expression> when_false)
: IfExpr(ifexpr_type,
std::move(test),
std::move(when_true),
std::move(when_false)) {}
};
} /*namespace ast*/ } /*namespace ast*/
} /*namespace xo*/ } /*namespace xo*/

53
xo-expression/src/expression/IfExpr.cpp
View file

@ -7,6 +7,21 @@
namespace xo { namespace xo {
namespace ast { namespace ast {
auto IfExpr::check_consistent_valuetype(const rp<Expression> & when_true,
const rp<Expression> & when_false) -> TypeDescr
{
if (when_true->valuetype() != when_false->valuetype())
return nullptr;
return when_true->valuetype();
}
void IfExpr::establish_valuetype()
{
if (this->when_true_.get() && this->when_false_.get())
this->assign_valuetype(check_consistent_valuetype(this->when_true_, this->when_false_));
}
rp<IfExpr> rp<IfExpr>
IfExpr::make(const rp<Expression> & test, IfExpr::make(const rp<Expression> & test,
const rp<Expression> & when_true, const rp<Expression> & when_true,
@ -36,9 +51,10 @@ namespace xo {
IfExpr::display(std::ostream & os) const { IfExpr::display(std::ostream & os) const {
os << "<IfExpr" os << "<IfExpr"
<< xtag("test", test_) << xtag("test", test_)
<< xtag("when_true", when_true_) << xtag("when_true", when_true_);
<< xtag("when_false", when_false_) if (when_false_)
<< ">"; os << xtag("when_false", when_false_);
os << ">";
} /*display*/ } /*display*/
std::uint32_t std::uint32_t
@ -48,6 +64,37 @@ namespace xo {
refrtag("when_true", when_true_), refrtag("when_true", when_true_),
refrtag("when_false", when_false_)); refrtag("when_false", when_false_));
} }
rp<IfExprAccess>
IfExprAccess::make(rp<Expression> test,
rp<Expression> when_true,
rp<Expression> when_false)
{
auto ifexpr_type = check_consistent_valuetype(when_true, when_false);
return new IfExprAccess(ifexpr_type, std::move(test), std::move(when_true), std::move(when_false));
}
rp<IfExprAccess>
IfExprAccess::make_empty()
{
return new IfExprAccess(nullptr /*ifexpr_valuetype*/,
nullptr /*test*/,
nullptr /*when_true*/,
nullptr /*when_false*/);
}
void
IfExprAccess::assign_when_true(rp<Expression> x)
{
this->when_true_ = std::move(x);
}
void
IfExprAccess::assign_when_false(rp<Expression> x)
{
this->when_false_ = std::move(x);
}
} /*namespace ast*/ } /*namespace ast*/
} /*namespace xo*/ } /*namespace xo*/

4
xo-reader/examples/exprrepl/exprrepl.cpp
View file

@ -47,7 +47,9 @@ main() {
bool interactive = isatty(STDIN_FILENO); bool interactive = isatty(STDIN_FILENO);
reader rdr; bool c_debug_flag = false;
reader rdr(c_debug_flag);
rdr.begin_interactive_session(); rdr.begin_interactive_session();
string input_str; string input_str;

4
xo-reader/examples/exprreplxx/exprreplxx.cpp
View file

@ -77,7 +77,9 @@ main() {
// rx.bind_key_internal(Replxx::KEY::control('p'), "history_previous"); // rx.bind_key_internal(Replxx::KEY::control('p'), "history_previous");
// rx.bind_key_internal(Replxx::KEY::control('n'), "history_next"); // rx.bind_key_internal(Replxx::KEY::control('n'), "history_next");
reader rdr; constexpr bool c_debug_flag = true;
reader rdr(c_debug_flag);
rdr.begin_interactive_session(); rdr.begin_interactive_session();
string input_str; string input_str;

17
xo-reader/include/xo/reader/expect_expr_xs.hpp
View file

@ -12,6 +12,18 @@ namespace xo {
/** @class expect_expr_xs /** @class expect_expr_xs
* @brief state machine to expect + capture an expression * @brief state machine to expect + capture an expression
*
* Examples:
* @text
* def x : i64 = 5 ; // with allow_defs
* lambda (f : x64) { ... } ;
* if (prime(x)) { ... } ;
* 5 + x ;
* @endtext
*
* top exprstate when expect_expr_xs::start() invoked
* will receive parsed expression via
* exprstate::on_expr() or exprstate::on_expr_with_semicolon().
**/ **/
class expect_expr_xs : public exprstate { class expect_expr_xs : public exprstate {
public: public:
@ -41,6 +53,9 @@ namespace xo {
virtual void on_symbol_token(const token_type & tk, virtual void on_symbol_token(const token_type & tk,
parserstatemachine * p_psm) override; parserstatemachine * p_psm) override;
virtual void on_bool_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_i64_token(const token_type & tk, virtual void on_i64_token(const token_type & tk,
parserstatemachine * p_psm) override; parserstatemachine * p_psm) override;
@ -50,7 +65,7 @@ namespace xo {
/** update exprstate in response to a successfully-parsed subexpression **/ /** update exprstate in response to a successfully-parsed subexpression **/
virtual void on_expr(bp<Expression> expr, virtual void on_expr(bp<Expression> expr,
parserstatemachine * p_psm) override; parserstatemachine * p_psm) override;
/** update exprstate in response to a successfully-parsed subexpression, /** update exprstate in response to a successfully-parsed subexpression
* that's terminated by semicolon ';' * that's terminated by semicolon ';'
**/ **/
virtual void on_expr_with_semicolon(bp<Expression> expr, virtual void on_expr_with_semicolon(bp<Expression> expr,

20
xo-reader/include/xo/reader/exprseq_xs.hpp
View file

@ -17,8 +17,6 @@ namespace xo {
* This used for top-level expressions in a translation unit. * This used for top-level expressions in a translation unit.
**/ **/
toplevel_batch, toplevel_batch,
/** nested sequence, for example in function body **/
nested,
}; };
/** @class exprseq_xs /** @class exprseq_xs
@ -27,14 +25,14 @@ namespace xo {
* expression sequences come in several types: * expression sequences come in several types:
* 1. top-level interactive * 1. top-level interactive
* 2. top-level batch * 2. top-level batch
* 3. nested
* *
* @text * @text
* 1 2 3 * 1 2
* +-------- * +-----
* def | y y y * def | y y
* symbol | y n n 1: evaluate as variable * if | y n 1: eval as expression
* i64 | y n n 1: evaluate as constant * symbol | y n 1: evaluate as variable
* i64 | y n 1: evaluate as constant
* *
* @endtext * @endtext
**/ **/
@ -45,12 +43,18 @@ namespace xo {
static void start(exprseqtype seqtype, parserstatemachine * p_psm); static void start(exprseqtype seqtype, parserstatemachine * p_psm);
public: public:
virtual const char * get_expect_str() const override;
// ----- token input methods ----- // ----- token input methods -----
virtual void on_def_token(const token_type & tk, virtual void on_def_token(const token_type & tk,
parserstatemachine * p_psm) override; parserstatemachine * p_psm) override;
virtual void on_if_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_symbol_token(const token_type & tk, virtual void on_symbol_token(const token_type & tk,
parserstatemachine * p_psm) override; parserstatemachine * p_psm) override;
virtual void on_bool_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_i64_token(const token_type & tk, virtual void on_i64_token(const token_type & tk,
parserstatemachine * p_psm) override; parserstatemachine * p_psm) override;

21
xo-reader/include/xo/reader/exprstate.hpp
View file

@ -45,6 +45,11 @@ namespace xo {
**/ **/
let1expr, let1expr,
/** handle if-else expression
* see @ref if_else_xs
**/
ifexpr,
expect_rhs_expression, expect_rhs_expression,
expect_symbol, expect_symbol,
expect_type, expect_type,
@ -177,6 +182,22 @@ namespace xo {
virtual void on_operator_token(const token_type & tk, virtual void on_operator_token(const token_type & tk,
parserstatemachine * p_psm); parserstatemachine * p_psm);
/** handle incoming if-keyword token **/
virtual void on_if_token(const token_type & tk,
parserstatemachine * p_psm);
/** handle incoming then-keyword token **/
virtual void on_then_token(const token_type & tk,
parserstatemachine * p_psm);
/** handle incoming then-keyword token **/
virtual void on_else_token(const token_type & tk,
parserstatemachine * p_psm);
/** handle incoming bool-literal token **/
virtual void on_bool_token(const token_type & tk,
parserstatemachine * p_psm);
/** handle incoming integer-literal token **/ /** handle incoming integer-literal token **/
virtual void on_i64_token(const token_type & tk, virtual void on_i64_token(const token_type & tk,
parserstatemachine * p_psm); parserstatemachine * p_psm);

97
xo-reader/include/xo/reader/if_else_xs.hpp Normal file
View file

@ -0,0 +1,97 @@
/** @file if_else_xs.hpp
*
* author: Roland Conybeare, Jul 2025
**/
#pragma once
#include "xo/expression/IfExpr.hpp"
#include "exprstate.hpp"
#include "xo/indentlog/print/ppdetail_atomic.hpp"
namespace xo {
namespace scm {
/**
* if test-expr then then-expr else else-expr ;
* ^ ^ ^ ^ ^ ^ ^
* | | | | | | |
* | if_1 if_2 if_3 if_4 if_5 if_6
* if_0
*
* if_0 --on_if_token()--> if_1
* if_1 --on_expr()--> if_2
* if_2 --on_then_token()--> if_3
* if_3 --on_expr()--> if_4
* if_4 --on_else_token()--> if_5
* --on_semicolon_token()--> (done)
* if_5 --on_expr()-->if_6
* if_6 --on_semicolon_token()--> (done)
**/
enum class ifexprstatetype {
invalid = -1,
if_0,
if_1,
if_2,
if_3,
if_4,
if_5,
if_6,
n_ifexprstatetype,
};
extern const char * ifexprstatetype_descr(ifexprstatetype x);
std::ostream &
operator<<(std::ostream & os, ifexprstatetype x);
/** @class if_else_xs
* @brief state to provide parsing of a conditional expression
**/
class if_else_xs : public exprstate {
public:
using IfExprAccess = xo::ast::IfExprAccess;
public:
if_else_xs(rp<IfExprAccess> if_expr);
virtual ~if_else_xs() = default;
static const if_else_xs * from(const exprstate * x) { return dynamic_cast<const if_else_xs *>(x); }
static void start(parserstatemachine * p_psm);
// ----- inherited from exprstate -----
virtual const char * get_expect_str() const override;
virtual void on_if_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_then_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_else_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_semicolon_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_expr(bp<Expression> expr,
parserstatemachine * p_psm) override;
virtual void on_expr_with_semicolon(bp<Expression> expr,
parserstatemachine * p_psm) override;
virtual void print(std::ostream & os) const override;
private:
static std::unique_ptr<if_else_xs> make();
private:
ifexprstatetype ifxs_type_;
/** scaffold output expression here **/
rp<IfExprAccess> if_expr_;
};
} /*namespace scm*/
} /*namespace xo*/
/** end if_else_xs.hpp **/

10
xo-reader/include/xo/reader/parser.hpp
View file

@ -112,6 +112,9 @@ namespace xo {
* | symbol-literal * | symbol-literal
* | struct-literal * | struct-literal
* *
*
* boolean-literal = true | false
*
* variable-expr = $varname * variable-expr = $varname
* apply-expr = fn-expr(arg-expr(1), .., arg-expr(n)) * apply-expr = fn-expr(arg-expr(1), .., arg-expr(n))
* fn-expr = expression * fn-expr = expression
@ -155,8 +158,10 @@ namespace xo {
public: public:
/** create parser in initial state; /** create parser in initial state;
* parser is ready to receive tokens via @ref include_token * parser is ready to receive tokens via @ref include_token
*
* @p debug_flag true to enable debug logging
**/ **/
parser(); explicit parser(bool debug_flag);
/** true if parser is at top-level, i.e. ready for next top-level expression **/ /** true if parser is at top-level, i.e. ready for next top-level expression **/
bool is_at_toplevel() const { return stack_size() == 0; } bool is_at_toplevel() const { return stack_size() == 0; }
@ -243,6 +248,9 @@ namespace xo {
/** parser result state **/ /** parser result state **/
parser_result result_; parser_result result_;
/** enable/disable debug logging **/
bool debug_flag_;
}; /*parser*/ }; /*parser*/
inline std::ostream & inline std::ostream &

16
xo-reader/include/xo/reader/parserstatemachine.hpp
View file

@ -28,10 +28,12 @@ namespace xo {
public: public:
parserstatemachine(exprstatestack * p_stack, parserstatemachine(exprstatestack * p_stack,
envframestack * p_env_stack, envframestack * p_env_stack,
parser_result * p_result) parser_result * p_result,
bool debug_flag)
: p_stack_{p_stack}, : p_stack_{p_stack},
p_env_stack_{p_env_stack}, p_env_stack_{p_env_stack},
p_result_{p_result} p_result_{p_result},
debug_flag_{debug_flag}
{} {}
//const parser_result & result() const { return result_; } //const parser_result & result() const { return result_; }
@ -44,6 +46,8 @@ namespace xo {
exprstate & top_exprstate(); exprstate & top_exprstate();
void push_exprstate(std::unique_ptr<exprstate> x); void push_exprstate(std::unique_ptr<exprstate> x);
bool debug_flag() const { return debug_flag_; }
/** lookup variable name in lexical context represented by /** lookup variable name in lexical context represented by
* this psm. nullptr if not found * this psm. nullptr if not found
**/ **/
@ -54,9 +58,13 @@ namespace xo {
**/ **/
void upsert_var(bp<Variable> x); void upsert_var(bp<Variable> x);
/** @return available variable bindings in current parsing state **/
bp<LocalEnv> top_envframe() const; bp<LocalEnv> top_envframe() const;
/** push frame @p x (with new variable bindings) onto environment stack **/
void push_envframe(const rp<LocalEnv> & x); void push_envframe(const rp<LocalEnv> & x);
/** @return pop innermost environment frame and return it **/
rp<LocalEnv> pop_envframe(); rp<LocalEnv> pop_envframe();
/** @return number of stacked environment frames **/
size_t env_stack_size() const { return p_env_stack_->size(); } size_t env_stack_size() const { return p_env_stack_->size(); }
// ----- parsing outputs ----- // ----- parsing outputs -----
@ -71,6 +79,8 @@ namespace xo {
void on_operator_token(const token_type & tk); void on_operator_token(const token_type & tk);
void on_leftbrace_token(const token_type & tk); void on_leftbrace_token(const token_type & tk);
void on_rightbrace_token(const token_type & tk); void on_rightbrace_token(const token_type & tk);
void on_then_token(const token_type & tk);
void on_else_token(const token_type & tk);
// ----- parsing error ----- // ----- parsing error -----
@ -91,6 +101,8 @@ namespace xo {
envframestack * p_env_stack_ = nullptr; envframestack * p_env_stack_ = nullptr;
/** parser result object **/ /** parser result object **/
parser_result * p_result_ = nullptr; parser_result * p_result_ = nullptr;
/** enable debug logging **/
bool debug_flag_ = false;
}; };
inline std::ostream & inline std::ostream &

6
xo-reader/include/xo/reader/progress_xs.hpp
View file

@ -95,6 +95,10 @@ namespace xo {
parserstatemachine * p_psm) override; parserstatemachine * p_psm) override;
virtual void on_rightparen_token(const token_type & tk, virtual void on_rightparen_token(const token_type & tk,
parserstatemachine * p_psm) override; parserstatemachine * p_psm) override;
virtual void on_then_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_else_token(const token_type & tk,
parserstatemachine * p_psm) override;
/* entry point for an infix operator token */ /* entry point for an infix operator token */
virtual void on_operator_token(const token_type & tk, virtual void on_operator_token(const token_type & tk,
@ -121,7 +125,7 @@ namespace xo {
* *
* where f determined by @ref op_type_ * where f determined by @ref op_type_
**/ **/
rp<Expression> assemble_expr(); rp<Expression> assemble_expr(parserstatemachine * p_psm);
private: private:
/** populate an expression here, may be followed by an operator **/ /** populate an expression here, may be followed by an operator **/

2
xo-reader/include/xo/reader/reader.hpp
View file

@ -78,7 +78,7 @@ namespace xo {
using span_type = tokenizer_type::span_type; using span_type = tokenizer_type::span_type;
public: public:
reader() = default; explicit reader(bool debug_flag);
/** call once before calling .read_expr() /** call once before calling .read_expr()
* for a new interactive session * for a new interactive session

1
xo-reader/src/reader/CMakeLists.txt
View file

@ -9,6 +9,7 @@ set(SELF_SRCS
exprstate.cpp exprstate.cpp
exprstatestack.cpp exprstatestack.cpp
define_xs.cpp define_xs.cpp
if_else_xs.cpp
progress_xs.cpp progress_xs.cpp
paren_xs.cpp paren_xs.cpp
sequence_xs.cpp sequence_xs.cpp

4
xo-reader/src/reader/define_xs.cpp
View file

@ -14,7 +14,7 @@ namespace xo {
const char * const char *
defexprstatetype_descr(defexprstatetype x) { defexprstatetype_descr(defexprstatetype x) {
switch(x) { switch (x) {
case defexprstatetype::invalid: return "invalid"; case defexprstatetype::invalid: return "invalid";
case defexprstatetype::def_0: return "def_0"; case defexprstatetype::def_0: return "def_0";
case defexprstatetype::def_1: return "def_1"; case defexprstatetype::def_1: return "def_1";
@ -208,7 +208,7 @@ namespace xo {
return; return;
} }
constexpr const char * c_self_name = "define_xs::on_symbol"; constexpr const char * c_self_name = "define_xs::on_def_token";
const char * exp = this->get_expect_str(); const char * exp = this->get_expect_str();
this->illegal_input_on_token(c_self_name, tk, exp, p_psm); this->illegal_input_on_token(c_self_name, tk, exp, p_psm);

22
xo-reader/src/reader/expect_expr_xs.cpp
View file

@ -129,6 +129,8 @@ namespace xo {
log && log(xtag("tk", tk)); log && log(xtag("tk", tk));
constexpr const char * c_self_name = "expect_expr_xs::on_symbol_token";
/* various possibilities when looking for rhs expression: /* various possibilities when looking for rhs expression:
* *
* x := y // (1) * x := y // (1)
@ -143,10 +145,8 @@ namespace xo {
bp<Variable> var = p_psm->lookup_var(tk.text()); bp<Variable> var = p_psm->lookup_var(tk.text());
if (!var) { if (!var) {
throw std::runtime_error this->unknown_variable_error(c_self_name, tk, p_psm);
(tostr("expect_expr_xs::on_symbol_token", return;
": unknown symbol",
xtag("name", tk.text())));
} }
/* e.g. /* e.g.
@ -173,12 +173,22 @@ namespace xo {
return; return;
} }
void
expect_expr_xs::on_bool_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
progress_xs::start
(Constant<bool>::make(tk.bool_value()),
p_psm);
}
void void
expect_expr_xs::on_i64_token(const token_type & tk, expect_expr_xs::on_i64_token(const token_type & tk,
parserstatemachine * p_psm) parserstatemachine * p_psm)
{ {
constexpr bool c_debug_flag = true; scope log(XO_DEBUG(p_psm->debug_flag()));
scope log(XO_DEBUG(c_debug_flag));
progress_xs::start progress_xs::start
(Constant<int64_t>::make(tk.i64_value()), (Constant<int64_t>::make(tk.i64_value()),

80
xo-reader/src/reader/exprseq_xs.cpp
View file

@ -7,6 +7,7 @@
//#include "expect_expr_xs.hpp" //#include "expect_expr_xs.hpp"
#include "progress_xs.hpp" #include "progress_xs.hpp"
#include "define_xs.hpp" #include "define_xs.hpp"
#include "if_else_xs.hpp"
#include "expect_symbol_xs.hpp" #include "expect_symbol_xs.hpp"
#include "xo/expression/Constant.hpp" #include "xo/expression/Constant.hpp"
@ -30,6 +31,24 @@ namespace xo {
{ {
} }
const char *
exprseq_xs::get_expect_str() const
{
/*
* def...
* ^
* exprseq_xs
*/
switch (this->xseqtype_) {
case exprseqtype::toplevel_interactive:
return "def|expression";
case exprseqtype::toplevel_batch:
return "def";
}
return "?expect";
}
void void
exprseq_xs::on_def_token(const token_type & /*tk*/, exprseq_xs::on_def_token(const token_type & /*tk*/,
parserstatemachine * p_psm) parserstatemachine * p_psm)
@ -45,6 +64,25 @@ namespace xo {
*/ */
} }
void
exprseq_xs::on_if_token(const token_type & tk,
parserstatemachine * p_psm)
{
if (xseqtype_ == exprseqtype::toplevel_interactive)
{
/* in interactive session, allow top-level if-expressions.
* Could be:
* if sometest() do_something() do_otherthing();
*/
if_else_xs::start(p_psm);
} else {
constexpr const char * c_self_name = "exprseq_xs::on_if_token";
const char * exp = get_expect_str();
this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
}
}
void void
exprseq_xs::on_symbol_token(const token_type & tk, exprseq_xs::on_symbol_token(const token_type & tk,
parserstatemachine * p_psm) parserstatemachine * p_psm)
@ -72,7 +110,39 @@ namespace xo {
/* policy: don't allow variable references as toplevel expressions /* policy: don't allow variable references as toplevel expressions
* unless interactive session * unless interactive session
*/ */
this->illegal_input_error(c_self_name, tk); const char * exp = get_expect_str();
this->illegal_input_on_token(c_self_name,
tk,
exp,
p_psm);
}
}
void
exprseq_xs::on_bool_token(const token_type & tk,
parserstatemachine * p_psm)
{
using xo::ast::Constant;
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
constexpr const char * c_self_name = "exprseq_xs::on_bool_token";
if (xseqtype_ == exprseqtype::toplevel_interactive)
{
progress_xs::start(Constant<bool>::make(tk.bool_value()), p_psm);
} else {
/* policy: don't allow literals as toplevel expressions
* unless interactive session
*/
const char * exp = get_expect_str();
this->illegal_input_on_token(c_self_name,
tk,
exp,
p_psm);
} }
} }
@ -94,7 +164,12 @@ namespace xo {
/* policy: don't allow literals as toplevel expressions /* policy: don't allow literals as toplevel expressions
* unless interactive session. * unless interactive session.
*/ */
this->illegal_input_error(c_self_name, tk); const char * exp = get_expect_str();
this->illegal_input_on_token(c_self_name,
tk,
exp,
p_psm);
} }
} }
@ -115,7 +190,6 @@ namespace xo {
* arbitrary number of expressions. * arbitrary number of expressions.
*/ */
*(p_psm->p_result_) = parser_result::expression(expr.promote()); *(p_psm->p_result_) = parser_result::expression(expr.promote());
} }

69
xo-reader/src/reader/exprstate.cpp
View file

@ -40,6 +40,8 @@ namespace xo {
return "sequenceexpr"; return "sequenceexpr";
case exprstatetype::let1expr: case exprstatetype::let1expr:
return "let1expr"; return "let1expr";
case exprstatetype::ifexpr:
return "ifexpr";
case exprstatetype::expect_rhs_expression: case exprstatetype::expect_rhs_expression:
return "expect_rhs_expression"; return "expect_rhs_expression";
case exprstatetype::expect_symbol: case exprstatetype::expect_symbol:
@ -279,6 +281,55 @@ namespace xo {
this->illegal_input_on_token(c_self_name, tk, exp, p_psm); this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
} }
void
exprstate::on_if_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
constexpr const char * c_self_name = "exprstate::on_if_token";
const char * exp = get_expect_str();
this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
}
void
exprstate::on_then_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
constexpr const char * c_self_name = "exprstate::on_then_token";
const char * exp = get_expect_str();
this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
}
void
exprstate::on_else_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
constexpr const char * c_self_name = "exprstate::on_else_token";
const char * exp = get_expect_str();
this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
}
void
exprstate::on_bool_token(const token_type & tk,
parserstatemachine * p_psm)
{
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
constexpr const char * c_self_name = "exprstate::on_bool";
const char * exp = get_expect_str();
this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
}
void void
exprstate::on_i64_token(const token_type & tk, exprstate::on_i64_token(const token_type & tk,
parserstatemachine * p_psm) parserstatemachine * p_psm)
@ -326,6 +377,10 @@ namespace xo {
this->on_lambda_token(tk, p_psm); this->on_lambda_token(tk, p_psm);
return; return;
case tokentype::tk_bool:
this->on_bool_token(tk, p_psm);
return;
case tokentype::tk_i64: case tokentype::tk_i64:
this->on_i64_token(tk, p_psm); this->on_i64_token(tk, p_psm);
return; return;
@ -402,7 +457,21 @@ namespace xo {
return; return;
case tokentype::tk_type: case tokentype::tk_type:
assert(false);
return;
case tokentype::tk_if: case tokentype::tk_if:
this->on_if_token(tk, p_psm);
return;
case tokentype::tk_then:
this->on_then_token(tk, p_psm);
return;
case tokentype::tk_else:
this->on_else_token(tk, p_psm);
return;
case tokentype::tk_let: case tokentype::tk_let:
case tokentype::tk_in: case tokentype::tk_in:

4
xo-reader/src/reader/exprstatestack.cpp
View file

@ -89,7 +89,7 @@ namespace xo {
/** always multiple lines if more than one element in stack **/ /** always multiple lines if more than one element in stack **/
if ((stack_.size() > 0) if ((stack_.size() > 0)
&& !pps->print_upto_tag("[0]", *stack_[0].get())) && !pps->print_upto_tag("[0]", stack_[0].get()))
{ {
return false; return false;
} }
@ -105,7 +105,7 @@ namespace xo {
for (std::size_t i = 0, z = stack_.size(); i < z; ++i) { for (std::size_t i = 0, z = stack_.size(); i < z; ++i) {
std::string i_str = tostr("[", z-i-1, "]"); std::string i_str = tostr("[", z-i-1, "]");
pps->newline_pretty_tag(ppii.ci1(), i_str, *stack_[i].get()); pps->newline_pretty_tag(ppii.ci1(), i_str, stack_[i].get());
} }
pps->write(">"); pps->write(">");

269
xo-reader/src/reader/if_else_xs.cpp Normal file
View file

@ -0,0 +1,269 @@
/** @file if_else_xs.cpp
*
* author: Roland Conybeare, Jul 2025
**/
#include "if_else_xs.hpp"
//#include "exprstatestack.hpp"
#include "parserstatemachine.hpp"
#include "expect_expr_xs.hpp"
namespace xo {
namespace scm {
// ----- ifexprstatetype -----
const char *
ifexprstatetype_descr(ifexprstatetype x) {
switch (x) {
case ifexprstatetype::invalid: return "invalid";
case ifexprstatetype::if_0: return "if_0";
case ifexprstatetype::if_1: return "if_1";
case ifexprstatetype::if_2: return "if_2";
case ifexprstatetype::if_3: return "if_3";
case ifexprstatetype::if_4: return "if_4";
case ifexprstatetype::if_5: return "if_5";
case ifexprstatetype::if_6: return "if_6";
case ifexprstatetype::n_ifexprstatetype: break;
}
return "???ifexprstatetype";
}
std::ostream &
operator<<(std::ostream & os, ifexprstatetype x) {
os << ifexprstatetype_descr(x);
return os;
}
// ----- if_else_xs -----
std::unique_ptr<if_else_xs>
if_else_xs::make() {
return std::make_unique<if_else_xs>(if_else_xs(IfExprAccess::make_empty()));
}
void
if_else_xs::start(parserstatemachine * p_psm)
{
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
p_psm->push_exprstate(if_else_xs::make());
p_psm->top_exprstate().on_if_token(token_type::if_token(), p_psm);
}
if_else_xs::if_else_xs(rp<IfExprAccess> if_expr)
: exprstate(exprstatetype::ifexpr),
ifxs_type_{ifexprstatetype::if_0},
if_expr_{std::move(if_expr)}
{}
const char *
if_else_xs::get_expect_str() const
{
/**
* if test-expr then then-expr else else-expr ;
* ^ ^ ^ ^ ^ ^ ^
* | | | | | | |
* | if_1 if_2 if_3 if_4 if_5 if_6
* if_0
*
* if_0 --on_if_token()--> if_1
* if_1 --on_expr()--> if_2
* if_2 --on_then_token()--> if_3
* if_3 --on_expr()--> if_4
* if_4 --on_else_token()--> if_5
* --on_semicolon_token()--> (done)
* if_5 --on_expr()-->if_6
* if_6 --on_semicolon_token()--> (done)
**/
switch (this->ifxs_type_) {
case ifexprstatetype::invalid:
case ifexprstatetype::if_0:
case ifexprstatetype::n_ifexprstatetype:
assert(false); // unreachable
return nullptr;
case ifexprstatetype::if_1:
return "expression";
case ifexprstatetype::if_2:
return "then";
case ifexprstatetype::if_3:
return "expression";
case ifexprstatetype::if_4:
return "else|semicolon";
case ifexprstatetype::if_5:
return "expression";
case ifexprstatetype::if_6:
return "semicolon";
}
return "?expect";
}
void
if_else_xs::on_if_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
log && log("ifxs_type", ifxs_type_);
if (this->ifxs_type_ == ifexprstatetype::if_0) {
this->ifxs_type_ = ifexprstatetype::if_1;
expect_expr_xs::start(p_psm);
return;
}
constexpr const char * c_self_name = "if_else_xs::on_if_token";
const char * exp = this->get_expect_str();
this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
}
void
if_else_xs::on_then_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
log && log("ifxs_type", ifxs_type_);
if (this->ifxs_type_ == ifexprstatetype::if_2) {
this->ifxs_type_ = ifexprstatetype::if_3;
expect_expr_xs::start(p_psm);
return;
}
constexpr const char * c_self_name = "if_else_xs::on_then_token";
const char * exp = this->get_expect_str();
this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
}
void
if_else_xs::on_else_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
log && log("ifxs_type", ifxs_type_);
if (this->ifxs_type_ == ifexprstatetype::if_4) {
this->ifxs_type_ = ifexprstatetype::if_5;
expect_expr_xs::start(p_psm);
return;
}
constexpr const char * c_self_name = "if_else_xs::on_else_token";
const char * exp = this->get_expect_str();
this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
}
void
if_else_xs::on_semicolon_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
log && log("ifxs_type", ifxs_type_);
const char * c_self_name = "if_else_xs::on_semicolon_token";
switch (this->ifxs_type_) {
case ifexprstatetype::invalid:
case ifexprstatetype::if_0:
case ifexprstatetype::n_ifexprstatetype:
// unreachable
assert(false);
return;
case ifexprstatetype::if_1:
case ifexprstatetype::if_2:
case ifexprstatetype::if_3:
case ifexprstatetype::if_5:
this->illegal_input_on_token(c_self_name, tk, get_expect_str(), p_psm);
return;
case ifexprstatetype::if_4:
case ifexprstatetype::if_6: {
rp<IfExprAccess> if_expr = this->if_expr_;
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
TypeDescr td = nullptr;
if_expr->assign_valuetype(td);
p_psm->top_exprstate().on_expr(if_expr, p_psm);
return;
}
}
}
void
if_else_xs::on_expr(bp<Expression> expr,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
log && log(xtag("ifxs_type", ifxs_type_));
switch (this->ifxs_type_) {
case ifexprstatetype::invalid:
case ifexprstatetype::if_0:
case ifexprstatetype::n_ifexprstatetype:
assert(false); // unreachable
return;
case ifexprstatetype::if_1:
if_expr_->assign_test(expr.promote());
ifxs_type_ = ifexprstatetype::if_2;
return;
case ifexprstatetype::if_2:
/** error: expecting 'then' **/
break;
case ifexprstatetype::if_3:
if_expr_->assign_when_true(expr.promote());
ifxs_type_ = ifexprstatetype::if_4;
return;
case ifexprstatetype::if_4:
/** error: expecting 'else' or ';' **/
break;
case ifexprstatetype::if_5:
if_expr_->assign_when_false(expr.promote());
ifxs_type_ = ifexprstatetype::if_6;
return;
case ifexprstatetype::if_6:
/** error: expecting ';' **/
break;
}
constexpr const char* c_self_name = "if_else_xs::on_expr";
const char * exp = get_expect_str();
this->illegal_input_on_expr(c_self_name, expr, exp, p_psm);
}
void
if_else_xs::on_expr_with_semicolon(bp<Expression> expr,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
log && log(xtag("ifxs_type", ifxs_type_));
this->on_expr(expr, p_psm);
this->on_semicolon_token(token_type::semicolon(), p_psm);
}
void
if_else_xs::print(std::ostream & os) const {
os << "<if_else_xs"
<< xtag("this", (void*)this)
<< xtag("ifxs_type", ifxs_type_)
<< ">";
}
} /*namespace scm*/
} /*namespace xo*/

12
xo-reader/src/reader/parser.cpp
View file

@ -29,8 +29,8 @@ namespace xo {
namespace scm { namespace scm {
// ----- parser ----- // ----- parser -----
parser::parser() parser::parser(bool debug_flag)
: xs_stack_{}, env_stack_{}, result_{} : xs_stack_{}, env_stack_{}, result_{}, debug_flag_{debug_flag}
{ {
/* top-level environment. initially empty */ /* top-level environment. initially empty */
rp<LocalEnv> toplevel_env = LocalEnv::make_empty(); rp<LocalEnv> toplevel_env = LocalEnv::make_empty();
@ -48,7 +48,8 @@ namespace xo {
parser::begin_interactive_session() { parser::begin_interactive_session() {
parserstatemachine psm(&xs_stack_, parserstatemachine psm(&xs_stack_,
&env_stack_, &env_stack_,
&result_); &result_,
debug_flag_);
exprseq_xs::start(exprseqtype::toplevel_interactive, &psm); exprseq_xs::start(exprseqtype::toplevel_interactive, &psm);
} }
@ -57,7 +58,8 @@ namespace xo {
parser::begin_translation_unit() { parser::begin_translation_unit() {
parserstatemachine psm(&xs_stack_, parserstatemachine psm(&xs_stack_,
&env_stack_, &env_stack_,
&result_); &result_,
debug_flag_);
exprseq_xs::start(exprseqtype::toplevel_batch, &psm); exprseq_xs::start(exprseqtype::toplevel_batch, &psm);
} }
@ -79,7 +81,7 @@ namespace xo {
log && log(xtag("top", xs_stack_.top_exprstate())); log && log(xtag("top", xs_stack_.top_exprstate()));
parserstatemachine psm(&xs_stack_, &env_stack_, &result_); parserstatemachine psm(&xs_stack_, &env_stack_, &result_, debug_flag_);
xs_stack_.top_exprstate().on_input(tk, &psm); xs_stack_.top_exprstate().on_input(tk, &psm);

27
xo-reader/src/reader/parserstatemachine.cpp
View file

@ -146,8 +146,7 @@ namespace xo {
void void
parserstatemachine::on_rightbrace_token(const token_type & tk) parserstatemachine::on_rightbrace_token(const token_type & tk)
{ {
constexpr bool c_debug_flag = true; scope log(XO_DEBUG(debug_flag_));
scope log(XO_DEBUG(c_debug_flag));
log && log(xtag("tk", tk), log && log(xtag("tk", tk),
xtag("psm", *this)); xtag("psm", *this));
@ -156,6 +155,30 @@ namespace xo {
->top_exprstate().on_rightbrace_token(tk, this); ->top_exprstate().on_rightbrace_token(tk, this);
} }
void
parserstatemachine::on_then_token(const token_type & tk)
{
scope log(XO_DEBUG(debug_flag_));
log && log(xtag("tk", tk),
xtag("psm", *this));
this->p_stack_
->top_exprstate().on_then_token(tk, this);
}
void
parserstatemachine::on_else_token(const token_type & tk)
{
scope log(XO_DEBUG(debug_flag_));
log && log(xtag("tk", tk),
xtag("psm", *this));
this->p_stack_
->top_exprstate().on_else_token(tk, this);
}
void void
parserstatemachine::on_error(const char * self_name, std::string errmsg) parserstatemachine::on_error(const char * self_name, std::string errmsg)
{ {

69
xo-reader/src/reader/progress_xs.cpp
View file

@ -104,7 +104,7 @@ namespace xo {
} }
rp<Expression> rp<Expression>
progress_xs::assemble_expr() { progress_xs::assemble_expr(parserstatemachine * p_psm) {
/* need to defer building Apply incase expr followed by higher-precedence operator: /* need to defer building Apply incase expr followed by higher-precedence operator:
* consider input like * consider input like
* 3.14 + 2.0 * ... * 3.14 + 2.0 * ...
@ -113,10 +113,11 @@ namespace xo {
constexpr const char * c_self_name = "progress_xs::assemble_expr"; constexpr const char * c_self_name = "progress_xs::assemble_expr";
if ((op_type_ != optype::invalid) && (rhs_.get() == nullptr)) { if ((op_type_ != optype::invalid) && (rhs_.get() == nullptr)) {
throw std::runtime_error(tostr(c_self_name, std::string errmsg = tostr("expected expression on rhs of operator op",
": expected expr on rhs of operator", xtag("lhs", lhs_),
xtag("lhs", lhs_), xtag("op", op_type_));
xtag("op", op_type_)));
p_psm->on_error(c_self_name, errmsg);
} }
/* consecutive expressions not legal, e.g: /* consecutive expressions not legal, e.g:
@ -220,7 +221,7 @@ namespace xo {
// this->on_semicolon_token(token_type::semicolon(), p_psm); // this->on_semicolon_token(token_type::semicolon(), p_psm);
// INSTEAD, spell out the body // INSTEAD, spell out the body
rp<Expression> expr2 = this->assemble_expr(); rp<Expression> expr2 = this->assemble_expr(p_psm);
std::unique_ptr<exprstate> self = p_psm->pop_exprstate(); std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
@ -261,10 +262,9 @@ namespace xo {
{ {
/* note: implementation parallels .on_rightparen_token() */ /* note: implementation parallels .on_rightparen_token() */
constexpr bool c_debug_flag = true; scope log(XO_DEBUG(p_psm->debug_flag()));
scope log(XO_DEBUG(c_debug_flag));
rp<Expression> expr = this->assemble_expr(); rp<Expression> expr = this->assemble_expr(p_psm);
log && log(xtag("assembled-expr", expr)); log && log(xtag("assembled-expr", expr));
@ -316,9 +316,7 @@ namespace xo {
{ {
/* note: implementation parallels .on_semicolon_token() */ /* note: implementation parallels .on_semicolon_token() */
scope log(XO_DEBUG(p_psm->debug_flag()));
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
constexpr const char * self_name = "progress_xs::on_rightparen"; constexpr const char * self_name = "progress_xs::on_rightparen";
@ -336,7 +334,7 @@ namespace xo {
*/ */
/* right paren confirms stack expression */ /* right paren confirms stack expression */
rp<Expression> expr = this->assemble_expr(); rp<Expression> expr = this->assemble_expr(p_psm);
std::unique_ptr<exprstate> self = p_psm->pop_exprstate(); std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
@ -353,6 +351,49 @@ namespace xo {
p_psm->top_exprstate().on_rightparen_token(tk, p_psm); p_psm->top_exprstate().on_rightparen_token(tk, p_psm);
} }
void
progress_xs::on_then_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
rp<Expression> expr = this->assemble_expr(p_psm);
log && log(xtag("assembled-expr", expr));
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
p_psm->on_expr(expr);
p_psm->on_then_token(tk);
/* control here on input like:
*
* if a > b then..
*
*/
}
void
progress_xs::on_else_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
rp<Expression> expr = this->assemble_expr(p_psm);
log && log(xtag("assembled-expr", expr));
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
p_psm->on_expr(expr);
p_psm->on_else_token(tk);
/* control here on input like:
*
* if a > b then c else..
*/
}
namespace { namespace {
optype optype
tk2op(const tokentype & tktype) { tk2op(const tokentype & tktype) {
@ -406,7 +447,7 @@ namespace xo {
*/ */
/* 1. instantiate expression for *this */ /* 1. instantiate expression for *this */
auto expr = this->assemble_expr(); auto expr = this->assemble_expr(p_psm);
/* 2. remove from stack */ /* 2. remove from stack */
std::unique_ptr<exprstate> self = p_psm->pop_exprstate(); std::unique_ptr<exprstate> self = p_psm->pop_exprstate();

4
xo-reader/src/reader/reader.cpp
View file

@ -4,6 +4,10 @@
namespace xo { namespace xo {
namespace scm { namespace scm {
reader::reader(bool debug_flag)
: parser_{debug_flag}
{}
void void
reader::begin_interactive_session() { reader::begin_interactive_session() {
parser_.begin_interactive_session(); parser_.begin_interactive_session();

5
xo-reader/utest/parser.test.cpp
View file

@ -21,9 +21,10 @@ namespace xo {
namespace ut { namespace ut {
TEST_CASE("parser", "[parser]") { TEST_CASE("parser", "[parser]") {
for (std::size_t i_tc = 0; i_tc < 2; ++i_tc) { for (std::size_t i_tc = 0; i_tc < 2; ++i_tc) {
parser_type parser;
constexpr bool c_debug_flag = true; constexpr bool c_debug_flag = true;
parser_type parser(c_debug_flag);
scope log(XO_DEBUG(c_debug_flag), xtag("i_tc", i_tc)); scope log(XO_DEBUG(c_debug_flag), xtag("i_tc", i_tc));
parser.begin_translation_unit(); parser.begin_translation_unit();

2
xo-reader/utest/reader.test.cpp
View file

@ -30,7 +30,7 @@ namespace xo {
for (std::size_t i_tc = 0; i_tc < s_testcase_v.size(); ++i_tc) { for (std::size_t i_tc = 0; i_tc < s_testcase_v.size(); ++i_tc) {
const test_case & tc = s_testcase_v[i_tc]; const test_case & tc = s_testcase_v[i_tc];
reader rdr; reader rdr(c_debug_flag);
scope log(XO_ENTER2(always, c_debug_flag, "reader.testcase"), scope log(XO_ENTER2(always, c_debug_flag, "reader.testcase"),
xtag("i_tc", i_tc)); xtag("i_tc", i_tc));

38
xo-tokenizer/include/xo/tokenizer/token.hpp
View file

@ -60,6 +60,12 @@ namespace xo {
/** create invalid token (same as null ctor, but explicit) **/ /** create invalid token (same as null ctor, but explicit) **/
static token invalid() { return token(); } static token invalid() { return token(); }
/** Create token representing a boolean literal from text @p txt
* @p txt must be @c true or @c false
**/
static token bool_token(const std::string & txt) {
return token(tokentype::tk_bool, txt);
}
/** Create token representing 64-bit signed integer literal parsed from decimal @p txt. /** Create token representing 64-bit signed integer literal parsed from decimal @p txt.
* The string @p txt must be a decimal integer literal, since @ref i64_value re-parses @p txt. * The string @p txt must be a decimal integer literal, since @ref i64_value re-parses @p txt.
**/ **/
@ -132,6 +138,8 @@ namespace xo {
static token lambda() { return token(tokentype::tk_lambda); } static token lambda() { return token(tokentype::tk_lambda); }
/** token representing keyword @c if **/ /** token representing keyword @c if **/
static token if_token() { return token(tokentype::tk_if); } static token if_token() { return token(tokentype::tk_if); }
/** token representing keyword @c else **/
static token else_token() { return token(tokentype::tk_else); }
/** token representing keyword @c let **/ /** token representing keyword @c let **/
static token let() { return token(tokentype::tk_let); } static token let() { return token(tokentype::tk_let); }
/** token representing keyword @c in **/ /** token representing keyword @c in **/
@ -165,10 +173,13 @@ namespace xo {
|| tk_type_ == tokentype::tk_string || tk_type_ == tokentype::tk_string
|| tk_type_ == tokentype::tk_symbol); } || tk_type_ == tokentype::tk_symbol); }
/** expect input matching @c "[+|-][0-9][0-9]*" **/ /** expect input matching @c true or @c false **/
bool bool_value() const;
/** expect input matching @c [+|-][0-9][0-9]* **/
std::int64_t i64_value() const; std::int64_t i64_value() const;
/** expect input matching @c "[+|-][0-9]*[.][0-9]*[e|E][+|-][0-9]*" **/ /** expect input matching @c [+|-][0-9]*[.][0-9]*[e|E][+|-][0-9]* **/
double f64_value() const; double f64_value() const;
/** print human-readable token representation on stream @p os **/ /** print human-readable token representation on stream @p os **/
@ -196,6 +207,29 @@ namespace xo {
///@} ///@}
}; };
template <typename CharT>
bool
token<CharT>::bool_value() const {
if (tk_type_ != tokentype::tk_bool) {
throw (std::runtime_error
(tostr("token::bool_value",
": token with type tk found where tk_bool expected",
xtag("tk", tk_type_))));
}
if (text_ == "true")
return true;
if (text_ == "false")
return false;
throw (std::runtime_error
(tostr("token::bool_value",
": unexpected input string tk_bool token",
xtag("text", text_))));
return false;
}
template <typename CharT> template <typename CharT>
std::int64_t std::int64_t
token<CharT>::i64_value() const { token<CharT>::i64_value() const {

21
xo-tokenizer/include/xo/tokenizer/tokenizer.hpp
View file

@ -479,9 +479,6 @@ namespace xo {
(error_type(__FUNCTION__ /*src_function*/, (error_type(__FUNCTION__ /*src_function*/,
"expecting key following escape character \\", "expecting key following escape character \\",
input_state_, input_state_,
//current_line_,
//current_pos_,
//initial_whitespace,
(ix - tk_start))); (ix - tk_start)));
} }
@ -511,9 +508,6 @@ namespace xo {
(error_type(__FUNCTION__ /*src_function*/, (error_type(__FUNCTION__ /*src_function*/,
"expecting one of n|r|\"|\\ following escape \\", "expecting one of n|r|\"|\\ following escape \\",
input_state_, input_state_,
//current_line_,
//current_pos_,
//initial_whitespace,
(ix - tk_start))); (ix - tk_start)));
} }
break; break;
@ -531,9 +525,6 @@ namespace xo {
(error_type(__FUNCTION__ /*src_function*/, (error_type(__FUNCTION__ /*src_function*/,
"missing terminating '\"' to complete literal string", "missing terminating '\"' to complete literal string",
input_state_, input_state_,
//current_line_,
//current_pos_,
//initial_whitespace,
(ix - tk_start))); (ix - tk_start)));
} }
@ -683,7 +674,10 @@ namespace xo {
bool keep_text = false; bool keep_text = false;
if (tk_text == "type") { if ((tk_text == "true") || (tk_text == "false")) {
tk_type = tokentype::tk_bool;
keep_text = true;
} else if (tk_text == "type") {
tk_type = tokentype::tk_type; tk_type = tokentype::tk_type;
} else if (tk_text == "def") { } else if (tk_text == "def") {
tk_type = tokentype::tk_def; tk_type = tokentype::tk_def;
@ -691,6 +685,10 @@ namespace xo {
tk_type = tokentype::tk_lambda; tk_type = tokentype::tk_lambda;
} else if (tk_text == "if") { } else if (tk_text == "if") {
tk_type = tokentype::tk_if; tk_type = tokentype::tk_if;
} else if (tk_text == "then") {
tk_type = tokentype::tk_then;
} else if (tk_text == "else") {
tk_type = tokentype::tk_else;
} else if (tk_text == "let") { } else if (tk_text == "let") {
tk_type = tokentype::tk_let; tk_type = tokentype::tk_let;
} else if (tk_text == "in") { } else if (tk_text == "in") {
@ -842,9 +840,6 @@ namespace xo {
(error_type(__FUNCTION__ /*src_function*/, (error_type(__FUNCTION__ /*src_function*/,
"must use \\n or \\r to encode newline/cr in string literal", "must use \\n or \\r to encode newline/cr in string literal",
input_state_, input_state_,
//current_line_,
//current_pos_,
//whitespace.size(),
(ix - tk_start))); (ix - tk_start)));
} }

9
xo-tokenizer/include/xo/tokenizer/tokentype.hpp
View file

@ -49,6 +49,9 @@ namespace xo {
/** sentinel value **/ /** sentinel value **/
tk_invalid = -1, tk_invalid = -1,
/** a boolean constant **/
tk_bool,
/** an integer constant (signed 64-bit integer) **/ /** an integer constant (signed 64-bit integer) **/
tk_i64, tk_i64,
@ -135,6 +138,12 @@ namespace xo {
/** keyword @c 'if' **/ /** keyword @c 'if' **/
tk_if, tk_if,
/** keyworkd @c 'then' **/
tk_then,
/** keyword @c 'else' **/
tk_else,
/** keyword @c 'let' **/ /** keyword @c 'let' **/
tk_let, tk_let,

3
xo-tokenizer/src/tokenizer/tokentype.cpp
View file

@ -13,6 +13,7 @@ namespace xo {
#define CASE(x) case tokentype::x: return STRINGIFY(x) #define CASE(x) case tokentype::x: return STRINGIFY(x)
switch(tk_type) { switch(tk_type) {
CASE(tk_bool);
CASE(tk_i64); CASE(tk_i64);
CASE(tk_f64); CASE(tk_f64);
CASE(tk_string); CASE(tk_string);
@ -46,6 +47,8 @@ namespace xo {
CASE(tk_def); CASE(tk_def);
CASE(tk_lambda); CASE(tk_lambda);
CASE(tk_if); CASE(tk_if);
CASE(tk_then);
CASE(tk_else);
CASE(tk_let); CASE(tk_let);
CASE(tk_in); CASE(tk_in);