roland/xo-expression2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-interpreter: apply expressions + llvm builtins working!

Browse source
This commit is contained in:
Roland Conybeare 2025-11-28 19:32:56 -05:00
commit 1d1e72adf3
31 changed files with 531 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

3
xo-expression/include/xo/expression/Apply.hpp
View file

@ -80,6 +80,9 @@ namespace xo {
const rp<Expression> & fn() const { return fn_; }
const std::vector<rp<Expression>> & argv() const { return argv_; }
std::size_t n_arg() const { return argv_.size(); }
const rp<Expression> & lookup_arg(size_t i) const { return argv_.at(i); }
virtual std::set<std::string> get_free_variables() const override {
std::set<std::string> retval = fn_->get_free_variables();

14
xo-expression/include/xo/expression/PrimitiveExpr.hpp
View file

@ -47,9 +47,9 @@ namespace xo {
public:
static rp<PrimitiveExpr> make(const std::string & name,
FunctionPointer fnptr,
bool explicit_symbol_def,
llvmintrinsic intrinsic) {
FunctionPointer fnptr,
bool explicit_symbol_def,
llvmintrinsic intrinsic) {
TypeDescr fn_type = Reflect::require<FunctionPointer>();
return new PrimitiveExpr(fn_type, name, fnptr, explicit_symbol_def, intrinsic);
@ -60,16 +60,16 @@ namespace xo {
FunctionPointer value() const { return value_; }
TypeDescr value_td() const { return value_td_; }
TaggedPtr value_tp() const {
// ----- PrimitiveExprInterface -----
virtual TaggedPtr value_tp() const final override {
/* note: idk why, but need to spell this out in two steps with gcc 13.2 */
const void * erased_cptr = &value_;
void * erased_ptr = const_cast<void*>(erased_cptr);
return TaggedPtr(value_td_, erased_ptr);
}
// ----- PrimitiveExprInterface -----
virtual llvmintrinsic intrinsic() const override { return intrinsic_; }
virtual bool explicit_symbol_def() const override { return explicit_symbol_def_; }
virtual void_function_type function_address() const override { return reinterpret_cast<void_function_type>(value_); }

7
xo-expression/include/xo/expression/PrimitiveExprInterface.hpp
View file

@ -7,12 +7,14 @@
#include "ProcedureExprInterface.hpp"
#include "llvmintrinsic.hpp"
#include "xo/reflect/TaggedPtr.hpp"
#include <type_traits>
namespace xo {
namespace scm {
class PrimitiveExprInterface : public ProcedureExprInterface {
public:
using TaggedPtr = xo::reflect::TaggedPtr;
using void_function_type = void (*)();
public:
@ -24,6 +26,11 @@ namespace xo {
return bp<PrimitiveExprInterface>::from(x);
}
/** @return executable function as tagged pointer.
* Load-bearing for xo-interpreter.
**/
virtual TaggedPtr value_tp() const = 0;
/** if true, Jit will try to explicitly symbol for this primitive
* (instead of looking it up in host process).
* Don't know if this works.

7
xo-expression/src/expression/PrimitiveExpr.cpp
View file

@ -8,8 +8,11 @@ extern "C" {
* 1. Fallback implementation under llvm.
* In practice will use llvm intrinsic instead.
* See xo-jit/src/jit/MachPipeline.cpp
* 2. Schematika interpreter (aspirational asof jul 2025)
*
* 2. Schematika interpreter (aspirational asof jul 2025, wip nov 2025)
* For schematika interpreter need to uplift
* these to obj::Primitive.
* See BuiltinPrimitives::install_interpreter_conversions()
* in xo-interpreter/src/BuiltinPrimitives.cpp
**/
bool

1
xo-interpreter/example/replxx/replxx.cpp
View file

@ -9,6 +9,7 @@ main(int argc, char ** argv)
using xo::scm::Schematika;
Schematika::Config cfg;
cfg.debug_flag = true;
cfg.vsm_log_level_ = log_level::verbose;
Schematika scm = Schematika::make(cfg);

16
xo-interpreter/include/xo/interpreter/BuiltinPrimitives.hpp
View file

@ -3,6 +3,7 @@
* @author Roland Conybeare, Nov 2025
**/
#include "xo/object/ObjectConverter.hpp"
#include "xo/alloc/IAlloc.hpp"
#include "GlobalEnv.hpp"
@ -10,6 +11,21 @@ namespace xo {
namespace scm {
struct BuiltinPrimitives {
public:
using ObjectConverter = xo::obj::ObjectConverter;
/** install conversions for PrimitiveExpr<Fn> -> Primitive<Fn>
* for particular function pointer types Fn.
*
* Source type from xo-expression
* Dest type from xo-object.
*
* Module dependence goes the other way
* i.e. xo-interpreter -uses-> xo-expression
* -uses-> xo-object
* For this reason rejected adding a virtual method to PrimitiveExprInterface
**/
static void install_interpreter_conversions(ObjectConverter * target);
static void install(gc::IAlloc * mm, gp<GlobalEnv> env);
};
}

29
xo-interpreter/include/xo/interpreter/VirtualSchematikaMachine.hpp
View file

@ -75,14 +75,30 @@ namespace xo {
* - expr_ input, caller saves
* - env_ input, caller saves
* - cont_ input, caller saves
* - stack_ input, caller saves
* - value_ output
* - error_ output
*
**/
void execute_one();
/* design note:
* - eval_xxx_op() methods are primary VSM transitions,
* in the sense that they begin a sequence of transitions to interpret a
* particular kind of expression
* - do_xxx_op() methods represent secondary VSM transitions,
* that continue an instruction sequence that was initiated by a preceding
* eval_xxx_op() method
*/
/** interpret literal constant expression **/
void eval_constant_op();
/** interpreter literal primitive expression
* (these appear implicitly as result of builtin operators like {+, ==, ..})
**/
void eval_primitive_op();
/** execute define expression (finished in do_complete_assign_op()) **/
void eval_define_op();
/** execute assign expression (finishes in do_complete_assign_op()) **/
@ -98,11 +114,22 @@ namespace xo {
/** continue after establish value of test expression **/
void do_complete_ifexpr_op();
/** interprete sequence **/
/** interpret sequence **/
void eval_sequence_op();
/** continue after establishing value for a sequence element **/
void do_complete_sequence_op();
/** interpret apply-expression (i.e. function call) **/
void eval_apply_op();
/** continue assembling args for a function call;
* transition to (interpretation of) called function once all arguments
* are evaluated.
**/
void do_complete_evalargs_op();
/** execute function application, given actuals in top stack frame **/
void apply_op();
/** goto error state with message @p err **/
void report_error(const std::string & err);

11
xo-interpreter/include/xo/interpreter/VsmInstr.hpp
View file

@ -41,6 +41,17 @@ namespace xo {
**/
complete_sequence,
/** execute remainder of argument sequence evaluation;
* subsidiary to marshalling a function call
**/
complete_evalargs,
/** Call a function. Arguments have been evaluated
* and are in top stack frame, in order,
* starting with target function itself
**/
apply,
/** choose branch of if-expression + continue
*
* stack: frame with

7
xo-interpreter/include/xo/interpreter/VsmStackFrame.hpp
View file

@ -23,15 +23,13 @@ namespace xo {
public:
VsmStackFrame(gc::IAlloc * mm, gp<VsmStackFrame> p, std::size_t n, const VsmInstr * cont);
#ifdef NOT_YET
/** create frame using allocator @p mm,
* with parent @p p and exactly @p n_slot object pointers.
**/
static gp<VsmStackFrame> make(gc::IAlloc * mm,
gp<VsmStackFrame> p,
const VsmInstr * cont,
std::size_t n_slot);
#endif
std::size_t n_slot,
const VsmInstr * cont);
/** create new stack frame using allocator @p mm,
* with parent frame @p p; new frame contains values @p s0
@ -56,6 +54,7 @@ namespace xo {
gp<VsmStackFrame> parent() const { return parent_; }
std::size_t size() const { return slot_v_.size(); }
const obj::CVector<gp<Object>> & argv() const { return slot_v_; }
const VsmInstr * continuation() const { return cont_; }
gp<Object> operator[](std::size_t i) const { return slot_v_[i]; }

26
xo-interpreter/src/interpreter/BuiltinPrimitives.cpp
View file

@ -4,14 +4,16 @@
**/
#include "BuiltinPrimitives.hpp"
#include "ObjectConversion.hpp"
#include "Integer.hpp"
#include "Primitive.hpp"
#include "xo/expression/PrimitiveExpr.hpp"
#include "xo/object/ObjectConversion.hpp"
#include "xo/reflect/Reflect.hpp"
#include <cstdint>
namespace xo {
using xo::reflect::Reflect;
using xo::reflect::TaggedPtr;
using xo::reflect::TypeDescr;
namespace scm {
@ -21,6 +23,12 @@ namespace xo {
return x + y;
}
void
BuiltinPrimitives::install_interpreter_conversions(ObjectConverter * /*target*/)
{
/* abandoning this path */
}
void
BuiltinPrimitives::install(gc::IAlloc * mm, gp<GlobalEnv> env)
{
@ -29,12 +37,26 @@ namespace xo {
// add(x,y)
{
gp<Object> rhs = xo::obj::make_primitive(mm, "add", add64);
TypeDescr td = Reflect::require<decltype(add64)>();
TypeDescr td = Reflect::require_function<decltype(&add64)>();
rp<Variable> lhs = Variable::make("add", td);
gp<Object> * addr = env->establish_var(lhs.borrow());
*addr = rhs;
}
// add2_i64
{
auto pm_expr = PrimitiveExpr_i64::make_add2_i64();
gp<Object> rhs = xo::obj::make_primitive(mm, pm_expr->name(), pm_expr->value());
rp<Variable> lhs = Variable::make(pm_expr->name(), pm_expr->value_td());
gp<Object> * addr = env->establish_var(lhs.borrow());
*addr = rhs;
}
}
}
}

16
xo-interpreter/src/interpreter/Schematika.cpp
View file

@ -28,10 +28,7 @@ namespace xo {
* rather than VirtualSchematikaMachine to own allocator
* to preserve option to share it
**/
Impl(const Config & config, up<IAlloc> mm, gp<GlobalEnv> toplevel_env) :
config_{config},
mm_{std::move(mm)},
vsm_{mm_.get(), toplevel_env, config.vsm_log_level_} {}
Impl(const Config & config, up<IAlloc> mm, gp<GlobalEnv> toplevel_env);
~Impl();
/** create instance + allocator **/
@ -68,6 +65,14 @@ namespace xo {
VirtualSchematikaMachine vsm_;
};
Schematika::Impl::Impl(const Config & config, up<IAlloc> mm, gp<GlobalEnv> toplevel_env) :
config_{config},
mm_{std::move(mm)},
vsm_{mm_.get(), toplevel_env, config.vsm_log_level_}
{
}
Schematika::Impl::~Impl() = default;
up<Schematika::Impl>
@ -77,6 +82,9 @@ namespace xo {
rp<GlobalSymtab> symtab = GlobalSymtab::make_empty();
gp<GlobalEnv> env = GlobalEnv::make_empty(mm.get(), symtab);
/* also see VirtualSchematikaMachineFlyweight::Impl::Impl,
* for BuiltinPrimitives::install_interpreter_conversions()
*/
BuiltinPrimitives::install(mm.get(), env);
return std::make_unique<Impl>(cfg, std::move(mm), env);

237
xo-interpreter/src/interpreter/VirtualSchematikaMachine.cpp
View file

@ -2,13 +2,18 @@
#include "VirtualSchematikaMachine.hpp"
#include "VsmInstr.hpp"
#include "BuiltinPrimitives.hpp"
#include "ExpressionBoxed.hpp"
#include "xo/expression/Constant.hpp"
#include "xo/expression/PrimitiveExprInterface.hpp"
#include "xo/expression/DefineExpr.hpp"
#include "xo/expression/AssignExpr.hpp"
#include "xo/expression/Variable.hpp"
#include "xo/expression/IfExpr.hpp"
#include "xo/expression/Sequence.hpp"
#include "xo/expression/Apply.hpp"
#include "xo/object/Procedure.hpp"
#include "xo/object/Primitive.hpp"
#include "xo/object/Integer.hpp"
#include "xo/object/Boolean.hpp"
#include "xo/alloc/GC.hpp"
@ -26,6 +31,7 @@
namespace xo {
using xo::gc::GC;
using xo::obj::Procedure;
using xo::obj::Integer;
using xo::obj::Boolean;
@ -56,9 +62,22 @@ namespace xo {
/** proceed to next element of sequence-expr.
* - opcode is Opcode::complete_sequence
* - top stack fram contains {seq, next, cont}
* - top stack frame contains {seq, next, cont}
*/
static VsmInstr complete_sequence_op;
/** proceed to next argument in apply-expr
* - opcode is Opcode::eval_collect_args
* - top stack frame contains {apply, targetarg, cont}
*/
static VsmInstr complete_evalargs_op;
/** call a procedure, where evaluated arguments (including target function)
* are in top stack frame.
* - opcode is Opcode::apply
* - top stack frame contains evaluated arguments.
**/
static VsmInstr apply_op;
};
VsmInstr
@ -76,6 +95,12 @@ namespace xo {
VsmInstr
VsmOps::complete_sequence_op{VsmInstr::Opcode::complete_sequence, "complete-sequence"};
VsmInstr
VsmOps::complete_evalargs_op{VsmInstr::Opcode::complete_evalargs, "complete-evalargs"};
VsmInstr
VsmOps::apply_op{VsmInstr::Opcode::apply, "apply"};
// ----- VirtualSchematikaMachineFlyweight -----
VirtualSchematikaMachineFlyweight::VirtualSchematikaMachineFlyweight(gc::IAlloc * mm,
@ -84,7 +109,9 @@ namespace xo {
object_mm_{mm},
toplevel_env_{env},
log_level_{ll}
{}
{
BuiltinPrimitives::install_interpreter_conversions(&object_converter_);
}
// ----- VirtualSchematikaMachine -----
@ -188,6 +215,11 @@ namespace xo {
this->eval_constant_op();
break;
case exprtype::primitive:
log && log("eval -> primitive");
this->eval_primitive_op();
break;
case exprtype::define:
log && log("eval -> define");
this->eval_define_op();
@ -213,10 +245,13 @@ namespace xo {
this->eval_sequence_op();
break;
case exprtype::invalid:
case exprtype::primitive:
case exprtype::apply:
log && log("eval -> apply");
this->eval_apply_op();
break;
case exprtype::invalid:
case exprtype::lambda:
case exprtype::convert:
case exprtype::n_expr:
@ -242,6 +277,14 @@ namespace xo {
this->do_complete_sequence_op();
break;
case Opcode::complete_evalargs:
this->do_complete_evalargs_op();
break;
case Opcode::apply:
this->apply_op();
break;
case Opcode::N_Opcode:
assert(false);
break;
@ -286,7 +329,32 @@ namespace xo {
}
}
// placeholder: primitive_op
void
VirtualSchematikaMachine::eval_primitive_op()
{
using xo::obj::Primitive;
using xo::reflect::TaggedPtr;
scope log(XO_DEBUG(true));
bp<PrimitiveExprInterface> expr = PrimitiveExprInterface::from(expr_);
const gp<Object> * slot = env_->lookup_slot(expr->name());
if (slot) {
this->value_ = *slot;
this->pc_ = cont_;
} else {
std::string err = tostr("no binding for primitive", xtag("name", expr->name()));
this->value_ = nullptr;
this->error_ = SchematikaError(err);
/* note: poor man's exception */
this->pc_ = nullptr;
this->cont_ = nullptr;
}
}
void
VirtualSchematikaMachine::eval_define_op()
@ -626,6 +694,163 @@ namespace xo {
this->cont_ = &VsmOps::complete_sequence_op;
}
}
void
VirtualSchematikaMachine::eval_apply_op()
{
/* strategy:
* 1. calling sequence will involve two stack frames.
* 1a. the outer frame will hold 'final evaluated arguments'
* to the called function. When control transfers to that
* function, this frame will be at the top of stack_
* 1b. innert frame will be used by eval_apply_op() and
* helper do_eval_collect_args() to evaluate function
* arguments, and populate the outer frame.
*/
using xo::scm::Apply;
scope log(XO_DEBUG(true));
gc::IAlloc * mm = flyweight_.object_mm_;
/** must promote bp<Apply> -> gp<ExpressionBoxed> **/
gp<ExpressionBoxed> apply_boxed = ExpressionBoxed::make(mm, expr_);
bp<Apply> apply = Apply::from(expr_);
assert(apply.get());
size_t n = apply->n_arg() + 1;
/* reminder: argument 0 refers to the function being called */
gp<Integer> targetarg = Integer::make(mm, 0);
/* outer frame */
gp<VsmStackFrame> argstack = VsmStackFrame::make(mm, stack_, n, cont_);
/* scratch frame during call sequence.
* probably collect->cont_ will not be used?
*/
gp<VsmStackFrame> collect = VsmStackFrame::push2(mm,
argstack,
apply_boxed,
targetarg,
&VsmOps::complete_evalargs_op);
this->pc_ = &VsmOps::eval_op;
this->expr_ = apply->fn();
this->stack_ = collect;
this->cont_ = &VsmOps::complete_evalargs_op;
}
void
VirtualSchematikaMachine::do_complete_evalargs_op()
{
using xo::scm::Apply;
scope log(XO_DEBUG(true));
/* - stack: top frame has 2 slots
* [0] : apply (boxed Apply)
* [1] : targetarg index of next evaluated argument to deliver.
* (to corresponding slot in 2nd frame)
* - 2nd frame has n slots, where n = #of arguments at this site
* [0] : actual #0
* ..
* [targetarg-1] : actual #{targetarg-1}
*/
assert(value_.get());
assert(stack_.get());
gp<VsmStackFrame> sp0 = this->stack_;
assert(sp0.get());
assert(sp0->size() == 2);
bp<Apply> apply = Apply::from(ExpressionBoxed::from((*sp0)[0])->contents());
assert(apply.get());
gp<Integer> targetarg_obj = Integer::from((*stack_)[1]);
size_t targetarg = targetarg_obj->value();
/* note: apply->n_arg() doesn't count function itself */
assert(targetarg < apply->n_arg() + 1);
gp<VsmStackFrame> argstack = sp0->parent();
assert(argstack.get());
/* storing actual parameter in its correct position in call stackframe */
(*argstack)[targetarg] = value_;
++targetarg;
if (targetarg < apply->n_arg() + 1) {
/*
* arguments 0 .. #targetarg-1 already present in argstack
* arguments #targetarg .. #n still need to be evaluated
*/
/* ok to update in place, since Integer in sp0 is unique */
targetarg_obj->assign_value(targetarg);
rp<Expression> targetexpr = apply->lookup_arg(targetarg - 1);
this->pc_ = &VsmOps::eval_op;
this->expr_ = targetexpr;
assert(this->stack_.get() == sp0.get());
this->cont_ = &VsmOps::complete_evalargs_op;
} else {
/* all args evaluated: proceed to invoke evaluated function */
this->pc_ = &VsmOps::apply_op;
this->expr_ = nullptr;
this->stack_ = argstack;
/* unnecessary - will actually be set by apply_op() */
this->cont_ = argstack->continuation();
}
}
void
VirtualSchematikaMachine::apply_op()
{
scope log(XO_DEBUG(true));
auto mm = flyweight_.object_mm_;
// NOTE: Closures will have special handling.
// Could alternatively forward the whole problem
// (along with VSM state) to procedure implementation
/* stack: top frame has n slots for procedure with n canonical args */
gp<VsmStackFrame> sp0 = stack_;
assert(sp0->size() > 0);
gp<Procedure> fn = Procedure::from((*sp0)[0]);
if (fn->n_args() + 1 != sp0->size()) {
throw std::runtime_error(tostr("VirtualSchematikaMachine::apply_op:"
" argument mismatch in apply"
": k arguments supplied where n expected",
xtag("k", sp0->size() - 1),
xtag("n", fn->n_args())));
}
/* todo:
* check function signature?
* should have been guaranteed by expression parser,
* but complications in interactive session when variables redefined.
*/
gp<Object> retval = fn->apply_nocheck(mm, sp0->argv());
this->pc_ = this->cont_;
this->stack_ = sp0->parent();
this->value_ = retval;
}
} /*namespace scm*/
} /*namespace xo*/

14
xo-interpreter/src/interpreter/VsmStackFrame.cpp
View file

@ -34,6 +34,20 @@ namespace xo {
cont_{cont}
{}
gp<VsmStackFrame>
VsmStackFrame::make(gc::IAlloc * mm,
gp<VsmStackFrame> p,
std::size_t n,
const VsmInstr * cont)
{
gp<VsmStackFrame> retval = new (MMPtr(mm)) VsmStackFrame(mm, p, n, cont);
for (std::size_t i = 0; i < n; ++i)
(*retval)[i] = nullptr;
return retval;
}
gp<VsmStackFrame>
VsmStackFrame::push1(gc::IAlloc * mm,
gp<VsmStackFrame> p,

3
xo-object/include/xo/object/Primitive.hpp
View file

@ -56,6 +56,8 @@ namespace xo {
explicit Primitive(std::string_view name,
function_type fn) : PrimitiveBase<Ret (*)(Arg1, Arg2)>{name, fn} {}
/** see also AdoptPrimitiveExpr::adopt() in xo-interpreter/AdoptPrimitiveExpr.hp **/
// inherited from Procedure..
virtual std::size_t n_args() const final override { return 2; }
@ -100,6 +102,7 @@ namespace xo {
make_primitive(gc::IAlloc * mm, std::string_view name, Fn fn) {
return new (MMPtr(mm)) Primitive<Fn>(name, fn);
}
}
}

3
xo-object/include/xo/object/Procedure.hpp
View file

@ -17,6 +17,9 @@ namespace xo {
**/
class Procedure : public Object {
public:
/** downcast from @p x iff x is actually a Procedure. Otherwise nullptr **/
static gp<Procedure> from(gp<Object> x) { return gp<Procedure>::from(x); }
virtual std::size_t n_args() const = 0;
virtual gp<Object> apply_nocheck(gc::IAlloc * mm, const CVector<gp<Object>> & args) = 0;

2
xo-object/src/object/Integer.cpp
View file

@ -24,7 +24,7 @@ namespace xo {
gp<Integer>
Integer::from(gp<Object> x) {
return dynamic_cast<Integer*>(x.ptr());
return gp<Integer>::from(x);
}
TaggedPtr

1
xo-reader/include/xo/reader/apply_xs.hpp
View file

@ -94,6 +94,7 @@ namespace xo {
parserstatemachine * p_psm) override;
virtual void print(std::ostream & os) const override;
virtual bool pretty_print(const print::ppindentinfo & ppii) const final override;
private:
static std::unique_ptr<apply_xs> make();

1
xo-reader/include/xo/reader/expect_expr_xs.hpp
View file

@ -15,6 +15,7 @@ namespace xo {
*
* Examples:
* @text
* add (1, 2) ;
* def x : i64 = 5 ; // with allow_defs
* lambda (f : x64) { ... } ;
* if (prime(x)) { ... } ;

10
xo-reader/include/xo/reader/lambda_xs.hpp
View file

@ -55,8 +55,8 @@ namespace xo {
**/
class lambda_xs : public exprstate {
public:
using Environment = xo::scm::SymbolTable;
using LocalEnv = xo::scm::LocalSymtab;
using SymbolTable = xo::scm::SymbolTable;
using LocalSymtab = xo::scm::LocalSymtab;
public:
lambda_xs();
@ -81,6 +81,8 @@ namespace xo {
parserstatemachine * p_psm) override;
virtual void on_semicolon_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_f64_token(const token_type & tk,
parserstatemachine * p_psm) final override;
virtual void print(std::ostream & os) const override;
virtual bool pretty_print(const print::ppindentinfo & ppii) const override;
@ -93,7 +95,7 @@ namespace xo {
lambdastatetype lmxs_type_ = lambdastatetype::lm_0;
/** lambda environment (for formal parameters) **/
rp<LocalEnv> local_env_;
rp<LocalSymtab> local_env_;
/** explicit return type (if supplied) **/
TypeDescr explicit_return_td_ = nullptr;
@ -105,7 +107,7 @@ namespace xo {
rp<Expression> body_;
/** parent environment **/
rp<Environment> parent_env_;
rp<SymbolTable> parent_env_;
};
} /*namespace scm*/

1
xo-reader/include/xo/reader/parserstatemachine.hpp
View file

@ -83,6 +83,7 @@ namespace xo {
void on_rightbrace_token(const token_type & tk);
void on_then_token(const token_type & tk);
void on_else_token(const token_type & tk);
void on_f64_token(const token_type & tk);
// ----- parsing error -----

6
xo-reader/include/xo/reader/progress_xs.hpp
View file

@ -73,6 +73,10 @@ namespace xo {
* Deals with infix operators, handles operator precedence.
* Also generates argument-type-specific arithmetic expressions,
* for example using ``Apply::make_add2_f64()`` when adding floating-point numbers
*
* One reason for this to exist is to simulate one-token lookahead.
* To look at but not consume a token T, can push a progress_xs instance P,
* then send T to P.
**/
class progress_xs : public exprstate {
public:
@ -105,6 +109,8 @@ namespace xo {
parserstatemachine * p_psm) override;
virtual void on_symbol_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_comma_token(const token_type & tk,
parserstatemachine * p_psm) final override;
virtual void on_typedescr(TypeDescr td,
parserstatemachine * p_psm) override;

13
xo-reader/src/reader/apply_xs.cpp
View file

@ -76,13 +76,16 @@ namespace xo {
// unreachable
break;
case applyexprstatetype::apply_0:
log && log("stash fn -> new state apply_1");
this->fn_expr_ = expr.promote();
this->applyxs_type_ = applyexprstatetype::apply_1;
return;
case applyexprstatetype::apply_1:
log && log("error: was expecting lparen");
// error, expecting lparen
break;
case applyexprstatetype::apply_2:
log && log(xtag("expr", expr), xtag("do", "stash expr -> new state apply_3"));
this->args_expr_v_.push_back(expr.promote());
this->applyxs_type_ = applyexprstatetype::apply_3;
return;
@ -145,6 +148,7 @@ namespace xo {
if (this->applyxs_type_ == applyexprstatetype::apply_3) {
/* (done) state */
log("apply complete -> pop + send expr");
rp<Apply> apply_expr = Apply::make(this->fn_expr_, this->args_expr_v_);
@ -169,6 +173,15 @@ namespace xo {
os << ">";
}
bool
apply_xs::pretty_print(const xo::print::ppindentinfo & ppii) const
{
return ppii.pps()->pretty_struct(ppii, "apply_xs",
refrtag("applyxs_type", applyxs_type_),
refrtag("fn_expr", fn_expr_),
refrtag("args_expr_v", args_expr_v_));
}
} /*namespace scm*/
} /*namespace xo*/

6
xo-reader/src/reader/expect_expr_xs.cpp
View file

@ -201,7 +201,9 @@ namespace xo {
expect_expr_xs::on_i64_token(const token_type & tk,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()));
scope log(XO_DEBUG(p_psm->debug_flag()),
xtag("tk", tk),
xtag("do", "push progress xs w/ tk value"));
progress_xs::start
(Constant<int64_t>::make(tk.i64_value()),
@ -233,6 +235,7 @@ namespace xo {
log && log(xtag("exstype", this->exs_type_),
xtag("expr", expr.promote()));
log && log("pop expect_expr_xs, forward to parent");
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
@ -247,6 +250,7 @@ namespace xo {
log && log(xtag("exstype", this->exs_type_),
xtag("expr", expr.promote()));
log && log("pop expect_expr_xs, forward to parent");
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();

2
xo-reader/src/reader/expect_type_xs.cpp
View file

@ -59,7 +59,7 @@ namespace xo {
if (!td) {
const char * exp = get_expect_str();
std::string errmsg = tostr("unexpected token for parsing state",
std::string errmsg = tostr("expect_type_xs: unexpected token for parsing state",
xtag("expecting", exp),
xtag("token", tk.tk_type()),
xtag("text", tk.text()),

1
xo-reader/src/reader/exprseq_xs.cpp
View file

@ -97,6 +97,7 @@ namespace xo {
* a = 1; // single assignment
* a == 1; // rhs expression
* a + b; // rhs expression
* a(1,2); // apply expression
* Variable must have been defined!
*/
bp<Variable> var = p_psm->lookup_var(tk.text());

2
xo-reader/src/reader/exprstate.cpp
View file

@ -610,7 +610,7 @@ namespace xo {
const char * expect_str,
parserstatemachine * p_psm) const
{
std::string errmsg = tostr("unexpected token for parsing state",
std::string errmsg = tostr("exprstate::illegal_input_on_token: unexpected token for parsing state",
xtag("expecting", expect_str),
xtag("token", tk.tk_type()),
xtag("text", tk.text()),

30
xo-reader/src/reader/lambda_xs.cpp
View file

@ -102,7 +102,7 @@ namespace xo {
if (lmxs_type_ == lambdastatetype::lm_1) {
this->lmxs_type_ = lambdastatetype::lm_2;
this->parent_env_ = p_psm->top_envframe().promote();
this->local_env_ = LocalEnv::make(argl, parent_env_);
this->local_env_ = LocalSymtab::make(argl, parent_env_);
p_psm->push_envframe(local_env_);
@ -268,6 +268,31 @@ namespace xo {
exprstate::on_semicolon_token(tk, p_psm);
}
void
lambda_xs::on_f64_token(const token_type & tk,
parserstatemachine * p_psm)
{
constexpr const char * c_self_name = "lambda_xs::on_f64_token";
/* f64 literal can begin lambda body, otherwise illegal.
* for example:
* def foo = lambda (x: bool) 3.14;
*/
if (lmxs_type_ == lambdastatetype::lm_2) {
/* omitting return type.
* omitting left brace.
*/
this->lmxs_type_ = lambdastatetype::lm_4;
expect_expr_xs::start(p_psm);
p_psm->on_f64_token(tk);
} else {
this->illegal_input_on_token(c_self_name, tk, this->get_expect_str(), p_psm);
}
}
// TODO: on_i64_token, on_bool token
void
lambda_xs::print(std::ostream & os) const {
os << "<lambda_xs"
@ -279,7 +304,8 @@ namespace xo {
lambda_xs::pretty_print(const xo::print::ppindentinfo & ppii) const
{
return ppii.pps()->pretty_struct(ppii, "lambda_xs",
refrtag("lmxs_type", lmxs_type_));
refrtag("lmxs_type", lmxs_type_),
refrtag("body", body_));
}
} /*namespace scm*/

11
xo-reader/src/reader/parserstatemachine.cpp
View file

@ -172,6 +172,17 @@ namespace xo {
this->xs_stack_.top_exprstate().on_else_token(tk, this);
}
void
parserstatemachine::on_f64_token(const token_type & tk)
{
scope log(XO_DEBUG(debug_flag_));
log && log(xtag("tk", tk),
xtag("psm", this));
this->xs_stack_.top_exprstate().on_f64_token(tk, this);
}
void
parserstatemachine::on_error(const char * self_name, std::string errmsg)
{

96
xo-reader/src/reader/progress_xs.cpp
View file

@ -310,6 +310,8 @@ namespace xo {
progress_xs::on_expr(bp<Expression> expr,
parserstatemachine * p_psm)
{
scope log(XO_DEBUG(p_psm->debug_flag()), xtag("expr", expr));
/* note: previous token probably an operator,
* handled from progress_xs::on_operator_token(),
* which pushes expect_expr_xs::expect_rhs_expression()
@ -318,21 +320,33 @@ namespace xo {
constexpr const char * c_self_name = "progress_xs::on_expr";
const char * exp = get_expect_str();
if (op_type_ == optype::invalid) {
this->illegal_input_on_expr(c_self_name, expr, exp, p_psm);
}
if (lhs_) {
if (op_type_ == optype::invalid) {
/* two consecutive expression without an operator */
this->illegal_input_on_expr(c_self_name, expr, exp, p_psm);
}
#ifdef NOT_QUITE
assert(result.get());
assert(result.get());
/* this expression complete.. */
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
/* this expression complete.. */
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
/* ..but more operators could follow, so don't commit yet */
p_stack->push_exprstate(progress_xs::make(result));
/* ..but more operators could follow, so don't commit yet */
p_stack->push_exprstate(progress_xs::make(result));
#endif
this->rhs_ = expr.promote();
this->rhs_ = expr.promote();
} else {
/* control here on input like
* add(1,2)...
*
* add(1,2) needs to be handled inside a progress_xs
* instance because may be followed by an operator:
* add(1,2) + ...
*/
this->lhs_ = expr.promote();
}
}
void
@ -375,6 +389,46 @@ namespace xo {
assert(false);
}
void
progress_xs::on_comma_token(const token_type & tk,
parserstatemachine * p_psm)
{
/* note: implementation parllels .on_semicolon_token(), .on_rightparen_token() */
scope log(XO_DEBUG(p_psm->debug_flag()));
constexpr const char * self_name = "progress::xs::on_comma_token";
auto & xs_stack = p_psm->xs_stack_;
/* stack may be something like
*
* applyexpr
* expect_expr_xs
* progress_xs
* <-- comma
*
* 1. comma completes expression-in-progress
*/
/* comma confirms stack expression */
rp<Expression> expr = this->assemble_expr(p_psm);
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
if (xs_stack.empty()) {
throw std::runtime_error(tostr(self_name,
": expected non-empty parsing state"));
}
log && log(xtag("stack", &xs_stack));
p_psm->top_exprstate().on_expr(expr, p_psm);
/* now deliver comma */
p_psm->top_exprstate().on_comma_token(tk, p_psm);
}
void
progress_xs::on_typedescr(TypeDescr /*td*/,
parserstatemachine * /*p_psm*/)
@ -441,6 +495,7 @@ namespace xo {
this->on_operator_token(tk, p_psm);
}
/* editor bait: on_lparen */
void
progress_xs::on_leftparen_token(const token_type & tk,
parserstatemachine * p_psm)
@ -456,12 +511,24 @@ namespace xo {
/* start function call */
assert(rhs_.get() == nullptr);
/* unwind this progress_xs + replace with function call */
rp<Expression> fn_expr = lhs_;
/* reset this progress_xs back to empty state;
* apply_xs will be responsible for lhs_.
*/
lhs_ = nullptr;
#ifdef OBSOLETE
/* don't unwind! want to handle input like
* f(x,y)+g(z)
*/
/* unwind this progress_xs + replace with function call */
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
#endif
apply_xs::start(fn_expr, p_psm);
/* control will reenter progress_xs via .on_expr() */
return;
}
@ -486,7 +553,7 @@ namespace xo {
/* stack may be something like:
*
* lparen_0
* expect_rhs_expression
* expect_expr_xs
* expr_progress
* <-- rightparen
*
@ -497,6 +564,9 @@ namespace xo {
/* right paren confirms stack expression */
rp<Expression> expr = this->assemble_expr(p_psm);
log && log(xtag("expr", expr),
xtag("do", "pop self + send {expr, rparen} -> parent"));
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
if (xs_stack.empty()) {
@ -756,7 +826,7 @@ namespace xo {
&& (rhs_ ? ppii.pps()->print_upto(refrtag("rhs", rhs_)) : true)
&& ppii.pps()->print_upto(">"));
} else {
ppii.pps()->write("<progress_xs");
ppii.pps()->write("<progress_xs ");
if (lhs_)
ppii.pps()->pretty(refrtag("lhs", lhs_));
if (op_type_ != optype::invalid)

5
xo-reader/utest/reader.test.cpp
View file

@ -21,6 +21,7 @@ namespace xo {
{"def foo = lambda (x : f64, y : f64) 3.1415965;"},
{"def foo = lambda (x : f64) x;"},
{"def foo = lambda (x : f64) { def y = x * x; y; };"},
{"add(1,2);"},
};
}
@ -50,11 +51,13 @@ namespace xo {
log && log(xtag("expr", rr.expr_));
#ifdef OBSOLETE // input not consumed until next line presented
input = input.after_prefix(rr.rem_);
log && log(xtag("post.input", input));
#endif
REQUIRE(input.empty());
//REQUIRE(input.empty());
} catch (std::exception & ex) {
log && log(ex.what());

2
xo-tokenizer/include/xo/tokenizer/input_state.hpp
View file

@ -247,7 +247,7 @@ namespace xo {
input_state<CharT>::advance_until(const CharT * pos) {
scope log(XO_DEBUG(debug_flag_));
assert(current_line_.lo() <= pos && pos < current_line_.hi());
assert(current_line_.lo() <= pos && pos <= current_line_.hi());
this->current_pos_ = pos - current_line_.lo();