diff --git a/xo-expression/include/xo/expression/Apply.hpp b/xo-expression/include/xo/expression/Apply.hpp index 003b49fa..7dd66a7e 100644 --- a/xo-expression/include/xo/expression/Apply.hpp +++ b/xo-expression/include/xo/expression/Apply.hpp @@ -80,6 +80,9 @@ namespace xo { const rp & fn() const { return fn_; } const std::vector> & argv() const { return argv_; } + std::size_t n_arg() const { return argv_.size(); } + const rp & lookup_arg(size_t i) const { return argv_.at(i); } + virtual std::set get_free_variables() const override { std::set retval = fn_->get_free_variables(); diff --git a/xo-expression/include/xo/expression/PrimitiveExpr.hpp b/xo-expression/include/xo/expression/PrimitiveExpr.hpp index 6050e8db..86ca5e03 100644 --- a/xo-expression/include/xo/expression/PrimitiveExpr.hpp +++ b/xo-expression/include/xo/expression/PrimitiveExpr.hpp @@ -47,9 +47,9 @@ namespace xo { public: static rp make(const std::string & name, - FunctionPointer fnptr, - bool explicit_symbol_def, - llvmintrinsic intrinsic) { + FunctionPointer fnptr, + bool explicit_symbol_def, + llvmintrinsic intrinsic) { TypeDescr fn_type = Reflect::require(); return new PrimitiveExpr(fn_type, name, fnptr, explicit_symbol_def, intrinsic); @@ -60,16 +60,16 @@ namespace xo { FunctionPointer value() const { return value_; } TypeDescr value_td() const { return value_td_; } - TaggedPtr value_tp() const { + + // ----- PrimitiveExprInterface ----- + + virtual TaggedPtr value_tp() const final override { /* note: idk why, but need to spell this out in two steps with gcc 13.2 */ const void * erased_cptr = &value_; void * erased_ptr = const_cast(erased_cptr); return TaggedPtr(value_td_, erased_ptr); } - - // ----- PrimitiveExprInterface ----- - virtual llvmintrinsic intrinsic() const override { return intrinsic_; } virtual bool explicit_symbol_def() const override { return explicit_symbol_def_; } virtual void_function_type function_address() const override { return reinterpret_cast(value_); } diff --git a/xo-expression/include/xo/expression/PrimitiveExprInterface.hpp b/xo-expression/include/xo/expression/PrimitiveExprInterface.hpp index 9db7be14..0fbd0ae3 100644 --- a/xo-expression/include/xo/expression/PrimitiveExprInterface.hpp +++ b/xo-expression/include/xo/expression/PrimitiveExprInterface.hpp @@ -7,12 +7,14 @@ #include "ProcedureExprInterface.hpp" #include "llvmintrinsic.hpp" +#include "xo/reflect/TaggedPtr.hpp" #include namespace xo { namespace scm { class PrimitiveExprInterface : public ProcedureExprInterface { public: + using TaggedPtr = xo::reflect::TaggedPtr; using void_function_type = void (*)(); public: @@ -24,6 +26,11 @@ namespace xo { return bp::from(x); } + /** @return executable function as tagged pointer. + * Load-bearing for xo-interpreter. + **/ + virtual TaggedPtr value_tp() const = 0; + /** if true, Jit will try to explicitly symbol for this primitive * (instead of looking it up in host process). * Don't know if this works. diff --git a/xo-expression/src/expression/PrimitiveExpr.cpp b/xo-expression/src/expression/PrimitiveExpr.cpp index 09246a62..4d73b14c 100644 --- a/xo-expression/src/expression/PrimitiveExpr.cpp +++ b/xo-expression/src/expression/PrimitiveExpr.cpp @@ -8,8 +8,11 @@ extern "C" { * 1. Fallback implementation under llvm. * In practice will use llvm intrinsic instead. * See xo-jit/src/jit/MachPipeline.cpp - * 2. Schematika interpreter (aspirational asof jul 2025) - * + * 2. Schematika interpreter (aspirational asof jul 2025, wip nov 2025) + * For schematika interpreter need to uplift + * these to obj::Primitive. + * See BuiltinPrimitives::install_interpreter_conversions() + * in xo-interpreter/src/BuiltinPrimitives.cpp **/ bool diff --git a/xo-interpreter/example/replxx/replxx.cpp b/xo-interpreter/example/replxx/replxx.cpp index 5d9eb9ab..ef677594 100644 --- a/xo-interpreter/example/replxx/replxx.cpp +++ b/xo-interpreter/example/replxx/replxx.cpp @@ -9,6 +9,7 @@ main(int argc, char ** argv) using xo::scm::Schematika; Schematika::Config cfg; + cfg.debug_flag = true; cfg.vsm_log_level_ = log_level::verbose; Schematika scm = Schematika::make(cfg); diff --git a/xo-interpreter/include/xo/interpreter/BuiltinPrimitives.hpp b/xo-interpreter/include/xo/interpreter/BuiltinPrimitives.hpp index 3b23be69..300e7f08 100644 --- a/xo-interpreter/include/xo/interpreter/BuiltinPrimitives.hpp +++ b/xo-interpreter/include/xo/interpreter/BuiltinPrimitives.hpp @@ -3,6 +3,7 @@ * @author Roland Conybeare, Nov 2025 **/ +#include "xo/object/ObjectConverter.hpp" #include "xo/alloc/IAlloc.hpp" #include "GlobalEnv.hpp" @@ -10,6 +11,21 @@ namespace xo { namespace scm { struct BuiltinPrimitives { public: + using ObjectConverter = xo::obj::ObjectConverter; + + /** install conversions for PrimitiveExpr -> Primitive + * for particular function pointer types Fn. + * + * Source type from xo-expression + * Dest type from xo-object. + * + * Module dependence goes the other way + * i.e. xo-interpreter -uses-> xo-expression + * -uses-> xo-object + * For this reason rejected adding a virtual method to PrimitiveExprInterface + **/ + static void install_interpreter_conversions(ObjectConverter * target); + static void install(gc::IAlloc * mm, gp env); }; } diff --git a/xo-interpreter/include/xo/interpreter/VirtualSchematikaMachine.hpp b/xo-interpreter/include/xo/interpreter/VirtualSchematikaMachine.hpp index 5b4c29ed..a8a182d9 100644 --- a/xo-interpreter/include/xo/interpreter/VirtualSchematikaMachine.hpp +++ b/xo-interpreter/include/xo/interpreter/VirtualSchematikaMachine.hpp @@ -75,14 +75,30 @@ namespace xo { * - expr_ input, caller saves * - env_ input, caller saves * - cont_ input, caller saves + * - stack_ input, caller saves * - value_ output * - error_ output + * **/ void execute_one(); + /* design note: + * - eval_xxx_op() methods are primary VSM transitions, + * in the sense that they begin a sequence of transitions to interpret a + * particular kind of expression + * - do_xxx_op() methods represent secondary VSM transitions, + * that continue an instruction sequence that was initiated by a preceding + * eval_xxx_op() method + */ + /** interpret literal constant expression **/ void eval_constant_op(); + /** interpreter literal primitive expression + * (these appear implicitly as result of builtin operators like {+, ==, ..}) + **/ + void eval_primitive_op(); + /** execute define expression (finished in do_complete_assign_op()) **/ void eval_define_op(); /** execute assign expression (finishes in do_complete_assign_op()) **/ @@ -98,11 +114,22 @@ namespace xo { /** continue after establish value of test expression **/ void do_complete_ifexpr_op(); - /** interprete sequence **/ + /** interpret sequence **/ void eval_sequence_op(); /** continue after establishing value for a sequence element **/ void do_complete_sequence_op(); + /** interpret apply-expression (i.e. function call) **/ + void eval_apply_op(); + /** continue assembling args for a function call; + * transition to (interpretation of) called function once all arguments + * are evaluated. + **/ + void do_complete_evalargs_op(); + + /** execute function application, given actuals in top stack frame **/ + void apply_op(); + /** goto error state with message @p err **/ void report_error(const std::string & err); diff --git a/xo-interpreter/include/xo/interpreter/VsmInstr.hpp b/xo-interpreter/include/xo/interpreter/VsmInstr.hpp index 1f82fd7c..a1539bd0 100644 --- a/xo-interpreter/include/xo/interpreter/VsmInstr.hpp +++ b/xo-interpreter/include/xo/interpreter/VsmInstr.hpp @@ -41,6 +41,17 @@ namespace xo { **/ complete_sequence, + /** execute remainder of argument sequence evaluation; + * subsidiary to marshalling a function call + **/ + complete_evalargs, + + /** Call a function. Arguments have been evaluated + * and are in top stack frame, in order, + * starting with target function itself + **/ + apply, + /** choose branch of if-expression + continue * * stack: frame with diff --git a/xo-interpreter/include/xo/interpreter/VsmStackFrame.hpp b/xo-interpreter/include/xo/interpreter/VsmStackFrame.hpp index 222def45..0fc0dee7 100644 --- a/xo-interpreter/include/xo/interpreter/VsmStackFrame.hpp +++ b/xo-interpreter/include/xo/interpreter/VsmStackFrame.hpp @@ -23,15 +23,13 @@ namespace xo { public: VsmStackFrame(gc::IAlloc * mm, gp p, std::size_t n, const VsmInstr * cont); -#ifdef NOT_YET /** create frame using allocator @p mm, * with parent @p p and exactly @p n_slot object pointers. **/ static gp make(gc::IAlloc * mm, gp p, - const VsmInstr * cont, - std::size_t n_slot); -#endif + std::size_t n_slot, + const VsmInstr * cont); /** create new stack frame using allocator @p mm, * with parent frame @p p; new frame contains values @p s0 @@ -56,6 +54,7 @@ namespace xo { gp parent() const { return parent_; } std::size_t size() const { return slot_v_.size(); } + const obj::CVector> & argv() const { return slot_v_; } const VsmInstr * continuation() const { return cont_; } gp operator[](std::size_t i) const { return slot_v_[i]; } diff --git a/xo-interpreter/src/interpreter/BuiltinPrimitives.cpp b/xo-interpreter/src/interpreter/BuiltinPrimitives.cpp index 7224c92a..edc8bdae 100644 --- a/xo-interpreter/src/interpreter/BuiltinPrimitives.cpp +++ b/xo-interpreter/src/interpreter/BuiltinPrimitives.cpp @@ -4,14 +4,16 @@ **/ #include "BuiltinPrimitives.hpp" -#include "ObjectConversion.hpp" #include "Integer.hpp" #include "Primitive.hpp" +#include "xo/expression/PrimitiveExpr.hpp" +#include "xo/object/ObjectConversion.hpp" #include "xo/reflect/Reflect.hpp" #include namespace xo { using xo::reflect::Reflect; + using xo::reflect::TaggedPtr; using xo::reflect::TypeDescr; namespace scm { @@ -21,6 +23,12 @@ namespace xo { return x + y; } + void + BuiltinPrimitives::install_interpreter_conversions(ObjectConverter * /*target*/) + { + /* abandoning this path */ + } + void BuiltinPrimitives::install(gc::IAlloc * mm, gp env) { @@ -29,12 +37,26 @@ namespace xo { // add(x,y) { gp rhs = xo::obj::make_primitive(mm, "add", add64); - TypeDescr td = Reflect::require(); + + TypeDescr td = Reflect::require_function(); + rp lhs = Variable::make("add", td); gp * addr = env->establish_var(lhs.borrow()); *addr = rhs; } + + // add2_i64 + { + auto pm_expr = PrimitiveExpr_i64::make_add2_i64(); + + gp rhs = xo::obj::make_primitive(mm, pm_expr->name(), pm_expr->value()); + + rp lhs = Variable::make(pm_expr->name(), pm_expr->value_td()); + gp * addr = env->establish_var(lhs.borrow()); + + *addr = rhs; + } } } } diff --git a/xo-interpreter/src/interpreter/Schematika.cpp b/xo-interpreter/src/interpreter/Schematika.cpp index afa0a9ff..039b2e67 100644 --- a/xo-interpreter/src/interpreter/Schematika.cpp +++ b/xo-interpreter/src/interpreter/Schematika.cpp @@ -28,10 +28,7 @@ namespace xo { * rather than VirtualSchematikaMachine to own allocator * to preserve option to share it **/ - Impl(const Config & config, up mm, gp toplevel_env) : - config_{config}, - mm_{std::move(mm)}, - vsm_{mm_.get(), toplevel_env, config.vsm_log_level_} {} + Impl(const Config & config, up mm, gp toplevel_env); ~Impl(); /** create instance + allocator **/ @@ -68,6 +65,14 @@ namespace xo { VirtualSchematikaMachine vsm_; }; + Schematika::Impl::Impl(const Config & config, up mm, gp toplevel_env) : + config_{config}, + mm_{std::move(mm)}, + vsm_{mm_.get(), toplevel_env, config.vsm_log_level_} + { + + } + Schematika::Impl::~Impl() = default; up @@ -77,6 +82,9 @@ namespace xo { rp symtab = GlobalSymtab::make_empty(); gp env = GlobalEnv::make_empty(mm.get(), symtab); + /* also see VirtualSchematikaMachineFlyweight::Impl::Impl, + * for BuiltinPrimitives::install_interpreter_conversions() + */ BuiltinPrimitives::install(mm.get(), env); return std::make_unique(cfg, std::move(mm), env); diff --git a/xo-interpreter/src/interpreter/VirtualSchematikaMachine.cpp b/xo-interpreter/src/interpreter/VirtualSchematikaMachine.cpp index 0927e395..9a02f22b 100644 --- a/xo-interpreter/src/interpreter/VirtualSchematikaMachine.cpp +++ b/xo-interpreter/src/interpreter/VirtualSchematikaMachine.cpp @@ -2,13 +2,18 @@ #include "VirtualSchematikaMachine.hpp" #include "VsmInstr.hpp" +#include "BuiltinPrimitives.hpp" #include "ExpressionBoxed.hpp" #include "xo/expression/Constant.hpp" +#include "xo/expression/PrimitiveExprInterface.hpp" #include "xo/expression/DefineExpr.hpp" #include "xo/expression/AssignExpr.hpp" #include "xo/expression/Variable.hpp" #include "xo/expression/IfExpr.hpp" #include "xo/expression/Sequence.hpp" +#include "xo/expression/Apply.hpp" +#include "xo/object/Procedure.hpp" +#include "xo/object/Primitive.hpp" #include "xo/object/Integer.hpp" #include "xo/object/Boolean.hpp" #include "xo/alloc/GC.hpp" @@ -26,6 +31,7 @@ namespace xo { using xo::gc::GC; + using xo::obj::Procedure; using xo::obj::Integer; using xo::obj::Boolean; @@ -56,9 +62,22 @@ namespace xo { /** proceed to next element of sequence-expr. * - opcode is Opcode::complete_sequence - * - top stack fram contains {seq, next, cont} + * - top stack frame contains {seq, next, cont} */ static VsmInstr complete_sequence_op; + + /** proceed to next argument in apply-expr + * - opcode is Opcode::eval_collect_args + * - top stack frame contains {apply, targetarg, cont} + */ + static VsmInstr complete_evalargs_op; + + /** call a procedure, where evaluated arguments (including target function) + * are in top stack frame. + * - opcode is Opcode::apply + * - top stack frame contains evaluated arguments. + **/ + static VsmInstr apply_op; }; VsmInstr @@ -76,6 +95,12 @@ namespace xo { VsmInstr VsmOps::complete_sequence_op{VsmInstr::Opcode::complete_sequence, "complete-sequence"}; + VsmInstr + VsmOps::complete_evalargs_op{VsmInstr::Opcode::complete_evalargs, "complete-evalargs"}; + + VsmInstr + VsmOps::apply_op{VsmInstr::Opcode::apply, "apply"}; + // ----- VirtualSchematikaMachineFlyweight ----- VirtualSchematikaMachineFlyweight::VirtualSchematikaMachineFlyweight(gc::IAlloc * mm, @@ -84,7 +109,9 @@ namespace xo { object_mm_{mm}, toplevel_env_{env}, log_level_{ll} - {} + { + BuiltinPrimitives::install_interpreter_conversions(&object_converter_); + } // ----- VirtualSchematikaMachine ----- @@ -188,6 +215,11 @@ namespace xo { this->eval_constant_op(); break; + case exprtype::primitive: + log && log("eval -> primitive"); + this->eval_primitive_op(); + break; + case exprtype::define: log && log("eval -> define"); this->eval_define_op(); @@ -213,10 +245,13 @@ namespace xo { this->eval_sequence_op(); break; - case exprtype::invalid: - case exprtype::primitive: - case exprtype::apply: + log && log("eval -> apply"); + this->eval_apply_op(); + break; + + case exprtype::invalid: + case exprtype::lambda: case exprtype::convert: case exprtype::n_expr: @@ -242,6 +277,14 @@ namespace xo { this->do_complete_sequence_op(); break; + case Opcode::complete_evalargs: + this->do_complete_evalargs_op(); + break; + + case Opcode::apply: + this->apply_op(); + break; + case Opcode::N_Opcode: assert(false); break; @@ -286,7 +329,32 @@ namespace xo { } } - // placeholder: primitive_op + void + VirtualSchematikaMachine::eval_primitive_op() + { + using xo::obj::Primitive; + using xo::reflect::TaggedPtr; + + scope log(XO_DEBUG(true)); + + bp expr = PrimitiveExprInterface::from(expr_); + + const gp * slot = env_->lookup_slot(expr->name()); + + if (slot) { + this->value_ = *slot; + this->pc_ = cont_; + } else { + std::string err = tostr("no binding for primitive", xtag("name", expr->name())); + + this->value_ = nullptr; + this->error_ = SchematikaError(err); + + /* note: poor man's exception */ + this->pc_ = nullptr; + this->cont_ = nullptr; + } + } void VirtualSchematikaMachine::eval_define_op() @@ -626,6 +694,163 @@ namespace xo { this->cont_ = &VsmOps::complete_sequence_op; } } + + void + VirtualSchematikaMachine::eval_apply_op() + { + /* strategy: + * 1. calling sequence will involve two stack frames. + * 1a. the outer frame will hold 'final evaluated arguments' + * to the called function. When control transfers to that + * function, this frame will be at the top of stack_ + * 1b. innert frame will be used by eval_apply_op() and + * helper do_eval_collect_args() to evaluate function + * arguments, and populate the outer frame. + */ + + using xo::scm::Apply; + + scope log(XO_DEBUG(true)); + + gc::IAlloc * mm = flyweight_.object_mm_; + + /** must promote bp -> gp **/ + gp apply_boxed = ExpressionBoxed::make(mm, expr_); + bp apply = Apply::from(expr_); + + assert(apply.get()); + + size_t n = apply->n_arg() + 1; + + /* reminder: argument 0 refers to the function being called */ + gp targetarg = Integer::make(mm, 0); + + /* outer frame */ + gp argstack = VsmStackFrame::make(mm, stack_, n, cont_); + + /* scratch frame during call sequence. + * probably collect->cont_ will not be used? + */ + gp collect = VsmStackFrame::push2(mm, + argstack, + apply_boxed, + targetarg, + &VsmOps::complete_evalargs_op); + + this->pc_ = &VsmOps::eval_op; + this->expr_ = apply->fn(); + this->stack_ = collect; + this->cont_ = &VsmOps::complete_evalargs_op; + } + + void + VirtualSchematikaMachine::do_complete_evalargs_op() + { + using xo::scm::Apply; + + scope log(XO_DEBUG(true)); + + /* - stack: top frame has 2 slots + * [0] : apply (boxed Apply) + * [1] : targetarg index of next evaluated argument to deliver. + * (to corresponding slot in 2nd frame) + * - 2nd frame has n slots, where n = #of arguments at this site + * [0] : actual #0 + * .. + * [targetarg-1] : actual #{targetarg-1} + */ + + assert(value_.get()); + assert(stack_.get()); + + gp sp0 = this->stack_; + + assert(sp0.get()); + assert(sp0->size() == 2); + + bp apply = Apply::from(ExpressionBoxed::from((*sp0)[0])->contents()); + assert(apply.get()); + gp targetarg_obj = Integer::from((*stack_)[1]); + size_t targetarg = targetarg_obj->value(); + + /* note: apply->n_arg() doesn't count function itself */ + assert(targetarg < apply->n_arg() + 1); + + gp argstack = sp0->parent(); + + assert(argstack.get()); + + /* storing actual parameter in its correct position in call stackframe */ + (*argstack)[targetarg] = value_; + + ++targetarg; + + if (targetarg < apply->n_arg() + 1) { + /* + * arguments 0 .. #targetarg-1 already present in argstack + * arguments #targetarg .. #n still need to be evaluated + */ + + /* ok to update in place, since Integer in sp0 is unique */ + targetarg_obj->assign_value(targetarg); + + rp targetexpr = apply->lookup_arg(targetarg - 1); + + this->pc_ = &VsmOps::eval_op; + this->expr_ = targetexpr; + assert(this->stack_.get() == sp0.get()); + this->cont_ = &VsmOps::complete_evalargs_op; + } else { + /* all args evaluated: proceed to invoke evaluated function */ + + this->pc_ = &VsmOps::apply_op; + this->expr_ = nullptr; + this->stack_ = argstack; + /* unnecessary - will actually be set by apply_op() */ + this->cont_ = argstack->continuation(); + } + } + + void + VirtualSchematikaMachine::apply_op() + { + scope log(XO_DEBUG(true)); + + auto mm = flyweight_.object_mm_; + + // NOTE: Closures will have special handling. + // Could alternatively forward the whole problem + // (along with VSM state) to procedure implementation + + /* stack: top frame has n slots for procedure with n canonical args */ + + gp sp0 = stack_; + + assert(sp0->size() > 0); + + gp fn = Procedure::from((*sp0)[0]); + + if (fn->n_args() + 1 != sp0->size()) { + throw std::runtime_error(tostr("VirtualSchematikaMachine::apply_op:" + " argument mismatch in apply" + ": k arguments supplied where n expected", + xtag("k", sp0->size() - 1), + xtag("n", fn->n_args()))); + } + + /* todo: + * check function signature? + * should have been guaranteed by expression parser, + * but complications in interactive session when variables redefined. + */ + + gp retval = fn->apply_nocheck(mm, sp0->argv()); + + this->pc_ = this->cont_; + this->stack_ = sp0->parent(); + this->value_ = retval; + } + } /*namespace scm*/ } /*namespace xo*/ diff --git a/xo-interpreter/src/interpreter/VsmStackFrame.cpp b/xo-interpreter/src/interpreter/VsmStackFrame.cpp index 09af308a..9bca575e 100644 --- a/xo-interpreter/src/interpreter/VsmStackFrame.cpp +++ b/xo-interpreter/src/interpreter/VsmStackFrame.cpp @@ -34,6 +34,20 @@ namespace xo { cont_{cont} {} + gp + VsmStackFrame::make(gc::IAlloc * mm, + gp p, + std::size_t n, + const VsmInstr * cont) + { + gp retval = new (MMPtr(mm)) VsmStackFrame(mm, p, n, cont); + + for (std::size_t i = 0; i < n; ++i) + (*retval)[i] = nullptr; + + return retval; + } + gp VsmStackFrame::push1(gc::IAlloc * mm, gp p, diff --git a/xo-object/include/xo/object/Primitive.hpp b/xo-object/include/xo/object/Primitive.hpp index 3dc171fe..f6bf6b81 100644 --- a/xo-object/include/xo/object/Primitive.hpp +++ b/xo-object/include/xo/object/Primitive.hpp @@ -56,6 +56,8 @@ namespace xo { explicit Primitive(std::string_view name, function_type fn) : PrimitiveBase{name, fn} {} + /** see also AdoptPrimitiveExpr::adopt() in xo-interpreter/AdoptPrimitiveExpr.hp **/ + // inherited from Procedure.. virtual std::size_t n_args() const final override { return 2; } @@ -100,6 +102,7 @@ namespace xo { make_primitive(gc::IAlloc * mm, std::string_view name, Fn fn) { return new (MMPtr(mm)) Primitive(name, fn); } + } } diff --git a/xo-object/include/xo/object/Procedure.hpp b/xo-object/include/xo/object/Procedure.hpp index 2ab78ab2..1bd682f9 100644 --- a/xo-object/include/xo/object/Procedure.hpp +++ b/xo-object/include/xo/object/Procedure.hpp @@ -17,6 +17,9 @@ namespace xo { **/ class Procedure : public Object { public: + /** downcast from @p x iff x is actually a Procedure. Otherwise nullptr **/ + static gp from(gp x) { return gp::from(x); } + virtual std::size_t n_args() const = 0; virtual gp apply_nocheck(gc::IAlloc * mm, const CVector> & args) = 0; diff --git a/xo-object/src/object/Integer.cpp b/xo-object/src/object/Integer.cpp index 6a1abe87..5aee03f6 100644 --- a/xo-object/src/object/Integer.cpp +++ b/xo-object/src/object/Integer.cpp @@ -24,7 +24,7 @@ namespace xo { gp Integer::from(gp x) { - return dynamic_cast(x.ptr()); + return gp::from(x); } TaggedPtr diff --git a/xo-reader/include/xo/reader/apply_xs.hpp b/xo-reader/include/xo/reader/apply_xs.hpp index 9c4cf99f..14cf234c 100644 --- a/xo-reader/include/xo/reader/apply_xs.hpp +++ b/xo-reader/include/xo/reader/apply_xs.hpp @@ -94,6 +94,7 @@ namespace xo { parserstatemachine * p_psm) override; virtual void print(std::ostream & os) const override; + virtual bool pretty_print(const print::ppindentinfo & ppii) const final override; private: static std::unique_ptr make(); diff --git a/xo-reader/include/xo/reader/expect_expr_xs.hpp b/xo-reader/include/xo/reader/expect_expr_xs.hpp index b4c8b388..91cc7f31 100644 --- a/xo-reader/include/xo/reader/expect_expr_xs.hpp +++ b/xo-reader/include/xo/reader/expect_expr_xs.hpp @@ -15,6 +15,7 @@ namespace xo { * * Examples: * @text + * add (1, 2) ; * def x : i64 = 5 ; // with allow_defs * lambda (f : x64) { ... } ; * if (prime(x)) { ... } ; diff --git a/xo-reader/include/xo/reader/lambda_xs.hpp b/xo-reader/include/xo/reader/lambda_xs.hpp index 68dfc392..0d4d89cb 100644 --- a/xo-reader/include/xo/reader/lambda_xs.hpp +++ b/xo-reader/include/xo/reader/lambda_xs.hpp @@ -55,8 +55,8 @@ namespace xo { **/ class lambda_xs : public exprstate { public: - using Environment = xo::scm::SymbolTable; - using LocalEnv = xo::scm::LocalSymtab; + using SymbolTable = xo::scm::SymbolTable; + using LocalSymtab = xo::scm::LocalSymtab; public: lambda_xs(); @@ -81,6 +81,8 @@ namespace xo { parserstatemachine * p_psm) override; virtual void on_semicolon_token(const token_type & tk, parserstatemachine * p_psm) override; + virtual void on_f64_token(const token_type & tk, + parserstatemachine * p_psm) final override; virtual void print(std::ostream & os) const override; virtual bool pretty_print(const print::ppindentinfo & ppii) const override; @@ -93,7 +95,7 @@ namespace xo { lambdastatetype lmxs_type_ = lambdastatetype::lm_0; /** lambda environment (for formal parameters) **/ - rp local_env_; + rp local_env_; /** explicit return type (if supplied) **/ TypeDescr explicit_return_td_ = nullptr; @@ -105,7 +107,7 @@ namespace xo { rp body_; /** parent environment **/ - rp parent_env_; + rp parent_env_; }; } /*namespace scm*/ diff --git a/xo-reader/include/xo/reader/parserstatemachine.hpp b/xo-reader/include/xo/reader/parserstatemachine.hpp index 8988c6df..9cec6ee3 100644 --- a/xo-reader/include/xo/reader/parserstatemachine.hpp +++ b/xo-reader/include/xo/reader/parserstatemachine.hpp @@ -83,6 +83,7 @@ namespace xo { void on_rightbrace_token(const token_type & tk); void on_then_token(const token_type & tk); void on_else_token(const token_type & tk); + void on_f64_token(const token_type & tk); // ----- parsing error ----- diff --git a/xo-reader/include/xo/reader/progress_xs.hpp b/xo-reader/include/xo/reader/progress_xs.hpp index 780f5232..1884623a 100644 --- a/xo-reader/include/xo/reader/progress_xs.hpp +++ b/xo-reader/include/xo/reader/progress_xs.hpp @@ -73,6 +73,10 @@ namespace xo { * Deals with infix operators, handles operator precedence. * Also generates argument-type-specific arithmetic expressions, * for example using ``Apply::make_add2_f64()`` when adding floating-point numbers + * + * One reason for this to exist is to simulate one-token lookahead. + * To look at but not consume a token T, can push a progress_xs instance P, + * then send T to P. **/ class progress_xs : public exprstate { public: @@ -105,6 +109,8 @@ namespace xo { parserstatemachine * p_psm) override; virtual void on_symbol_token(const token_type & tk, parserstatemachine * p_psm) override; + virtual void on_comma_token(const token_type & tk, + parserstatemachine * p_psm) final override; virtual void on_typedescr(TypeDescr td, parserstatemachine * p_psm) override; diff --git a/xo-reader/src/reader/apply_xs.cpp b/xo-reader/src/reader/apply_xs.cpp index d6315a08..8f5b557a 100644 --- a/xo-reader/src/reader/apply_xs.cpp +++ b/xo-reader/src/reader/apply_xs.cpp @@ -76,13 +76,16 @@ namespace xo { // unreachable break; case applyexprstatetype::apply_0: + log && log("stash fn -> new state apply_1"); this->fn_expr_ = expr.promote(); this->applyxs_type_ = applyexprstatetype::apply_1; return; case applyexprstatetype::apply_1: + log && log("error: was expecting lparen"); // error, expecting lparen break; case applyexprstatetype::apply_2: + log && log(xtag("expr", expr), xtag("do", "stash expr -> new state apply_3")); this->args_expr_v_.push_back(expr.promote()); this->applyxs_type_ = applyexprstatetype::apply_3; return; @@ -145,6 +148,7 @@ namespace xo { if (this->applyxs_type_ == applyexprstatetype::apply_3) { /* (done) state */ + log("apply complete -> pop + send expr"); rp apply_expr = Apply::make(this->fn_expr_, this->args_expr_v_); @@ -169,6 +173,15 @@ namespace xo { os << ">"; } + bool + apply_xs::pretty_print(const xo::print::ppindentinfo & ppii) const + { + return ppii.pps()->pretty_struct(ppii, "apply_xs", + refrtag("applyxs_type", applyxs_type_), + refrtag("fn_expr", fn_expr_), + refrtag("args_expr_v", args_expr_v_)); + } + } /*namespace scm*/ } /*namespace xo*/ diff --git a/xo-reader/src/reader/expect_expr_xs.cpp b/xo-reader/src/reader/expect_expr_xs.cpp index 5bf7571a..c297b969 100644 --- a/xo-reader/src/reader/expect_expr_xs.cpp +++ b/xo-reader/src/reader/expect_expr_xs.cpp @@ -201,7 +201,9 @@ namespace xo { expect_expr_xs::on_i64_token(const token_type & tk, parserstatemachine * p_psm) { - scope log(XO_DEBUG(p_psm->debug_flag())); + scope log(XO_DEBUG(p_psm->debug_flag()), + xtag("tk", tk), + xtag("do", "push progress xs w/ tk value")); progress_xs::start (Constant::make(tk.i64_value()), @@ -233,6 +235,7 @@ namespace xo { log && log(xtag("exstype", this->exs_type_), xtag("expr", expr.promote())); + log && log("pop expect_expr_xs, forward to parent"); std::unique_ptr self = p_psm->pop_exprstate(); @@ -247,6 +250,7 @@ namespace xo { log && log(xtag("exstype", this->exs_type_), xtag("expr", expr.promote())); + log && log("pop expect_expr_xs, forward to parent"); std::unique_ptr self = p_psm->pop_exprstate(); diff --git a/xo-reader/src/reader/expect_type_xs.cpp b/xo-reader/src/reader/expect_type_xs.cpp index f986ab50..93fd1bbd 100644 --- a/xo-reader/src/reader/expect_type_xs.cpp +++ b/xo-reader/src/reader/expect_type_xs.cpp @@ -59,7 +59,7 @@ namespace xo { if (!td) { const char * exp = get_expect_str(); - std::string errmsg = tostr("unexpected token for parsing state", + std::string errmsg = tostr("expect_type_xs: unexpected token for parsing state", xtag("expecting", exp), xtag("token", tk.tk_type()), xtag("text", tk.text()), diff --git a/xo-reader/src/reader/exprseq_xs.cpp b/xo-reader/src/reader/exprseq_xs.cpp index f58f846d..7bbb30b6 100644 --- a/xo-reader/src/reader/exprseq_xs.cpp +++ b/xo-reader/src/reader/exprseq_xs.cpp @@ -97,6 +97,7 @@ namespace xo { * a = 1; // single assignment * a == 1; // rhs expression * a + b; // rhs expression + * a(1,2); // apply expression * Variable must have been defined! */ bp var = p_psm->lookup_var(tk.text()); diff --git a/xo-reader/src/reader/exprstate.cpp b/xo-reader/src/reader/exprstate.cpp index 47e368d4..3b631e26 100644 --- a/xo-reader/src/reader/exprstate.cpp +++ b/xo-reader/src/reader/exprstate.cpp @@ -610,7 +610,7 @@ namespace xo { const char * expect_str, parserstatemachine * p_psm) const { - std::string errmsg = tostr("unexpected token for parsing state", + std::string errmsg = tostr("exprstate::illegal_input_on_token: unexpected token for parsing state", xtag("expecting", expect_str), xtag("token", tk.tk_type()), xtag("text", tk.text()), diff --git a/xo-reader/src/reader/lambda_xs.cpp b/xo-reader/src/reader/lambda_xs.cpp index 3cb0e98e..ca070627 100644 --- a/xo-reader/src/reader/lambda_xs.cpp +++ b/xo-reader/src/reader/lambda_xs.cpp @@ -102,7 +102,7 @@ namespace xo { if (lmxs_type_ == lambdastatetype::lm_1) { this->lmxs_type_ = lambdastatetype::lm_2; this->parent_env_ = p_psm->top_envframe().promote(); - this->local_env_ = LocalEnv::make(argl, parent_env_); + this->local_env_ = LocalSymtab::make(argl, parent_env_); p_psm->push_envframe(local_env_); @@ -268,6 +268,31 @@ namespace xo { exprstate::on_semicolon_token(tk, p_psm); } + void + lambda_xs::on_f64_token(const token_type & tk, + parserstatemachine * p_psm) + { + constexpr const char * c_self_name = "lambda_xs::on_f64_token"; + + /* f64 literal can begin lambda body, otherwise illegal. + * for example: + * def foo = lambda (x: bool) 3.14; + */ + if (lmxs_type_ == lambdastatetype::lm_2) { + /* omitting return type. + * omitting left brace. + */ + this->lmxs_type_ = lambdastatetype::lm_4; + + expect_expr_xs::start(p_psm); + p_psm->on_f64_token(tk); + } else { + this->illegal_input_on_token(c_self_name, tk, this->get_expect_str(), p_psm); + } + } + + // TODO: on_i64_token, on_bool token + void lambda_xs::print(std::ostream & os) const { os << "pretty_struct(ppii, "lambda_xs", - refrtag("lmxs_type", lmxs_type_)); + refrtag("lmxs_type", lmxs_type_), + refrtag("body", body_)); } } /*namespace scm*/ diff --git a/xo-reader/src/reader/parserstatemachine.cpp b/xo-reader/src/reader/parserstatemachine.cpp index 101c2ba9..e27801e0 100644 --- a/xo-reader/src/reader/parserstatemachine.cpp +++ b/xo-reader/src/reader/parserstatemachine.cpp @@ -172,6 +172,17 @@ namespace xo { this->xs_stack_.top_exprstate().on_else_token(tk, this); } + void + parserstatemachine::on_f64_token(const token_type & tk) + { + scope log(XO_DEBUG(debug_flag_)); + + log && log(xtag("tk", tk), + xtag("psm", this)); + + this->xs_stack_.top_exprstate().on_f64_token(tk, this); + } + void parserstatemachine::on_error(const char * self_name, std::string errmsg) { diff --git a/xo-reader/src/reader/progress_xs.cpp b/xo-reader/src/reader/progress_xs.cpp index d47c8cf9..01cfff21 100644 --- a/xo-reader/src/reader/progress_xs.cpp +++ b/xo-reader/src/reader/progress_xs.cpp @@ -310,6 +310,8 @@ namespace xo { progress_xs::on_expr(bp expr, parserstatemachine * p_psm) { + scope log(XO_DEBUG(p_psm->debug_flag()), xtag("expr", expr)); + /* note: previous token probably an operator, * handled from progress_xs::on_operator_token(), * which pushes expect_expr_xs::expect_rhs_expression() @@ -318,21 +320,33 @@ namespace xo { constexpr const char * c_self_name = "progress_xs::on_expr"; const char * exp = get_expect_str(); - if (op_type_ == optype::invalid) { - this->illegal_input_on_expr(c_self_name, expr, exp, p_psm); - } + if (lhs_) { + if (op_type_ == optype::invalid) { + /* two consecutive expression without an operator */ + this->illegal_input_on_expr(c_self_name, expr, exp, p_psm); + } #ifdef NOT_QUITE - assert(result.get()); + assert(result.get()); - /* this expression complete.. */ - std::unique_ptr self = p_psm->pop_exprstate(); + /* this expression complete.. */ + std::unique_ptr self = p_psm->pop_exprstate(); - /* ..but more operators could follow, so don't commit yet */ - p_stack->push_exprstate(progress_xs::make(result)); + /* ..but more operators could follow, so don't commit yet */ + p_stack->push_exprstate(progress_xs::make(result)); #endif - this->rhs_ = expr.promote(); + this->rhs_ = expr.promote(); + } else { + /* control here on input like + * add(1,2)... + * + * add(1,2) needs to be handled inside a progress_xs + * instance because may be followed by an operator: + * add(1,2) + ... + */ + this->lhs_ = expr.promote(); + } } void @@ -375,6 +389,46 @@ namespace xo { assert(false); } + void + progress_xs::on_comma_token(const token_type & tk, + parserstatemachine * p_psm) + { + /* note: implementation parllels .on_semicolon_token(), .on_rightparen_token() */ + + scope log(XO_DEBUG(p_psm->debug_flag())); + + constexpr const char * self_name = "progress::xs::on_comma_token"; + + auto & xs_stack = p_psm->xs_stack_; + + /* stack may be something like + * + * applyexpr + * expect_expr_xs + * progress_xs + * <-- comma + * + * 1. comma completes expression-in-progress + */ + + /* comma confirms stack expression */ + rp expr = this->assemble_expr(p_psm); + + std::unique_ptr self = p_psm->pop_exprstate(); + + if (xs_stack.empty()) { + throw std::runtime_error(tostr(self_name, + ": expected non-empty parsing state")); + } + + log && log(xtag("stack", &xs_stack)); + + p_psm->top_exprstate().on_expr(expr, p_psm); + + /* now deliver comma */ + p_psm->top_exprstate().on_comma_token(tk, p_psm); + } + void progress_xs::on_typedescr(TypeDescr /*td*/, parserstatemachine * /*p_psm*/) @@ -441,6 +495,7 @@ namespace xo { this->on_operator_token(tk, p_psm); } + /* editor bait: on_lparen */ void progress_xs::on_leftparen_token(const token_type & tk, parserstatemachine * p_psm) @@ -456,12 +511,24 @@ namespace xo { /* start function call */ assert(rhs_.get() == nullptr); - /* unwind this progress_xs + replace with function call */ - rp fn_expr = lhs_; + + /* reset this progress_xs back to empty state; + * apply_xs will be responsible for lhs_. + */ + lhs_ = nullptr; + +#ifdef OBSOLETE + /* don't unwind! want to handle input like + * f(x,y)+g(z) + */ + /* unwind this progress_xs + replace with function call */ std::unique_ptr self = p_psm->pop_exprstate(); +#endif apply_xs::start(fn_expr, p_psm); + + /* control will reenter progress_xs via .on_expr() */ return; } @@ -486,7 +553,7 @@ namespace xo { /* stack may be something like: * * lparen_0 - * expect_rhs_expression + * expect_expr_xs * expr_progress * <-- rightparen * @@ -497,6 +564,9 @@ namespace xo { /* right paren confirms stack expression */ rp expr = this->assemble_expr(p_psm); + log && log(xtag("expr", expr), + xtag("do", "pop self + send {expr, rparen} -> parent")); + std::unique_ptr self = p_psm->pop_exprstate(); if (xs_stack.empty()) { @@ -756,7 +826,7 @@ namespace xo { && (rhs_ ? ppii.pps()->print_upto(refrtag("rhs", rhs_)) : true) && ppii.pps()->print_upto(">")); } else { - ppii.pps()->write("write("pretty(refrtag("lhs", lhs_)); if (op_type_ != optype::invalid) diff --git a/xo-reader/utest/reader.test.cpp b/xo-reader/utest/reader.test.cpp index 11d84edb..0bccf3b3 100644 --- a/xo-reader/utest/reader.test.cpp +++ b/xo-reader/utest/reader.test.cpp @@ -21,6 +21,7 @@ namespace xo { {"def foo = lambda (x : f64, y : f64) 3.1415965;"}, {"def foo = lambda (x : f64) x;"}, {"def foo = lambda (x : f64) { def y = x * x; y; };"}, + {"add(1,2);"}, }; } @@ -50,11 +51,13 @@ namespace xo { log && log(xtag("expr", rr.expr_)); +#ifdef OBSOLETE // input not consumed until next line presented input = input.after_prefix(rr.rem_); log && log(xtag("post.input", input)); +#endif - REQUIRE(input.empty()); + //REQUIRE(input.empty()); } catch (std::exception & ex) { log && log(ex.what()); diff --git a/xo-tokenizer/include/xo/tokenizer/input_state.hpp b/xo-tokenizer/include/xo/tokenizer/input_state.hpp index 0cea1155..fbff7e57 100644 --- a/xo-tokenizer/include/xo/tokenizer/input_state.hpp +++ b/xo-tokenizer/include/xo/tokenizer/input_state.hpp @@ -247,7 +247,7 @@ namespace xo { input_state::advance_until(const CharT * pos) { scope log(XO_DEBUG(debug_flag_)); - assert(current_line_.lo() <= pos && pos < current_line_.hi()); + assert(current_line_.lo() <= pos && pos <= current_line_.hi()); this->current_pos_ = pos - current_line_.lo();