xo-reader2: more work on apply expressions [WIP]

xo-reader2/include/xo/reader2/DApplySsm.hpp

@@ -46,8 +46,8 @@ namespace xo {
 
             apply_0,
             apply_1,
-            apply_2,
+            apply_2,  // rename -> apply_2a
-            apply_3,
+            apply_3,  // rename -> apply_2b
 
             N
         };
@@ -81,7 +81,8 @@ namespace xo {
              *  that we parse f before parser knows that it will be
              *  followed by leftparen
              **/
-            explicit DApplySsm(obj<AExpression> fn_expr);
+            explicit DApplySsm(obj<AExpression> fn_expr,
+                               DArray * args);
 
             /** create instance using memory from @p parser_mm.
              *  with function to be called supplied by @p fn_expr.
@@ -103,9 +104,8 @@ namespace xo {
              **/
             static void start(obj<AExpression> fnex,
                               ParserStateMachine * p_psm);
-
             ///@}
-            /** @defgroup scm-applyssm-access methods **/
+            /** @defgroup scm-applyssm-access-methods access methods **/
             ///@{
 
             /** identify this nested state machine **/
@@ -114,6 +114,16 @@ namespace xo {
             /** current internal state **/
             applyexprstatetype applystate() const noexcept { return applystate_; }
 
+            ///@}
+            /** @defgroup scm-applyssm-methods general methods **/
+            ///@{
+
+            /** handle leftparen token @p tk for this ssm,
+             *  with overall parser state in @p p_psm
+             **/
+            void on_leftparen_token(const Token & tk,
+                                    ParserStateMachine * p_psm);
+
             ///@}
             /** @defgroup ssm-applyssm-facet syntaxstatemachine facet methods **/
             ///@{
@@ -124,6 +134,12 @@ namespace xo {
             /** mnemonic for expected remaining syntax for current parsing state **/
             std::string_view get_expect_str() const noexcept;
 
+            /** update this apply-ssm for incoming token @p tk,
+             *  with overall parsing state in @p p_psm
+             **/
+            void on_token(const Token & tk,
+                          ParserStateMachine * p_psm);
+
             ///@}
 
 #ifdef NOT_YET
@@ -133,14 +149,9 @@ namespace xo {
 
             virtual void on_comma_token(const token_type & tk,
                                         parserstatemachine * p_psm) override;
-            virtual void on_leftparen_token(const token_type & tk,
-                                             parserstatemachine * p_psm) override;
             virtual void on_rightparen_token(const token_type & tk,
                                              parserstatemachine * p_psm) override;
 
-            virtual void print(std::ostream & os) const override;
-            virtual bool pretty_print(const print::ppindentinfo & ppii) const final override;
-
         private:
             static std::unique_ptr<apply_xs> make();
 #endif
@@ -158,10 +169,14 @@ namespace xo {
             applyexprstatetype applystate_ = applyexprstatetype::apply_0;
             /** evaluates to function to be invoked **/
             obj<AExpression> fn_expr_;
-#ifdef NOT_YET
+            /** args_expr_v_[i] evaluates to the i'th argument to call.
-            /** evaluates to the arguments to pass to @ref fn_ **/
+             *  Not using flexible array here since we don't know size at
-            std::vector<rp<Expression>> args_expr_v_;
+             *  construction time
-#endif
+             *
+             *  (though could revisit +  realloc this DApplySsm instance in
+             *  place to optimize)
+             **/
+            DArray * args_expr_v_ = nullptr;
         };
     } /*namespace scm */
 

xo-reader2/include/xo/reader2/DProgressSsm.hpp

@@ -137,6 +137,8 @@ namespace xo {
             /** handle leftparen token @p tk. Overall parser state in @p p_psm **/
             void on_leftparen_token(const Token & tk,
                                     ParserStateMachine * p_psm);
+            void on_rightparen_token(const Token & tk,
+                                     ParserStateMachine * p_psm);
 
             void on_symbol_token(const Token & tk,
                                  ParserStateMachine * p_psm);

xo-reader2/src/reader2/DApplySsm.cpp

@@ -4,6 +4,7 @@
  **/
 
 #include "ApplySsm.hpp"
+#include "ExpectExprSsm.hpp"
 #include <xo/reflectutil/typeseq.hpp>
 
 //#include "parserstatemachine.hpp"
@@ -39,13 +40,18 @@ namespace xo {
 
         // ----- DApplySsm -----
 
-        DApplySsm::DApplySsm(obj<AExpression> fn_expr)
+        DApplySsm::DApplySsm(applyexprstatetype applystate,
-            : applystate_{applyexprstatetype::apply_0},
+                             obj<AExpression> fn_expr,
-              fn_expr_{fn_expr}
+                             DArray * args)
+        : applystate_{applystate},
+          fn_expr_{fn_expr},
+          arg_expr_v_{args}
         {
             if (fn_expr) {
                 this->applystate_ = applyexprstatetype::apply_1;
             }
+
+            assert(args->empty());
         }
 
         DApplySsm *
@@ -55,10 +61,21 @@ namespace xo {
             void * mem = mm.alloc(typeseq::id<DApplySsm>(),
                                   sizeof(DApplySsm));
 
-            // TODO: revisit if we use flexible array for
+            /* allocate room for 8 arguments (during parsing)
-            //   arguments
+             *  will reallocate to expand if needed.
+             *
+             *  See similar code in DExpectFormalArglistSsm::_make
+             */
+            DArray * args = DArray::empty(mm, 8);
 
-            return new (mem) DApplySsm(fn_expr);
+            applyexprstatetype applystate
+                = (fn_expr
+                   ? applyexprstatetype::apply_1
+                   : applyexprstatetype::apply_0);
+
+            // TODO: revisit if we decide to use flexible array for arguments
+
+            return new (mem) DApplySsm(applystate, fn_expr, args);
         }
 
         void
@@ -98,6 +115,124 @@ namespace xo {
             return "?expect";
         }
 
+        void
+        DApplySsm::on_token(const Token & tk,
+                            ParserStateMachine * p_psm)
+        {
+            scope log(XO_DEBUG(p_psm->debug_flag()), xtag("tk", tk));
+
+            switch (tk.tk_type()) {
+            case tokentype::tk_leftparen:
+                this->on_leftparen_token(tk, p_psm);
+                return;
+            case tokentype::tk_symbol:
+            case tokentype::tk_def:
+            case tokentype::tk_if:
+            case tokentype::tk_then:
+            case tokentype::tk_else:
+            case tokentype::tk_semicolon:
+            case tokentype::tk_colon:
+            case tokentype::tk_singleassign:
+            case tokentype::tk_string:
+            case tokentype::tk_f64:
+            case tokentype::tk_i64:
+            case tokentype::tk_bool:
+            case tokentype::tk_invalid:
+            case tokentype::tk_rightparen:
+            case tokentype::tk_leftbracket:
+            case tokentype::tk_rightbracket:
+            case tokentype::tk_leftbrace:
+            case tokentype::tk_rightbrace:
+            case tokentype::tk_leftangle:
+            case tokentype::tk_rightangle:
+            case tokentype::tk_lessequal:
+            case tokentype::tk_greatequal:
+            case tokentype::tk_dot:
+            case tokentype::tk_comma:
+            case tokentype::tk_doublecolon:
+            case tokentype::tk_assign:
+            case tokentype::tk_yields:
+            case tokentype::tk_plus:
+            case tokentype::tk_minus:
+            case tokentype::tk_star:
+            case tokentype::tk_slash:
+            case tokentype::tk_cmpeq:
+            case tokentype::tk_cmpne:
+            case tokentype::tk_type:
+            case tokentype::tk_lambda:
+            case tokentype::tk_let:
+            case tokentype::tk_in:
+            case tokentype::tk_end:
+            case tokentype::N:
+                break;
+            }
+
+            Super::on_token(tk, p_psm);
+        }
+
+        void
+        DApplySsm::on_leftparen_token(const Token & tk,
+                                      ParserStateMachine * p_psm)
+        {
+            if (applystate_ == applyexprstatetype::apply_1) {
+                this->applystate_ = applyexprstatetype::apply_2;
+
+                DExpectExprSsm::start(p_psm);
+                return;
+            }
+
+            Super::on_token(tk, p_psm);
+        }
+
+        void
+        DApplySsm::on_parsed_expression_with_token(obj<AExpression> expr,
+                                                   const Token & tk)
+        {
+            if (applystate_ == applyexprstatetype::apply_2) {
+                if (args_expr_v_->size() == args_expr_v_->capacity()) {
+                    // need to expand .args_expr_v_ capacity.
+                    // Could use DArena checkpoint to redo this in place,
+                    // since argument array must be on the top of the stack.
+
+                    obj<AAllocator,DArena> mm(&(p_psm->parser_alloc()));
+
+                    DArray * argv_2x = DArray::empty(mm, 2 * args_expr_v_->capacity());
+
+                    for (DArray::size_type i = 0, n = arg_expr_v_->size(); i < n; ++i) {
+                        argv_2x->push_back((*argv_)[i]);
+                    }
+
+                    this->args_expr_v_ = argv_2x;
+                }
+
+                if (args_expr_v_->size() < args_expr_v_->capacity()) {
+                    args_expr_v_->push_back(expr);
+
+                }
+
+                if (tk.tk_type() == tokentype::tk_rightparen) {
+                    // expression completes apply
+
+                    // TODO: assemble apply expression..
+
+                    assert(false);
+                }
+
+                if ((tk.tk_type() == tokentype::tk_comma)
+                    || (tk.tk_type() == tokentype::tk_rightparen)) {
+
+                    // do stuff,
+                    //return;
+                }
+
+                 // complain
+
+                assert(false);
+            }
+
+            Super::on_parsed_expression_with_token(expr, tk);
+        }
+
 #ifdef NOT_YET
         void
         apply_xs::on_expr(bp<Expression> expr,
@@ -154,25 +289,6 @@ namespace xo {
             }
         }
 
-        void
-        apply_xs::on_leftparen_token(const token_type & tk,
-                                     parserstatemachine * p_psm)
-        {
-            scope log(XO_DEBUG(p_psm->debug_flag()));
-
-            log && log("applyxs_type", applyxs_type_);
-
-            if (this->applyxs_type_ == applyexprstatetype::apply_1) {
-                this->applyxs_type_ = applyexprstatetype::apply_2;
-                expect_expr_xs::start(p_psm);
-            } else {
-                constexpr const char * c_self_name = "apply_xs::on_leftparen_token";
-                const char * exp = this->get_expect_str();
-
-                this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
-            }
-        }
-
         void
         apply_xs::on_rightparen_token(const token_type & tk,
                                       parserstatemachine * p_psm)
@@ -214,26 +330,6 @@ namespace xo {
                                              refrtag("fn_expr", fn_expr, fn_expr_present));
         }
 
-#ifdef NOT_YET
-        void
-        apply_xs::print(std::ostream & os) const
-        {
-            os << "<apply_xs"
-               << xtag("this", (void*)this)
-               << xtag("applyxs_type", applyxs_type_);
-            os << ">";
-        }
-
-        bool
-        apply_xs::pretty_print(const xo::print::ppindentinfo & ppii) const
-        {
-            return ppii.pps()->pretty_struct(ppii, "apply_xs",
-                                             refrtag("applyxs_type", applyxs_type_),
-                                             refrtag("fn_expr", fn_expr_),
-                                             refrtag("args_expr_v", args_expr_v_));
-        }
-
-#endif
     } /*namespace scm*/
 } /*namespace xo*/
 

xo-reader2/src/reader2/DExpectFormalArglistSsm.cpp

@@ -182,7 +182,8 @@ namespace xo {
             if (fastate_ == formalarglstatetype::argl_1a) {
                 this->fastate_ = formalarglstatetype::argl_1b;
 
-                TypeRef typeref = TypeRef::dwim(TypeRef::prefix_type::from_chars("formal"), param_type);
+                TypeRef typeref
+                    = TypeRef::dwim(TypeRef::prefix_type::from_chars("formal"), param_type);
 
                 DVariable * var = DVariable::make(p_psm->expr_alloc(),
                                                   param_name,

xo-reader2/src/reader2/DProgressSsm.cpp

@@ -267,11 +267,14 @@ namespace xo {
                 this->on_leftparen_token(tk, p_psm);
                 return;
 
+            case tokentype::tk_rightparen:
+                this->on_rightparen_token(tk, p_psm);
+                return;
+
             // all the not-yet handled cases
             case tokentype::tk_invalid:
             case tokentype::tk_def:
             case tokentype::tk_if:
-            case tokentype::tk_rightparen:
             case tokentype::tk_leftbracket:
             case tokentype::tk_rightbracket:
             case tokentype::tk_leftbrace:
@@ -937,6 +940,9 @@ namespace xo {
                 this->lhs_ = obj<AExpression>();
 
                 DApplySsm::start(fn_expr, p_psm);
+                // + send leftparen to just-pushed apply
+                p_psm->on_token(tk);
+
                 return;
             }
 
@@ -986,7 +992,40 @@ namespace xo {
 
             this->illegal_input_on_token(c_self_name, tk, exp, p_psm);
         }
+#endif
 
+        void
+        DProgressSsm::on_rightparen_token(const Token & tk,
+                                          ParserStateMachine * p_psm)
+        {
+            /* note: implementation parallels .on_semicolon_token() */
+
+            scope log(XO_DEBUG(p_psm->debug_flag()));
+
+            /* stack may be something like:
+             *
+             *   [0] DProgressSsm
+             *   [1] DExpectExprSsm
+             *   [2] DApplySsm
+             *
+             * where we want rightparen to resolve in [2] DApplySsm,
+             * after triggering completion of [0] and [1]
+             */
+            auto expr = this->assemble_expr(p_psm);
+
+            if (expr) {
+                /* 1. popping self from parser stack.. */
+                p_psm->pop_ssm();
+                /* 2. report parsed subexpr to parent ssm, along with the triggering rightparen **/
+                p_psm->on_parsed_expression_with_token(expr, tk);
+
+                return;
+            }
+
+            Super::on_token(tk, p_psm);
+        }
+
+#ifdef NOT_YET
          void
          progress_xs::on_rightparen_token(const token_type & tk,
                                           parserstatemachine * p_psm)

xo-reader2/utest/SchematikaParser.test.cpp

@@ -1204,6 +1204,30 @@ namespace xo {
                 REQUIRE(!result.is_error());
                 REQUIRE(result.is_incomplete());
             }
+
+            {
+                auto & result = parser.on_token(Token::i64_token("13"));
+
+                log && log("after i64(13) token:");
+                log && log(xtag("parser", &parser));
+                log && log(xtag("result", result));
+
+                REQUIRE(parser.has_incomplete_expr() == true);
+                REQUIRE(!result.is_error());
+                REQUIRE(result.is_incomplete());
+            }
+
+            {
+                auto & result = parser.on_token(Token::rightparen_token());
+
+                log && log("after rightparen token:");
+                log && log(xtag("parser", &parser));
+                log && log(xtag("result", result));
+
+                REQUIRE(parser.has_incomplete_expr() == true);
+                REQUIRE(!result.is_error());
+                REQUIRE(result.is_incomplete());
+            }
         }
     } /*namespace ut*/
 } /*namespace xo*/