xo-reader2: top-level string literal + on_string_token() in SSM

idl/SyntaxStateMachine.json5

@@ -117,6 +117,15 @@
                 {type: "ParserStateMachine *", name: "p_psm"},
             ],
         },
+        {
+            name: "on_string_token",
+            doc: ["update state machine for incoming string-token @p tk"],
+            return_type: "void",
+            args: [
+                {type: "const Token &", name: "tk"},
+                {type: "ParserStateMachine *", name: "p_psm"},
+            ],
+        },
         {
             name: "on_semicolon_token",
             doc: ["update state machine for incoming semicolon-token @p tk"],

include/xo/reader2/DDefineSsm.hpp

@@ -147,6 +147,12 @@ namespace xo {
             void on_singleassign_token(const Token & tk,
                                        ParserStateMachine * p_psm);
 
+            /** update state for this syntax on incoming string tokne @p tk,
+             *  overall parser state in @p p_psm
+             **/
+            void on_string_token(const Token & tk,
+                                 ParserStateMachine * p_psm);
+
             /** update state for this syntax on incoming f64 token @p tk,
              *  overall parser state in @p p_psm
              **/

include/xo/reader2/DExpectExprSsm.hpp

@@ -83,6 +83,12 @@ namespace xo {
             void on_singleassign_token(const Token & tk,
                                        ParserStateMachine * p_psm);
 
+            /** update state for this syntax on incoming string token @p tk,
+             *  overall parser state in @p p_psm
+             **/
+            void on_string_token(const Token & tk,
+                                 ParserStateMachine * p_psm);
+
             /** update state for this syntax on incoming f64 token @p tk,
              *  overall parser state in @p p_psm
              **/

include/xo/reader2/DExpectSymbolSsm.hpp

@@ -117,6 +117,12 @@ namespace xo {
             void on_singleassign_token(const Token & tk,
                                        ParserStateMachine * p_psm);
 
+            /** update state for this syntax on incoming string token @p tk,
+             *  overall parser state in @p p_psm
+             **/
+            void on_string_token(const Token & tk,
+                                 ParserStateMachine * p_psm);
+
             /** update state for this syntax on incoming f64 token @p tk,
              *  overall parser state in @p p_psm
              **/

include/xo/reader2/DExpectTypeSsm.hpp

@@ -77,6 +77,12 @@ namespace xo {
             void on_singleassign_token(const Token & tk,
                                        ParserStateMachine * p_psm);
 
+            /** update state for this syntax on incoming string token @p tk,
+             *  overall parser state in @p p_psm
+             **/
+            void on_string_token(const Token & tk,
+                                 ParserStateMachine * p_psm);
+
             /** update state for this syntax on incoming f64 token @p tk,
              *  overall parser state in @p p_psm
              **/

include/xo/reader2/DExprSeqState.hpp

@@ -90,6 +90,11 @@ namespace xo {
              **/
             void on_singleassign_token(const Token & tk, ParserStateMachine * p_psm);
 
+            /** update state for this syntax on incoming string token @p tk,
+             *  overall parser state in @p p_psm
+             **/
+            void on_string_token(const Token & tk, ParserStateMachine * p_psm);
+
             /** update state for this syntax on incoming f64 token @p tk,
              *  overall parser state in @p p_psm
              **/

include/xo/reader2/DProgressSsm.hpp

@@ -141,6 +141,8 @@ namespace xo {
                                 ParserStateMachine * p_psm);
             void on_singleassign_token(const Token & tk,
                                        ParserStateMachine * p_psm);
+            void on_string_token(const Token & tk,
+                                 ParserStateMachine * p_psm);
             void on_f64_token(const Token & tk,
                               ParserStateMachine * p_psm);
             void on_i64_token(const Token & tk,

include/xo/reader2/ParserStateMachine.hpp

@@ -138,6 +138,9 @@ namespace xo {
             /** operate state machine for incoming singleassign-token @p tk **/
             void on_singleassign_token(const Token & tk);
 
+            /** operate state machine for incoming string-otoken @p tk **/
+            void on_string_token(const Token & tk);
+
             /** operate state machine for incoming f64-token @p tk **/
             void on_f64_token(const Token & tk);
 

include/xo/reader2/ssm/ASyntaxStateMachine.hpp

@@ -73,6 +73,8 @@ public:
     virtual void on_i64_token(Opaque data, const Token & tk, ParserStateMachine * p_psm)  = 0;
     /** update state machine for incoming bool-token @p tk **/
     virtual void on_bool_token(Opaque data, const Token & tk, ParserStateMachine * p_psm)  = 0;
+    /** update state machine for incoming string-token @p tk **/
+    virtual void on_string_token(Opaque data, const Token & tk, ParserStateMachine * p_psm)  = 0;
     /** update state machine for incoming semicolon-token @p tk **/
     virtual void on_semicolon_token(Opaque data, const Token & tk, ParserStateMachine * p_psm)  = 0;
     /** update  stat machine for incoming parsed symbol @p sym **/

include/xo/reader2/ssm/ISyntaxStateMachine_Any.hpp

@@ -68,6 +68,7 @@ namespace scm {
         [[noreturn]] void on_f64_token(Opaque, const Token &, ParserStateMachine *)  override;
         [[noreturn]] void on_i64_token(Opaque, const Token &, ParserStateMachine *)  override;
         [[noreturn]] void on_bool_token(Opaque, const Token &, ParserStateMachine *)  override;
+        [[noreturn]] void on_string_token(Opaque, const Token &, ParserStateMachine *)  override;
         [[noreturn]] void on_semicolon_token(Opaque, const Token &, ParserStateMachine *)  override;
         [[noreturn]] void on_parsed_symbol(Opaque, std::string_view, ParserStateMachine *)  override;
         [[noreturn]] void on_parsed_typedescr(Opaque, TypeDescr, ParserStateMachine *)  override;

include/xo/reader2/ssm/ISyntaxStateMachine_DDefineSsm.hpp

@@ -70,6 +70,8 @@ namespace xo {
             static void on_i64_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming bool-token @p tk **/
             static void on_bool_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm);
+            /** update state machine for incoming string-token @p tk **/
+            static void on_string_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming semicolon-token @p tk **/
             static void on_semicolon_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update  stat machine for incoming parsed symbol @p sym **/

include/xo/reader2/ssm/ISyntaxStateMachine_DExpectExprSsm.hpp

@@ -70,6 +70,8 @@ namespace xo {
             static void on_i64_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming bool-token @p tk **/
             static void on_bool_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm);
+            /** update state machine for incoming string-token @p tk **/
+            static void on_string_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming semicolon-token @p tk **/
             static void on_semicolon_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update  stat machine for incoming parsed symbol @p sym **/

include/xo/reader2/ssm/ISyntaxStateMachine_DExpectSymbolSsm.hpp

@@ -70,6 +70,8 @@ namespace xo {
             static void on_i64_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming bool-token @p tk **/
             static void on_bool_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm);
+            /** update state machine for incoming string-token @p tk **/
+            static void on_string_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming semicolon-token @p tk **/
             static void on_semicolon_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update  stat machine for incoming parsed symbol @p sym **/

include/xo/reader2/ssm/ISyntaxStateMachine_DExpectTypeSsm.hpp

@@ -70,6 +70,8 @@ namespace xo {
             static void on_i64_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming bool-token @p tk **/
             static void on_bool_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm);
+            /** update state machine for incoming string-token @p tk **/
+            static void on_string_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming semicolon-token @p tk **/
             static void on_semicolon_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update  stat machine for incoming parsed symbol @p sym **/

include/xo/reader2/ssm/ISyntaxStateMachine_DExprSeqState.hpp

@@ -70,6 +70,8 @@ namespace xo {
             static void on_i64_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming bool-token @p tk **/
             static void on_bool_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm);
+            /** update state machine for incoming string-token @p tk **/
+            static void on_string_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming semicolon-token @p tk **/
             static void on_semicolon_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm);
             /** update  stat machine for incoming parsed symbol @p sym **/

include/xo/reader2/ssm/ISyntaxStateMachine_DProgressSsm.hpp

@@ -70,6 +70,8 @@ namespace xo {
             static void on_i64_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming bool-token @p tk **/
             static void on_bool_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm);
+            /** update state machine for incoming string-token @p tk **/
+            static void on_string_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update state machine for incoming semicolon-token @p tk **/
             static void on_semicolon_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm);
             /** update  stat machine for incoming parsed symbol @p sym **/

include/xo/reader2/ssm/ISyntaxStateMachine_Xfer.hpp

@@ -76,6 +76,9 @@ namespace scm {
         void on_bool_token(Opaque data, const Token & tk, ParserStateMachine * p_psm)  override {
             return I::on_bool_token(_dcast(data), tk, p_psm);
         }
+        void on_string_token(Opaque data, const Token & tk, ParserStateMachine * p_psm)  override {
+            return I::on_string_token(_dcast(data), tk, p_psm);
+        }
         void on_semicolon_token(Opaque data, const Token & tk, ParserStateMachine * p_psm)  override {
             return I::on_semicolon_token(_dcast(data), tk, p_psm);
         }

include/xo/reader2/ssm/RSyntaxStateMachine.hpp

@@ -80,6 +80,9 @@ public:
     void on_bool_token(const Token & tk, ParserStateMachine * p_psm)  {
         return O::iface()->on_bool_token(O::data(), tk, p_psm);
     }
+    void on_string_token(const Token & tk, ParserStateMachine * p_psm)  {
+        return O::iface()->on_string_token(O::data(), tk, p_psm);
+    }
     void on_semicolon_token(const Token & tk, ParserStateMachine * p_psm)  {
         return O::iface()->on_semicolon_token(O::data(), tk, p_psm);
     }

src/reader2/DDefineSsm.cpp

@@ -567,6 +567,15 @@ namespace xo {
                                           this->get_expect_str());
         }
 
+        void
+        DDefineSsm::on_string_token(const Token & tk,
+                                    ParserStateMachine * p_psm)
+        {
+            p_psm->illegal_input_on_token("DDefineSsm::on_string_token",
+                                          tk,
+                                          this->get_expect_str());
+        }
+
         void
         DDefineSsm::on_f64_token(const Token & tk,
                                  ParserStateMachine * p_psm)

src/reader2/DExpectExprSsm.cpp

@@ -147,6 +147,15 @@ namespace xo {
                                           this->get_expect_str());
         }
 
+        void
+        DExpectExprSsm::on_string_token(const Token & tk,
+                                        ParserStateMachine * p_psm)
+        {
+            p_psm->illegal_input_on_token("DExpectExprSsm::on_string_token",
+                                          tk,
+                                          this->get_expect_str());
+        }
+
         void
         DExpectExprSsm::on_f64_token(const Token & tk,
                                      ParserStateMachine * p_psm)

src/reader2/DExpectSymbolSsm.cpp

@@ -149,6 +149,15 @@ namespace xo {
                                           this->get_expect_str());
         }
 
+        void
+        DExpectSymbolSsm::on_string_token(const Token & tk,
+                                          ParserStateMachine * p_psm)
+        {
+            p_psm->illegal_input_on_token("DExpectSymbolSsm::on_string_token",
+                                          tk,
+                                          this->get_expect_str());
+        }
+
         void
         DExpectSymbolSsm::on_f64_token(const Token & tk,
                                        ParserStateMachine * p_psm)

src/reader2/DExpectTypeSsm.cpp

@@ -92,6 +92,15 @@ namespace xo {
                                           this->get_expect_str());
         }
 
+        void
+        DExpectTypeSsm::on_string_token(const Token & tk,
+                                        ParserStateMachine * p_psm)
+        {
+            p_psm->illegal_input_on_token("DExpectTypeSsm",
+                                          tk,
+                                          this->get_expect_str());
+        }
+
         void
         DExpectTypeSsm::on_f64_token(const Token & tk,
                                      ParserStateMachine * p_psm)

src/reader2/DExprSeqState.cpp

@@ -9,6 +9,8 @@
 #include <xo/reader2/DProgressSsm.hpp>
 #include <xo/expression2/DConstant.hpp>
 #include <xo/expression2/detail/IExpression_DConstant.hpp>
+#include <xo/object2/DString.hpp>
+#include <xo/object2/string/IGCObject_DString.hpp>
 #include <xo/object2/DFloat.hpp>
 #include <xo/object2/number/IGCObject_DFloat.hpp>
 #include <xo/object2/DBoolean.hpp>
@@ -192,6 +194,36 @@ namespace xo {
                                           this->get_expect_str());
         }
 
+        void
+        DExprSeqState::on_string_token(const Token & tk,
+                                       ParserStateMachine * p_psm)
+        {
+            switch (seqtype_) {
+            case exprseqtype::toplevel_interactive:
+                {
+                    DString * dstr = DString::from_cstr(p_psm->expr_alloc(),
+                                                        tk.text().c_str());
+                    obj<AGCObject,DString> str(dstr);
+                    auto * dconst = DConstant::make(p_psm->expr_alloc(), str);
+                    obj<AExpression,DConstant> expr(dconst);
+
+                    DProgressSsm::start(p_psm->parser_alloc(),
+                                        expr,
+                                        p_psm);
+                    return;
+                }
+            case exprseqtype::toplevel_batch:
+                break;
+            case exprseqtype::N:
+                assert(false); // unreachable
+                break;
+            }
+
+            p_psm->illegal_input_on_token("DExprSeqState::on_string_token",
+                                          tk,
+                                          this->get_expect_str());
+        }
+
         void
         DExprSeqState::on_f64_token(const Token & tk,
                                     ParserStateMachine * p_psm)

src/reader2/DProgressSsm.cpp

@@ -203,6 +203,15 @@ namespace xo {
                                           this->get_expect_str());
         }
 
+        void
+        DProgressSsm::on_string_token(const Token & tk,
+                                      ParserStateMachine * p_psm)
+        {
+            p_psm->illegal_input_on_token("DProgressSsm::on_string_token",
+                                          tk,
+                                          this->get_expect_str());
+        }
+
         void
         DProgressSsm::on_f64_token(const Token & tk,
                                    ParserStateMachine * p_psm)

src/reader2/ISyntaxStateMachine_Any.cpp

@@ -82,6 +82,12 @@ ISyntaxStateMachine_Any::on_bool_token(Opaque, const Token &, ParserStateMachine
     _fatal();
 }
 
+auto
+ISyntaxStateMachine_Any::on_string_token(Opaque, const Token &, ParserStateMachine *)  -> void
+{
+    _fatal();
+}
+
 auto
 ISyntaxStateMachine_Any::on_semicolon_token(Opaque, const Token &, ParserStateMachine *)  -> void
 {

src/reader2/ISyntaxStateMachine_DDefineSsm.cpp

@@ -68,6 +68,11 @@ namespace xo {
             self.on_bool_token(tk, p_psm);
         }
         auto
+        ISyntaxStateMachine_DDefineSsm::on_string_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
+        {
+            self.on_string_token(tk, p_psm);
+        }
+        auto
         ISyntaxStateMachine_DDefineSsm::on_semicolon_token(DDefineSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
         {
             self.on_semicolon_token(tk, p_psm);

src/reader2/ISyntaxStateMachine_DExpectExprSsm.cpp

@@ -68,6 +68,11 @@ namespace xo {
             self.on_bool_token(tk, p_psm);
         }
         auto
+        ISyntaxStateMachine_DExpectExprSsm::on_string_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
+        {
+            self.on_string_token(tk, p_psm);
+        }
+        auto
         ISyntaxStateMachine_DExpectExprSsm::on_semicolon_token(DExpectExprSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
         {
             self.on_semicolon_token(tk, p_psm);

src/reader2/ISyntaxStateMachine_DExpectSymbolSsm.cpp

@@ -68,6 +68,11 @@ namespace xo {
             self.on_bool_token(tk, p_psm);
         }
         auto
+        ISyntaxStateMachine_DExpectSymbolSsm::on_string_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
+        {
+            self.on_string_token(tk, p_psm);
+        }
+        auto
         ISyntaxStateMachine_DExpectSymbolSsm::on_semicolon_token(DExpectSymbolSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
         {
             self.on_semicolon_token(tk, p_psm);

src/reader2/ISyntaxStateMachine_DExpectTypeSsm.cpp

@@ -68,6 +68,11 @@ namespace xo {
             self.on_bool_token(tk, p_psm);
         }
         auto
+        ISyntaxStateMachine_DExpectTypeSsm::on_string_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
+        {
+            self.on_string_token(tk, p_psm);
+        }
+        auto
         ISyntaxStateMachine_DExpectTypeSsm::on_semicolon_token(DExpectTypeSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
         {
             self.on_semicolon_token(tk, p_psm);

src/reader2/ISyntaxStateMachine_DExprSeqState.cpp

@@ -68,6 +68,11 @@ namespace xo {
             self.on_bool_token(tk, p_psm);
         }
         auto
+        ISyntaxStateMachine_DExprSeqState::on_string_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm) -> void
+        {
+            self.on_string_token(tk, p_psm);
+        }
+        auto
         ISyntaxStateMachine_DExprSeqState::on_semicolon_token(DExprSeqState & self, const Token & tk, ParserStateMachine * p_psm) -> void
         {
             self.on_semicolon_token(tk, p_psm);

src/reader2/ISyntaxStateMachine_DProgressSsm.cpp

@@ -68,6 +68,11 @@ namespace xo {
             self.on_bool_token(tk, p_psm);
         }
         auto
+        ISyntaxStateMachine_DProgressSsm::on_string_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
+        {
+            self.on_string_token(tk, p_psm);
+        }
+        auto
         ISyntaxStateMachine_DProgressSsm::on_semicolon_token(DProgressSsm & self, const Token & tk, ParserStateMachine * p_psm) -> void
         {
             self.on_semicolon_token(tk, p_psm);

src/reader2/ParserStateMachine.cpp

@@ -183,6 +183,10 @@ namespace xo {
                 this->on_singleassign_token(tk);
                 break;
 
+            case tokentype::tk_string:
+                this->on_string_token(tk);
+                break;
+
             case tokentype::tk_f64:
                 this->on_f64_token(tk);
                 break;
@@ -201,7 +205,6 @@ namespace xo {
 
             // all the not-yet handled cases
             case tokentype::tk_invalid:
-            case tokentype::tk_string:
             case tokentype::tk_leftparen:
             case tokentype::tk_rightparen:
             case tokentype::tk_leftbracket:
@@ -278,6 +281,14 @@ namespace xo {
             stack_->top().on_singleassign_token(tk, this);
         }
 
+        void
+        ParserStateMachine::on_string_token(const Token & tk)
+        {
+            scope log(XO_DEBUG(debug_flag_), xtag("tk", tk));
+
+            stack_->top().on_string_token(tk, this);
+        }
+
         void
         ParserStateMachine::on_f64_token(const Token & tk)
         {