xo-tokenizer: feat: + basic arithmetic operators

include/xo/tokenizer/span.hpp

@@ -62,7 +62,7 @@ namespace xo {
 
             /** @brief create span representing prefix up to (but not including) @p *p
              **/
-            span prefix(CharT * p) const {
+            span prefix_upto(CharT * p) const {
                 if (p <= hi_)
                     return span(lo_, p);
                 else

include/xo/tokenizer/tokenizer.hpp

@@ -560,13 +560,13 @@ namespace xo {
                 /* no-op */
                 return {
                     token_type::invalid(),
-                    input.prefix(ix)
+                    input.prefix_upto(ix)
                 };
             }
 
             /* here: *ix is not whitespace */
 
-            auto whitespace = input.prefix(ix);
+            auto whitespace = input.prefix_upto(ix);
 
             log && log(xtag("whitespace.size", whitespace.size()));
 
@@ -630,7 +630,7 @@ namespace xo {
                 }
             }
 
-            auto token_span = input.after_prefix(whitespace).prefix(ix);
+            auto token_span = input.after_prefix(whitespace).prefix_upto(ix);
 
             token tk
                 = (this->prefix_.empty()

include/xo/tokenizer/tokentype.hpp

@@ -106,6 +106,20 @@ namespace xo {
             /** '->' **/
             tk_yields,
 
+            /** note: operators not treated as punctuation
+             *  'do-always' is a legal variable name,
+             *  as is 'maybe*2', 'maybe+1', 'path/to/foo'
+             **/
+
+            /** operator '+' **/
+            tk_plus,
+            /** operator '-' **/
+            tk_minus,
+            /** operator '*' **/
+            tk_star,
+            /** operator '/' **/
+            tk_slash,
+
             /** keyworkd 'type' **/
             tk_type,
 

src/tokenizer/tokentype.cpp

@@ -37,6 +37,11 @@ namespace xo {
                 CASE(tk_assign);
                 CASE(tk_yields);
 
+                CASE(tk_plus);
+                CASE(tk_minus);
+                CASE(tk_star);
+                CASE(tk_slash);
+
                 CASE(tk_type);
                 CASE(tk_def);
                 CASE(tk_lambda);