From fa335ee523cdc1b54075b9da2bfc7d75d21704b7 Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Wed, 14 Aug 2024 15:44:08 -0400
Subject: [PATCH] xo-tokenizer: feat: + basic arithmetic operators

---
 include/xo/tokenizer/span.hpp      |  2 +-
 include/xo/tokenizer/tokenizer.hpp |  6 +++---
 include/xo/tokenizer/tokentype.hpp | 14 ++++++++++++++
 src/tokenizer/tokentype.cpp        |  5 +++++
 4 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/include/xo/tokenizer/span.hpp b/include/xo/tokenizer/span.hpp
index ca63759f..241f52aa 100644
--- a/include/xo/tokenizer/span.hpp
+++ b/include/xo/tokenizer/span.hpp
@@ -62,7 +62,7 @@ namespace xo {
 
             /** @brief create span representing prefix up to (but not including) @p *p
              **/
-            span prefix(CharT * p) const {
+            span prefix_upto(CharT * p) const {
                 if (p <= hi_)
                     return span(lo_, p);
                 else
diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp
index 09bb4d97..11ee5aca 100644
--- a/include/xo/tokenizer/tokenizer.hpp
+++ b/include/xo/tokenizer/tokenizer.hpp
@@ -560,13 +560,13 @@ namespace xo {
                 /* no-op */
                 return {
                     token_type::invalid(),
-                    input.prefix(ix)
+                    input.prefix_upto(ix)
                 };
             }
 
             /* here: *ix is not whitespace */
 
-            auto whitespace = input.prefix(ix);
+            auto whitespace = input.prefix_upto(ix);
 
             log && log(xtag("whitespace.size", whitespace.size()));
 
@@ -630,7 +630,7 @@ namespace xo {
                 }
             }
 
-            auto token_span = input.after_prefix(whitespace).prefix(ix);
+            auto token_span = input.after_prefix(whitespace).prefix_upto(ix);
 
             token tk
                 = (this->prefix_.empty()
diff --git a/include/xo/tokenizer/tokentype.hpp b/include/xo/tokenizer/tokentype.hpp
index bd20e8eb..6da013d9 100644
--- a/include/xo/tokenizer/tokentype.hpp
+++ b/include/xo/tokenizer/tokentype.hpp
@@ -106,6 +106,20 @@ namespace xo {
             /** '->' **/
             tk_yields,
 
+            /** note: operators not treated as punctuation
+             *  'do-always' is a legal variable name,
+             *  as is 'maybe*2', 'maybe+1', 'path/to/foo'
+             **/
+
+            /** operator '+' **/
+            tk_plus,
+            /** operator '-' **/
+            tk_minus,
+            /** operator '*' **/
+            tk_star,
+            /** operator '/' **/
+            tk_slash,
+
             /** keyworkd 'type' **/
             tk_type,
 
diff --git a/src/tokenizer/tokentype.cpp b/src/tokenizer/tokentype.cpp
index 08dbba84..b7172118 100644
--- a/src/tokenizer/tokentype.cpp
+++ b/src/tokenizer/tokentype.cpp
@@ -37,6 +37,11 @@ namespace xo {
                 CASE(tk_assign);
                 CASE(tk_yields);
 
+                CASE(tk_plus);
+                CASE(tk_minus);
+                CASE(tk_star);
+                CASE(tk_slash);
+
                 CASE(tk_type);
                 CASE(tk_def);
                 CASE(tk_lambda);