From e149e85910c6968177233c156b03491b52e78653 Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Wed, 14 Aug 2024 16:27:44 -0400
Subject: [PATCH] xo-tokenizer: recognize +,-,*,/ tokens

---
 include/xo/tokenizer/tokenizer.hpp | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp
index 11ee5aca..f2eff1b8 100644
--- a/include/xo/tokenizer/tokenizer.hpp
+++ b/include/xo/tokenizer/tokenizer.hpp
@@ -147,6 +147,10 @@ namespace xo {
             case '+':
                 /* can't be punctuation -- can appear inside f64 token */
                 return false;
+            case '*':
+            case '/':
+                /* not punctuation -- for symmetry with +,- */
+                return false;
             case '.':
                 /* can't be punctuation -- can appear inside f64 token */
                 return false;
@@ -179,6 +183,16 @@ namespace xo {
             switch (*ix) {
             case '-':
             case '+':
+                if (token_text.size() == 1) {
+                    /* standalone '+' or '-' */
+                    if (*ix == '+')
+                        tk_type = tokentype::tk_plus;
+                    else if(*ix == '-')
+                        tk_type = tokentype::tk_minus;
+                }
+
+                /** fall through to numeric literal code below **/
+                ;
             case '.':
             case '0':
             case '1':
@@ -306,6 +320,18 @@ namespace xo {
 
                 break;
             }
+            case '*':
+                if (token_text.size() == 1) {
+                    /* standalone '*' */
+                    tk_type = tokentype::tk_star;
+                }
+                break;
+            case '/':
+                if (token_text.size() == 1) {
+                    /* standalone '/' */
+                    tk_type = tokentype::tk_slash;
+                }
+                break;
             case '"':
             {
                 log && log("recognize string-token");