From 75b0383e668597fdadf8bfc307d6538563f10a59 Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Sat, 24 Aug 2024 12:30:34 -0400
Subject: [PATCH] xo-tokenizer: * token

---
 include/xo/tokenizer/token.hpp     |  2 ++
 include/xo/tokenizer/tokenizer.hpp | 10 ++++++++++
 utest/tokenizer.test.cpp           | 23 +++++++++++++++++++++++
 3 files changed, 35 insertions(+)

diff --git a/include/xo/tokenizer/token.hpp b/include/xo/tokenizer/token.hpp
index 9944cb3d..988b4976 100644
--- a/include/xo/tokenizer/token.hpp
+++ b/include/xo/tokenizer/token.hpp
@@ -80,6 +80,8 @@ namespace xo {
             static token assign_token() { return token(tokentype::tk_assign); }
             static token yields() { return token(tokentype::tk_yields); }
 
+            static token star_token() { return token(tokentype::tk_star); }
+
             static token type() { return token(tokentype::tk_type); }
             static token def() { return token(tokentype::tk_def); }
             static token lambda() { return token(tokentype::tk_lambda); }
diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp
index bb67eb13..329f5226 100644
--- a/include/xo/tokenizer/tokenizer.hpp
+++ b/include/xo/tokenizer/tokenizer.hpp
@@ -154,6 +154,9 @@ namespace xo {
             case '+':
                 /* can't be punctuation -- can appear inside f64 token: e.g. 1.23e+4 */
                 return false;
+            case '*':
+                /* not punctuation -- allowed in symbol */
+                return false;
             case '.':
                 /* can't be punctuation -- can appear inside f64 token: e.g. 1.23 */
                 return false;
@@ -501,6 +504,13 @@ namespace xo {
                 tk_type = tokentype::tk_semicolon;
                 ++ix;
                 break;
+            case '*':
+                /* '*' isn't punctuation, since can appear within symbol.
+                 * However it cannot begin a symbol..
+                 */
+                tk_type = tokentype::tk_star;
+                ++ix;
+                break;
             case ':':
             {
                 log && log("colon or assignment token");
diff --git a/utest/tokenizer.test.cpp b/utest/tokenizer.test.cpp
index 8a821b96..e0ec6a56 100644
--- a/utest/tokenizer.test.cpp
+++ b/utest/tokenizer.test.cpp
@@ -183,6 +183,29 @@ namespace xo {
                   token::singleassign(),
                   token::f64_token("3.141"),
                   token::semicolon()
+                 }},
+                {"def foo = lambda (x : f64) { def y = x * x; y; }",
+                 false,
+                 {token::def(),
+                  token::symbol_token("foo"),
+                  token::singleassign(),
+                  token::lambda(),
+                  token::leftparen(),
+                  token::symbol_token("x"),
+                  token::colon(),
+                  token::symbol_token("f64"),
+                  token::rightparen(),
+                  token::leftbrace(),
+                  token::def(),
+                  token::symbol_token("y"),
+                  token::singleassign(),
+                  token::symbol_token("x"),
+                  token::star_token(),
+                  token::symbol_token("x"),
+                  token::semicolon(),
+                  token::symbol_token("y"),
+                  token::semicolon(),
+                  token::rightbrace()
                  }}
             };
         }