diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp index 329f5226..d73e163f 100644 --- a/include/xo/tokenizer/tokenizer.hpp +++ b/include/xo/tokenizer/tokenizer.hpp @@ -157,6 +157,10 @@ namespace xo { case '*': /* not punctuation -- allowed in symbol */ return false; + case '*': + case '/': + /* not punctuation -- for symmetry with +,- */ + return false; case '.': /* can't be punctuation -- can appear inside f64 token: e.g. 1.23 */ return false; @@ -201,6 +205,16 @@ namespace xo { switch (*ix) { case '-': case '+': + if (token_text.size() == 1) { + /* standalone '+' or '-' */ + if (*ix == '+') + tk_type = tokentype::tk_plus; + else if(*ix == '-') + tk_type = tokentype::tk_minus; + } + + /** fall through to numeric literal code below **/ + ; case '.': case '0': case '1': @@ -328,6 +342,18 @@ namespace xo { break; } + case '*': + if (token_text.size() == 1) { + /* standalone '*' */ + tk_type = tokentype::tk_star; + } + break; + case '/': + if (token_text.size() == 1) { + /* standalone '/' */ + tk_type = tokentype::tk_slash; + } + break; case '"': { log && log("recognize string-token");