roland/xo-alloc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: recognize +,-,*,/ tokens

Browse source
This commit is contained in:
Roland Conybeare 2024-08-14 16:27:44 -04:00
commit e149e85910
1 changed files with 26 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

26
include/xo/tokenizer/tokenizer.hpp
View file

@ -147,6 +147,10 @@ namespace xo {
case '+':
/* can't be punctuation -- can appear inside f64 token */
return false;
case '*':
case '/':
/* not punctuation -- for symmetry with +,- */
return false;
case '.':
/* can't be punctuation -- can appear inside f64 token */
return false;
@ -179,6 +183,16 @@ namespace xo {
switch (*ix) {
case '-':
case '+':
if (token_text.size() == 1) {
/* standalone '+' or '-' */
if (*ix == '+')
tk_type = tokentype::tk_plus;
else if(*ix == '-')
tk_type = tokentype::tk_minus;
}
/** fall through to numeric literal code below **/
;
case '.':
case '0':
case '1':
@ -306,6 +320,18 @@ namespace xo {
break;
}
case '*':
if (token_text.size() == 1) {
/* standalone '*' */
tk_type = tokentype::tk_star;
}
break;
case '/':
if (token_text.size() == 1) {
/* standalone '/' */
tk_type = tokentype::tk_slash;
}
break;
case '"':
{
log && log("recognize string-token");