roland/xo-alloc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'main' of github.com:Rconybea/xo-tokenizer

Browse source
This commit is contained in:
Roland Conybeare 2024-08-27 16:27:08 -04:00
commit 9ea386ecc4
1 changed files with 26 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

26
include/xo/tokenizer/tokenizer.hpp
View file

@ -157,6 +157,10 @@ namespace xo {
case '*':
/* not punctuation -- allowed in symbol */
return false;
case '*':
case '/':
/* not punctuation -- for symmetry with +,- */
return false;
case '.':
/* can't be punctuation -- can appear inside f64 token: e.g. 1.23 */
return false;
@ -201,6 +205,16 @@ namespace xo {
switch (*ix) {
case '-':
case '+':
if (token_text.size() == 1) {
/* standalone '+' or '-' */
if (*ix == '+')
tk_type = tokentype::tk_plus;
else if(*ix == '-')
tk_type = tokentype::tk_minus;
}
/** fall through to numeric literal code below **/
;
case '.':
case '0':
case '1':
@ -328,6 +342,18 @@ namespace xo {
break;
}
case '*':
if (token_text.size() == 1) {
/* standalone '*' */
tk_type = tokentype::tk_star;
}
break;
case '/':
if (token_text.size() == 1) {
/* standalone '/' */
tk_type = tokentype::tk_slash;
}
break;
case '"':
{
log && log("recognize string-token");