roland/xo-alloc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: * token

Browse source
This commit is contained in:
Roland Conybeare 2024-08-24 12:30:34 -04:00
commit 75b0383e66
3 changed files with 35 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
include/xo/tokenizer/token.hpp
View file

@ -80,6 +80,8 @@ namespace xo {
static token assign_token() { return token(tokentype::tk_assign); }
static token yields() { return token(tokentype::tk_yields); }
static token star_token() { return token(tokentype::tk_star); }
static token type() { return token(tokentype::tk_type); }
static token def() { return token(tokentype::tk_def); }
static token lambda() { return token(tokentype::tk_lambda); }

10
include/xo/tokenizer/tokenizer.hpp
View file

@ -154,6 +154,9 @@ namespace xo {
case '+':
/* can't be punctuation -- can appear inside f64 token: e.g. 1.23e+4 */
return false;
case '*':
/* not punctuation -- allowed in symbol */
return false;
case '.':
/* can't be punctuation -- can appear inside f64 token: e.g. 1.23 */
return false;
@ -501,6 +504,13 @@ namespace xo {
tk_type = tokentype::tk_semicolon;
++ix;
break;
case '*':
/* '*' isn't punctuation, since can appear within symbol.
* However it cannot begin a symbol..
*/
tk_type = tokentype::tk_star;
++ix;
break;
case ':':
{
log && log("colon or assignment token");