roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: * token

Browse source
This commit is contained in:
Roland Conybeare 2024-08-24 12:30:34 -04:00
commit 75b0383e66
3 changed files with 35 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

2
include/xo/tokenizer/token.hpp
View file

@ -80,6 +80,8 @@ namespace xo {
static token assign_token() { return token(tokentype::tk_assign); }
static token yields() { return token(tokentype::tk_yields); }
static token star_token() { return token(tokentype::tk_star); }
static token type() { return token(tokentype::tk_type); }
static token def() { return token(tokentype::tk_def); }
static token lambda() { return token(tokentype::tk_lambda); }

10
include/xo/tokenizer/tokenizer.hpp
View file

@ -154,6 +154,9 @@ namespace xo {
case '+':
/* can't be punctuation -- can appear inside f64 token: e.g. 1.23e+4 */
return false;
case '*':
/* not punctuation -- allowed in symbol */
return false;
case '.':
/* can't be punctuation -- can appear inside f64 token: e.g. 1.23 */
return false;
@ -501,6 +504,13 @@ namespace xo {
tk_type = tokentype::tk_semicolon;
++ix;
break;
case '*':
/* '*' isn't punctuation, since can appear within symbol.
* However it cannot begin a symbol..
*/
tk_type = tokentype::tk_star;
++ix;
break;
case ':':
{
log && log("colon or assignment token");

23
utest/tokenizer.test.cpp
View file

@ -183,6 +183,29 @@ namespace xo {
token::singleassign(),
token::f64_token("3.141"),
token::semicolon()
}},
{"def foo = lambda (x : f64) { def y = x * x; y; }",
false,
{token::def(),
token::symbol_token("foo"),
token::singleassign(),
token::lambda(),
token::leftparen(),
token::symbol_token("x"),
token::colon(),
token::symbol_token("f64"),
token::rightparen(),
token::leftbrace(),
token::def(),
token::symbol_token("y"),
token::singleassign(),
token::symbol_token("x"),
token::star_token(),
token::symbol_token("x"),
token::semicolon(),
token::symbol_token("y"),
token::semicolon(),
token::rightbrace()
}}
};
}