roland/xo-alloc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: mvp: recognize keywords

Browse source
This commit is contained in:
Roland Conybeare 2024-08-06 11:37:41 -04:00
commit 5d31ac7a43
3 changed files with 37 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
include/xo/tokenizer/token.hpp
View file

@ -86,6 +86,7 @@ namespace xo {
static token if_token() { return token(tokentype::tk_if); }
static token let() { return token(tokentype::tk_let); }
static token in() { return token(tokentype::tk_in); }
static token end() { return token(tokentype::tk_end); }
tokentype tk_type() const { return tk_type_; }
const std::string & text() const { return text_; }

28
include/xo/tokenizer/tokenizer.hpp
View file

@ -510,6 +510,34 @@ namespace xo {
; /* nothing to do here -- desired tk_text already constructed */
}
if (tk_type == tokentype::tk_symbol) {
/* check for keywords */
bool keep_text = false;
if (tk_text == "type") {
tk_type = tokentype::tk_type;
} else if (tk_text == "def") {
tk_type = tokentype::tk_def;
} else if (tk_text == "lambda") {
tk_type = tokentype::tk_lambda;
} else if (tk_text == "if") {
tk_type = tokentype::tk_if;
} else if (tk_text == "let") {
tk_type = tokentype::tk_let;
} else if (tk_text == "in") {
tk_type = tokentype::tk_in;
} else if (tk_text == "end") {
tk_type = tokentype::tk_end;
} else {
/* keep as symbol */
keep_text = true;
}
if (!keep_text)
tk_text.clear();
}
return token_type(tk_type, std::move(tk_text));
} /*assemble_token*/

8
utest/tokenizer.test.cpp
View file

@ -100,6 +100,14 @@ namespace xo {
token::string_token("tab to the right [\t], to the right [\t]"), true},
{"symbol", false, token::symbol_token("symbol"), true},
{"type", false, token::type(), true},
{"def", false, token::def(), true},
{"lambda", false, token::lambda(), true},
{"if", false, token::if_token(), true},
{"let", false, token::let(), true},
{"in", false, token::in(), true},
{"end", false, token::end(), true},
};
}