roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: feat: + basic arithmetic operators

Browse source
This commit is contained in:
Roland Conybeare 2024-08-14 15:44:08 -04:00
commit fa335ee523
4 changed files with 23 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
include/xo/tokenizer/span.hpp
View file

@ -62,7 +62,7 @@ namespace xo {
/** @brief create span representing prefix up to (but not including) @p *p
**/
span prefix(CharT * p) const {
span prefix_upto(CharT * p) const {
if (p <= hi_)
return span(lo_, p);
else

6
include/xo/tokenizer/tokenizer.hpp
View file

@ -560,13 +560,13 @@ namespace xo {
/* no-op */
return {
token_type::invalid(),
input.prefix(ix)
input.prefix_upto(ix)
};
}
/* here: *ix is not whitespace */
auto whitespace = input.prefix(ix);
auto whitespace = input.prefix_upto(ix);
log && log(xtag("whitespace.size", whitespace.size()));
@ -630,7 +630,7 @@ namespace xo {
}
}
auto token_span = input.after_prefix(whitespace).prefix(ix);
auto token_span = input.after_prefix(whitespace).prefix_upto(ix);
token tk
= (this->prefix_.empty()

14
include/xo/tokenizer/tokentype.hpp
View file

@ -106,6 +106,20 @@ namespace xo {
/** '->' **/
tk_yields,
/** note: operators not treated as punctuation
* 'do-always' is a legal variable name,
* as is 'maybe*2', 'maybe+1', 'path/to/foo'
**/
/** operator '+' **/
tk_plus,
/** operator '-' **/
tk_minus,
/** operator '*' **/
tk_star,
/** operator '/' **/
tk_slash,
/** keyworkd 'type' **/
tk_type,

5
src/tokenizer/tokentype.cpp
View file

@ -37,6 +37,11 @@ namespace xo {
CASE(tk_assign);
CASE(tk_yields);
CASE(tk_plus);
CASE(tk_minus);
CASE(tk_star);
CASE(tk_slash);
CASE(tk_type);
CASE(tk_def);
CASE(tk_lambda);