roland/xo-alloc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: feat: + basic arithmetic operators

Browse source
This commit is contained in:
Roland Conybeare 2024-08-14 15:44:08 -04:00
commit fa335ee523
4 changed files with 23 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

6
include/xo/tokenizer/tokenizer.hpp
View file

@ -560,13 +560,13 @@ namespace xo {
/* no-op */
return {
token_type::invalid(),
input.prefix(ix)
input.prefix_upto(ix)
};
}
/* here: *ix is not whitespace */
auto whitespace = input.prefix(ix);
auto whitespace = input.prefix_upto(ix);
log && log(xtag("whitespace.size", whitespace.size()));
@ -630,7 +630,7 @@ namespace xo {
}
}
auto token_span = input.after_prefix(whitespace).prefix(ix);
auto token_span = input.after_prefix(whitespace).prefix_upto(ix);
token tk
= (this->prefix_.empty()