roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: bugfix: yields token works + 2phase utest

Browse source
This commit is contained in:
Roland Conybeare 2025-06-22 16:16:23 -05:00
commit 27ef5701ac
8 changed files with 722 additions and 312 deletions
Download patch file Download diff file Expand all files Collapse all files

6
xo-reader/src/reader/reader.cpp
View file

@ -32,10 +32,10 @@ namespace xo {
while (!input.empty()) {
/* read one token from input */
auto sr = this->tokenizer_.scan2(input, eof);
const auto & tk = sr.first;
const span_type & used_span = sr.second;
const auto & tk = sr.get_token();
const span_type & used_span = sr.consumed();
log && log(xtag("used_span", used_span));
log && log(xtag("consumed", used_span));
log && log(xtag("input.pre", input));
input = input.after_prefix(used_span);