roland/xo-numeric
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2: + example app 'readerreplxx'

Browse source
This commit is contained in:
Roland Conybeare 2026-01-23 11:54:32 -05:00
commit 258d0823f5
16 changed files with 258 additions and 131 deletions
Download patch file Download diff file Expand all files Collapse all files

45
xo-tokenizer2/example/tokenrepl/tokenrepl.cpp
View file

@ -85,32 +85,33 @@ main() {
{
//cout << "input: " << input << endl;
auto input_ext = Tokenizer::span_type::from_cstr(input_cstr);
// reminder: input may contain multiple tokens
if (input_cstr && *input_cstr) {
auto [error, input] = tkz.buffer_input_line(input_cstr, false /*!eof*/);
auto [error, input] = tkz.buffer_input_line(input_ext, false /*!eof*/);
if (log) {
log(xtag("msg", "buffered input line"));
log(xtag("input", input));
if (log) {
log(xtag("msg", "buffered input line"));
log(xtag("input", input));
}
while (!input.empty())
{
auto [tk, consumed, error] = tkz.scan(input);
log && log(xtag("consumed", consumed), xtag("tk", tk));
if (tk.is_valid()) {
cout << tk << endl;
} else if (error.is_error()) {
cout << "tokenizer error: " << endl;
error.report(cout);
break;
}
while (!input.empty())
{
auto [tk, consumed, error] = tkz.scan(input);
log && log(xtag("consumed", consumed), xtag("tk", tk));
if (tk.is_valid()) {
cout << tk << endl;
} else if (error.is_error()) {
cout << "tokenizer error: " << endl;
error.report(cout);
break;
}
input = input.after_prefix(consumed);
}
input = input.after_prefix(consumed);
}
/* here: input.empty() or error encountered */