xo-tokenizer: + doc for tokenizer + other doc-related improvements

2025-06-25 23:50:30 -05:00 · 2025-06-25 23:50:30 -05:00 · 3b073d013f
commit 3b073d013f
parent 0bd77b3c52
11 changed files with 145 additions and 31 deletions
--- a/xo-tokenizer/example/tokenrepl/tokenrepl.cpp
+++ b/xo-tokenizer/example/tokenrepl/tokenrepl.cpp
@ -48,7 +48,6 @@ main() {
            }

            input = tkz.consume(consumed, input);
-            //input = input.after_prefix(consumed.size());
        }

        /* here: input.empty() or error encountered */