From a7ed10c16a6011cc59ef0e4ce30da9bbfdfac4a0 Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Sun, 11 Jan 2026 19:10:42 -0500
Subject: [PATCH] xo-tokenizer: example tokenrepl restored to wokring order

Now with CBufferedInput in Tokenizer
---
 example/tokenrepl/tokenrepl.cpp | 15 ++++++++++++++-
 src/tokenizer2/Tokenizer.cpp    | 14 ++++----------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/example/tokenrepl/tokenrepl.cpp b/example/tokenrepl/tokenrepl.cpp
index 0852f028..1cf02244 100644
--- a/example/tokenrepl/tokenrepl.cpp
+++ b/example/tokenrepl/tokenrepl.cpp
@@ -51,6 +51,8 @@ main() {
     using xo::scm::operator<<;
     using xo::mm::CircularBufferConfig;
     using xo::mm::span;
+    using xo::scope;
+    using xo::xtag;
     using replxx::Replxx;
 
     using namespace std;
@@ -65,10 +67,13 @@ main() {
     rx.set_max_history_size(1000);
     rx.history_load("repl_history.txt");
 
+    constexpr bool c_debug_flag = true;
+    scope log(XO_DEBUG(c_debug_flag));
+
     Tokenizer tkz(CircularBufferConfig{.name_ = "tokenrepl-input",
                                        .max_capacity_ = 4*1024,
                                        .max_captured_span_ = 128},
-                  true /*debug_flag*/);
+                  c_debug_flag);
 
     const char * input_cstr = nullptr;;
 
@@ -84,9 +89,17 @@ main() {
         if (input_cstr && *input_cstr) {
             auto [error, input] = tkz.buffer_input_line(input_cstr, false /*!eof*/);
 
+            if (log) {
+                log(xtag("msg", "buffered input line"));
+                log(xtag("input", input));
+            }
+
+            while (!input.empty())
             {
                 auto [tk, consumed, error] = tkz.scan(input);
 
+                log && log(xtag("consumed", consumed), xtag("tk", tk));
+
                 if (tk.is_valid()) {
                     cout << tk << endl;
                 } else if (error.is_error()) {
diff --git a/src/tokenizer2/Tokenizer.cpp b/src/tokenizer2/Tokenizer.cpp
index 888a0c43..4fa98a97 100644
--- a/src/tokenizer2/Tokenizer.cpp
+++ b/src/tokenizer2/Tokenizer.cpp
@@ -622,15 +622,9 @@ namespace xo {
             auto buf_input_0 = input_buffer_.input_range().hi();
 
             auto remainder = input_buffer_.append
-                                 (DCircularBuffer::const_span_type
-                                      ((const byte *)input_cstr,
-                                       (const byte *)input_cstr + strlen(input_cstr)));
-
-            const char * newline_cstr = "\n";
+                                 (DCircularBuffer::const_span_type::from_cstr(input_cstr));
             auto remainder2 = input_buffer_.append
-                                  (DCircularBuffer::const_span_type
-                                       ((const byte *)newline_cstr,
-                                        (const byte *)newline_cstr + strlen(newline_cstr)));
+                                  (DCircularBuffer::const_span_type::from_cstr("\n"));
 
             if (!remainder.empty() || !remainder2.empty()) {
                 throw std::runtime_error(tostr("Tokenizer::buffer_line: line too long!",
@@ -639,8 +633,8 @@ namespace xo {
 
             auto buf_input_1 = input_buffer_.input_range().hi();
 
-            span_type input = span_type((const char *)buf_input_0,
-                                        (const char *)buf_input_1);
+            span_type input = span_type(buf_input_0,
+                                        buf_input_1);
 
             return this->input_state_.capture_current_line(input, eof_flag);
         }