From 415a382442e496d2ac04fd2942aa68f84e9ee1a0 Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Mon, 2 Feb 2026 21:55:34 -0500
Subject: [PATCH] xo-interpreter2: scaffold repl + alloc measurement frameowkr

---
 include/xo/tokenizer2/Tokenizer.hpp |  6 ++++++
 src/tokenizer2/Tokenizer.cpp        | 13 +++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/include/xo/tokenizer2/Tokenizer.hpp b/include/xo/tokenizer2/Tokenizer.hpp
index 3dc6da11..83015c03 100644
--- a/include/xo/tokenizer2/Tokenizer.hpp
+++ b/include/xo/tokenizer2/Tokenizer.hpp
@@ -61,6 +61,7 @@ namespace xo {
             using error_type = TokenizerError;
             using DCircularBuffer = xo::mm::DCircularBuffer;
             using CircularBufferConfig = xo::mm::CircularBufferConfig;
+            using MemorySizeInfo = xo::mm::MemorySizeInfo;
             using span_type = xo::mm::span<const CharT>;
             //using input_state_type = TkInputState;
             using result_type = scan_result;
@@ -90,6 +91,11 @@ namespace xo {
             const TkInputState & input_state() const { return input_state_; }
 #pragma GCC diagnostic pop
 
+            /** number of distinct memory pools owned by tokenizer **/
+            std::size_t _n_store() const noexcept;
+            /** memory consumption for i'th memory pool **/
+            MemorySizeInfo _store_info(std::size_t i) const noexcept;
+
             ///@}
 
             /** @defgroup tokenizer-general-methods tokenizer methods **/
diff --git a/src/tokenizer2/Tokenizer.cpp b/src/tokenizer2/Tokenizer.cpp
index c79e10c3..c36d85a5 100644
--- a/src/tokenizer2/Tokenizer.cpp
+++ b/src/tokenizer2/Tokenizer.cpp
@@ -6,6 +6,7 @@
 #include "Tokenizer.hpp"
 
 namespace xo {
+    using xo::mm::MemorySizeInfo;
     using std::byte;
 
     namespace scm {
@@ -21,6 +22,18 @@ namespace xo {
             this->input_state_.discard_current_line();
         }
 
+        std::size_t
+        Tokenizer::_n_store() const noexcept
+        {
+            return input_buffer_._n_store();
+        }
+
+        MemorySizeInfo
+        Tokenizer::_store_info(std::size_t i) const noexcept
+        {
+            return input_buffer_._store_info(i);
+        }
+
         bool
         Tokenizer::is_1char_punctuation(CharT ch)
         {