From 0bd77b3c529e925c8ad30e783298e4fecd8fa8be Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Wed, 25 Jun 2025 21:45:44 -0500
Subject: [PATCH] xo-tokenizer: + missed doc files prev commit

---
 xo-tokenizer/docs/input-state-class.rst | 29 +++++++++++++++++++++++++
 xo-tokenizer/docs/scan-result-class.rst | 29 +++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 xo-tokenizer/docs/input-state-class.rst
 create mode 100644 xo-tokenizer/docs/scan-result-class.rst

diff --git a/xo-tokenizer/docs/input-state-class.rst b/xo-tokenizer/docs/input-state-class.rst
new file mode 100644
index 00000000..01d14267
--- /dev/null
+++ b/xo-tokenizer/docs/input-state-class.rst
@@ -0,0 +1,29 @@
+
+.. _input-state-class:
+
+Input State
+===========
+
+Track detailed state of input stream to collect information useful for detailed error reporting
+
+Context
+-------
+
+.. ditaa::
+    :--scale: 0.85
+
+    +-----------------------------------------+----------+
+    |                tokenizer                |          |
+    +-----------------------------------------+          |
+    |               scan_result               |          |
+    +-----------------+-----------------------+          |
+    |                 |    tokenizer_error    |  buffer  |
+    |     token       +-----------------------+          |
+    |                 |cBLU  input_state      |          |
+    +-----------------+-----------------------+          |
+    |    tokentype    |          span         |          |
+    +-----------------+-----------------------+----------+
+
+.. code-block:: cpp
+
+    #include <xo/tokenizer/input_state.hpp>
diff --git a/xo-tokenizer/docs/scan-result-class.rst b/xo-tokenizer/docs/scan-result-class.rst
new file mode 100644
index 00000000..6581a839
--- /dev/null
+++ b/xo-tokenizer/docs/scan-result-class.rst
@@ -0,0 +1,29 @@
+
+.. _scan-result-class:
+
+Scan Result
+===========
+
+Represent the result of a tokenizer scan call
+
+Context
+-------
+
+.. ditaa::
+    :--scale: 0.85
+
+    +-----------------------------------------+----------+
+    |                tokenizer                |          |
+    +-----------------------------------------+          |
+    |cBLU           scan_result               |          |
+    +-----------------+-----------------------+          |
+    |                 |    tokenizer_error    |  buffer  |
+    |     token       +-----------------------+          |
+    |                 |      input_state      |          |
+    +-----------------+-----------------------+          |
+    |    tokentype    |          span         |          |
+    +-----------------+-----------------------+----------+
+
+.. code-block:: cpp
+
+    #include <xo/tokenizer/input_state.hpp>