From fbfd5b286105dd3e802456f888961d290ab130e9 Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Wed, 25 Jun 2025 21:45:44 -0500
Subject: [PATCH] xo-tokenizer: + missed doc files prev commit

---
 docs/input-state-class.rst | 29 +++++++++++++++++++++++++++++
 docs/scan-result-class.rst | 29 +++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)
 create mode 100644 docs/input-state-class.rst
 create mode 100644 docs/scan-result-class.rst

diff --git a/docs/input-state-class.rst b/docs/input-state-class.rst
new file mode 100644
index 00000000..01d14267
--- /dev/null
+++ b/docs/input-state-class.rst
@@ -0,0 +1,29 @@
+
+.. _input-state-class:
+
+Input State
+===========
+
+Track detailed state of input stream to collect information useful for detailed error reporting
+
+Context
+-------
+
+.. ditaa::
+    :--scale: 0.85
+
+    +-----------------------------------------+----------+
+    |                tokenizer                |          |
+    +-----------------------------------------+          |
+    |               scan_result               |          |
+    +-----------------+-----------------------+          |
+    |                 |    tokenizer_error    |  buffer  |
+    |     token       +-----------------------+          |
+    |                 |cBLU  input_state      |          |
+    +-----------------+-----------------------+          |
+    |    tokentype    |          span         |          |
+    +-----------------+-----------------------+----------+
+
+.. code-block:: cpp
+
+    #include <xo/tokenizer/input_state.hpp>
diff --git a/docs/scan-result-class.rst b/docs/scan-result-class.rst
new file mode 100644
index 00000000..6581a839
--- /dev/null
+++ b/docs/scan-result-class.rst
@@ -0,0 +1,29 @@
+
+.. _scan-result-class:
+
+Scan Result
+===========
+
+Represent the result of a tokenizer scan call
+
+Context
+-------
+
+.. ditaa::
+    :--scale: 0.85
+
+    +-----------------------------------------+----------+
+    |                tokenizer                |          |
+    +-----------------------------------------+          |
+    |cBLU           scan_result               |          |
+    +-----------------+-----------------------+          |
+    |                 |    tokenizer_error    |  buffer  |
+    |     token       +-----------------------+          |
+    |                 |      input_state      |          |
+    +-----------------+-----------------------+          |
+    |    tokentype    |          span         |          |
+    +-----------------+-----------------------+----------+
+
+.. code-block:: cpp
+
+    #include <xo/tokenizer/input_state.hpp>