From dabc642ca4f4d0b9f3b4a4276d48984c26ac3f78 Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Tue, 6 Aug 2024 09:33:53 -0400
Subject: [PATCH] xo-tokenizer: feat: + tokenizer.scan2()

---
 include/xo/tokenizer/tokenizer.hpp | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp
index bc0507e0..7f34b2ad 100644
--- a/include/xo/tokenizer/tokenizer.hpp
+++ b/include/xo/tokenizer/tokenizer.hpp
@@ -83,6 +83,12 @@ namespace xo {
              **/
             scan_result scan(const span_type & input);
 
+            /** When eof is false, same as scan(input).
+             *  When eof is true and scan(input) does not report a token,
+             *  return notify_eof()
+             **/
+            scan_result scan2(const span_type & input, bool eof);
+
             /** notify end of input,  resolve any stored input **/
             token_type notify_eof();
 
@@ -607,6 +613,18 @@ namespace xo {
                 { tk, input.prefix(whitespace.size() + token_span.size()) };
         } /*scan*/
 
+        template <typename CharT>
+        auto
+        tokenizer<CharT>::scan2(const span_type & input, bool eof) -> scan_result {
+            auto sr = this->scan(input);
+
+            if (!sr.first.is_valid() && eof) {
+                sr.first = this->notify_eof();
+            }
+
+            return sr;
+        }
+
         template <typename CharT>
         auto
         tokenizer<CharT>::notify_eof() -> token_type {