From 3746f325297ff44d363cbfabe15a5efe0f3b972a Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Tue, 6 Aug 2024 23:11:17 -0400
Subject: [PATCH] xo-tokenizer: fix: missing assignment

---
 include/xo/tokenizer/tokenizer.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp
index 4fad37ba..09bb4d97 100644
--- a/include/xo/tokenizer/tokenizer.hpp
+++ b/include/xo/tokenizer/tokenizer.hpp
@@ -648,6 +648,10 @@ namespace xo {
 
             if (!sr.first.is_valid() && eof) {
                 sr.first = this->notify_eof();
+                /* always consume remainder of input here.
+                 * ambiguous prefix can represent at most one token
+                 */
+                sr.second = input;
             }
 
             return sr;