From 7c82821461224e8bea938fdfa2264e29023c5f05 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 22 Jun 2025 16:16:23 -0500 Subject: [PATCH] xo-tokenizer: bugfix: yields token works + 2phase utest --- src/reader/reader.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/reader/reader.cpp b/src/reader/reader.cpp index d08b9552..eb012541 100644 --- a/src/reader/reader.cpp +++ b/src/reader/reader.cpp @@ -32,10 +32,10 @@ namespace xo { while (!input.empty()) { /* read one token from input */ auto sr = this->tokenizer_.scan2(input, eof); - const auto & tk = sr.first; - const span_type & used_span = sr.second; + const auto & tk = sr.get_token(); + const span_type & used_span = sr.consumed(); - log && log(xtag("used_span", used_span)); + log && log(xtag("consumed", used_span)); log && log(xtag("input.pre", input)); input = input.after_prefix(used_span);