From 9d4b50ede4d4b94fd78309ef58cede5b5fd8e603 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 19 Jan 2026 11:33:14 -0500 Subject: [PATCH] xo-tokenizer2: cosmetic / minor --- src/tokenizer2/Tokenizer.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tokenizer2/Tokenizer.cpp b/src/tokenizer2/Tokenizer.cpp index 4fa98a97..7076a95d 100644 --- a/src/tokenizer2/Tokenizer.cpp +++ b/src/tokenizer2/Tokenizer.cpp @@ -595,10 +595,13 @@ namespace xo { tk_text.clear(); } + // TOOD: report tk_text as span, + // but must pin / unpin + /* input.prefix(0): * require caller preserves current input line until it's entirely exhausted */ - return result_type(token_type(tk_type, std::move(tk_text)), + return result_type(Token(tk_type, std::move(tk_text)), p_input_state->current_line().prefix(0)); } /*assemble_token*/