roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2: readerreplxx works + streamline debugging

Browse source
This commit is contained in:
Roland Conybeare 2026-01-23 14:57:43 -05:00
commit 7432a0bd1d
2 changed files with 25 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

8
include/xo/tokenizer2/Tokenizer.hpp
View file

@ -109,19 +109,19 @@ namespace xo {
static bool is_2char_punctuation(CharT ch);
/** assemble token from text @p token_text.
* @p initial_whitespace Amount of whitespace input being consumed from input.
* @p ws_span whitespace preceding token
* @p token_text subset of input_line representing a single token.
* @p p_input_state input state containing input_line. On exit current line cleared
* if error
*
* retval.consumed will represent some possibly-empty prefix of @p input
**/
static scan_result assemble_token(std::size_t initial_whitespace,
const span_type & token_text,
static scan_result assemble_token( span_type ws_span,
span_type token_text,
TkInputState * p_input_state);
/** degenerate version of assemble_token() on reaching end-of-file **/
static scan_result assemble_final_token(const span_type & token_text,
static scan_result assemble_final_token(span_type token_text,
TkInputState * p_input_state);
/** true if tokenizer contains stored prefix of