roland/xo-tokenizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-interpreter: setting up for gc in interactive interpreter

Browse source
This commit is contained in:
Roland Conybeare 2025-11-20 21:26:18 -05:00
commit 9a7e1a8bd3
2 changed files with 9 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
include/xo/tokenizer/tokenizer.hpp
View file

@ -166,7 +166,10 @@ namespace xo {
/** @defgroup tokenizer-instance-vars tokenizer instance variables **/
///@{
/** track input state (line#,pos,..) for error messages **/
/** track input state (line#,pos,..) for error messages.
* There's an ordering problem here:
* 1. input_state_.skip_leading_whitespace() advances current line when it sees newline.
**/
input_state_type input_state_;
/** Accumulate partial token here.
* This will happen if input sent to @ref tokenizer::scan