diff --git a/include/xo/tokenizer/input_state.hpp b/include/xo/tokenizer/input_state.hpp index f7f7293f..8e73321f 100644 --- a/include/xo/tokenizer/input_state.hpp +++ b/include/xo/tokenizer/input_state.hpp @@ -177,14 +177,18 @@ namespace xo { ++eol; this->current_line_ = span_type(sol, eol); + this->current_pos_ = 0; - log && log(xtag("current_line", print::printspan(current_line_))); + log && log(xtag("current_line", print::printspan(current_line_)), + xtag("current_pos", current_pos_)); } template const CharT * input_state::skip_leading_whitespace(const span_type & input) { + scope log(XO_DEBUG(debug_flag_)); + const CharT * ix = input.lo(); if (this->current_line().is_null()) { diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp index 7984126c..8d6ac215 100644 --- a/include/xo/tokenizer/tokenizer.hpp +++ b/include/xo/tokenizer/tokenizer.hpp @@ -166,7 +166,10 @@ namespace xo { /** @defgroup tokenizer-instance-vars tokenizer instance variables **/ ///@{ - /** track input state (line#,pos,..) for error messages **/ + /** track input state (line#,pos,..) for error messages. + * There's an ordering problem here: + * 1. input_state_.skip_leading_whitespace() advances current line when it sees newline. + **/ input_state_type input_state_; /** Accumulate partial token here. * This will happen if input sent to @ref tokenizer::scan