diff --git a/xo-reader/examples/exprreplxx/exprreplxx.cpp b/xo-reader/examples/exprreplxx/exprreplxx.cpp index 51d6e8d9..ced1a40b 100644 --- a/xo-reader/examples/exprreplxx/exprreplxx.cpp +++ b/xo-reader/examples/exprreplxx/exprreplxx.cpp @@ -96,7 +96,7 @@ main() welcome(cerr); while (replxx_getline(interactive, parser_stack_size, rx, input_str)) { - input = span_type::from_string(input_str); + input = span_type::from_string(input_str); while (!input.empty()) { auto [expr, consumed, psz, error] = rdr.read_expr(input, eof); diff --git a/xo-reader/src/reader/reader.cpp b/xo-reader/src/reader/reader.cpp index 2ea9b4fd..b19ea718 100644 --- a/xo-reader/src/reader/reader.cpp +++ b/xo-reader/src/reader/reader.cpp @@ -53,7 +53,8 @@ namespace xo { * tokenizer stashes one line at a time, but used_span only * reports in used_span the portion representing the first token. */ - auto [tk, used_span, error1] = this->tokenizer_.scan(input, eof_flag); + auto [tk, used_span, error1] + = this->tokenizer_.scan(input, eof_flag); log && log(xtag("consumed", used_span)); log && log(xtag("input.pre", input)); @@ -101,7 +102,7 @@ namespace xo { * input span may contain more tokens -> iterate */ } - } else { + } else /*invalid token*/ { if (error1.is_error()) { /* tokenizer detected an error */ @@ -113,15 +114,19 @@ namespace xo { error1.error_description(), error1.input_state(), error1.error_pos())); + } else if (used_span.size() > 0) { + /* control here on reaching the last token in input line; + * used_span will be the entire line + */ } else { - /* control should not come here */ - + /* control should not come here: invalid token with no error + */ assert(input.empty()); + break; } - - /* need more tokens in input */ - break; } + + input = input.after_prefix(used_span); } /* control here: either