diff --git a/example/tokenrepl/tokenrepl.cpp b/example/tokenrepl/tokenrepl.cpp index 1cf02244..d8ddbd7f 100644 --- a/example/tokenrepl/tokenrepl.cpp +++ b/example/tokenrepl/tokenrepl.cpp @@ -85,32 +85,33 @@ main() { { //cout << "input: " << input << endl; + auto input_ext = Tokenizer::span_type::from_cstr(input_cstr); + // reminder: input may contain multiple tokens - if (input_cstr && *input_cstr) { - auto [error, input] = tkz.buffer_input_line(input_cstr, false /*!eof*/); + auto [error, input] = tkz.buffer_input_line(input_ext, false /*!eof*/); - if (log) { - log(xtag("msg", "buffered input line")); - log(xtag("input", input)); + if (log) { + log(xtag("msg", "buffered input line")); + log(xtag("input", input)); + } + + while (!input.empty()) + { + auto [tk, consumed, error] = tkz.scan(input); + + log && log(xtag("consumed", consumed), xtag("tk", tk)); + + if (tk.is_valid()) { + cout << tk << endl; + } else if (error.is_error()) { + cout << "tokenizer error: " << endl; + + error.report(cout); + + break; } - while (!input.empty()) - { - auto [tk, consumed, error] = tkz.scan(input); - - log && log(xtag("consumed", consumed), xtag("tk", tk)); - - if (tk.is_valid()) { - cout << tk << endl; - } else if (error.is_error()) { - cout << "tokenizer error: " << endl; - error.report(cout); - - break; - } - - input = input.after_prefix(consumed); - } + input = input.after_prefix(consumed); } /* here: input.empty() or error encountered */ diff --git a/include/xo/tokenizer2/Tokenizer.hpp b/include/xo/tokenizer2/Tokenizer.hpp index 40a98cd9..69843a5a 100644 --- a/include/xo/tokenizer2/Tokenizer.hpp +++ b/include/xo/tokenizer2/Tokenizer.hpp @@ -129,10 +129,11 @@ namespace xo { **/ bool has_prefix() const { return !prefix_.empty(); } - /** buffer contents of input_cstr. + /** copy into buffer the contents of @p input. * May throw if buffer space exhausted **/ - std::pair buffer_input_line(const char * input_cstr, bool eof_flag); + std::pair buffer_input_line(span_type input, + bool eof_flag); /** scan for next input token, given @p input. * Note: diff --git a/include/xo/tokenizer2/TokenizerError.hpp b/include/xo/tokenizer2/TokenizerError.hpp index b8a50988..bf7702b1 100644 --- a/include/xo/tokenizer2/TokenizerError.hpp +++ b/include/xo/tokenizer2/TokenizerError.hpp @@ -32,7 +32,7 @@ namespace xo { * @p tk_start current position on entry to scanner * @p error_pos error location relative to token start **/ - TokenizerError(const char * src_function, + TokenizerError(std::string_view src_function, std::string error_description, const TkInputState & input_state, size_t error_pos) @@ -46,12 +46,20 @@ namespace xo { log && log(xtag("input_state.current_pos", input_state.current_pos()), xtag("error_pos", error_pos)); } + + TokenizerError with_error(std::string_view error_src_fn, + std::string error_msg) { + return TokenizerError(error_src_fn, + std::string(error_msg), + this->input_state_, + 0 /*error_pos*/); + } ///@} /** @defgroup tokenizer-error-access-methods **/ ///@{ - const char * src_function() const { return src_function_; } + std::string_view src_function() const { return src_function_; } const std::string & error_description() const { return error_description_; } #pragma GCC diagnostic push #ifndef __APPLE__ @@ -88,8 +96,8 @@ namespace xo { ///@{ /** source location (in tokenizer) at which error identified **/ - char const * src_function_ = nullptr; - /** static error description **/ + std::string_view src_function_; + /** error description **/ std::string error_description_; /** input state associated with this error. * Sufficient to precisely locate it with context. diff --git a/src/tokenizer2/Tokenizer.cpp b/src/tokenizer2/Tokenizer.cpp index 7076a95d..2784072a 100644 --- a/src/tokenizer2/Tokenizer.cpp +++ b/src/tokenizer2/Tokenizer.cpp @@ -615,19 +615,18 @@ namespace xo { } auto - Tokenizer::buffer_input_line(const char * input_cstr, + Tokenizer::buffer_input_line(span_type input_ext, bool eof_flag) -> std::pair { scope log(XO_DEBUG(input_state_.debug_flag())); - log && log(xtag("input", input_cstr)); + log && log(xtag("input_ext", input_ext)); auto buf_input_0 = input_buffer_.input_range().hi(); - auto remainder = input_buffer_.append - (DCircularBuffer::const_span_type::from_cstr(input_cstr)); - auto remainder2 = input_buffer_.append - (DCircularBuffer::const_span_type::from_cstr("\n")); + auto remainder = input_buffer_.append(input_ext); + auto remainder2 = input_buffer_.append(span_type::from_cstr("\n")); + //(DCircularBuffer::const_span_type::from_cstr("\n")); if (!remainder.empty() || !remainder2.empty()) { throw std::runtime_error(tostr("Tokenizer::buffer_line: line too long!", @@ -636,10 +635,10 @@ namespace xo { auto buf_input_1 = input_buffer_.input_range().hi(); - span_type input = span_type(buf_input_0, - buf_input_1); + span_type input_ours = span_type(buf_input_0, + buf_input_1); - return this->input_state_.capture_current_line(input, eof_flag); + return this->input_state_.capture_current_line(input_ours, eof_flag); } auto