diff --git a/example/tokenrepl/tokenrepl.cpp b/example/tokenrepl/tokenrepl.cpp index 9afe0059..d6eacfea 100644 --- a/example/tokenrepl/tokenrepl.cpp +++ b/example/tokenrepl/tokenrepl.cpp @@ -41,7 +41,7 @@ main() { if (tk.is_valid()) { cout << tk << endl; } else if (error.is_error()) { - cout << "parsing error: " << endl; + cout << "tokenizer error: " << endl; error.report(cout); break; diff --git a/include/xo/tokenizer/span.hpp b/include/xo/tokenizer/span.hpp index bb1eeccb..8cf7a4a7 100644 --- a/include/xo/tokenizer/span.hpp +++ b/include/xo/tokenizer/span.hpp @@ -104,6 +104,18 @@ namespace xo { /** @defgroup span-general-methods **/ ///@{ + /** @brief strip prefix until first occurence of '\n', including the newline **/ + void discard_until_newline() { + for (const CharT * p = lo_; p < hi_; ++p) { + if (*p == '\n') { + lo_ = p + 1; + return; + } + } + + lo_ = hi_; + } + /** Create new span over supplied type, * with identical (possibly misaligned) endpoints. * @@ -142,8 +154,7 @@ namespace xo { /** @brief create span with @p prefix of this span removed **/ span after_prefix(const span & prefix) const { - assert(prefix.lo() == lo_); - if (prefix.lo() != lo_) { + if (!prefix.is_null() && (prefix.lo() != lo_)) { throw std::runtime_error ("after_prefix: expected prefix of this span"); } @@ -174,7 +185,7 @@ namespace xo { span & operator+=(const span & x) { if (hi_ == x.lo_) { hi_ = x.hi_; - } else { + } else if (!x.is_null()) { assert(false); } diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp index 75e4d4d1..74cc43f6 100644 --- a/include/xo/tokenizer/tokenizer.hpp +++ b/include/xo/tokenizer/tokenizer.hpp @@ -31,12 +31,16 @@ namespace xo { * span_type input = ...; * * while (!input.empty()) { - * auto res = tkz.scan(input); - * auto [tk, consumed, error] = res.first; + * auto [tk, consumed, error] = tkz.scan(input); * - * // do something with tk if tk.is_valid() + * if (tk.is_valid()) { + * // do something with tk + * } else if (error.is_error()) { + * error.report(cout); + * break; + * } * - * input = tkz.consume(res.second, input); + * input = tkz.consume(consumed, input); * } * * if endofinput { diff --git a/include/xo/tokenizer/tokenizer_error.hpp b/include/xo/tokenizer/tokenizer_error.hpp index 9488b62b..743e3faf 100644 --- a/include/xo/tokenizer/tokenizer_error.hpp +++ b/include/xo/tokenizer/tokenizer_error.hpp @@ -27,7 +27,7 @@ namespace xo { /** @defgroup tokenizer-error-ctors **/ ///@{ - /** Default ctor represent a not-an-error sentinel object **/ + /** Default ctor represents a not-an-error sentinel object **/ tokenizer_error() = default; /** Constructor to capture parsing error context * @p tk_start current position on entry to scanner @@ -69,7 +69,7 @@ namespace xo { /** true, except for a sentinel error object **/ bool is_error() const { return error_description_ != nullptr; } - /** true except for object in sentinel state **/ + /** false except for object in sentinel state **/ bool is_not_an_error() const { return error_description_ == nullptr; } /** Print representation to stream @p os. Intended for tokenizer diagnostics.