roland/xo-tokenizer
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

reader reports tokenizer errors through normal return

Browse source
This commit is contained in:
Roland Conybeare 2025-07-19 16:47:59 -05:00
commit dbd59ada5f
4 changed files with 25 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

2
example/tokenrepl/tokenrepl.cpp
View file

@ -41,7 +41,7 @@ main() {
if (tk.is_valid()) {
cout << tk << endl;
} else if (error.is_error()) {
cout << "parsing error: " << endl;
cout << "tokenizer error: " << endl;
error.report(cout);
break;

17
include/xo/tokenizer/span.hpp
View file

@ -104,6 +104,18 @@ namespace xo {
/** @defgroup span-general-methods **/
///@{
/** @brief strip prefix until first occurence of '\n', including the newline **/
void discard_until_newline() {
for (const CharT * p = lo_; p < hi_; ++p) {
if (*p == '\n') {
lo_ = p + 1;
return;
}
}
lo_ = hi_;
}
/** Create new span over supplied type,
* with identical (possibly misaligned) endpoints.
*
@ -142,8 +154,7 @@ namespace xo {
/** @brief create span with @p prefix of this span removed **/
span after_prefix(const span & prefix) const {
assert(prefix.lo() == lo_);
if (prefix.lo() != lo_) {
if (!prefix.is_null() && (prefix.lo() != lo_)) {
throw std::runtime_error
("after_prefix: expected prefix of this span");
}
@ -174,7 +185,7 @@ namespace xo {
span & operator+=(const span & x) {
if (hi_ == x.lo_) {
hi_ = x.hi_;
} else {
} else if (!x.is_null()) {
assert(false);
}

12
include/xo/tokenizer/tokenizer.hpp
View file

@ -31,12 +31,16 @@ namespace xo {
* span_type input = ...;
*
* while (!input.empty()) {
* auto res = tkz.scan(input);
* auto [tk, consumed, error] = res.first;
* auto [tk, consumed, error] = tkz.scan(input);
*
* // do something with tk if tk.is_valid()
* if (tk.is_valid()) {
* // do something with tk
* } else if (error.is_error()) {
* error.report(cout);
* break;
* }
*
* input = tkz.consume(res.second, input);
* input = tkz.consume(consumed, input);
* }
*
* if endofinput {

4
include/xo/tokenizer/tokenizer_error.hpp
View file

@ -27,7 +27,7 @@ namespace xo {
/** @defgroup tokenizer-error-ctors **/
///@{
/** Default ctor represent a not-an-error sentinel object **/
/** Default ctor represents a not-an-error sentinel object **/
tokenizer_error() = default;
/** Constructor to capture parsing error context
* @p tk_start current position on entry to scanner
@ -69,7 +69,7 @@ namespace xo {
/** true, except for a sentinel error object **/
bool is_error() const { return error_description_ != nullptr; }
/** true except for object in sentinel state **/
/** false except for object in sentinel state **/
bool is_not_an_error() const { return error_description_ == nullptr; }
/** Print representation to stream @p os. Intended for tokenizer diagnostics.