roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2: + example app 'readerreplxx'

Browse source
This commit is contained in:
Roland Conybeare 2026-01-23 11:54:32 -05:00
commit 9044e1d196
4 changed files with 42 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

45
example/tokenrepl/tokenrepl.cpp
View file

@ -85,32 +85,33 @@ main() {
{ {
//cout << "input: " << input << endl; //cout << "input: " << input << endl;
auto input_ext = Tokenizer::span_type::from_cstr(input_cstr);
// reminder: input may contain multiple tokens // reminder: input may contain multiple tokens
if (input_cstr && *input_cstr) { auto [error, input] = tkz.buffer_input_line(input_ext, false /*!eof*/);
auto [error, input] = tkz.buffer_input_line(input_cstr, false /*!eof*/);
if (log) { if (log) {
log(xtag("msg", "buffered input line")); log(xtag("msg", "buffered input line"));
log(xtag("input", input)); log(xtag("input", input));
}
while (!input.empty())
{
auto [tk, consumed, error] = tkz.scan(input);
log && log(xtag("consumed", consumed), xtag("tk", tk));
if (tk.is_valid()) {
cout << tk << endl;
} else if (error.is_error()) {
cout << "tokenizer error: " << endl;
error.report(cout);
break;
} }
while (!input.empty()) input = input.after_prefix(consumed);
{
auto [tk, consumed, error] = tkz.scan(input);
log && log(xtag("consumed", consumed), xtag("tk", tk));
if (tk.is_valid()) {
cout << tk << endl;
} else if (error.is_error()) {
cout << "tokenizer error: " << endl;
error.report(cout);
break;
}
input = input.after_prefix(consumed);
}
} }
/* here: input.empty() or error encountered */ /* here: input.empty() or error encountered */

5
include/xo/tokenizer2/Tokenizer.hpp
View file

@ -129,10 +129,11 @@ namespace xo {
**/ **/
bool has_prefix() const { return !prefix_.empty(); } bool has_prefix() const { return !prefix_.empty(); }
/** buffer contents of input_cstr. /** copy into buffer the contents of @p input.
* May throw if buffer space exhausted * May throw if buffer space exhausted
**/ **/
std::pair<input_error, span_type> buffer_input_line(const char * input_cstr, bool eof_flag); std::pair<input_error, span_type> buffer_input_line(span_type input,
bool eof_flag);
/** scan for next input token, given @p input. /** scan for next input token, given @p input.
* Note: * Note:

16
include/xo/tokenizer2/TokenizerError.hpp
View file

@ -32,7 +32,7 @@ namespace xo {
* @p tk_start current position on entry to scanner * @p tk_start current position on entry to scanner
* @p error_pos error location relative to token start * @p error_pos error location relative to token start
**/ **/
TokenizerError(const char * src_function, TokenizerError(std::string_view src_function,
std::string error_description, std::string error_description,
const TkInputState & input_state, const TkInputState & input_state,
size_t error_pos) size_t error_pos)
@ -46,12 +46,20 @@ namespace xo {
log && log(xtag("input_state.current_pos", input_state.current_pos()), log && log(xtag("input_state.current_pos", input_state.current_pos()),
xtag("error_pos", error_pos)); xtag("error_pos", error_pos));
} }
TokenizerError with_error(std::string_view error_src_fn,
std::string error_msg) {
return TokenizerError(error_src_fn,
std::string(error_msg),
this->input_state_,
0 /*error_pos*/);
}
///@} ///@}
/** @defgroup tokenizer-error-access-methods **/ /** @defgroup tokenizer-error-access-methods **/
///@{ ///@{
const char * src_function() const { return src_function_; } std::string_view src_function() const { return src_function_; }
const std::string & error_description() const { return error_description_; } const std::string & error_description() const { return error_description_; }
#pragma GCC diagnostic push #pragma GCC diagnostic push
#ifndef __APPLE__ #ifndef __APPLE__
@ -88,8 +96,8 @@ namespace xo {
///@{ ///@{
/** source location (in tokenizer) at which error identified **/ /** source location (in tokenizer) at which error identified **/
char const * src_function_ = nullptr; std::string_view src_function_;
/** static error description **/ /** error description **/
std::string error_description_; std::string error_description_;
/** input state associated with this error. /** input state associated with this error.
* Sufficient to precisely locate it with context. * Sufficient to precisely locate it with context.

17
src/tokenizer2/Tokenizer.cpp
View file

@ -615,19 +615,18 @@ namespace xo {
} }
auto auto
Tokenizer::buffer_input_line(const char * input_cstr, Tokenizer::buffer_input_line(span_type input_ext,
bool eof_flag) -> std::pair<input_error, span_type> bool eof_flag) -> std::pair<input_error, span_type>
{ {
scope log(XO_DEBUG(input_state_.debug_flag())); scope log(XO_DEBUG(input_state_.debug_flag()));
log && log(xtag("input", input_cstr)); log && log(xtag("input_ext", input_ext));
auto buf_input_0 = input_buffer_.input_range().hi(); auto buf_input_0 = input_buffer_.input_range().hi();
auto remainder = input_buffer_.append auto remainder = input_buffer_.append(input_ext);
(DCircularBuffer::const_span_type::from_cstr(input_cstr)); auto remainder2 = input_buffer_.append(span_type::from_cstr("\n"));
auto remainder2 = input_buffer_.append //(DCircularBuffer::const_span_type::from_cstr("\n"));
(DCircularBuffer::const_span_type::from_cstr("\n"));
if (!remainder.empty() || !remainder2.empty()) { if (!remainder.empty() || !remainder2.empty()) {
throw std::runtime_error(tostr("Tokenizer::buffer_line: line too long!", throw std::runtime_error(tostr("Tokenizer::buffer_line: line too long!",
@ -636,10 +635,10 @@ namespace xo {
auto buf_input_1 = input_buffer_.input_range().hi(); auto buf_input_1 = input_buffer_.input_range().hi();
span_type input = span_type(buf_input_0, span_type input_ours = span_type(buf_input_0,
buf_input_1); buf_input_1);
return this->input_state_.capture_current_line(input, eof_flag); return this->input_state_.capture_current_line(input_ours, eof_flag);
} }
auto auto