roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2: + example app 'readerreplxx'

Browse source
This commit is contained in:
Roland Conybeare 2026-01-23 11:54:32 -05:00
commit 9044e1d196
4 changed files with 42 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

7
example/tokenrepl/tokenrepl.cpp
View file

@ -85,9 +85,10 @@ main() {
{
//cout << "input: " << input << endl;
auto input_ext = Tokenizer::span_type::from_cstr(input_cstr);
// reminder: input may contain multiple tokens
if (input_cstr && *input_cstr) {
auto [error, input] = tkz.buffer_input_line(input_cstr, false /*!eof*/);
auto [error, input] = tkz.buffer_input_line(input_ext, false /*!eof*/);
if (log) {
log(xtag("msg", "buffered input line"));
@ -104,6 +105,7 @@ main() {
cout << tk << endl;
} else if (error.is_error()) {
cout << "tokenizer error: " << endl;
error.report(cout);
break;
@ -111,7 +113,6 @@ main() {
input = input.after_prefix(consumed);
}
}
/* here: input.empty() or error encountered */

5
include/xo/tokenizer2/Tokenizer.hpp
View file

@ -129,10 +129,11 @@ namespace xo {
**/
bool has_prefix() const { return !prefix_.empty(); }
/** buffer contents of input_cstr.
/** copy into buffer the contents of @p input.
* May throw if buffer space exhausted
**/
std::pair<input_error, span_type> buffer_input_line(const char * input_cstr, bool eof_flag);
std::pair<input_error, span_type> buffer_input_line(span_type input,
bool eof_flag);
/** scan for next input token, given @p input.
* Note:

16
include/xo/tokenizer2/TokenizerError.hpp
View file

@ -32,7 +32,7 @@ namespace xo {
* @p tk_start current position on entry to scanner
* @p error_pos error location relative to token start
**/
TokenizerError(const char * src_function,
TokenizerError(std::string_view src_function,
std::string error_description,
const TkInputState & input_state,
size_t error_pos)
@ -46,12 +46,20 @@ namespace xo {
log && log(xtag("input_state.current_pos", input_state.current_pos()),
xtag("error_pos", error_pos));
}
TokenizerError with_error(std::string_view error_src_fn,
std::string error_msg) {
return TokenizerError(error_src_fn,
std::string(error_msg),
this->input_state_,
0 /*error_pos*/);
}
///@}
/** @defgroup tokenizer-error-access-methods **/
///@{
const char * src_function() const { return src_function_; }
std::string_view src_function() const { return src_function_; }
const std::string & error_description() const { return error_description_; }
#pragma GCC diagnostic push
#ifndef __APPLE__
@ -88,8 +96,8 @@ namespace xo {
///@{
/** source location (in tokenizer) at which error identified **/
char const * src_function_ = nullptr;
/** static error description **/
std::string_view src_function_;
/** error description **/
std::string error_description_;
/** input state associated with this error.
* Sufficient to precisely locate it with context.

15
src/tokenizer2/Tokenizer.cpp
View file

@ -615,19 +615,18 @@ namespace xo {
}
auto
Tokenizer::buffer_input_line(const char * input_cstr,
Tokenizer::buffer_input_line(span_type input_ext,
bool eof_flag) -> std::pair<input_error, span_type>
{
scope log(XO_DEBUG(input_state_.debug_flag()));
log && log(xtag("input", input_cstr));
log && log(xtag("input_ext", input_ext));
auto buf_input_0 = input_buffer_.input_range().hi();
auto remainder = input_buffer_.append
(DCircularBuffer::const_span_type::from_cstr(input_cstr));
auto remainder2 = input_buffer_.append
(DCircularBuffer::const_span_type::from_cstr("\n"));
auto remainder = input_buffer_.append(input_ext);
auto remainder2 = input_buffer_.append(span_type::from_cstr("\n"));
//(DCircularBuffer::const_span_type::from_cstr("\n"));
if (!remainder.empty() || !remainder2.empty()) {
throw std::runtime_error(tostr("Tokenizer::buffer_line: line too long!",
@ -636,10 +635,10 @@ namespace xo {
auto buf_input_1 = input_buffer_.input_range().hi();
span_type input = span_type(buf_input_0,
span_type input_ours = span_type(buf_input_0,
buf_input_1);
return this->input_state_.capture_current_line(input, eof_flag);
return this->input_state_.capture_current_line(input_ours, eof_flag);
}
auto