roland/xo-expression2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: doc: ++ comments

Browse source
This commit is contained in:
Roland Conybeare 2024-08-06 09:33:36 -04:00
commit 52b522c8b1
1 changed files with 8 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

11
include/xo/tokenizer/tokenizer.hpp
View file

@ -22,7 +22,7 @@ namespace xo {
* span_type input = ...; * span_type input = ...;
* *
* while !input.empty() { * while !input.empty() {
* auto res = tkz.assemble_scan(input); * auto res = tkz.scan(input);
* const auto & tk = res.first; * const auto & tk = res.first;
* *
* // do something with tk if tk.is_valid() * // do something with tk if tk.is_valid()
@ -75,7 +75,12 @@ namespace xo {
**/ **/
token_type assemble_token(const span_type & token_text) const; token_type assemble_token(const span_type & token_text) const;
/** scan for next input token, given @p input **/ /** scan for next input token, given @p input.
* Note tokenizer can consume input (e.g. whitespace)
* without completing a token
*
* @return {parsed token, consumed span}
**/
scan_result scan(const span_type & input); scan_result scan(const span_type & input);
/** notify end of input, resolve any stored input **/ /** notify end of input, resolve any stored input **/
@ -568,7 +573,7 @@ namespace xo {
} }
if (!complete_flag) { if (!complete_flag) {
/* need more input to know if/when tokken complete */ /* need more input to know if/when token complete */
this->prefix_ += std::string(tk_start, input.hi()); this->prefix_ += std::string(tk_start, input.hi());
log && log(xtag("captured-prefix", this->prefix_)); log && log(xtag("captured-prefix", this->prefix_));