Examples

See xo-tokenizer/examples/tokenrepl for (slighly elaborated) version of code below

 1 #include "xo/tokenizer/tokenizer.hpp"
 2
 3 int
 4 main() {
 5     using namespace xo::scm;
 6     using namespace std;
 7
 8     using tokenizer_type = tokenizer<char>;
 9     using span_type = tokenizer_type::span_type;
10
11     tokenizer_type tkz;
12     string input_str;
13
14     while (getline(cin, input_str)) {
15         // we want tokenizer to see newline, it's syntax
16         input_str.push_back('\n');
17         span_type input(input_str.begin(), input_str.end());
18
19         // input may contain multiple tokens
20         while (!input.empty()) {
21             auto [tk, consumed, error] = tkz.scan(input);
22
23             if (tk.is_valid()) {
24                 cout << tk;
25             }
26
27             input = input.after_prefix(consumed.size());
28         }
29     }
30
31     auto [tk, consumed, error] = tkz.notify_eof(spxn_type::from_string(input_str));
32
33     if (tk.is_valid()) {
34         cout << tk;
35     } else if (error.is_error()) {
36         cout << "parsing error: " << endl;
37         error.report(cout);
38     }
39 }

Reminder: enable building examples with cmake -DXO_ENABLE_EXAMPLES=1 ..

 1 $ .build/xo-tokenizer/example/tokenrepl/xo_tokenizer_repl
 2 > 123
 3 <token :type tk_i64 :text 123>
 4 > 123e5
 5 <token :type tk_f64 :text 123e5>
 6 > def sq(x: i64) -> i64 { x * x }
 7 <token :type tk_def :text "">
 8 <token :type tk_symbol :text sq>
 9 <token :type tk_leftparen :text "">
10 <token :type tk_symbol :text x>
11 <token :type tk_colon :text "">
12 <token :type tk_symbol :text i64>
13 <token :type tk_rightparen :text "">
14 <token :type tk_yields :text "">
15 <token :type tk_symbol :text i64>
16 <token :type tk_leftbrace :text "">
17 <token :type tk_symbol :text x>
18 <token :type tk_star :text "">
19 <token :type tk_symbol :text x>
20 <token :type tk_rightbrace :text "">

Example of error reporting (via error.report(cout) above)

 1 $ .build/xo-tokenizer/example/tokenrepl/xo_tokenizer_repl
 2
 3 > 123q
 4 parsing error:
 5 char: 4
 6 input: 123q
 7        ---^
 8 unexpected character in numeric constant
 9
10 > (8 * 8 * 123fd)
11 parsing error:
12 char: 13
13 input: (8 * 8 * 123fd)
14                 ---^
15 unexpected character in numeric constant