xo-tokenizer: + doc for tokenizer + other doc-related improvements

docs/examples.rst

@@ -45,14 +45,55 @@ See ``xo-tokenizer/examples/tokenrepl`` for (slighly elaborated) version of code
 
         if (tk.is_valid()) {
             cout << tk;
+        } else if (error.is_error()) {
+            cout << "parsing error: " << endl;
+            error.report(cout);
         }
     }
 
+Reminder: enable building examples with ``cmake -DXO_ENABLE_EXAMPLES=1 ..``
+
 .. code-block::
    :linenos:
 
-    $ .build/xo-tokenizer/utest/utest.tokenizer
+    $ .build/xo-tokenizer/example/tokenrepl/xo_tokenizer_repl
     > 123
     <token :type tk_i64 :text 123>
     > 123e5
     <token :type tk_f64 :text 123e5>
+    > def sq(x: i64) -> i64 { x * x }
+    <token :type tk_def :text "">
+    <token :type tk_symbol :text sq>
+    <token :type tk_leftparen :text "">
+    <token :type tk_symbol :text x>
+    <token :type tk_colon :text "">
+    <token :type tk_symbol :text i64>
+    <token :type tk_rightparen :text "">
+    <token :type tk_yields :text "">
+    <token :type tk_symbol :text i64>
+    <token :type tk_leftbrace :text "">
+    <token :type tk_symbol :text x>
+    <token :type tk_star :text "">
+    <token :type tk_symbol :text x>
+    <token :type tk_rightbrace :text "">
+
+Example of error reporting (via ``error.report(cout)`` above)
+
+.. code-block::
+   :linenos:
+
+    $ .build/xo-tokenizer/example/tokenrepl/xo_tokenizer_repl
+
+    > 123q
+    parsing error:
+    char: 4
+    input: 123q
+           ---^
+    unexpected character in numeric constant
+
+    > (8 * 8 * 123fd)
+    parsing error:
+    char: 13
+    input: (8 * 8 * 123fd)
+                    ---^
+    unexpected character in numeric constant

docs/input-state-class.rst

@@ -27,3 +27,51 @@ Context
 .. code-block:: cpp
 
     #include <xo/tokenizer/input_state.hpp>
+
+.. uml::
+    :scale: 99%
+    :align: center
+
+    allowmixing
+
+    object in1<<input_state>>
+    in1 : current_line = input
+    in1 : current_pos
+    in1 : whitespace
+    in1 : debug_flag
+
+    object input
+    input : (x * y * 123d)
+
+    input o-- sp1
+
+
+Class
+-----
+
+.. doxygenclass:: xo::scm::input_state
+
+Instance Variables
+------------------
+
+.. doxygengroup:: input-state-instance-vars
+
+Constructors
+------------
+
+.. doxygengroup:: input-state-ctors
+
+Static Methods
+--------------
+
+.. doxygengroup:: input-state-static-methods
+
+Access Methods
+--------------
+
+.. doxygengroup:: input-state-access-methods
+
+General Methods
+---------------
+
+.. doxygengroup:: input-state-general-methods

docs/tokenizer-class.rst

@@ -35,9 +35,10 @@ Context
     allowmixing
 
     object tkz1<<tokenizer>>
-    tkz : input_state
+    tkz1 : input_state = ins1
 
     object ins1<<input_state>>
+    ins1 : current_line = (9 * 8)
 
     tkz1 o-- ins1
 
@@ -54,9 +55,14 @@ Class
 Instance Variables
 ------------------
 
-.. doxygenclass:: tokenizer-instance-vars
+.. doxygengroup:: tokenizer-instance-vars
 
 Constructors
 ------------
 
 .. doxygengroup:: tokenizer-ctors
+
+Methods
+-------
+
+.. doxygengroup:: tokenizer-general-methods