xo-tokenizer: streamline input_state/tokenizer interaction + docs

xo-tokenizer/example/tokenrepl/tokenrepl.cpp

@@ -51,6 +51,8 @@ main() {
             //input = input.after_prefix(consumed.size());
         }
 
+        /* here: input.empty() or error encountered */
+
         /* discard stashed remainder of input line
          * (for nicely-formatted errors)
          */