diff --git a/include/xo/tokenizer/span.hpp b/include/xo/tokenizer/span.hpp index 241f52aa..22695ec5 100644 --- a/include/xo/tokenizer/span.hpp +++ b/include/xo/tokenizer/span.hpp @@ -71,7 +71,7 @@ namespace xo { /** @brief create span with first @p z members of this span removed **/ span after_prefix(size_type z) const { - if (z > hi_ - lo_) + if (lo_ + z > hi_) z = hi_ - lo_; return span(lo_ + z, hi_); diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp index 8caa1440..ba340b1a 100644 --- a/include/xo/tokenizer/tokenizer.hpp +++ b/include/xo/tokenizer/tokenizer.hpp @@ -646,6 +646,8 @@ namespace xo { } else if (is_2char_punctuation(*ix)) { CharT ch1 = *ix; + (void)ch1; + ++ix; if (ix == input.hi()) { diff --git a/utest/tokenizer.test.cpp b/utest/tokenizer.test.cpp index e0ec6a56..44600e7f 100644 --- a/utest/tokenizer.test.cpp +++ b/utest/tokenizer.test.cpp @@ -111,6 +111,8 @@ namespace xo { {"let", false, token::let(), true}, {"in", false, token::in(), true}, {"end", false, token::end(), true}, + + {"*", false, token::star_token(), true}, }; }