diff --git a/include/xo/tokenizer2/Token.hpp b/include/xo/tokenizer2/Token.hpp index b211f967..9ddd0181 100644 --- a/include/xo/tokenizer2/Token.hpp +++ b/include/xo/tokenizer2/Token.hpp @@ -87,6 +87,8 @@ namespace xo { static Token symbol_token(const std::string & txt) { return Token(tokentype::tk_symbol, txt); } + /** token representing quote @c "'" **/ + static Token quote() { return Token(tokentype::tk_quote); } /** token representing left angle bracket @c "<" **/ static Token leftangle() { return Token(tokentype::tk_leftangle); } /** token representing right angle bracket @c ">" **/ diff --git a/include/xo/tokenizer2/tokentype.hpp b/include/xo/tokenizer2/tokentype.hpp index 91cb3622..3f259f8d 100644 --- a/include/xo/tokenizer2/tokentype.hpp +++ b/include/xo/tokenizer2/tokentype.hpp @@ -64,6 +64,9 @@ namespace xo { /** a symbol **/ tk_symbol, + /** quote @c ' **/ + tk_quote, + /** left-hand parenthesis @c '(' **/ tk_leftparen, diff --git a/src/tokenizer2/Tokenizer.cpp b/src/tokenizer2/Tokenizer.cpp index f176a88f..8821cd65 100644 --- a/src/tokenizer2/Tokenizer.cpp +++ b/src/tokenizer2/Tokenizer.cpp @@ -32,6 +32,8 @@ namespace xo { Tokenizer::is_1char_punctuation(CharT ch) { switch(ch) { + case '\'': + return true; case '(': return true; case ')': @@ -418,6 +420,15 @@ namespace xo { break; } + case '\'': + { + log && log("quote token"); + + tk_type = tokentype::tk_quote; + ++ix; + + break; + } case 'a': case 'A': case 'b': case 'B': case 'c': case 'C': @@ -593,6 +604,8 @@ namespace xo { tk_type = tokentype::tk_in; } else if (tk_text == "end") { tk_type = tokentype::tk_end; + } else if (tk_text == "#q") { + tk_type = tokentype::tk_quote; } else { /* keep as symbol */ keep_text = true; diff --git a/src/tokenizer2/tokentype.cpp b/src/tokenizer2/tokentype.cpp index 40c2dbfb..0831940f 100644 --- a/src/tokenizer2/tokentype.cpp +++ b/src/tokenizer2/tokentype.cpp @@ -18,16 +18,17 @@ namespace xo { CASE(tk_f64); CASE(tk_string); CASE(tk_symbol); - CASE(tk_leftparen); + CASE(tk_quote); + CASE(tk_leftparen); CASE(tk_rightparen); CASE(tk_leftbracket); CASE(tk_rightbracket); CASE(tk_leftbrace); CASE(tk_rightbrace); - CASE(tk_leftangle); CASE(tk_rightangle); + CASE(tk_lessequal); CASE(tk_greatequal); CASE(tk_dot);