roland/xo-expression2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2 stack: + #q token + QuoteSsm [WIP - not functional]

Browse source
This commit is contained in:
Roland Conybeare 2026-03-01 13:06:57 +11:00
commit 9920812d42
4 changed files with 21 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
include/xo/tokenizer2/Token.hpp
View file

@ -87,6 +87,8 @@ namespace xo {
static Token symbol_token(const std::string & txt) { static Token symbol_token(const std::string & txt) {
return Token(tokentype::tk_symbol, txt); return Token(tokentype::tk_symbol, txt);
} }
/** token representing quote @c "'" **/
static Token quote() { return Token(tokentype::tk_quote); }
/** token representing left angle bracket @c "<" **/ /** token representing left angle bracket @c "<" **/
static Token leftangle() { return Token(tokentype::tk_leftangle); } static Token leftangle() { return Token(tokentype::tk_leftangle); }
/** token representing right angle bracket @c ">" **/ /** token representing right angle bracket @c ">" **/

3
include/xo/tokenizer2/tokentype.hpp
View file

@ -64,6 +64,9 @@ namespace xo {
/** a symbol **/ /** a symbol **/
tk_symbol, tk_symbol,
/** quote @c ' **/
tk_quote,
/** left-hand parenthesis @c '(' **/ /** left-hand parenthesis @c '(' **/
tk_leftparen, tk_leftparen,

13
src/tokenizer2/Tokenizer.cpp
View file

@ -32,6 +32,8 @@ namespace xo {
Tokenizer::is_1char_punctuation(CharT ch) Tokenizer::is_1char_punctuation(CharT ch)
{ {
switch(ch) { switch(ch) {
case '\'':
return true;
case '(': case '(':
return true; return true;
case ')': case ')':
@ -418,6 +420,15 @@ namespace xo {
break; break;
} }
case '\'':
{
log && log("quote token");
tk_type = tokentype::tk_quote;
++ix;
break;
}
case 'a': case 'A': case 'a': case 'A':
case 'b': case 'B': case 'b': case 'B':
case 'c': case 'C': case 'c': case 'C':
@ -593,6 +604,8 @@ namespace xo {
tk_type = tokentype::tk_in; tk_type = tokentype::tk_in;
} else if (tk_text == "end") { } else if (tk_text == "end") {
tk_type = tokentype::tk_end; tk_type = tokentype::tk_end;
} else if (tk_text == "#q") {
tk_type = tokentype::tk_quote;
} else { } else {
/* keep as symbol */ /* keep as symbol */
keep_text = true; keep_text = true;

5
src/tokenizer2/tokentype.cpp
View file

@ -18,16 +18,17 @@ namespace xo {
CASE(tk_f64); CASE(tk_f64);
CASE(tk_string); CASE(tk_string);
CASE(tk_symbol); CASE(tk_symbol);
CASE(tk_leftparen);
CASE(tk_quote);
CASE(tk_leftparen);
CASE(tk_rightparen); CASE(tk_rightparen);
CASE(tk_leftbracket); CASE(tk_leftbracket);
CASE(tk_rightbracket); CASE(tk_rightbracket);
CASE(tk_leftbrace); CASE(tk_leftbrace);
CASE(tk_rightbrace); CASE(tk_rightbrace);
CASE(tk_leftangle); CASE(tk_leftangle);
CASE(tk_rightangle); CASE(tk_rightangle);
CASE(tk_lessequal); CASE(tk_lessequal);
CASE(tk_greatequal); CASE(tk_greatequal);
CASE(tk_dot); CASE(tk_dot);