roland/xo-reader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader2 stack: + #q token + QuoteSsm [WIP - not functional]

Browse source
This commit is contained in:
Roland Conybeare 2026-03-01 13:06:57 +11:00
commit 9920812d42
4 changed files with 21 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

13
src/tokenizer2/Tokenizer.cpp
View file

@ -32,6 +32,8 @@ namespace xo {
Tokenizer::is_1char_punctuation(CharT ch)
{
switch(ch) {
case '\'':
return true;
case '(':
return true;
case ')':
@ -418,6 +420,15 @@ namespace xo {
break;
}
case '\'':
{
log && log("quote token");
tk_type = tokentype::tk_quote;
++ix;
break;
}
case 'a': case 'A':
case 'b': case 'B':
case 'c': case 'C':
@ -593,6 +604,8 @@ namespace xo {
tk_type = tokentype::tk_in;
} else if (tk_text == "end") {
tk_type = tokentype::tk_end;
} else if (tk_text == "#q") {
tk_type = tokentype::tk_quote;
} else {
/* keep as symbol */
keep_text = true;