roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: fix duplicate case statements

Browse source
This commit is contained in:
Roland Conybeare 2024-09-15 12:56:36 -05:00
commit 4fb94bc2d6
1 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

16
include/xo/tokenizer/tokenizer.hpp
View file

@ -157,7 +157,6 @@ namespace xo {
case '*':
/* not punctuation -- allowed in symbol */
return false;
case '*':
case '/':
/* not punctuation -- for symmetry with +,- */
return false;
@ -346,12 +345,20 @@ namespace xo {
if (token_text.size() == 1) {
/* standalone '*' */
tk_type = tokentype::tk_star;
++ix;
} else {
/* '*' isn't punctuation -- but may allow appearance in a longer token
*
* thinking that x*y is a symbol with an embedded '*' character;
* in particular want to support kebab-case symbols like 'foo-config'
*/
}
break;
case '/':
if (token_text.size() == 1) {
/* standalone '/' */
tk_type = tokentype::tk_slash;
++ix;
}
break;
case '"':
@ -530,13 +537,6 @@ namespace xo {
tk_type = tokentype::tk_semicolon;
++ix;
break;
case '*':
/* '*' isn't punctuation, since can appear within symbol.
* However it cannot begin a symbol..
*/
tk_type = tokentype::tk_star;
++ix;
break;
case ':':
{
log && log("colon or assignment token");