roland/xo-alloc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add 'xo-tokenizer/' from commit '830c6ebe55'

Browse source 
git-subtree-dir: xo-tokenizer
git-subtree-mainline: e8f65f88cf
git-subtree-split: 830c6ebe55
This commit is contained in:
Roland Conybeare 2025-05-11 01:36:13 -05:00
commit 6395a7a285
17 changed files with 2547 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

14
xo-tokenizer/src/tokenizer/CMakeLists.txt Normal file
View file

@ -0,0 +1,14 @@
# tokenizer/CMakeLists.txt
set(SELF_LIB xo_tokenizer)
set(SELF_SRCS
tokentype.cpp
token.cpp)
xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS})
#xo_dependency(${SELF_LIB} refcnt)
xo_dependency(${SELF_LIB} indentlog)
#xo_dependency(${SELF_LIB} subsys)
#xo_boost_dependency(${SELF_LIB})
# end CMakeLists.txt

9
xo-tokenizer/src/tokenizer/token.cpp Normal file
View file

@ -0,0 +1,9 @@
/** @file token.cpp
*
* author: Roland Conybeare
**/
#include "token.hpp"
#include "xo/indentlog/print/tag.hpp"
/** end token.cpp **/

67
xo-tokenizer/src/tokenizer/tokentype.cpp Normal file
View file

@ -0,0 +1,67 @@
/* file tokentype.cpp
*
* author: Roland Conybeare
*/
#include "tokentype.hpp"
namespace xo {
namespace scm {
char const *
tokentype_descr(tokentype tk_type)
{
#define CASE(x) case tokentype::x: return STRINGIFY(x)
switch(tk_type) {
CASE(tk_i64);
CASE(tk_f64);
CASE(tk_string);
CASE(tk_symbol);
CASE(tk_leftparen);
CASE(tk_rightparen);
CASE(tk_leftbracket);
CASE(tk_rightbracket);
CASE(tk_leftbrace);
CASE(tk_rightbrace);
CASE(tk_leftangle);
CASE(tk_rightangle);
CASE(tk_dot);
CASE(tk_comma);
CASE(tk_colon);
CASE(tk_doublecolon);
CASE(tk_semicolon);
CASE(tk_singleassign);
CASE(tk_assign);
CASE(tk_yields);
CASE(tk_plus);
CASE(tk_minus);
CASE(tk_star);
CASE(tk_slash);
CASE(tk_type);
CASE(tk_def);
CASE(tk_lambda);
CASE(tk_if);
CASE(tk_let);
CASE(tk_in);
CASE(tk_end);
case tokentype::tk_invalid:
case tokentype::n_tokentype:
return "?tokentype";
}
#undef CASE
return "???";
} /*tokentype_descr*/
} /*namespace scm*/
} /*namespace xo*/
/* end tokentype.cpp */