From 2d0336058e845acdb98a249a1812d1651d536b70 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 5 Aug 2024 14:44:39 -0400 Subject: [PATCH] xo-tokenizer: move to scm ns + print() diagnostic --- include/xo/tokenizer/buffer.hpp | 4 ++-- include/xo/tokenizer/span.hpp | 4 ++-- include/xo/tokenizer/token.hpp | 26 ++++++++++++++++++++++++-- include/xo/tokenizer/tokenizer.hpp | 4 ++-- include/xo/tokenizer/tokentype.hpp | 8 +++++--- src/tokenizer/tokentype.cpp | 10 ++++++++-- utest/token.test.cpp | 4 ++-- utest/tokenizer.test.cpp | 8 ++++---- 8 files changed, 49 insertions(+), 19 deletions(-) diff --git a/include/xo/tokenizer/buffer.hpp b/include/xo/tokenizer/buffer.hpp index eb206eac..bc3621f2 100644 --- a/include/xo/tokenizer/buffer.hpp +++ b/include/xo/tokenizer/buffer.hpp @@ -9,7 +9,7 @@ #include namespace xo { - namespace tok { + namespace scm { /** * @class buffer buffer.hpp * @@ -318,7 +318,7 @@ namespace xo { swap(buffer & lhs, buffer & rhs) { lhs.swap(rhs); } - } /*namespace tok*/ + } /*namespace scm*/ } /*namespace xo*/ /* end buffer.hpp */ diff --git a/include/xo/tokenizer/span.hpp b/include/xo/tokenizer/span.hpp index 2f847f6a..36353a63 100644 --- a/include/xo/tokenizer/span.hpp +++ b/include/xo/tokenizer/span.hpp @@ -7,7 +7,7 @@ #include namespace xo { - namespace tok { + namespace scm { /** @class span compression/span.hpp * * @brief Represents a contiguous memory range, without ownership. @@ -137,5 +137,5 @@ namespace xo { x.print(os); return os; } - } /*namespace tok*/ + } /*namespace scm*/ } /*namespace xo*/ diff --git a/include/xo/tokenizer/token.hpp b/include/xo/tokenizer/token.hpp index 643a5143..3883e15a 100644 --- a/include/xo/tokenizer/token.hpp +++ b/include/xo/tokenizer/token.hpp @@ -8,11 +8,12 @@ #include "tokentype.hpp" #include "xo/indentlog/print/tag.hpp" #include +#include #include #include namespace xo { - namespace tok { + namespace scm { namespace detail { /* compute a * b^p, p >= 0 */ constexpr double @@ -101,6 +102,9 @@ namespace xo { **/ double f64_value() const; + /** print human-readable token representation on stream @p os **/ + void print(std::ostream & os) const; + private: /** category for this token **/ tokentype tk_type_ = tokentype::tk_invalid; @@ -327,7 +331,25 @@ namespace xo { return retval; } /*f64_value*/ - } /*Namespace tok*/ + + template + void + token::print(std::ostream & os) const { + os << ""; + } /*print*/ + + template + inline std::ostream & + operator<< (std::ostream & os, + const token & tk) + { + tk.print(os); + return os; + } + } /*Namespace scm*/ } /*namespace xo*/ diff --git a/include/xo/tokenizer/tokenizer.hpp b/include/xo/tokenizer/tokenizer.hpp index bc5164f7..c37a4656 100644 --- a/include/xo/tokenizer/tokenizer.hpp +++ b/include/xo/tokenizer/tokenizer.hpp @@ -11,7 +11,7 @@ #include namespace xo { - namespace tok { + namespace scm { /** * Use: * @code @@ -619,7 +619,7 @@ namespace xo { return tk; } /*notify_eof*/ - } /*namespace tok*/ + } /*namespace scm*/ } /*namespace xo*/ /* end tokenizer.hpp */ diff --git a/include/xo/tokenizer/tokentype.hpp b/include/xo/tokenizer/tokentype.hpp index cd890e38..bd20e8eb 100644 --- a/include/xo/tokenizer/tokentype.hpp +++ b/include/xo/tokenizer/tokentype.hpp @@ -9,7 +9,7 @@ #include namespace xo { - namespace tok { + namespace scm { /** @enum tokentype * @brief enum to identify different schematica input token types * @@ -124,6 +124,9 @@ namespace xo { /** keyword 'in' **/ tk_in, + /** keyword 'end' **/ + tk_end, + n_tokentype /* comes last, counts #of entries */ }; /*tokentype*/ @@ -135,8 +138,7 @@ namespace xo { os << tokentype_descr(tk_type); return os; } - } /*namespace tok*/ + } /*namespace scm*/ } /*namespace xo*/ - /* end tokentype.hpp */ diff --git a/src/tokenizer/tokentype.cpp b/src/tokenizer/tokentype.cpp index 228790ec..08dbba84 100644 --- a/src/tokenizer/tokentype.cpp +++ b/src/tokenizer/tokentype.cpp @@ -6,7 +6,7 @@ #include "tokentype.hpp" namespace xo { - namespace tok { + namespace scm { char const * tokentype_descr(tokentype tk_type) { @@ -18,27 +18,33 @@ namespace xo { CASE(tk_string); CASE(tk_symbol); CASE(tk_leftparen); + CASE(tk_rightparen); CASE(tk_leftbracket); CASE(tk_rightbracket); CASE(tk_leftbrace); CASE(tk_rightbrace); + CASE(tk_leftangle); CASE(tk_rightangle); CASE(tk_dot); CASE(tk_comma); CASE(tk_colon); + CASE(tk_doublecolon); CASE(tk_semicolon); CASE(tk_singleassign); CASE(tk_assign); CASE(tk_yields); + CASE(tk_type); CASE(tk_def); CASE(tk_lambda); CASE(tk_if); CASE(tk_let); + CASE(tk_in); + CASE(tk_end); case tokentype::tk_invalid: case tokentype::n_tokentype: @@ -49,7 +55,7 @@ namespace xo { return "???"; } /*tokentype_descr*/ - } /*namespace tok*/ + } /*namespace scm*/ } /*namespace xo*/ diff --git a/utest/token.test.cpp b/utest/token.test.cpp index 16b30399..160420b0 100644 --- a/utest/token.test.cpp +++ b/utest/token.test.cpp @@ -8,8 +8,8 @@ #include namespace xo { - using token = xo::tok::token; - using xo::tok::tokentype; + using token = xo::scm::token; + using xo::scm::tokentype; namespace ut { struct testcase_i64 { diff --git a/utest/tokenizer.test.cpp b/utest/tokenizer.test.cpp index cb796f47..03cd71ad 100644 --- a/utest/tokenizer.test.cpp +++ b/utest/tokenizer.test.cpp @@ -7,9 +7,9 @@ #include namespace xo { - using xo::tok::tokentype; - using token = xo::tok::token; - using xo::tok::span; + using xo::scm::tokentype; + using token = xo::scm::token; + using xo::scm::span; namespace ut { namespace { @@ -111,7 +111,7 @@ namespace xo { INFO(xtag("i_tc", i_tc)); using tokenizer - = xo::tok::tokenizer; + = xo::scm::tokenizer; tokenizer tkz; tokenizer::span_type