diff --git a/example/readerreplxx/readerreplxx.cpp b/example/readerreplxx/readerreplxx.cpp index c95dd101..292c3a43 100644 --- a/example/readerreplxx/readerreplxx.cpp +++ b/example/readerreplxx/readerreplxx.cpp @@ -2,8 +2,9 @@ #include #include -#include +#include #include +#include #include #include #include @@ -137,6 +138,7 @@ namespace { struct AppConfig { using ReaderConfig = xo::scm::ReaderConfig; using X1CollectorConfig = xo::mm::X1CollectorConfig; + using ArenaConfig = xo::mm::ArenaConfig; AppConfig() { rdr_config_.reader_debug_flag_ = true; @@ -147,17 +149,24 @@ struct AppConfig { std::size_t max_history_size_ = 1000; std::string repl_history_fname_ = "repl_history.txt";; ReaderConfig rdr_config_; - X1CollectorConfig x1_config_ = (X1CollectorConfig().with_size(4*1024*1024)); + X1CollectorConfig x1_config_ = (X1CollectorConfig().with_name("gc").with_size(4*1024*1024)); + ArenaConfig fixed_config_ = (ArenaConfig().with_name("fixed").with_size(4*1024)); }; struct AppContext { + using AAllocator = xo::mm::AAllocator; using DX1Collector = xo::mm::DX1Collector; using X1CollectorConfig = xo::mm::X1CollectorConfig; + using DArena = xo::mm::DArena; + using ArenaConfig = xo::mm::ArenaConfig; using Replxx = replxx::Replxx; AppContext(const AppConfig & cfg = AppConfig()) : config_{cfg}, - x1_{X1CollectorConfig().with_size(4*1024*1024)}, - rdr_{config_.rdr_config_, x1_.ref()} + x1_{cfg.x1_config_}, + fixed_{cfg.fixed_config_}, + rdr_{config_.rdr_config_, + x1_.ref(), + obj(&fixed_)} { rx_.set_max_history_size(config_.max_history_size_); rx_.history_load(config_.repl_history_fname_); @@ -167,9 +176,12 @@ struct AppContext { AppConfig config_; Replxx rx_; + /** collector/allocator for schematika expressions **/ DX1Collector x1_; + /** e.g. for DArenaHashMap within global symtab **/ + DArena fixed_; SchematikaReader rdr_; -}; +}; int main() diff --git a/include/xo/reader2/ParserStateMachine.hpp b/include/xo/reader2/ParserStateMachine.hpp index d8dc8da9..74a3e9bb 100644 --- a/include/xo/reader2/ParserStateMachine.hpp +++ b/include/xo/reader2/ParserStateMachine.hpp @@ -41,9 +41,21 @@ namespace xo { using size_type = std::size_t; public: + /** + * @p config arena configuration for parser state + * @p max_stringtable_capacity + * hard max size for unique stringtable + * @p expr_alloc allocator for schematika expressions. + * Probably shared with execution. + * @p aux_alloc auxiliary allocator for non-copyable memory + * (e.g. DArenaHashMap for global symtable). + * If not using X1Collector, this can be the + * same as @p expr_alloc. + **/ ParserStateMachine(const ArenaConfig & config, size_type max_stringtable_capacity, - obj expr_alloc); + obj expr_alloc, + obj aux_alloc); /** @defgroup scm-parserstatemachine-accessors accessor methods **/ ///@{ @@ -280,6 +292,19 @@ namespace xo { **/ obj expr_alloc_; + /** Allocator for data with lifetime bounded by this ParserStateMachine + * + * Cannot be DX1Collector; for example DArenaHashMap will + * for global symtab will be allocated from here, + * and does not support gc. + * + * If @ref expr_alloc_ is an ordinary arena (e.g. DArenaAlloc) + * can have aux_alloc_ = expr_alloc_. + * When expr_alloc_ is a garbage collector (e.g. DX1Collector) + * this needs to be distinct. + **/ + obj aux_alloc_; + /** symbol table with local bindings. * non-null during parsing of lambda expressions. * Always allocated from @p expr_alloc_. diff --git a/include/xo/reader2/SchematikaParser.hpp b/include/xo/reader2/SchematikaParser.hpp index 4be59f04..13cb18ee 100644 --- a/include/xo/reader2/SchematikaParser.hpp +++ b/include/xo/reader2/SchematikaParser.hpp @@ -164,14 +164,18 @@ namespace xo { /** create parser in initial state; * parser is ready to receive tokens via @ref include_token * - * @p config arena configuration for parser stack - * @p expr_alloc allocator for schematika expressions. - * Probably shared with execution. + * @p config arena configuration for parser stack + * @p expr_alloc allocator for schematika expressions. + * Probably shared with execution. + * @p aux_alloc aux allocator for non-copyable memory + * with lifetime bounded by this + * SchematikeParser itself * @p debug_flag true to enable debug logging **/ SchematikaParser(const ArenaConfig & config, size_t max_stringtable_capacity, obj expr_alloc, + obj aux_alloc, bool debug_flag); /** scm-schematikaparser-access-methods **/ diff --git a/include/xo/reader2/SchematikaReader.hpp b/include/xo/reader2/SchematikaReader.hpp index ee3de91f..6f2b325e 100644 --- a/include/xo/reader2/SchematikaReader.hpp +++ b/include/xo/reader2/SchematikaReader.hpp @@ -41,8 +41,15 @@ namespace xo { using size_type = std::size_t; public: + /** + * @p expr_alloc. allocator for Schematika expressions + * @p aux_alloc. allocator for miscellaneous objects + * (e.g. DArenaHashMap for global symtab) + * that have lifetime bounded by Schematika reader itself. + **/ SchematikaReader(const ReaderConfig & config, - obj expr_alloc); + obj expr_alloc, + obj fixed_alloc); /** visit reader-owned memory pools; call visitor(info) for each. * Specifically exclude expr_alloc, since we don't consider diff --git a/src/reader2/ParserStateMachine.cpp b/src/reader2/ParserStateMachine.cpp index 58119789..5541e8a0 100644 --- a/src/reader2/ParserStateMachine.cpp +++ b/src/reader2/ParserStateMachine.cpp @@ -24,13 +24,14 @@ namespace xo { namespace scm { ParserStateMachine::ParserStateMachine(const ArenaConfig & config, size_type max_stringtable_capacity, - obj expr_alloc) + obj expr_alloc, + obj aux_alloc) : stringtable_{max_stringtable_capacity}, parser_alloc_{DArena::map(config)}, expr_alloc_{expr_alloc}, + aux_alloc_{aux_alloc}, debug_flag_{config.debug_flag_} { - } bool @@ -60,8 +61,8 @@ namespace xo { stringtable_.visit_pools(visitor); parser_alloc_.visit_pools(visitor); - // not counting expr_alloc_. We don't consider - // that to be owned by ParserStateMachine + // not counting {expr_alloc_, fixed_alloc_}. We don't consider + // either to be owned by ParserStateMachine } void diff --git a/src/reader2/SchematikaParser.cpp b/src/reader2/SchematikaParser.cpp index 1973bbe7..1b8b1904 100644 --- a/src/reader2/SchematikaParser.cpp +++ b/src/reader2/SchematikaParser.cpp @@ -23,9 +23,10 @@ namespace xo { SchematikaParser::SchematikaParser(const ArenaConfig & config, size_t max_stringtable_capacity, obj expr_alloc, + obj fixed_alloc, bool debug_flag) - : psm_{config, max_stringtable_capacity, expr_alloc}, - debug_flag_{debug_flag} + : psm_{config, max_stringtable_capacity, expr_alloc, fixed_alloc}, + debug_flag_{debug_flag} { } diff --git a/src/reader2/SchematikaReader.cpp b/src/reader2/SchematikaReader.cpp index 47140d63..f2e70824 100644 --- a/src/reader2/SchematikaReader.cpp +++ b/src/reader2/SchematikaReader.cpp @@ -10,12 +10,14 @@ namespace xo { namespace scm { SchematikaReader::SchematikaReader(const ReaderConfig & config, - obj expr_alloc) + obj expr_alloc, + obj fixed_alloc) : tokenizer_{config.tk_buffer_config_, config.tk_debug_flag_}, parser_{config.parser_arena_config_, config.max_stringtable_cap_, expr_alloc, + fixed_alloc, config.parser_debug_flag_}, debug_flag_{config.reader_debug_flag_} { diff --git a/utest/SchematikaParser.test.cpp b/utest/SchematikaParser.test.cpp index d6f74edc..94e8218d 100644 --- a/utest/SchematikaParser.test.cpp +++ b/utest/SchematikaParser.test.cpp @@ -88,8 +88,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); REQUIRE(parser.debug_flag() == false); REQUIRE(parser.is_at_toplevel() == true); @@ -103,8 +104,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -121,8 +123,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_batch_session(); @@ -133,17 +136,20 @@ namespace xo { TEST_CASE("SchematikaParser-batch-def", "[reader2][SchematikaParser]") { + const auto & testname = Catch::getResultCapture().getCurrentTestName(); + constexpr bool c_debug_flag = false; - scope log(XO_DEBUG(c_debug_flag)); + scope log(XO_DEBUG(c_debug_flag), xtag("test", testname)); ArenaConfig config; - config.name_ = "test-arena"; + config.name_ = testname; config.size_ = 16 * 1024; DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_batch_session(); @@ -185,8 +191,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -250,8 +257,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -315,8 +323,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -380,8 +389,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -435,16 +445,17 @@ namespace xo { const auto & testname = Catch::getResultCapture().getCurrentTestName(); constexpr bool c_debug_flag = true; - scope log(XO_DEBUG(c_debug_flag), xtag("test", testname)); + scope log(XO_DEBUG(c_debug_flag), + xtag("test", testname)); - ArenaConfig config; - config.name_ = "test-arena"; - config.size_ = 16 * 1024; + ArenaConfig config + = (ArenaConfig().with_name(testname).with_size(16 * 1024)); DArena expr_arena = DArena::map(config); - obj expr_alloc = with_facet::mkobj(&expr_arena); + auto expr_alloc = obj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -507,8 +518,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -517,7 +529,7 @@ namespace xo { * lambda (n : i64, r : i64) -> i64 { 123 } * ^ ^^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ * 0 1| 3 4 5 6 7 8 9 a b c d e - * 2 + * 2 **/ std::vector tk_v{ @@ -549,13 +561,14 @@ namespace xo { scope log(XO_DEBUG(c_debug_flag), xtag("test", testname)); ArenaConfig config; - config.name_ = "test-arena"; + config.name_ = testname; config.size_ = 16 * 1024; DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -592,8 +605,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -632,13 +646,14 @@ namespace xo { scope log(XO_DEBUG(c_debug_flag), xtag("test", testname)); ArenaConfig config; - config.name_ = "test-arena"; + config.name_ = testname; config.size_ = 16 * 1024; DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -689,8 +704,9 @@ namespace xo { DArena expr_arena = DArena::map(config); obj expr_alloc = with_facet::mkobj(&expr_arena); + auto aux_alloc = expr_alloc; - SchematikaParser parser(config, 4096, expr_alloc, false /*debug_flag*/); + SchematikaParser parser(config, 4096, expr_alloc, aux_alloc, false /*debug_flag*/); parser.begin_interactive_session(); @@ -699,7 +715,7 @@ namespace xo { * (lambda (x : i64, y : i64) { x * y })(13, 15) ; * ^^ ^^ ^ ^ ^ ^^ ^ ^ ^ ^ ^ ^ ^^^^ ^ ^ ^ ^ * 0| 2| 4 5 6 7| 9 a b c d e f|h| j k l m - * 1 3 8 g i + * 1 3 8 g i **/ std::vector tk_v{