From 6dc2bf1e939e99dedb341bc330371041a2e5d9eb Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 24 Mar 2026 22:06:20 -0400 Subject: [PATCH] xo-gc: X1Collector.assign_member() + GCRoots + use ArenaVector Using ArenaVector for mlog. --- include/xo/gc/DX1Collector.hpp | 115 ++++- include/xo/gc/MutationLogEntry.hpp | 2 +- include/xo/gc/X1CollectorConfig.hpp | 28 +- .../xo/gc/detail/ICollector_DX1Collector.hpp | 2 + include/xo/gc/object_age.hpp | 12 + src/gc/CMakeLists.txt | 6 + src/gc/DX1Collector.cpp | 445 ++++++++++++++---- src/gc/ICollector_DX1Collector.cpp | 9 + utest/Collector.test.cpp | 40 +- utest/DX1CollectorIterator.test.cpp | 16 +- 10 files changed, 559 insertions(+), 116 deletions(-) diff --git a/include/xo/gc/DX1Collector.hpp b/include/xo/gc/DX1Collector.hpp index 539c77e..cf21cf8 100644 --- a/include/xo/gc/DX1Collector.hpp +++ b/include/xo/gc/DX1Collector.hpp @@ -7,11 +7,13 @@ #include "X1CollectorConfig.hpp" #include "GCObject.hpp" +#include "MutationLogEntry.hpp" #include "generation.hpp" #include "object_age.hpp" #include "role.hpp" #include #include +#include #include #include #include @@ -64,9 +66,35 @@ namespace xo { struct DX1CollectorIterator; + /** @brief GC root struct + * + * A root is traversed much like other gc-owned value: + * a. if root is in GC from-space, move it to to-space. + * b. if root is in GC to-space, skip. + * e.g. root belongs to generation not subject to collection this cycle. + * c. if root it not allocated by GC, still do in-place forward on its + * children. This is load-bearing for ParserStateMachine, for example. + * Allows non-GC object to refer to a dynamic set of gc-owned objects + **/ + struct GCRoot { + public: + GCRoot() = default; + explicit GCRoot(obj * x) : root_{x} {} + + obj * root() { return root_; } + + private: + obj * root_ = nullptr; + }; + // ----- DX1Collector ----- + /** @brief garbage collector 'X1' + **/ struct DX1Collector { + public: + using RootSet = DArenaVector; + using MutationLog = DArenaVector; using typeseq = xo::facet::typeseq; using size_type = DArena::size_type; using value_type = DArena::value_type; @@ -75,6 +103,7 @@ namespace xo { /** hard max typeseq for collector-registered types **/ static constexpr size_t c_max_typeseq = 4096; + public: /** Create X1 collector instance. **/ explicit DX1Collector(const X1CollectorConfig & cfg); @@ -92,7 +121,7 @@ namespace xo { std::string_view name() const { return config_.name_; } const DArena * get_object_types() const noexcept { return &object_types_; } - const DArena * get_roots() const noexcept { return &roots_; } + const RootSet * get_root_set() const noexcept { return &root_set_; } const DArena * get_space(role r, generation g) const noexcept { return space_[r][g]; } DArena * get_space(role r, generation g) noexcept { return space_[r][g]; } DArena * from_space(generation g) noexcept { return get_space(role::from_space(), g); } @@ -120,6 +149,16 @@ namespace xo { **/ bool contains(role r, const void * addr) const noexcept; + /** true iff address @p addr allocated from this collector and currently live + * in role @p r (according to current GC state) + **/ + bool contains_allocated(role r, const void * addr) const noexcept; + + /** generation to which pointer @p addr belongs, given role @p r; + * sentinel if not found in this collector + **/ + generation generation_of(role r, const void * addr) const noexcept; + /** return details from last error (will be in gen0 to-space) **/ AllocError last_error() const noexcept; @@ -224,6 +263,21 @@ namespace xo { **/ bool expand(size_type z) noexcept; + // ----- mutation ----- + + /** Modify a gc-owned pointer @p *p_lhs, within allocation @p parent, + * to point to @p rhs. + * + * Motivation: need special handling for cross-generational pointers with + * incremental gc. + * + * Require: + * - if parent is owned by this collector, it has it's own allocation + * (alloc header immediately precedes object address @c parent.data_) + * - address @p p_lhs falls within extent of allocation for @c parent.data_ + **/ + void assign_member(void * parent, obj * p_lhs, obj rhs); + // ----- iteration ----- /** alloc iterator at begin position **/ @@ -242,16 +296,36 @@ namespace xo { void clear() noexcept; private: + /** aux init function: initialize @ref object_types_ arena **/ + void _init_object_types(const X1CollectorConfig & cfg, std::size_t page_z); + /** aux init function: initialize @ref roots_ arena **/ + void _init_gc_roots(const X1CollectorConfig & cfg, std::size_t page_z); + /** aux init function: initialize @ref mlog_storage_[][] arenas **/ + void _init_mlogs(const X1CollectorConfig & cfg, std::size_t page_z); + /** aux init function: create mutation log **/ + MutationLog _make_mlog(uint32_t igen, char tag_char, size_t mlog_z, std::size_t page_z); + /** aux init function: initialize @ref space_storage_[][] arenas **/ + void _init_space(const X1CollectorConfig & cfg); + /** swap from- and to- roles for all generations < @p upto **/ void swap_roles(generation upto) noexcept; /** copy roots + everything reachable from them, to to-space **/ void copy_roots(generation upto) noexcept; - /** move subgraph at @p from_src to to-space. - * + /** cleanup after gc **/ + void cleanup_phase(generation upto); + + /** move root subgraph at @p from_src to to-space. + * If not in gc-space, visit immediate children and move them. * Require: runstate_.is_running() **/ - void * deep_move(void * from_src, generation upto); + void * _deep_move_root(obj from_src, generation upto); + /** move interior subgraph at @p from_src to to-space. + * no-op if not in gc-space. + **/ + void * _deep_move_interior(void * from_src, generation upto); + /** common driver for _deep_move_root(), _deep_move_interior() **/ + void * _deep_move_gc_owned(void * from_src, generation upto); public: /** garbage collector configuration **/ @@ -270,15 +344,42 @@ namespace xo { /** if > 0: need gc for all generations < gc_pending_upto_ **/ generation gc_pending_upto_; - /** (ab)using arena to get extensible list of root objects. + /** using arena to get extensible list of root objects. * For each root store one address (type obj*) + * + * An Object x that supports AGCObject, but doesn't live in gc-space, + * will get special treatment if it appears in root_set_: + * collector will traverse x to forward pointers to gc-owned + * targets. In other contexts collector doesn't look inside + * non-gc-owned objects + * + * editor bait: root_v **/ - DArena roots_; + RootSet root_set_; + + /** Cross-generational mutations tracked in MutationLogs. + * We need three logs per generation: + * A. one to observe and remember mutations in to-space + * during normal operation (between GC cycles) + * B. during GC: 2nd mlog to hold entries from from-mlog + * that will still be needed post-GC (because ptr direction + * from higher gen to lower gen after cycle). + * C. during GC: 3rd mlog to triage entries for which + * liveness of pointer source isn't yet established. + * + * NOTE: indexed on generation of pointer *destination* + **/ + std::array mlog_storage_[c_n_role + 1]; + + /** mlog pointers. The roles of mlog_storage_[*][g] get permuted + * as each collection cycle proceeds + **/ + std::array mlog_[c_n_role + 1]; /** collector-managed memory here. * - space_[1] is from-space * - space_[0] is to-space - * coordinates with role ingc/role.hpp, see also. + * coordinates with role in gc/role.hpp, see also. **/ /** arena objects for collector managed memory diff --git a/include/xo/gc/MutationLogEntry.hpp b/include/xo/gc/MutationLogEntry.hpp index 01c3666..7459827 100644 --- a/include/xo/gc/MutationLogEntry.hpp +++ b/include/xo/gc/MutationLogEntry.hpp @@ -5,7 +5,7 @@ #pragma once -#include "GCObject.hpp" +#include namespace xo { namespace mm { diff --git a/include/xo/gc/X1CollectorConfig.hpp b/include/xo/gc/X1CollectorConfig.hpp index dec9545..db96a43 100644 --- a/include/xo/gc/X1CollectorConfig.hpp +++ b/include/xo/gc/X1CollectorConfig.hpp @@ -26,6 +26,16 @@ namespace xo { **/ X1CollectorConfig with_size(std::size_t gen_z); + /** copy of this config, + * but with @c debug_flag_ set to @p x + **/ + X1CollectorConfig with_debug_flag(bool x); + + /** copy of this config, + * but with @c sanitize_flag_ set to @p x + **/ + X1CollectorConfig with_sanitize_flag(bool x); + generation age2gen(object_age age) const noexcept { return generation(age % n_survive_threshold_); } @@ -37,13 +47,19 @@ namespace xo { std::string name_; /** Configuration for collector spaces. - * Will have at least {nursery,tenured} x {from,to} spaces. + * Will have (2 x G) of these, + * where G is @ref n_generation_. * Not using name_ member. * * REQUIRE: * - arena_config_.store_header_flag_ must be true **/ - ArenaConfig arena_config_; + ArenaConfig arena_config_ = ArenaConfig().with_store_header_flag(true); + + /** storage for xgen pointer bookkeeping (aka remembered sets). + * Use 3x this value per generation + **/ + std::size_t mutation_log_z_ = 1024; /** storage for N object types requires 8*N bytes **/ std::size_t object_types_z_ = 2*1024*1024; @@ -85,13 +101,17 @@ namespace xo { **/ uint32_t stats_history_z_ = false; + /** true to enable sanitize features: + * 1. zero out from-space at end of GC cycle + **/ + bool sanitize_flag_ = false; + /** true to enable debug logging **/ bool debug_flag_ = false; }; - + } /*namespace mm*/ } /*namespace xo*/ /* end X1CollectorConfig.hpp */ - diff --git a/include/xo/gc/detail/ICollector_DX1Collector.hpp b/include/xo/gc/detail/ICollector_DX1Collector.hpp index 9e4c56a..6984cd2 100644 --- a/include/xo/gc/detail/ICollector_DX1Collector.hpp +++ b/include/xo/gc/detail/ICollector_DX1Collector.hpp @@ -49,6 +49,8 @@ namespace xo { static void add_gc_root_poly(DX1Collector & d, obj * p_root); static void remove_gc_root_poly(DX1Collector & d, obj * p_root); static void request_gc(DX1Collector & d, generation upto); + static void assign_member(DX1Collector & d, void * parent, + obj * p_lhs, obj & rhs); static void forward_inplace(DX1Collector & d, AGCObject * lhs_iface, void ** lhs_data); static int32_t s_typeseq; diff --git a/include/xo/gc/object_age.hpp b/include/xo/gc/object_age.hpp index ae876bd..fe0a48a 100644 --- a/include/xo/gc/object_age.hpp +++ b/include/xo/gc/object_age.hpp @@ -26,6 +26,18 @@ namespace xo { std::uint32_t value_; }; + + inline bool operator==(object_age lhs, object_age rhs) { + return lhs.value_ == rhs.value_; + } + + inline bool operator<(object_age lhs, object_age rhs) { + return lhs.value_ < rhs.value_; + } + + inline bool operator>(object_age lhs, object_age rhs) { + return lhs.value_ > rhs.value_; + } } } diff --git a/src/gc/CMakeLists.txt b/src/gc/CMakeLists.txt index 6690932..eee6864 100644 --- a/src/gc/CMakeLists.txt +++ b/src/gc/CMakeLists.txt @@ -3,6 +3,9 @@ set(SELF_LIB xo_gc) set(SELF_SRCS + init_gc.cpp + SetupGc.cpp + IAllocator_DX1Collector.cpp ICollector_DX1Collector.cpp IAllocIterator_DX1CollectorIterator.cpp @@ -10,6 +13,9 @@ set(SELF_SRCS DX1Collector.cpp DX1CollectorIterator.cpp + X1CollectorConfig.cpp + MutationLogEntry.cpp + ) xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS}) diff --git a/src/gc/DX1Collector.cpp b/src/gc/DX1Collector.cpp index 25665d7..96f65a0 100644 --- a/src/gc/DX1Collector.cpp +++ b/src/gc/DX1Collector.cpp @@ -21,27 +21,12 @@ namespace xo { using xo::mm::AAllocator; + using xo::facet::TypeRegistry; using xo::facet::typeseq; using xo::facet::with_facet; namespace mm { - X1CollectorConfig - X1CollectorConfig::with_name(std::string name) - { - X1CollectorConfig copy = *this; - copy.name_ = std::move(name); - return copy; - } - - X1CollectorConfig - X1CollectorConfig::with_size(std::size_t gen_z) - { - X1CollectorConfig copy = *this; - copy.arena_config_ = arena_config_.with_size(gen_z); - return copy; - } - // ----- GCRunState ----- GCRunState::GCRunState(generation gc_upto) @@ -72,45 +57,109 @@ namespace xo { size_t page_z = getpagesize(); + this->_init_object_types(cfg, page_z); + this->_init_gc_roots(cfg, page_z); + this->_init_mlogs(cfg, page_z); + this->_init_space(cfg); + } + + void + DX1Collector::_init_object_types(const X1CollectorConfig & cfg, std::size_t page_z) + { /* 1MB reserved address space enough for up to 128k distinct types. * In this case don't want to use hugepages since actual #of types * likely << .size/8 */ - object_types_ = DArena::map( - ArenaConfig{ - .name_ = "x1-object-types", - .size_ = cfg.object_types_z_, - .hugepage_z_ = page_z, - .store_header_flag_ = false}); + this->object_types_ + = DArena::map(ArenaConfig{.name_ = "x1-object-types", + .size_ = cfg.object_types_z_, + .hugepage_z_ = page_z, + .store_header_flag_ = false}); + } - roots_ = DArena::map( - ArenaConfig{ - .name_ = "x1-object-roots", - .size_ = cfg.object_roots_z_, - .hugepage_z_ = page_z, - .store_header_flag_ = false}); + void + DX1Collector::_init_gc_roots(const X1CollectorConfig & cfg, std::size_t page_z) + { + this->root_set_ + = RootSet::map(ArenaConfig{.name_ = "x1-object-roots", + .size_ = cfg.object_roots_z_, + .hugepage_z_ = page_z, + .store_header_flag_ = false}); + } + + void + DX1Collector::_init_mlogs(const X1CollectorConfig & cfg, std::size_t page_z) + { + for (uint32_t igen = 0, ngen = cfg.n_generation_; igen + 1 < ngen; ++igen) { + // special case: no use for mutation log for youngest generation, + // so don't trouble to allocate one + + if (igen + 1 < c_max_generation) { + this->mlog_storage_[0][igen] = _make_mlog(igen, 'a', cfg.mutation_log_z_, page_z); + this->mlog_storage_[1][igen] = _make_mlog(igen, 'b', cfg.mutation_log_z_, page_z); + this->mlog_storage_[2][igen] = _make_mlog(igen, 'c', cfg.mutation_log_z_, page_z); + + this->mlog_[0][igen] = &mlog_storage_[0][igen]; + this->mlog_[1][igen] = &mlog_storage_[1][igen]; + this->mlog_[2][igen] = &mlog_storage_[2][igen]; + } else { + assert(false); + } + } + + if (cfg.n_generation_ > 0) { + for (uint32_t igen = cfg.n_generation_ - 1; igen + 1 < c_max_generation; ++igen) { + this->mlog_[0][igen] = nullptr; + this->mlog_[1][igen] = nullptr; + this->mlog_[2][igen] = nullptr; + } + } else { + assert(false); + } + } + + auto + DX1Collector::_make_mlog(uint32_t igen, char tag_char, size_t mlog_z, size_t page_z) -> MutationLog + { + char buf[40]; + snprintf(buf, sizeof(buf), "x1-mlog-G%u-%c", igen, tag_char); + + return MutationLog::map(ArenaConfig{.name_ = std::string(buf), + .size_ = mlog_z, + .hugepage_z_ = page_z, + .store_header_flag_ = false}); + } + + void + DX1Collector::_init_space(const X1CollectorConfig & cfg) + { + assert(c_n_role == 2); for (uint32_t igen = 0, ngen = cfg.n_generation_; igen < ngen; ++igen) { - { - char buf[40]; - snprintf(buf, sizeof(buf), "x1-space-G%u-a", igen); + if (igen < c_max_generation) { + { + char buf[40]; + snprintf(buf, sizeof(buf), "x1-space-G%u-a", igen); - space_storage_[0][igen] = DArena::map(cfg.arena_config_.with_name(std::string(buf))); + this->space_storage_[0][igen] = DArena::map(cfg.arena_config_.with_name(std::string(buf))); + } + { + char buf[40]; + snprintf(buf, sizeof(buf), "x1-space-G%u-b", igen); + + this->space_storage_[1][igen] = DArena::map(cfg.arena_config_.with_name(std::string(buf))); + } + + this->space_[role::to_space()][igen] = &space_storage_[0][igen]; + this->space_[role::from_space()][igen] = &space_storage_[1][igen]; + } else { + assert(false); } - { - char buf[40]; - snprintf(buf, sizeof(buf), "x1-space-G%u-b", igen); - - space_storage_[1][igen] = DArena::map(cfg.arena_config_.with_name(std::string(buf))); - } - - space_[role::to_space()][igen] = &space_storage_[0][igen]; - space_[role::from_space()][igen] = &space_storage_[1][igen]; } for (uint32_t igen = cfg.n_generation_; igen < c_max_generation; ++igen) { - space_[role::to_space()][igen] = nullptr; - space_[role::from_space()][igen] = nullptr; + this->space_[role::to_space()][igen] = nullptr; + this->space_[role::from_space()][igen] = nullptr; } } @@ -118,26 +167,49 @@ namespace xo { DX1Collector::visit_pools(const MemorySizeVisitor & visitor) const { object_types_.visit_pools(visitor); - roots_.visit_pools(visitor); + root_set_.visit_pools(visitor); - for (uint32_t i = 0; i < c_n_role; ++i) { - for (uint32_t j = 0; j < config_.n_generation_; ++j) { + for (uint32_t j = 0; j < config_.n_generation_; ++j) { + for (uint32_t i = 0; i < c_n_role; ++i) { space_storage_[i][j].visit_pools(visitor); } } + + for (uint32_t j = 1; j < config_.n_generation_; ++j) { + for (uint32_t i = 0; i < c_n_role + 1; ++i) { + mlog_storage_[i][j].visit_pools(visitor); + } + } } bool DX1Collector::contains(role r, const void * addr) const noexcept + { + return !(this->generation_of(r, addr).is_sentinel()); + } + + bool + DX1Collector::contains_allocated(role r, const void * addr) const noexcept + { + generation g = this->generation_of(r, addr); + + if (g.is_sentinel()) + return false; + + return this->get_space(r, g)->contains_allocated(addr); + } + + generation + DX1Collector::generation_of(role r, const void * addr) const noexcept { for (generation gi{0}; gi < config_.n_generation_; ++gi) { const DArena * arena = get_space(r, gi); if (arena->contains(addr)) - return true; + return gi; } - return false; + return generation::sentinel(); } AllocError @@ -156,7 +228,7 @@ namespace xo { size_t (DArena::* get_stat_fn)() const) noexcept { size_t z1 = (d.object_types_.*get_stat_fn)(); - size_t z2 = (d.roots_.*get_stat_fn)(); + size_t z2 = (d.root_set_.store()->*get_stat_fn)(); size_t z3 = 0; @@ -255,7 +327,11 @@ namespace xo { { AGCObject * v = reinterpret_cast(object_types_.lo_); - return &(v[tseq.seqno()]); + const AGCObject * target = &(v[tseq.seqno()]); + + assert(reinterpret_cast(target) < object_types_.limit_); + + return target; } /* editor bait: register_type */ @@ -279,12 +355,7 @@ namespace xo { void DX1Collector::add_gc_root_poly(obj * p_root) noexcept { - std::byte * mem - = roots_.alloc(typeseq::sentinel(), - sizeof(obj*)); - assert(mem); - - *(obj **)mem = p_root; + root_set_.push_back(GCRoot(p_root)); } void @@ -337,7 +408,10 @@ namespace xo { log && log("step 2b : [STUB] copy pinned"); log && log("step 3a : [STUB] run destructors"); log && log("step 3b : [STUB] keep reachable weak pointers"); + log && log("step 4 : [STUB] cleanup"); + this->cleanup_phase(upto); + } void @@ -352,6 +426,69 @@ namespace xo { } } + void + DX1Collector::cleanup_phase(generation upto) + { + scope log(XO_DEBUG(true), xtag("upto", upto)); + + // everything live has been copied out of from-space + // -> now set to empty + // + for (generation g = generation{0}; g < upto; ++g) { + if (config_.sanitize_flag_) { + space_[role::from_space()][g]->scrub(); + } + + space_[role::from_space()][g]->clear(); + } + + this->runstate_ = GCRunState::gc_not_running(); + } + + void * + DX1Collector::_deep_move_root(obj from_src, + generation upto) + { + // NOTE: + // Some roots are non-gc-owned nodes. + // GC must still visit immediate children of these nodes + // to move gc-owned children. + // This implements virtual root node feature, + // intended to mitigate mutation log churn. + + scope log(XO_DEBUG(config_.debug_flag_)); + + if (!from_src) + return nullptr; + + bool src_in_from_space = this->contains(role::from_space(), from_src.data()); + + if (src_in_from_space) { + return _deep_move_gc_owned(from_src.data(), upto); + } else { + auto self = this->ref(); + from_src.forward_children(self); + return from_src.data(); + } + } + + void * + DX1Collector::_deep_move_interior(void * from_src, + generation upto) + { + scope log(XO_DEBUG(config_.debug_flag_)); + + if (!from_src) + return nullptr; + + bool src_in_from_space = this->contains(role::from_space(), from_src); + + if (!src_in_from_space) + return from_src; + + return _deep_move_gc_owned(from_src, upto); + } + /* * rules: * - from_src must be in from-space @@ -364,24 +501,17 @@ namespace xo { * EDITOR: gc -> self */ void * - DX1Collector::deep_move(void * from_src, generation upto) + DX1Collector::_deep_move_gc_owned(void * from_src, + generation upto) { scope log(XO_DEBUG(config_.debug_flag_)); - if (!from_src) - return nullptr; - - if (!this->contains(role::from_space(), from_src)) { - /* presumeably memory not owned by collector - * (e.g. DBoolean {true, false}, DUniqueString {owned by StringTable} etc.) - */ - return from_src; - } - AllocInfo info = this->alloc_info((std::byte *)from_src); AllocHeader hdr = info.header(); typeseq tseq(info.tseq()); + assert(this->contains_allocated(role::from_space(), from_src)); + if (is_forwarding_header(hdr)) { /* already forwarded - pickup destination * @@ -475,6 +605,9 @@ namespace xo { obj alloc(this); const AGCObject * iface = lookup_type(tseq); + + assert(iface->_has_null_vptr() == false); + void * to_dest = this->shallow_move(iface, from_src); std::size_t fixup_work = 0; @@ -487,23 +620,37 @@ namespace xo { fixup_work = 0; for (generation g = generation{0}; g < upto; ++g) { + /** object index for this pass **/ + size_t i_obj = 0; + /* TODO: use AllocIterator here */ while(gray_lo_v[g] < to_space(g)->free_) { AllocHeader * hdr = (AllocHeader *)gray_lo_v[g]; void * src = (hdr + 1); - log && log("fwd children", xtag("src", src)); - const auto & hdr_cfg = config_.arena_config_.header_; typeseq tseq = typeseq(hdr_cfg.tseq(*hdr)); size_t z = hdr_cfg.size_with_padding(*hdr); + log && log("deep_move_gc_owned: fwd to-space children", + xtag("g", g), + xtag("i_obj", i_obj), + xtag("src", src), + xtag("tseq", tseq), + xtag("tname", TypeRegistry::id2name(tseq)), + xtag("z", z)); + const AGCObject * iface = this->lookup_type(tseq); + + assert(iface->_has_null_vptr() == false); + obj gc(this); iface->forward_children(src, gc); gray_lo_v[g] = ((std::byte *)src) + z; + + ++i_obj; ++fixup_work; } } @@ -512,21 +659,25 @@ namespace xo { log && log(xtag("to_dest", to_dest)); return to_dest; - } + } /*_deep_move_gc_owned*/ void DX1Collector::copy_roots(generation upto) noexcept { scope log(XO_DEBUG(true)); - for (obj ** p_root = (obj **)roots_.lo_; - p_root < (obj **)roots_.free_; ++p_root) - { - log && log("copy root", xtag("**p_root.data.pre", (**p_root).data_)); + for (RootSet::size_type i = 0, n = root_set_.size(); i < n; ++i) { + GCRoot & slot = root_set_[i]; - (*p_root)->reset_opaque(this->deep_move((*p_root)->data_, upto)); + log && log("copy root", + xtag("slot.root()", slot.root()), + xtag("slot.root()->data_", slot.root()->data_)); - log && log(xtag("**p_root.data.post", (**p_root).data_)); + void * root_to = this->_deep_move_root(*slot.root(), upto); + + slot.root()->reset_opaque(root_to); + + log && log(xtag("slot.root()->data_", slot.root()->data_)); } } @@ -590,9 +741,18 @@ namespace xo { /* recover allocation size */ std::size_t alloc_z = some_arena->config_.header_.size_with_padding(alloc_hdr); - log && log(xtag("some_arena.lo", some_arena->lo_), - xtag("p_header", p_header), - xtag("alloc_z", alloc_z)); + if (log) { + log(xtag("some_arena.lo", some_arena->lo_), + xtag("p_header", p_header), + xtag("alloc_z", alloc_z)); + + AllocInfo info = this->alloc_info((std::byte *)object_data); + log(xtag("tseq", info.tseq()), + xtag("tname", TypeRegistry::id2name(typeseq(info.tseq()))), + xtag("is_forwarding_tseq", info.is_forwarding_tseq()), + xtag("age", info.age()), + xtag("size", info.size())); + } /* need to be able to fit forwarding pointer * in place of forwarded object. @@ -616,9 +776,9 @@ namespace xo { * | * (to-space) | * +---+-+----+ | - * |FWD|G|size|<--/ - * +---+-+----+ - * | | + * |TSQ|G|size| | + * +---+-+----+ | + * | | <-/ * | | * | | * +----------+ @@ -638,22 +798,33 @@ namespace xo { * | * | (to-space) * | +---+-+----+ - * \---->|FWD|G|size| - * +---+-+----+ - * | | + * | |TSQ|G|size| + * | +---+-+----+ + * \---> | | * | | * | | * +----------+ */ + + if (log) { + log("lhs_data already forwarded", xtag("dest", dest)); + + AllocInfo info = this->alloc_info((std::byte *)dest); + log(xtag("tseq", info.tseq()), + xtag("tname", TypeRegistry::id2name(typeseq(info.tseq()))), + xtag("age", info.age()), xtag("size", info.size())); + } } else if (this->check_move_policy(alloc_hdr, object_data)) { /* copy object *lhs + replace with forwarding pointer */ + log && log("forward object now"); + /* * lhs obj (from-space) * | +---------+ +---+-+----+ - * \--->| .iface | |FWD|G|size| alloc_hdr + * \--->| .iface | |TSQ|G|size| alloc_hdr * +---------+ object_data +---+-+----+ - * | .data x----------------->| | + * | .data x----------------> | | * +---------+ | | * | | * +----------+ @@ -673,14 +844,16 @@ namespace xo { * | | * | (to-space) | * | +---+-+----+ | - * \---->|FWD|G|size|<--/ - * +---+-+----+ - * | | + * | |TSQ|G|size| | + * | +---+-+----+ | + * \---> | | <-/ * | | * | | * +----------+ */ } else { + log && log("object not eligible/required to forward"); + /* object doesn't need to move. * e.g. incremental collection + object is tenured */ @@ -698,6 +871,9 @@ namespace xo { void * to_dest = iface->shallow_copy(from_src, alloc); log && log(xtag("from_src", from_src), xtag("to_dest", to_dest)); + log && log(xtag("tseq", info.tseq()), + xtag("tname", TypeRegistry::id2name(typeseq(info.tseq()))), + xtag("age", info.age()), xtag("size", info.size())); if(to_dest == from_src) { assert(false); @@ -781,6 +957,95 @@ namespace xo { return this->get_space(role::to_space(), generation{0})->alloc_info(mem); } + // editor bait: write barrier + void + DX1Collector::assign_member(void * parent, obj * p_lhs, obj rhs) + { + scope log(XO_DEBUG(config_.debug_flag_), + xtag("parent", parent), xtag("lhs", p_lhs), xtag("rhs", rhs.data())); + + // ++ stats.n_mutation_; + + *p_lhs = rhs; + + if (runstate_.is_running()) { + // for removal of all doubt: + // don't log mutations during GC cycle + return; + } + + if (!config_.allow_incremental_gc_) { + // only need to log mutations when incremental gc is enabled + return; + } + + // logging policy depends on: + // 1. generation of lhs + // 2. generation of rhs + + generation src_g = this->generation_of(role::to_space(), p_lhs); + + if (src_g.is_sentinel()) { + // only need mlog entries for gc-owned pointers. + // In this case pointer does not originate in gc-owned space + return; + } + + generation dest_g = this->generation_of(role::to_space(), rhs.data()); + + if (dest_g.is_sentinel()) { + // similarly, don't need mlog entry to non-gc-owned destination + return; + } + + if (src_g < dest_g) { + // young-to-old pointers don't need to be remembered, + // since a GC cycle that collects the old generation is guarnatted + // to also collect the young generation. + return; + } + + if (src_g == dest_g) { + // for pointers within the same generation, need to log + // if source is older than destination. + + const DArena * arena = this->get_space(role::to_space(), src_g); + + const DArena::header_type * src_hdr = arena->obj2hdr(parent); + const DArena::header_type * dest_hdr = arena->obj2hdr(rhs.data()); + + assert(src_hdr && dest_hdr); + + if (header2age(*src_hdr) <= header2age(*dest_hdr)) { + // source and destination have the same age; + // therefore are always collected on the same set of GC cycles + // -> no need to remember separately. + return; + } else { + // even though {src,dest} belong to the same generation: + // source will be eligible for promotion before destination. + // At that point pointer would become a cross-generational pointer, + // so need to track it now. + + log && log("xage ptr -> must log"); + } + } else { + log && log("xgen ptr -> must log"); + } + + // control here: we have an older->younger pointer, need to log it + + // mlog keyed by generation in which pointer _destination_ resides: + // collection that moves destination generation around needs to also + // update pointers such as this one + // + MutationLog * mlog = this->mlog_[role::to_space()][dest_g]; + + mlog->push_back(MutationLogEntry(parent, + reinterpret_cast(&(p_lhs->data_)), + rhs)); + } + DX1CollectorIterator DX1Collector::begin() const noexcept { diff --git a/src/gc/ICollector_DX1Collector.cpp b/src/gc/ICollector_DX1Collector.cpp index 781fcfe..fd0c811 100644 --- a/src/gc/ICollector_DX1Collector.cpp +++ b/src/gc/ICollector_DX1Collector.cpp @@ -81,6 +81,15 @@ namespace xo { d.request_gc(upto); } + void + ICollector_DX1Collector::assign_member(DX1Collector & d, + void * parent, + obj * p_lhs, + obj & rhs) + { + d.assign_member(parent, p_lhs, rhs); + } + void ICollector_DX1Collector::forward_inplace(DX1Collector & d, AGCObject * lhs_iface, diff --git a/utest/Collector.test.cpp b/utest/Collector.test.cpp index 318c9c2..cf82fa9 100644 --- a/utest/Collector.test.cpp +++ b/utest/Collector.test.cpp @@ -63,10 +63,14 @@ namespace xo { 16 /*size_bits*/), }; X1CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 2, - .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0, + .gc_trigger_v_ = {{64*1024, 1024*1024, +#ifdef NOPE + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0}} }; + 0, 0, 0, 0 +#endif + }} }; DX1Collector gc = DX1Collector{cfg}; @@ -111,10 +115,14 @@ namespace xo { 16 /*size_bits*/), }; X1CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 2, - .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0, + .gc_trigger_v_ = {{64*1024, 1024*1024, +#ifdef NOPE + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0}} }; + 0, 0, 0, 0 +#endif + }} }; DX1Collector gc = DX1Collector{cfg}; @@ -137,10 +145,14 @@ namespace xo { 16 /*size-bits*/), }; X1CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 2, - .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0, + .gc_trigger_v_ = {{64*1024, 1024*1024, +#ifdef NOPE + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0}} }; + 0, 0, 0, 0 +#endif + }} }; DX1Collector gc = DX1Collector{cfg}; @@ -167,10 +179,14 @@ namespace xo { /* collector with one generation collapses to a non-generational copying collector */ X1CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 1, - .gc_trigger_v_ = {{64*1024, 0, 0, 0, + .gc_trigger_v_ = {{64*1024, 0, +#ifdef NOPE + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0}} }; + 0, 0, 0, 0 +#endif + }} }; DX1Collector x1state = DX1Collector{cfg}; @@ -211,10 +227,14 @@ namespace xo { /* collector with one generation collapses to a non-generational copying collector */ X1CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 1, - .gc_trigger_v_ = {{64*1024, 0, 0, 0, + .gc_trigger_v_ = {{64*1024, 0, +#ifdef NOPE + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0}} }; + 0, 0, 0, 0 +#endif + }} }; /* X1 allocator+collector */ DX1Collector x1state = DX1Collector{cfg}; diff --git a/utest/DX1CollectorIterator.test.cpp b/utest/DX1CollectorIterator.test.cpp index 137c07f..66a9e41 100644 --- a/utest/DX1CollectorIterator.test.cpp +++ b/utest/DX1CollectorIterator.test.cpp @@ -53,10 +53,14 @@ namespace xo { 16 /*size_bits*/), }; X1CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 2, - .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0, + .gc_trigger_v_ = {{64*1024, 1024*1024, +#ifdef NOPE + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0}} }; + 0, 0, 0, 0 +#endif + }} }; DX1Collector gc = DX1Collector{cfg}; @@ -96,10 +100,14 @@ namespace xo { 16 /*size_bits*/), }; X1CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 2, - .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0, + .gc_trigger_v_ = {{64*1024, 1024*1024, +#ifdef NOPE + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0}} }; + 0, 0, 0, 0 +#endif + }} }; DX1Collector gc = DX1Collector{cfg}; obj a1o{&gc};