diff --git a/include/xo/gc/DX1Collector.hpp b/include/xo/gc/DX1Collector.hpp index fa7dcdd..135ebc1 100644 --- a/include/xo/gc/DX1Collector.hpp +++ b/include/xo/gc/DX1Collector.hpp @@ -7,7 +7,8 @@ #include "X1CollectorConfig.hpp" #include "GCObject.hpp" -#include "MutationLogEntry.hpp" +#include "MutationLogState.hpp" +#include "X1VerifyStats.hpp" #include "generation.hpp" #include "object_age.hpp" #include "role.hpp" @@ -129,68 +130,11 @@ namespace xo { alignas(AGCObject) std::byte iface_[sizeof(AGCObject)]; }; - /** @brief statistics from mlog forwarding **/ - class MutationLogStatistics { - public: - MutationLogStatistics() = default; - - MutationLogStatistics & operator+=(const MutationLogStatistics & x) { - n_stale_ += x.n_stale_; - n_live_parent_ += x.n_live_parent_; - n_rescue_ += x.n_rescue_; - n_triage_ += x.n_triage_; - - return *this; - } - - public: - /** count superseded mlog entries **/ - std::size_t n_stale_ = 0; - /** count live parents encountered during mlog scan **/ - std::size_t n_live_parent_ = 0; - /** count child subgraphs rescued during mlog scan **/ - std::size_t n_rescue_ = 0; - /** count triaged mlog entries **/ - std::size_t n_triage_ = 0; - }; - - /** @brief info collected during a @ref DX1Collector::verify_ok call - * - **/ - struct VerifyStats { - bool is_ok() const noexcept { - return (n_from_ == 0) && (n_fwd_ == 0) && (n_no_iface_ == 0); - } - - void clear() { *this = VerifyStats(); } - - /** number of gc roots examined **/ - std::uint32_t n_gc_root_ = 0; - std::uint32_t n_ext_ = 0; - /** number of from-space objects encountered. Fatal if non-zero **/ - std::uint32_t n_from_ = 0; - /** number of to-space objects encountered. **/ - std::uint32_t n_to_ = 0; - /** counts forwarding object encountered in to-space scan. Fatal if non-zero **/ - std::uint32_t n_fwd_ = 0; - /** counts missing GCObject interface. Fatal if non-zero **/ - std::uint32_t n_no_iface_ = 0; - /** live mlog entry refers to to-space, as expected **/ - std::uint32_t n_mlog_vital_ = 0; - /** stale mlog entry. not troubling to verify these **/ - std::uint32_t n_mlog_stale_ = 0; - /** live mlog entry refers to from-space. Fatal if non-zero **/ - std::uint32_t n_mlog_from_ = 0; - /** live mlog entry refers to either some other generation or outside gc-space. Fatal if non-zero **/ - std::uint32_t n_mlog_wild_ = 0; - - }; - // ----- DX1Collector ----- /** @brief garbage collector 'X1' **/ - struct DX1Collector { + class DX1Collector { public: using RootSet = DArenaVector; using ObjectTypeTable = DArenaVector; @@ -215,6 +159,7 @@ namespace xo { // ----- access methods ----- + const X1CollectorConfig & config() const noexcept { return config_; } std::string_view name() const noexcept { return config_.name_; } GCRunState runstate() const noexcept { return runstate_; } const ObjectTypeTable * get_object_types() const noexcept { return &object_types_; } @@ -390,6 +335,11 @@ namespace xo { **/ bool check_move_policy(header_type alloc_hdr, void * object_data) const noexcept; + /** move interior subgraph at @p from_src to to-space. + * no-op if not in gc-space. + **/ + void * deep_move_interior(void * from_src, Generation upto); + // ----- allocation ----- /** simple allocation. allocate @p z bytes of memory @@ -460,8 +410,10 @@ namespace xo { void _init_gc_roots(const X1CollectorConfig & cfg, std::size_t page_z); /** aux init function: initialize @ref mlog_storage_[][] arenas **/ void _init_mlogs(const X1CollectorConfig & cfg, std::size_t page_z); +#ifdef MOVED /** aux init function: create mutation log **/ MutationLog _make_mlog(uint32_t igen, char tag_char, size_t mlog_z, std::size_t page_z); +#endif /** aux init function: initialize @ref space_storage_[][] arenas **/ void _init_space(const X1CollectorConfig & cfg); @@ -473,53 +425,14 @@ namespace xo { /** cureate new mutation log after copying roots **/ void forward_mutation_log(Generation upto); - /** Perform one pass over contents of @p *from_mlog for generation @p gen. - * @p *from_mlog contains all {xgen,xage} pointers that target generation @p gen. - * Surviving mlog entries are moved to either @p *to_mlog or @p *triage_mlog, - * (generation < @p upto being collected this cycle). - * - * Each mlog entry gets one of the following outcomes. - * 1. skip. mlog entry has been superseded by another mut at target site. - * 2. keep. mlog entry is live. destination has been evacuated, - * so source must be updated as well. - * 3. triage. source of incoming object belongs to a generation that was collected, - * and has not been evacuated. Although appears to be garbage, it may - * be live after all if reachable from the destination of some other - * mlog entry in @p *to_mlog. Store these mlog entries in @p *triage_mlog. - * - * @return number of mlog entries moved, whether to @p *to_mlog or @p *triage_mlog. - **/ - MutationLogStatistics _forward_mutation_log_phase(Generation upto, - Generation gen, - MutationLog * from_mlog, - MutationLog * to_mlog, - MutationLog * triage_mlog); - - MutationLogStatistics _preserve_child_of_live_parent(Generation upto, - Generation parent_gen, - const MutationLogEntry & from_entry, - MutationLog * keep_mlog); - - /** helper function to decide whether to keep a mutation log entry - * @return true iff mlog entry appended to @p keep_mlog - **/ - bool _check_keep_mutation_aux(const MutationLogEntry & from_entry, - Generation parent_gen_to, - void * child_to, - MutationLog * keep_mlog); - /** cleanup after gc **/ - void cleanup_phase(Generation upto); + void _cleanup_phase(Generation upto); /** move root subgraph at @p from_src to to-space. * If not in gc-space, visit immediate children and move them. * Require: runstate_.is_running() **/ void * _deep_move_root(obj from_src, Generation upto); - /** move interior subgraph at @p from_src to to-space. - * no-op if not in gc-space. - **/ - void * _deep_move_interior(void * from_src, Generation upto); /** Common driver for _deep_move_root(), _deep_move_interior() **/ void * _deep_move_gc_owned(void * from_src, Generation upto); /** snap checkpoint containing allocator state @@ -570,24 +483,15 @@ namespace xo { **/ RootSet root_set_; - /** Cross-generational mutations tracked in MutationLogs. - * We need three logs per generation: - * A. one to observe and remember mutations in to-space - * during normal operation (between GC cycles) - * B. during GC: 2nd mlog to hold entries from from-mlog - * that will still be needed post-GC (because ptr direction - * from higher gen to lower gen after cycle). - * C. during GC: 3rd mlog to triage entries for which - * liveness of pointer source isn't yet established. - * - * NOTE: indexed on generation of pointer *destination* + /** "remembered sets": track pointers P->C that require special handling + * during a gc cycle where either: + * 1. xgen pointers g(P) > g(C): + * P in a more senior generation than C + * 2. xage pointers g(P) = g(C), age(P) > age(C): + * {P,C} in same generation, but in fuutre suriving P would + * get promoted before C. **/ - std::array mlog_storage_[c_n_role + 1]; - - /** mlog pointers. The roles of mlog_storage_[*][g] get permuted - * as each collection cycle proceeds - **/ - std::array mlog_[c_n_role + 1]; + MutationLogState mlog_state_; /** collector-managed memory here. * - space_[1] is from-space diff --git a/include/xo/gc/DX1CollectorIterator.hpp b/include/xo/gc/DX1CollectorIterator.hpp index 82944ff..d96d9ab 100644 --- a/include/xo/gc/DX1CollectorIterator.hpp +++ b/include/xo/gc/DX1CollectorIterator.hpp @@ -12,7 +12,7 @@ namespace xo { namespace mm { - struct DX1Collector; + class DX1Collector; /** @class DX1CollectorIterator * @brief Representation for alloc iterator over X1 collector diff --git a/include/xo/gc/MutationLogState.hpp b/include/xo/gc/MutationLogState.hpp new file mode 100644 index 0000000..dd20655 --- /dev/null +++ b/include/xo/gc/MutationLogState.hpp @@ -0,0 +1,162 @@ +/** @file MutationLogState.hpp + * + * @author Roland Conybeare, Apr 2026 + **/ + +#pragma once + +#include "X1CollectorConfig.hpp" +#include "MutationLogStatistics.hpp" +#include "MutationLogEntry.hpp" +#include +#include + +namespace xo { + namespace mm { + class DX1Collector; + class VerifyStats; + + class MutationLogState { + public: + using MutationLog = DArenaVector; + using size_type = DArena::size_type; + + public: + MutationLogState(uint32_t ngen, bool debug_flag); + + /** Initialize mlog state for configuration @p cfg + * with o/s page size @p page_z + **/ + void init_mlogs(const X1CollectorConfig & cfg, std::size_t page_z); + + /** total number of active mlog entries (across all generations) + **/ + size_type mutation_log_entries() const noexcept; + + void visit_pools(const MemorySizeVisitor & visitor) const; + + /** verify consistent mlog state, + * on behalf of collector @p gc. + * (using gc to identify location of objects). + * Update counters in @p *p_verify_stats. + **/ + void verify_ok(DX1Collector * gc, + VerifyStats * p_verify_stats) noexcept; + + /** Append a single mutation to log for generation @p dest_g + * Mutation modifies @p parent at address @p addr, + * to refer to @p rhs. + * + * Require: mutation is from older->newer, + * see validation in DX1Collector::assign_member. + * + * NOTE: rhs can probably be dropped. Initially thought + * helpful to keep wrapped obj version. On closer look + * not necessary. Important to remember that gc can't change + * any interface pointers, it strictly preserves them. + * + * Since mutation log entries are specific to a particular + * rhs pointer value, they commit corresponding interface + * pointer. This means can alway recover that pointer + * by consulting the AllocHeader for the pointer target + */ + void append_mutation(Generation dest_g, + void * parent, + void ** addr, + obj rhs); + + /** swap {to, from} roles **/ + void swap_roles(Generation upto) noexcept; + + /** On behalf of collector @p gc: + * + * forward mutation logs, for generations 0 <= g < @p upto, + * from from-space to to-space. + **/ + void forward_mutation_log(DX1Collector * gc, + Generation upto); + + private: + /** aux init function: create mutation log **/ + MutationLog _make_mlog(uint32_t igen, char tag_char, + size_t mlog_z, std::size_t page_z); + + /** On behalf of collctor @p gc: + * + * Perform one pass over contents of @p *from_mlog for generation @p gen. + * @p *from_mlog contains all {xgen,xage} pointers that target generation @p gen. + * Surviving mlog entries are moved to either @p *to_mlog or @p *triage_mlog, + * (generation < @p upto being collected this cycle). + * + * Each mlog entry gets one of the following outcomes. + * 1. skip. mlog entry has been superseded by another mut at target site. + * 2. keep. mlog entry is live. destination has been evacuated, + * so source must be updated as well. + * 3. triage. source of incoming object belongs to a generation that was collected, + * and has not been evacuated. Although appears to be garbage, it may + * be live after all if reachable from the destination of some other + * mlog entry in @p *to_mlog. Store these mlog entries in @p *triage_mlog. + * + * @return number of mlog entries moved, whether to @p *to_mlog or @p *triage_mlog. + **/ + MutationLogStatistics _forward_mutation_log_phase(DX1Collector * gc, + Generation upto, + Generation gen, + MutationLog * from_mlog, + MutationLog * to_mlog, + MutationLog * triage_mlog); + + /** On behalf of collector @p gc: + * + * During gc of generations g < @p upto, + * with a P->C edge represented by mlog entry @p from_entry, + * with parent P in generation @p parent_gen: + * ensure child C is evacuated, and append @p from_entry to + * @p keep_mlog. + **/ + MutationLogStatistics _preserve_child_of_live_parent(DX1Collector * gc, + Generation upto, + Generation parent_gen, + const MutationLogEntry & from_entry, + MutationLog * keep_mlog); + + /** On behalf of collector @p gc: + * + * helper function to decide whether to keep a mutation log entry + * @return true iff mlog entry appended to @p keep_mlog + **/ + bool _check_keep_mutation_aux(DX1Collector * gc, + const MutationLogEntry & from_entry, + Generation parent_gen_to, + void * child_to, + MutationLog * keep_mlog); + + + public: + uint32_t n_generation_ = 0; + bool debug_flag_ = false; + + /** Cross-generational mutations tracked in MutationLogs. + * We need three logs per generation: + * A. one to observe and remember mutations in to-space + * during normal operation (between GC cycles) + * B. during GC: 2nd mlog to hold entries from from-mlog + * that will still be needed post-GC (because ptr direction + * from higher gen to lower gen after cycle). + * C. during GC: 3rd mlog to triage entries for which + * liveness of pointer source isn't yet established. + * + * NOTE: indexed on generation of pointer *destination* + **/ + std::array mlog_storage_[c_n_role + 1]; + + /** mlog pointers. The roles of mlog_storage_[*][g] get permuted + * as each collection cycle proceeds + **/ + std::array mlog_[c_n_role + 1]; + }; + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end MutationLogState.hpp */ diff --git a/include/xo/gc/MutationLogStatistics.hpp b/include/xo/gc/MutationLogStatistics.hpp new file mode 100644 index 0000000..986064f --- /dev/null +++ b/include/xo/gc/MutationLogStatistics.hpp @@ -0,0 +1,41 @@ +/** @file MutationLogStatistics.hpp + * + * @author Roland Conybeare, Apr 2026 + **/ + +#pragma once + +#include + +namespace xo { + namespace mm { + + /** @brief statistics from mlog forwarding **/ + class MutationLogStatistics { + public: + MutationLogStatistics() = default; + + MutationLogStatistics & operator+=(const MutationLogStatistics & x) { + n_stale_ += x.n_stale_; + n_live_parent_ += x.n_live_parent_; + n_rescue_ += x.n_rescue_; + n_triage_ += x.n_triage_; + + return *this; + } + + public: + /** count superseded mlog entries **/ + std::size_t n_stale_ = 0; + /** count live parents encountered during mlog scan **/ + std::size_t n_live_parent_ = 0; + /** count child subgraphs rescued during mlog scan **/ + std::size_t n_rescue_ = 0; + /** count triaged mlog entries **/ + std::size_t n_triage_ = 0; + }; + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end MutationLogStatistics.hpp */ diff --git a/include/xo/gc/X1CollectorConfig.hpp b/include/xo/gc/X1CollectorConfig.hpp index 155a1e5..1c79579 100644 --- a/include/xo/gc/X1CollectorConfig.hpp +++ b/include/xo/gc/X1CollectorConfig.hpp @@ -42,6 +42,7 @@ namespace xo { /** age threshold for promotion to generation @p g **/ uint32_t promotion_threshold(Generation g) const noexcept { + // TODO: may consider replacing with table-lookup // Require: if two distinct ages promote to some gen g at the same time, // then they also promote to gen g+k at the same time for all k>0. diff --git a/include/xo/gc/X1VerifyStats.hpp b/include/xo/gc/X1VerifyStats.hpp new file mode 100644 index 0000000..d5fbc5a --- /dev/null +++ b/include/xo/gc/X1VerifyStats.hpp @@ -0,0 +1,51 @@ +/** @file X1VerifyStats.hpp + * + * @author Roland Conybeare, Apr 2026 + **/ + +#pragma once + +#include + +namespace xo { + namespace mm { + + /** @brief info collected during a @ref DX1Collector::verify_ok call + * (or @ref MutationLogState::verify_ok call) + **/ + class VerifyStats { + public: + bool is_ok() const noexcept { + return (n_from_ == 0) && (n_fwd_ == 0) && (n_no_iface_ == 0); + } + + void clear() { *this = VerifyStats(); } + + /** number of gc roots examined **/ + std::uint32_t n_gc_root_ = 0; + std::uint32_t n_ext_ = 0; + /** number of from-space objects encountered. Fatal if non-zero **/ + std::uint32_t n_from_ = 0; + /** number of to-space objects encountered. **/ + std::uint32_t n_to_ = 0; + /** counts forwarding object encountered in to-space scan. Fatal if non-zero **/ + std::uint32_t n_fwd_ = 0; + /** counts missing GCObject interface. Fatal if non-zero **/ + std::uint32_t n_no_iface_ = 0; + /** live mlog entry refers to to-space, as expected **/ + std::uint32_t n_mlog_vital_ = 0; + /** stale mlog entry. not troubling to verify these **/ + std::uint32_t n_mlog_stale_ = 0; + /** live mlog entry refers to from-space. Fatal if non-zero **/ + std::uint32_t n_mlog_from_ = 0; + /** live mlog entry refers to either some other generation or outside gc-space. + * Fatal if non-zero + **/ + std::uint32_t n_mlog_wild_ = 0; + + }; + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end X1VerifyStats.hpp */ diff --git a/src/gc/CMakeLists.txt b/src/gc/CMakeLists.txt index 2dec873..d30acf0 100644 --- a/src/gc/CMakeLists.txt +++ b/src/gc/CMakeLists.txt @@ -15,6 +15,7 @@ set(SELF_SRCS DX1CollectorIterator.cpp X1CollectorConfig.cpp + MutationLogState.cpp MutationLogEntry.cpp ) diff --git a/src/gc/DX1Collector.cpp b/src/gc/DX1Collector.cpp index 9ad1948..342da1c 100644 --- a/src/gc/DX1Collector.cpp +++ b/src/gc/DX1Collector.cpp @@ -65,7 +65,8 @@ namespace xo { using size_type = xo::mm::DX1Collector::size_type; - DX1Collector::DX1Collector(const X1CollectorConfig & cfg) : config_{cfg} + DX1Collector::DX1Collector(const X1CollectorConfig & cfg) + : config_{cfg}, mlog_state_{cfg.n_generation_, cfg.debug_flag_} { assert(config_.arena_config_.header_.size_bits_ + config_.arena_config_.header_.age_bits_ + @@ -106,6 +107,9 @@ namespace xo { void DX1Collector::_init_mlogs(const X1CollectorConfig & cfg, std::size_t page_z) { + this->mlog_state_.init_mlogs(cfg, page_z); + +#ifdef MOVED for (uint32_t igen = 0, ngen = cfg.n_generation_; igen + 1 < ngen; ++igen) { // special case: no use for mutation log for youngest generation, // so don't trouble to allocate one @@ -132,8 +136,10 @@ namespace xo { } else { assert(false); } +#endif } +#ifdef MOVED auto DX1Collector::_make_mlog(uint32_t igen, char tag_char, size_t mlog_z, size_t page_z) -> MutationLog { @@ -145,6 +151,7 @@ namespace xo { .hugepage_z_ = page_z, .store_header_flag_ = false}); } +#endif void DX1Collector::_init_space(const X1CollectorConfig & cfg) @@ -195,11 +202,15 @@ namespace xo { } } + mlog_state_.visit_pools(visitor); + +#ifdef MOVED for (uint32_t j = 0; j + 1 < config_.n_generation_; ++j) { for (uint32_t i = 0; i < c_n_role + 1; ++i) { mlog_storage_[i][j].visit_pools(visitor); } } +#endif } bool @@ -299,6 +310,9 @@ namespace xo { size_type DX1Collector::mutation_log_entries() const noexcept { + return mlog_state_.mutation_log_entries(); + +#ifdef MOVED size_type z = 0; for (Generation gj{0}; gj + 1 < config_.n_generation_; ++gj) { @@ -306,6 +320,7 @@ namespace xo { } return z; +#endif } namespace { @@ -767,47 +782,8 @@ namespace xo { } // 4. scan mutation logs - for (Generation g(0); g + 1 < config_.n_generation_; ++g) { - const DArena * space = this->get_space(role::to_space(), g); - const DArena * from = this->get_space(role::from_space(), g); - - // mutation log for generation g records *incoming* pointers - // from more senior generations; includes objects from *this* - // generation that are older (track since source promotes before - // destination) - // - for (const MutationLogEntry & mrecd : *(mlog_[role::to_space()][g])) { - // mutation log entries are only valid until the next assignment - // at the source location. Superseded entry may now point - // somewhere else. The snapshot member must however point - // to this generation, since that's preserved as long as the - // log entry survives. - - void * orig_data = mrecd.snap().data(); - void * curr_data = *mrecd.p_data(); - - if (orig_data == curr_data) { - // live mlog entry must point to to-space - - if (space->contains_allocated(orig_data)) { - ++verify_stats_.n_mlog_vital_; - } else if (from->contains(curr_data)) { - // verify failure. - ++verify_stats_.n_mlog_from_; - } else { - // verify failure. - ++verify_stats_.n_mlog_wild_; - } - } else { - // requirements on superseded log entry: - // - snapshot refers to to-space - // - // no requirements on current data, entry is superseded anyway - // - ++verify_stats_.n_mlog_stale_; - } - } - } + mlog_state_.verify_ok(this, + &(this->verify_stats_)); } // restore run state at end of verify cycle @@ -957,7 +933,7 @@ namespace xo { log && log("step 4b : [STUB] keep reachable weak pointers"); log && log("step 5 : cleanup"); - this->cleanup_phase(upto); + this->_cleanup_phase(upto); if (config_.sanitize_flag_) { log && log("step 5b : verify"); @@ -987,203 +963,18 @@ namespace xo { log && log("swap roles", xtag("g", g)); std::swap(space_[role::to_space()][g], space_[role::from_space()][g]); - std::swap(mlog_[role::to_space()][g], mlog_[role::from_space()][g]); } + + mlog_state_.swap_roles(upto); } void DX1Collector::forward_mutation_log(Generation upto) { - /** non-zero if at least one object was rescued (from any generation) - * by mutation log scan - **/ - std::size_t work = 0; - - do { - // on 1st iteration, for all generations: - // - to_mlog, triage_mlog are empty - - for (Generation child_gen{0}; child_gen + 2 < config_.n_generation_; ++child_gen) { - - MutationLog * from_mlog = this->mlog_[role::from_space()][child_gen]; - - if (!from_mlog->empty()) { - MutationLog * to_mlog = this->mlog_[role::to_space()][child_gen]; - MutationLog * triage_mlog = this->mlog_[c_n_role][child_gen]; - - auto stats = this->_forward_mutation_log_phase(upto, - child_gen, - from_mlog, - to_mlog, - triage_mlog); - - from_mlog->clear(); - - // {from_mlog, triage_mlog} reverse roles - - std::swap(this->mlog_[role::from_space()][child_gen], - this->mlog_[c_n_role][child_gen]); - - work += stats.n_rescue_; - } - } - } while (work > 0); - - // here: reached fixpoints, any remaining triaged mlogs can be discarded - for (Generation child_gen{0}; child_gen + 2 < config_.n_generation_; ++child_gen) { - MutationLog * triage_mlog = this->mlog_[c_n_role][child_gen]; - - triage_mlog->clear(); - } - } - - MutationLogStatistics - DX1Collector::_forward_mutation_log_phase(Generation upto, - Generation child_gen, - MutationLog * from_mlog, - MutationLog * keep_mlog, - MutationLog * triage_mlog) - { - scope log(XO_DEBUG(config_.debug_flag_), - xtag("child_gen", child_gen), - xtag("mlog.size", from_mlog->size())); - - /* categorize each mlog entry based on combination of {src, dest}. - * In each case we care about {gen, age} of {src, dest} - * objects. - * - * Enough cases to deserve a table: - * - * Legend: - * - P : parent object - * - P' : parent object after this gc phase - * - g(P) : generation of parent P. - * '+' if gen > child_gen (parent gen not collected this cycle) - * - age(P) : age of parent P. - * - * - C : child object - * - C' : child object after this gc phase - * - g(C) : generation of child C. - * - age(C) : age of child C. - * - * - 0 : *from_mlog targets this object's generation. - * object not eligible for promotion. - * Write self* for objects eligible that promote - * if they survive this gc cycle. - * Write + for 'any generation senior to target' - * - 1 : *from_mlog target this object's generation; - * object promotes if it survives - * - * - role : 'to' this phase evacuated - * (or in generation not eligible for collection) - * 'fr' otherwise - * - * | mlog | par | | | mlog | upd - * case | cur | g(P) | P C | C' | action | P | move - * -------+------+-------+--------------+------+---------+--------+----- - * MLOG0 | no | | | | discard | | - - * | | | | | | | - * MLOG1 | yes | * | to:+ fwd:* | to | keep | P->C' | - - * MLOG2 | yes | | fr:0 | to | keep | P->C' | C->to - * MLOG3 | yes | * | fwd:* - | to | update | P'->C' | - - * MLOG4 | yes | | fr:* - | - | triage | - | - - - * notes: - * MLOG1 : child C already forwarded (whether or not promoted) - * MLOG2 : child C survives (and perhaps promoted). - * kept alive by parent in more-senior generation - * MLOG3 : parent has been forwarded. - * update mlog entry for new parent location - * MLOG4 : parent provisionally garbage. triage mlog entry until - * definite outcome. - */ - - MutationLogStatistics counters; - // index of current mlog entry during evac - std::uint32_t i_from = 0; - - for (MutationLogEntry & from_entry : *from_mlog) { - if (log) { - log(xtag("i_from", i_from)); - } - - if (from_entry.is_superseded()) { - // there must be a second mlog entry that refers to - // the new child. Rely on that second entry, - // skipping this one. - - // [MLOG0] obsolete mutation -> skip - ++counters.n_stale_; - continue; - } - - /* here: mlog current */ - - Generation parent_gen_to = this->generation_of(role::to_space(), - from_entry.parent()); - - if (parent_gen_to.is_sentinel()) { - void * parent_fr = *from_entry.p_data(); - - AllocInfo parent_info = this->alloc_info((std::byte *)parent_fr); - - if (parent_info.is_forwarding_tseq()) { - /* [MLOG3] */ - - ++counters.n_live_parent_; - - // new parent location in to-space - // TODO: method on AllocInfo to streamline this - void * parent_to = *(void **)parent_fr; - - parent_gen_to = this->generation_of(role::to_space(), - parent_to); - parent_info = this->alloc_info((std::byte *)parent_to); - - assert(!parent_gen_to.sentinel()); - - // Since parent already forwarded, we don't have to preserve child - // or update parent object. - // - // Do need to replace mlog entry to reflect new parent location. - - std::size_t offset - = ((std::byte *)from_entry.p_data() - - (std::byte *)from_entry.parent()); - - void ** p_data_to = (void **)((std::byte *)(parent_to) + offset); - void * child_to = *p_data_to; - - MutationLogEntry to_entry(parent_to, p_data_to, from_entry.snap()); - - this->_check_keep_mutation_aux(to_entry, - parent_gen_to, - child_to, - keep_mlog); - - - } else { - ++counters.n_triage_; - - // parent hasn't been collected and may be garbage. - // However this is only provisional, since - // parent could turn out to be reachable via some other mutation. - - triage_mlog->push_back(from_entry); - } - } else { - /* [MLOG1, MLOG2] */ - - counters += this->_preserve_child_of_live_parent(upto, - parent_gen_to, - from_entry, - keep_mlog); - } - } - - return counters; + mlog_state_.forward_mutation_log(this, upto); } +#ifdef MOVED MutationLogStatistics DX1Collector::_preserve_child_of_live_parent(Generation upto, Generation parent_gen, @@ -1233,7 +1024,9 @@ namespace xo { return counters; } +#endif +#ifdef MOVED bool DX1Collector::_check_keep_mutation_aux(const MutationLogEntry & from_entry, Generation parent_gen_to, @@ -1265,9 +1058,10 @@ namespace xo { return false; } } +#endif void - DX1Collector::cleanup_phase(Generation upto) + DX1Collector::_cleanup_phase(Generation upto) { scope log(XO_DEBUG(true), xtag("upto", upto)); @@ -1328,7 +1122,7 @@ namespace xo { } void * - DX1Collector::_deep_move_interior(void * from_src, + DX1Collector::deep_move_interior(void * from_src, Generation upto) { scope log(XO_DEBUG(config_.debug_flag_)); @@ -1948,16 +1742,10 @@ namespace xo { // control here: we have an older->younger pointer, need to log it - // mlog keyed by generation in which pointer _destination_ resides: - // collection that moves destination generation around needs to also - // update pointers such as this one - // - MutationLog * mlog = this->mlog_[role::to_space()][dest_g]; + void ** lhs_addr = reinterpret_cast(&(p_lhs->data_)); - mlog->push_back(MutationLogEntry(parent, - reinterpret_cast(&(p_lhs->data_)), - rhs)); - } + mlog_state_.append_mutation(dest_g, parent, lhs_addr, rhs); + } /*assign_member*/ DX1CollectorIterator DX1Collector::begin() const noexcept diff --git a/src/gc/MutationLogState.cpp b/src/gc/MutationLogState.cpp new file mode 100644 index 0000000..18c22d3 --- /dev/null +++ b/src/gc/MutationLogState.cpp @@ -0,0 +1,444 @@ +/** @file MutationLogState.cpp + * + * @author Roland Conybeare, Apr 2026 + **/ + +#include "MutationLogState.hpp" +#include "DX1Collector.hpp" + +namespace xo { + namespace mm { + + MutationLogState::MutationLogState(uint32_t ngen, bool debug_flag) + : n_generation_{ngen}, debug_flag_{debug_flag} + {} + + void + MutationLogState::init_mlogs(const X1CollectorConfig & cfg, + std::size_t page_z) + { + for (uint32_t igen = 0, ngen = cfg.n_generation_; igen + 1 < ngen; ++igen) { + // special case: no use for mutation log for youngest generation, + // so don't trouble to allocate one + + if (igen + 1 < c_max_generation) { + this->mlog_storage_[0][igen] = _make_mlog(igen, 'a', cfg.mutation_log_z_, page_z); + this->mlog_storage_[1][igen] = _make_mlog(igen, 'b', cfg.mutation_log_z_, page_z); + this->mlog_storage_[2][igen] = _make_mlog(igen, 'c', cfg.mutation_log_z_, page_z); + + this->mlog_[0][igen] = &mlog_storage_[0][igen]; + this->mlog_[1][igen] = &mlog_storage_[1][igen]; + this->mlog_[2][igen] = &mlog_storage_[2][igen]; + } else { + assert(false); + } + } + + if (cfg.n_generation_ > 0) { + for (uint32_t igen = cfg.n_generation_ - 1; igen + 1 < c_max_generation; ++igen) { + this->mlog_[0][igen] = nullptr; + this->mlog_[1][igen] = nullptr; + this->mlog_[2][igen] = nullptr; + } + } else { + assert(false); + } + } + + auto + MutationLogState::_make_mlog(uint32_t igen, char tag_char, + size_t mlog_z, size_t page_z) -> MutationLog + { + char buf[40]; + snprintf(buf, sizeof(buf), "x1-mlog-G%u-%c", igen, tag_char); + + return MutationLog::map(ArenaConfig{.name_ = std::string(buf), + .size_ = mlog_z, + .hugepage_z_ = page_z, + .store_header_flag_ = false}); + } + + auto + MutationLogState::mutation_log_entries() const noexcept -> size_type + { + size_type z = 0; + + for (Generation gj{0}; gj + 1 < n_generation_; ++gj) { + z += mlog_[role::to_space()][gj]->size(); + } + + return z; + } + + void + MutationLogState::visit_pools(const MemorySizeVisitor & visitor) const + { + for (uint32_t j = 0; j + 1 < n_generation_; ++j) { + for (uint32_t i = 0; i < c_n_role + 1; ++i) { + mlog_storage_[i][j].visit_pools(visitor); + } + } + } + + void + MutationLogState::verify_ok(DX1Collector * gc, + VerifyStats * p_verify_stats) noexcept + { + // 4. scan mutation logs + for (Generation g(0); g + 1 < n_generation_; ++g) { + const DArena * space = gc->get_space(role::to_space(), g); + const DArena * from = gc->get_space(role::from_space(), g); + + // mutation log for generation g records *incoming* pointers + // from more senior generations; includes objects from *this* + // generation that are older (track since source promotes before + // destination) + // + for (const MutationLogEntry & mrecd : *(mlog_[role::to_space()][g])) { + // mutation log entries are only valid until the next assignment + // at the source location. Superseded entry may now point + // somewhere else. The snapshot member must however point + // to this generation, since that's preserved as long as the + // log entry survives. + + void * orig_data = mrecd.snap().data(); + void * curr_data = *mrecd.p_data(); + + if (orig_data == curr_data) { + // live mlog entry must point to to-space + + if (space->contains_allocated(orig_data)) { + ++(p_verify_stats->n_mlog_vital_); + } else if (from->contains(curr_data)) { + // verify failure. + ++(p_verify_stats->n_mlog_from_); + } else { + // verify failure. + ++(p_verify_stats->n_mlog_wild_); + } + } else { + // requirements on superseded log entry: + // - snapshot refers to to-space + // + // no requirements on current data, entry is superseded anyway + // + ++(p_verify_stats->n_mlog_stale_); + } + } + } + } /*verify_ok*/ + + void + MutationLogState::append_mutation(Generation dest_g, + void * parent, + void ** addr, + obj rhs) + { + // mlog keyed by generation in which pointer _destination_ resides: + // collection that moves destination generation around needs to also + // update pointers such as this one + // + MutationLog * mlog = this->mlog_[role::to_space()][dest_g]; + + mlog->push_back(MutationLogEntry(parent, addr, rhs)); + } + + void + MutationLogState::swap_roles(Generation upto) noexcept + { + scope log(XO_DEBUG(true), xtag("upto", upto)); + + for (Generation g = Generation{0}; g < upto; ++g) { + log && log("swap roles", xtag("g", g)); + + std::swap(mlog_[role::to_space()][g], mlog_[role::from_space()][g]); + } + } + + void + MutationLogState::forward_mutation_log(DX1Collector * gc, + Generation upto) + { + /** non-zero if at least one object was rescued (from any generation) + * by mutation log scan + **/ + std::size_t work = 0; + + do { + // on 1st iteration, for all generations: + // - to_mlog, triage_mlog are empty + + for (Generation child_gen{0}; child_gen + 2 < n_generation_; ++child_gen) { + + MutationLog * from_mlog = this->mlog_[role::from_space()][child_gen]; + + if (!from_mlog->empty()) { + MutationLog * to_mlog = this->mlog_[role::to_space()][child_gen]; + MutationLog * triage_mlog = this->mlog_[c_n_role][child_gen]; + + auto stats = this->_forward_mutation_log_phase(gc, + upto, + child_gen, + from_mlog, + to_mlog, + triage_mlog); + + from_mlog->clear(); + + // {from_mlog, triage_mlog} reverse roles + + std::swap(this->mlog_[role::from_space()][child_gen], + this->mlog_[c_n_role][child_gen]); + + work += stats.n_rescue_; + } + } + } while (work > 0); + + // here: reached fixpoints, any remaining triaged mlogs can be discarded + for (Generation child_gen{0}; child_gen + 2 < n_generation_; ++child_gen) { + MutationLog * triage_mlog = this->mlog_[c_n_role][child_gen]; + + triage_mlog->clear(); + } + } + + MutationLogStatistics + MutationLogState::_forward_mutation_log_phase(DX1Collector * gc, + Generation upto, + Generation child_gen, + MutationLog * from_mlog, + MutationLog * keep_mlog, + MutationLog * triage_mlog) + { + scope log(XO_DEBUG(debug_flag_), + xtag("child_gen", child_gen), + xtag("mlog.size", from_mlog->size())); + + /* categorize each mlog entry based on combination of {src, dest}. + * In each case we care about {gen, age} of {src, dest} + * objects. + * + * Enough cases to deserve a table: + * + * Legend: + * - P : parent object + * - P' : parent object after this gc phase + * - g(P) : generation of parent P. + * '+' if gen > child_gen (parent gen not collected this cycle) + * - age(P) : age of parent P. + * + * - C : child object + * - C' : child object after this gc phase + * - g(C) : generation of child C. + * - age(C) : age of child C. + * + * - 0 : *from_mlog targets this object's generation. + * object not eligible for promotion. + * Write self* for objects eligible that promote + * if they survive this gc cycle. + * Write + for 'any generation senior to target' + * - 1 : *from_mlog target this object's generation; + * object promotes if it survives + * + * - role : 'to' this phase evacuated + * (or in generation not eligible for collection) + * 'fr' otherwise + * + * | mlog | par | | | mlog | upd + * case | cur | g(P) | P C | C' | action | P | move + * -------+------+-------+--------------+------+---------+--------+----- + * MLOG0 | no | | | | discard | | - + * | | | | | | | + * MLOG1 | yes | * | to:+ fwd:* | to | keep | P->C' | - + * MLOG2 | yes | | fr:0 | to | keep | P->C' | C->to + * MLOG3 | yes | * | fwd:* - | to | update | P'->C' | - + * MLOG4 | yes | | fr:* - | - | triage | - | - + + * notes: + * MLOG1 : child C already forwarded (whether or not promoted) + * MLOG2 : child C survives (and perhaps promoted). + * kept alive by parent in more-senior generation + * MLOG3 : parent has been forwarded. + * update mlog entry for new parent location + * MLOG4 : parent provisionally garbage. triage mlog entry until + * definite outcome. + */ + + MutationLogStatistics counters; + // index of current mlog entry during evac + std::uint32_t i_from = 0; + + for (MutationLogEntry & from_entry : *from_mlog) { + if (log) { + log(xtag("i_from", i_from)); + } + + if (from_entry.is_superseded()) { + // there must be a second mlog entry that refers to + // the new child. Rely on that second entry, + // skipping this one. + + // [MLOG0] obsolete mutation -> skip + ++counters.n_stale_; + continue; + } + + /* here: mlog current */ + + Generation parent_gen_to = gc->generation_of(role::to_space(), + from_entry.parent()); + + if (parent_gen_to.is_sentinel()) { + void * parent_fr = *from_entry.p_data(); + + AllocInfo parent_info = gc->alloc_info((std::byte *)parent_fr); + + if (parent_info.is_forwarding_tseq()) { + /* [MLOG3] */ + + ++counters.n_live_parent_; + + // new parent location in to-space + // TODO: method on AllocInfo to streamline this + void * parent_to = *(void **)parent_fr; + + parent_gen_to = gc->generation_of(role::to_space(), + parent_to); + parent_info = gc->alloc_info((std::byte *)parent_to); + + assert(!parent_gen_to.sentinel()); + + // Since parent already forwarded, we don't have to preserve child + // or update parent object. + // + // Do need to replace mlog entry to reflect new parent location. + + std::size_t offset + = ((std::byte *)from_entry.p_data() + - (std::byte *)from_entry.parent()); + + void ** p_data_to = (void **)((std::byte *)(parent_to) + offset); + void * child_to = *p_data_to; + + MutationLogEntry to_entry(parent_to, p_data_to, from_entry.snap()); + + this->_check_keep_mutation_aux(gc, + to_entry, + parent_gen_to, + child_to, + keep_mlog); + + + } else { + ++counters.n_triage_; + + // parent hasn't been collected and may be garbage. + // However this is only provisional, since + // parent could turn out to be reachable via some other mutation. + + triage_mlog->push_back(from_entry); + } + } else { + /* [MLOG1, MLOG2] */ + + counters += this->_preserve_child_of_live_parent(gc, + upto, + parent_gen_to, + from_entry, + keep_mlog); + } + } + + return counters; + } + + MutationLogStatistics + MutationLogState::_preserve_child_of_live_parent(DX1Collector * gc, + Generation upto, + Generation parent_gen, + const MutationLogEntry & from_entry, + MutationLog * keep_mlog) + { + void * child_fr = *from_entry.p_data(); + AllocInfo child_info = gc->alloc_info((std::byte *)(child_fr)); + + MutationLogStatistics counters; + + // if child collected: new child location in to-space + void * child_to = nullptr; + + // parent is alive: gc must ensure child remains alive + + ++counters.n_live_parent_; + + // Parent already recognized as alive. Either not subject to collection + // or already evacuated. + // (+ remember this need not be 1st pass over mlog entries) + + if (child_info.is_forwarding_tseq()) { + // [MLOG1] + + // child already forwarded. + // TODO: make this a method on AllocInfo + child_to = *(void **)child_fr; + + // assigning through address of P->C pointer + // also makes mlog entry current + + } else { + // [MLOG2] + + ++counters.n_rescue_; + + child_to = gc->deep_move_interior(child_fr, upto); + + // update child pointer in parent object + *from_entry.p_data() = child_to; + } + + // child_to generation in {gen, gen+1} + + this->_check_keep_mutation_aux(gc, from_entry, parent_gen, child_to, keep_mlog); + + return counters; + } + + bool + MutationLogState::_check_keep_mutation_aux(DX1Collector * gc, + const MutationLogEntry & from_entry, + Generation parent_gen_to, + void * child_to, + MutationLog * keep_mlog) + { + Generation child_gen_to + = gc->generation_of(role::to_space(), child_to); + + bool need_mlog_entry + = ((child_gen_to + 1 < n_generation_) + && (gc->config().promotion_threshold(parent_gen_to) + > gc->config().promotion_threshold(child_gen_to))); + + if (need_mlog_entry) { + // 1. P->C pointer is still cross-age (xage), and + // 2. this matters; in future P will promote before C + // + // Need to keep entry because parent will be eligible for promotion + // before child + + keep_mlog->push_back(from_entry); + + return true; + } else { + // child now in final generation, + // no longer need to track incoming mutations. + + return false; + } + } + + + } /*namespace mm*/ +} /*namespace xo*/ + +/* end MutationLogState.cpp */