/** @file DX1Collector.cpp * * @author Roland Conybeare, Dec 2025 **/ #include "X1Collector.hpp" #include #include #include #include #include #include #include #include #include #include "object_age.hpp" #include #include #include #include #include #include // for ::getpagesize() namespace xo { // for report_statistics(), report_object_types() using xo::scm::DDictionary; using xo::scm::DArray; using xo::scm::DString; using xo::scm::DInteger; using xo::scm::DBoolean; using xo::mm::AAllocator; using xo::facet::TypeRegistry; using xo::facet::typeseq; using xo::facet::with_facet; namespace mm { // ----- GCRunState ----- GCRunState::GCRunState(Mode mode, Generation gc_upto) : mode_{mode}, gc_upto_{gc_upto} {} GCRunState GCRunState::idle() { return GCRunState(Mode::idle, Generation::sentinel()); } GCRunState GCRunState::verify() { return GCRunState(Mode::verify, Generation::sentinel()); } GCRunState GCRunState::gc_upto(Generation g) { return GCRunState(Mode::gc, Generation(g + 1)); } // ----- DX1Collector ----- using size_type = xo::mm::DX1Collector::size_type; DX1Collector::DX1Collector(const X1CollectorConfig & cfg) : config_{cfg}, mlog_store_{cfg.mlog_config()}, gco_store_{cfg.gco_store_config()} { assert(config_.arena_config_.header_.size_bits_ + config_.arena_config_.header_.age_bits_ + config_.arena_config_.header_.tseq_bits_ <= 64); size_t page_z = getpagesize(); //this->_init_object_types(cfg, page_z); this->_init_gc_roots(cfg, page_z); this->_init_mlogs(page_z); } void DX1Collector::_init_gc_roots(const X1CollectorConfig & cfg, std::size_t page_z) { this->root_set_ = RootSet::map(ArenaConfig{.name_ = "x1-object-roots", .size_ = cfg.object_roots_z_, .hugepage_z_ = page_z, .store_header_flag_ = false}); } void DX1Collector::_init_mlogs(std::size_t page_z) { this->mlog_store_.init_mlogs(page_z); } void DX1Collector::visit_pools(const MemorySizeVisitor & visitor) const { //object_types_.visit_pools(visitor); root_set_.visit_pools(visitor); gco_store_.visit_pools(visitor); mlog_store_.visit_pools(visitor); } bool DX1Collector::contains(role r, const void * addr) const noexcept { return gco_store_.contains(r, addr); } bool DX1Collector::contains_allocated(role r, const void * addr) const noexcept { return gco_store_.contains_allocated(r, addr); } Generation DX1Collector::generation_of(role r, const void * addr) const noexcept { return gco_store_.generation_of(r, addr); } AllocError DX1Collector::last_error() const noexcept { // TODO: // need to adjust here if runtime errors // encountered during gc. return get_space(role::to_space(), Generation::nursery())->last_error_; } namespace { size_type accumulate_total_aux(const DX1Collector & d, size_t (DArena::* get_stat_fn)() const) noexcept { //size_t z1 = (d.object_types_.store()->*get_stat_fn)(); size_t z1 = (d.gco_store_.get_object_types()->store()->*get_stat_fn)(); size_t z2 = (d.root_set_.store()->*get_stat_fn)(); size_t z3 = 0; for (role ri : role::all()) { for (Generation gj{0}; gj < d.config_.n_generation_; ++gj) { const DArena * arena = d.get_space(ri, gj); assert(arena); z3 += (arena->*get_stat_fn)(); } } return z1 + z2 + z3; } } size_type DX1Collector::reserved() const noexcept { return accumulate_total_aux(*this, &DArena::reserved); } size_type DX1Collector::size_total() const noexcept { return this->committed(); } size_type DX1Collector::committed() const noexcept { return accumulate_total_aux(*this, &DArena::committed); } size_type DX1Collector::available() const noexcept { return accumulate_total_aux(*this, &DArena::available); } size_type DX1Collector::allocated() const noexcept { return accumulate_total_aux(*this, &DArena::allocated); } size_type DX1Collector::mutation_log_entries() const noexcept { return mlog_store_.mutation_log_entries(); } namespace { size_type stat_helper(const DX1Collector & d, size_type (DArena::* getter)() const, Generation g, role r) { const DArena * arena = d.get_space(r, g); if (arena) [[likely]] return (arena->*getter)(); return 0; } } size_type DX1Collector::allocated(Generation g, role r) const noexcept { return stat_helper(*this, &DArena::allocated, g, r); } size_type DX1Collector::committed(Generation g, role r) const noexcept { return stat_helper(*this, &DArena::committed, g, r); } size_type DX1Collector::reserved(Generation g, role r) const noexcept { return stat_helper(*this, &DArena::reserved, g, r); } std::int32_t DX1Collector::locate_address(const void * addr) const noexcept { Generation g; g = this->generation_of(role::to_space(), addr); if (!g.is_sentinel()) return g; g = this->generation_of(role::from_space(), addr); if (!g.is_sentinel()) { // use negative values for return -1 - g; } return -1; } // editor bait: report-gc-statistics bool DX1Collector::report_statistics(obj mm, obj error_mm, obj * p_output) const noexcept { (void)error_mm; DDictionary * rpt = DDictionary::make(mm); if (!rpt) return false; bool ok = true; // note: totals taken across both roles and generations, // so counts both from-space and to-space // ok &= rpt->upsert_cstr(mm, "n-generation", DInteger::box(mm, config_.n_generation_)); ok &= rpt->upsert_cstr(mm, "n-survive-threshold", DInteger::box(mm, config_.n_survive_threshold_)); ok &= rpt->upsert_cstr(mm, "allow-incremental-gc", DBoolean::box(mm, config_.allow_incremental_gc_)); ok &= rpt->upsert_cstr(mm, "sanitize", DBoolean::box(mm, config_.sanitize_flag_)); ok &= rpt->upsert_cstr(mm, "allocated", DInteger::box(mm, this->allocated())); ok &= rpt->upsert_cstr(mm, "committed", DInteger::box(mm, this->committed())); ok &= rpt->upsert_cstr(mm, "reserved", DInteger::box(mm, this->reserved())); ok &= rpt->upsert_cstr(mm, "n-mlog-entry", DInteger::box(mm, this->mutation_log_entries())); // per-(generation,role) info { for (Generation gi{0}; gi < config_.n_generation_; ++gi) { for (role rj : role::all()) { const DArena * arena = this->get_space(rj, gi); DDictionary * arena_d = DDictionary::make(mm); auto lo = reinterpret_cast(arena->lo_); auto free = reinterpret_cast(arena->free_); auto limit = reinterpret_cast(arena->limit_); auto hi = reinterpret_cast(arena->hi_); ok &= arena_d->upsert_cstr(mm, "lo", DInteger::box(mm, lo)); ok &= arena_d->upsert_cstr(mm, "d-free", DInteger::box(mm, free - lo)); ok &= arena_d->upsert_cstr(mm, "d-limit", DInteger::box(mm, limit - lo)); ok &= arena_d->upsert_cstr(mm, "d-hi", DInteger::box(mm, hi - lo)); const DString * key = DString::from_str(mm, arena->config_.name_); rpt->upsert(mm, std::make_pair(key, obj(arena_d))); } } } *p_output = obj(rpt); return ok; } bool DX1Collector::report_object_types(obj mm, obj error_mm, obj * p_output) const noexcept { return gco_store_.report_object_types(mm, error_mm, p_output); } bool DX1Collector::report_object_ages(obj mm, obj error_mm, obj * p_output) const noexcept { return gco_store_.report_object_types(mm, error_mm, p_output); } size_type DX1Collector::header2size(header_type hdr) const noexcept { return gco_store_.header2size(hdr); } object_age DX1Collector::header2age(header_type hdr) const noexcept { return gco_store_.header2age(hdr); } uint32_t DX1Collector::header2tseq(header_type hdr) const noexcept { return gco_store_.header2tseq(hdr); } bool DX1Collector::is_forwarding_header(header_type hdr) const noexcept { return gco_store_.is_forwarding_header(hdr); } AllocInfo DX1Collector::alloc_info(value_type mem) const noexcept { return gco_store_.alloc_info(mem); } bool DX1Collector::is_type_installed(typeseq tseq) const noexcept { return gco_store_.is_type_installed(tseq); } bool DX1Collector::verify_ok() noexcept { // 1. visit space pointers // - verify space_[*] points to space_storage_[*] // - verify mlog_[*] points to mlog_storage_[*] // // 2. visit roots: // for each root, verify that immediate child pointers are in to-space // // 3. scan to-space: // for each object, verify that immediate children are also in to-space // // 4. scan mutation logs: // verify that entries refer to to-space // Each AGCObject impl provides a forward_children() method, // that calls DX1Collector::forward_inplace(iface, &data) // // tactical plan: hijack forward_children. // Add run state so DX1Collector can recognize forward_inplace() // calls made for the purpose of checking child pointers. auto self = this->ref(); GCRunState saved_runstate = runstate_; { this->runstate_ = GCRunState::verify(); this->verify_stats_.clear(); // 2. visit roots for (GCRoot & root_slot : root_set_) { VerifyStats pre = verify_stats_; auto gco = *root_slot.root(); if (gco) { // forward_children is hijacked here to verify // pointer validity. // // Nested control re-enters // - X1Collector::forward_inplace() -> _verify_aux() // gco.forward_children(self); } VerifyStats post = verify_stats_; // assert fail -> root contains ptr to from-space assert(pre.n_from_ == post.n_from_); ++verify_stats_.n_gc_root_; } // 3. scan to-space for each generation for (Generation g(0); g < config_.n_generation_; ++g) { const DArena * space = this->get_space(role::to_space(), g); for (const AllocInfo & info : *space) { if (info.is_forwarding_tseq()) { ++verify_stats_.n_fwd_; } else { typeseq tseq(info.tseq()); const AGCObject * iface = this->lookup_type(tseq); if (iface && !(iface->_has_null_vptr())) { const void * data = info.payload().first; // assembled fop for gc-aware object obj gco(iface, const_cast(data)); // forward_children is hijacked here to verify // child pointer validity. // // Nested control reenters // X1Collector::forward_inplace() -> _verify_aux() // gco.forward_children(self); } else { ++verify_stats_.n_no_iface_; continue; } } } } // 4. scan mutation logs mlog_store_.verify_ok(&gco_store_, &(this->verify_stats_)); } // restore run state at end of verify cycle this->runstate_ = saved_runstate; bool ok = verify_stats_.is_ok(); return ok; } const AGCObject * DX1Collector::lookup_type(typeseq tseq) const noexcept { return gco_store_.lookup_type(tseq); } /* editor bait: register_type */ bool DX1Collector::install_type(const AGCObject & meta) noexcept { return gco_store_.install_type(meta); } void DX1Collector::add_gc_root_poly(obj * p_root) noexcept { root_set_.push_back(GCRoot(p_root)); } void DX1Collector::remove_gc_root_poly(obj * p_root) noexcept { // iterate over roots_, find p_root and drop it (void)p_root; } void DX1Collector::request_gc(Generation upto) noexcept { if (gc_blocked_ > 0) { if (gc_pending_upto_ < upto) { this->gc_pending_upto_ = upto; } /* intend collecting later */ } else { this->execute_gc(upto); } } void DX1Collector::execute_gc(Generation upto) noexcept { scope log(XO_DEBUG(true), xtag("upto", upto)); assert(!runstate_.is_running()); //auto t0 = std::chrono::steady_clock::now(); log && log("memory"); auto visitor = [&log](const MemorySizeInfo & info) { log && log(xtag("resource", info.resource_name_), xtag("used", info.used_), xtag("alloc", info.allocated_), xtag("commit", info.committed_), xtag("resv", info.reserved_), xtag("lo", info.lo_), xtag("hi", info.hi_)); }; this->visit_pools(visitor); if (config_.sanitize_flag_) { log && log("step 0a : verify"); this->verify_ok(); } log && log("step 0b : update run state"); this->runstate_ = GCRunState::gc_upto(upto); log && log("step 0c : [STUB] snapshot alloc state"); log && log("step 0d : [STUB] scan for object statistics"); log && log("step 1 : swap from/to roles (now to-space is empty)"); this->swap_roles(upto); log && log(xtag("from_0", get_space(role::from_space(), Generation{0})->lo_), xtag("to_0", get_space(role::to_space(), Generation{0})->lo_)); log && log("step 2a : copy roots"); this->copy_roots(upto); log && log("step 2b : [STUB] copy pinned"); log && log("step 3 : [STUB] forward mutation log"); mlog_store_.forward_mutation_log(this, upto); log && log("step 4a : [STUB] run destructors"); log && log("step 4b : [STUB] keep reachable weak pointers"); log && log("step 5 : cleanup"); this->_cleanup_phase(upto); if (config_.sanitize_flag_) { log && log("step 5b : verify"); bool ok = this->verify_ok(); log && log(xtag("n-gc-root", verify_stats_.n_gc_root_), xtag("n-ext", verify_stats_.n_ext_), xtag("n-from", verify_stats_.n_from_), xtag("n-to", verify_stats_.n_to_), xtag("n-fwd", verify_stats_.n_fwd_), xtag("n-no-iface", verify_stats_.n_no_iface_), xtag("n-mlog-vital", verify_stats_.n_mlog_vital_), xtag("n-mlog-stale", verify_stats_.n_mlog_stale_), xtag("n-mlog-from", verify_stats_.n_mlog_from_), xtag("n-mlog-wild", verify_stats_.n_mlog_wild_)); assert(ok); } } void DX1Collector::swap_roles(Generation upto) noexcept { scope log(XO_DEBUG(true), xtag("upto", upto)); gco_store_.swap_roles(upto); mlog_store_.swap_roles(upto); } void DX1Collector::_cleanup_phase(Generation upto) { scope log(XO_DEBUG(true), xtag("upto", upto)); this->gco_store_.cleanup_phase(upto, config_.sanitize_flag_); this->runstate_ = GCRunState::idle(); } void DX1Collector::copy_roots(Generation upto) noexcept { scope log(XO_DEBUG(true)); for (RootSet::size_type i = 0, n = root_set_.size(); i < n; ++i) { GCRoot & slot = root_set_[i]; log && log("copy root", xtag("slot.root()", slot.root()), xtag("slot.root()->data_", slot.root()->data_)); void * root_to = gco_store_._deep_move_root(this, *slot.root(), upto); slot.root()->reset_opaque(root_to); log && log(xtag("slot.root()->data_", slot.root()->data_)); } } void DX1Collector::forward_inplace(AGCObject * lhs_iface, void ** lhs_data) { Generation upto = runstate_.gc_upto(); if (runstate_.is_running()) { // called during collection phase gco_store_.forward_inplace_aux(this, lhs_iface, lhs_data, upto); } else if (runstate_.is_verify()) { // called during verify_ok this->_verify_aux(lhs_iface, *lhs_data); } else { // should be unreachable assert(false); } } void DX1Collector::_verify_aux(AGCObject * iface, void * data) { //scope log(XO_DEBUG(config_.debug_flag_), xtag("data", data)); (void)iface; (void)data; Generation g1 = this->generation_of(role::to_space(), data); if (g1.is_sentinel()) { assert(this->contains(role::to_space(), data) == false); Generation g2 = this->generation_of(role::from_space(), data); if (!g2.is_sentinel()) { // verify failure - live pointer still refers to from-space ++(verify_stats_.n_from_); } else { ++(verify_stats_.n_ext_); } } else { assert(this->contains(role::to_space(), data)); ++(verify_stats_.n_to_); } } bool DX1Collector::check_move_policy(header_type alloc_hdr, void * object_data) const noexcept { assert(runstate_.is_running()); return gco_store_._check_move_policy(alloc_hdr, object_data, runstate_.gc_upto()); } auto DX1Collector::alloc(typeseq t, size_type z) noexcept -> value_type { return with_facet::mkobj(new_space()).alloc(t, z); } auto DX1Collector::super_alloc(typeseq t, size_type z) noexcept -> value_type { return with_facet::mkobj(new_space()).super_alloc(t, z); } auto DX1Collector::sub_alloc(size_type z, bool complete) noexcept -> value_type { return with_facet::mkobj(new_space()).sub_alloc(z, complete); } auto DX1Collector::alloc_copy(value_type src) noexcept -> value_type { return with_facet::mkobj(new_space()).alloc_copy(src); } bool DX1Collector::expand(size_type z) noexcept { if (with_facet::mkobj(to_space(Generation{0})).expand(z)) return with_facet::mkobj(from_space(Generation{0})).expand(z); return false; } // editor bait: write barrier void DX1Collector::assign_member(void * parent, obj * p_lhs, obj rhs) { scope log(XO_DEBUG(config_.debug_flag_), xtag("parent", parent), xtag("lhs", p_lhs), xtag("rhs", rhs.data())); // ++ stats.n_mutation_; *p_lhs = rhs; if (runstate_.is_running()) { // for removal of all doubt: // don't log mutations during GC cycle return; } if (!config_.allow_incremental_gc_) { // only need to log mutations when incremental gc is enabled return; } // logging policy depends on: // 1. generation of lhs // 2. generation of rhs Generation src_g = this->generation_of(role::to_space(), p_lhs); if (src_g.is_sentinel()) { // only need mlog entries for gc-owned pointers. // In this case pointer does not originate in gc-owned space return; } Generation dest_g = this->generation_of(role::to_space(), rhs.data()); if (dest_g.is_sentinel()) { // similarly, don't need mlog entry to non-gc-owned destination return; } if (src_g < dest_g) { // young-to-old pointers don't need to be remembered, // since a GC cycle that collects an (old) generation is guarnatted // to also collect all younger generations. return; } if (src_g == dest_g) { // for pointers within the same generation, need to log // if source is older than destination. const DArena * arena = this->get_space(role::to_space(), src_g); const DArena::header_type * src_hdr = arena->obj2hdr(parent); const DArena::header_type * dest_hdr = arena->obj2hdr(rhs.data()); assert(src_hdr && dest_hdr); if (this->header2age(*src_hdr) <= this->header2age(*dest_hdr)) { // source and destination have the same age; // therefore are always collected on the same set of GC cycles // -> no need to remember separately. return; } else { // even though {src,dest} belong to the same generation: // source will be eligible for promotion before destination. // At that point pointer would become a cross-generational pointer, // so need to track it now. log && log("xage ptr -> must log"); } } else { log && log("xgen ptr -> must log"); } // control here: we have an older->younger pointer, need to log it void ** lhs_addr = reinterpret_cast(&(p_lhs->data_)); mlog_store_.append_mutation(dest_g, parent, lhs_addr, rhs); } /*assign_member*/ DX1CollectorIterator DX1Collector::begin() const noexcept { scope log(XO_DEBUG(false)); const DArena * arena = get_space(role::to_space(), Generation{0}); return DX1CollectorIterator(this, Generation{0}, Generation{config_.n_generation_}, arena->begin(), arena->end()); } DX1CollectorIterator DX1Collector::end() const noexcept { scope log(XO_DEBUG(false)); Generation gen_hi = Generation{config_.n_generation_}; /** valid iterator for end points to end of last DArena. * otherwise will interfere with working compare * (since invalid iterators are incomparable) **/ const DArena * arena = get_space(role::to_space(), Generation(config_.n_generation_ - 1)); DArenaIterator arena_end = arena->end(); return DX1CollectorIterator(this, gen_hi, gen_hi, arena_end, arena_end); } #ifdef MOVED void DX1Collector::reverse_roles(Generation g) noexcept { assert(g < config_.n_generation_); std::swap(space_[role::from_space()][g], space_[role::to_space()][g]); } #endif void DX1Collector::clear() noexcept { for (role ri : role::all()) { for (Generation gj{0}; gj < config_.n_generation_; ++gj) { DArena * arena = this->get_space(ri, gj); assert(arena); arena->clear(); } } } } /*namespace mm*/ } /*namespace xo*/ /* end DX1Collector.cpp */