/** @file GCObjectStore.cpp * * @author Roland Conybeare, Apr 2026 **/ #include "GCObjectStore.hpp" #include #include #include #include #include #include #include #include #include #include // for ::getpagesize() namespace xo { using xo::scm::DDictionary; using xo::scm::DArray; using xo::scm::DString; using xo::scm::DInteger; using xo::mm::DArena; using xo::facet::TypeRegistry; using xo::reflect::typeseq; namespace mm { GCObjectStore::GCObjectStore(const GCObjectStoreConfig & cfg) : config_{cfg} { assert(config_.arena_config_.header_.size_bits_ + config_.arena_config_.header_.age_bits_ + config_.arena_config_.header_.tseq_bits_ <= 64); size_t page_z = getpagesize(); this->_init_object_types(page_z); this->_init_space(); } void GCObjectStore::_init_object_types(std::size_t page_z) { /* 1MB reserved address space enough for up to 128k distinct types. * In this case don't want to use hugepages since actual #of types * likely << .size/8 */ this->object_types_ = ObjectTypeTable::map(ArenaConfig{.name_ = "x1-object-types", .size_ = config_.object_types_z_, .hugepage_z_ = page_z, .store_header_flag_ = false}); } void GCObjectStore::_init_space() { assert(c_n_role == 2); for (uint32_t igen = 0, ngen = config_.n_generation_; igen < ngen; ++igen) { if (igen < c_max_generation) { { char buf[40]; snprintf(buf, sizeof(buf), "x1-space-G%u-a", igen); this->space_storage_[0][igen] = DArena::map(config_.arena_config_.with_name(std::string(buf))); } { char buf[40]; snprintf(buf, sizeof(buf), "x1-space-G%u-b", igen); this->space_storage_[1][igen] = DArena::map(config_.arena_config_.with_name(std::string(buf))); } this->space_[role::to_space()][igen] = &space_storage_[0][igen]; this->space_[role::from_space()][igen] = &space_storage_[1][igen]; } else { assert(false); } } for (uint32_t igen = config_.n_generation_; igen < c_max_generation; ++igen) { this->space_[role::to_space()][igen] = nullptr; this->space_[role::from_space()][igen] = nullptr; } if (config_.n_generation_ == 2) { assert(this->get_space(role::to_space(), Generation{2}) == nullptr); } } bool GCObjectStore::is_type_installed(typeseq tseq) const noexcept { if (tseq.is_sentinel() || static_cast(tseq.seqno()) > object_types_.size()) { return false; } const ObjectTypeSlot & slot = object_types_[tseq.seqno()]; return slot.is_occupied(); } const AGCObject * GCObjectStore::lookup_type(typeseq tseq) const noexcept { scope log(XO_DEBUG(false)); if (tseq.is_sentinel() || (static_cast(tseq.seqno()) > object_types_.size())) { log.retroactively_enable("out-of-bounds", xtag("tseq", tseq), xtag("tname", TypeRegistry::id2name(tseq))); log(xtag("types.size", object_types_.size()), xtag("types.allocated", object_types_.store()->allocated()), xtag("types.committed", object_types_.store()->committed()), xtag("types.lo", object_types_.store()->lo_), xtag("types.limit", object_types_.store()->limit_), xtag("types.hi", object_types_.store()->hi_)); assert(false); return nullptr; } const ObjectTypeSlot & slot = object_types_[tseq.seqno()]; if (slot.is_null()) { log.retroactively_enable("null-vtable", xtag("tseq", tseq), xtag("tname", TypeRegistry::id2name(tseq))); assert(false); return nullptr; } return slot.iface(); } Generation GCObjectStore::generation_of(role r, const void * addr) const noexcept { for (Generation gi{0}; gi < config_.n_generation_; ++gi) { const DArena * arena = this->get_space(r, gi); if (arena->contains(addr)) return gi; } return Generation::sentinel(); } auto GCObjectStore::header2size(header_type hdr) const noexcept -> size_type { uint32_t z = config_.arena_config_.header_.size(hdr); return z; } object_age GCObjectStore::header2age(header_type hdr) const noexcept { uint32_t age = config_.arena_config_.header_.age(hdr); assert(age < c_max_object_age); return object_age(age); } uint32_t GCObjectStore::header2tseq(header_type hdr) const noexcept { uint32_t tseq = config_.arena_config_.header_.tseq(hdr); return tseq; } bool GCObjectStore::is_forwarding_header(header_type hdr) const noexcept { /** forwarding pointer encoded as sentinel tseq **/ return config_.arena_config_.header_.is_forwarding_tseq(hdr); } AllocInfo GCObjectStore::alloc_info(value_type mem) const noexcept { for (role ri : role::all()) { for (Generation gj{0}; gj < config_.n_generation_; ++gj) { const DArena * arena = this->get_space(ri, gj); assert(arena); if (arena->contains(mem)) { return arena->alloc_info(mem); } } } // deliberately attempt on nursery to-space, to capture error info + return sentinel return this->get_space(role::to_space(), Generation{0})->alloc_info(mem); } void GCObjectStore::visit_pools(const MemorySizeVisitor & visitor) const { for (uint32_t j = 0; j < config_.n_generation_; ++j) { for (uint32_t i = 0; i < c_n_role; ++i) { space_storage_[i][j].visit_pools(visitor); } } object_types_.visit_pools(visitor); } bool GCObjectStore::contains(role r, const void * addr) const noexcept { return !(this->generation_of(r, addr).is_sentinel()); } bool GCObjectStore::contains_allocated(role r, const void * addr) const noexcept { Generation g = this->generation_of(r, addr); if (g.is_sentinel()) return false; return this->get_space(r, g)->contains_allocated(addr); } bool GCObjectStore::report_object_types(obj mm, obj error_mm, obj * p_output) const noexcept { scope log(XO_DEBUG(true)); (void)error_mm; bool ok = true; // stats, indexed by tseq // could use c++ vector in scratch space instead of running on // boxed types. // DArray * stats_v = DArray::empty(mm, object_types_.size()); if (!stats_v) return false; stats_v->resize(stats_v->capacity()); log && log(xtag("object_types_.size", object_types_.size()), xtag("stats_v.capacity", stats_v->capacity()), xtag("stats_v.size", stats_v->size())); // count #of occupied type slots std::uint32_t n_tseq_present = 0; // largest tseq present with non-null AGCObject* iface std::int32_t max_tseq = 0; for (const ObjectTypeSlot & slot : object_types_) { AGCObject * iface = slot.iface(); if (iface) { typeseq tseq = iface->_typeseq(); ++n_tseq_present; if (max_tseq < tseq.seqno()) max_tseq = tseq.seqno(); assert(tseq.seqno() >= 0); auto tname_sv = TypeRegistry::id2name(tseq); DString * tname = DString::from_view(mm, tname_sv); DDictionary * recd = DDictionary::make(mm); if (!recd) return false; recd->upsert_cstr(mm, "name", obj(tname)); recd->upsert_cstr(mm, "tseq", DInteger::box(mm, tseq.seqno())); recd->upsert_cstr(mm, "n-live", DInteger::box(mm, 0)); recd->upsert_cstr(mm, "bytes", DInteger::box(mm, 0)); stats_v->assign_at(tseq.seqno(), obj(recd)); } } // scan to-space, count objects by type for (Generation g{0}; g < config_.n_generation_; ++g) { const DArena * arena = this->get_space(role::to_space(), g); for (AllocInfo info : *arena) { if (info.is_forwarding_tseq()) { assert(false); return false; } uint32_t ix = info.tseq(); size_t z = info.size(); auto recd = obj::from(stats_v->at(ix)); assert(recd); auto n_live_opt = recd->lookup_cstr("n-live"); assert(n_live_opt); auto bytes_opt = recd->lookup_cstr("bytes"); assert(bytes_opt); if (n_live_opt && bytes_opt) { auto n_live_gco = obj::from(n_live_opt.value()); auto bytes_gco = obj::from(bytes_opt.value()); n_live_gco->assign_value(n_live_gco->value() + 1); bytes_gco->assign_value(bytes_gco->value() + z); } } } stats_v->resize(max_tseq + 1); DArray * final_stats_v = DArray::empty(mm, n_tseq_present); for (std::size_t i = 0, n = stats_v->size(); i < n; ++i) { auto recd = stats_v->at(i); if (recd) { bool ok = final_stats_v->push_back(recd); assert(ok); } } *p_output = obj(final_stats_v); return ok; } bool GCObjectStore::_check_move_policy(header_type alloc_hdr, void * object_data, Generation upto) const noexcept { (void)object_data; // when gc is moving objects, to- and from- spaces have been // reversed: forwarding pointers are located in from-space and // refer to to-space. object_age age = this->header2age(alloc_hdr); Generation g = config_.age2gen(age); //assert(runstate_.is_running()); return (g < upto); } void GCObjectStore::swap_roles(Generation upto) noexcept { scope log(XO_DEBUG(config_.debug_flag_), xtag("upto", upto)); for (Generation g = Generation{0}; g < upto; ++g) { log && log("swap roles", xtag("g", g)); std::swap(space_[role::to_space()][g], space_[role::from_space()][g]); } } void GCObjectStore::cleanup_phase(Generation upto, bool sanitize_flag) { scope log(XO_DEBUG(config_.debug_flag_), xtag("upto", upto)); // everything live has been copied out of from-space // -> now set to empty // for (Generation g = Generation{0}; g < upto; ++g) { if (sanitize_flag) { space_[role::from_space()][g]->scrub(); } space_[role::from_space()][g]->clear(); } } auto GCObjectStore::snap_move_checkpoint(Generation upto) -> GCMoveCheckpoint { GCMoveCheckpoint gray_lo_v; for (uint32_t g = 0; g < upto; ++g) { gray_lo_v[g] = this->to_space(Generation{g})->free_; } return gray_lo_v; } /* editor bait: register_type */ bool GCObjectStore::install_type(const AGCObject & meta) noexcept { typeseq tseq = meta._typeseq(); assert(tseq.seqno() > 0); auto ix = static_cast(tseq.seqno()); if (ix >= object_types_.size()) { if (!object_types_.resize(std::max(2 * object_types_.size(), ix + 1))) return false; } assert(ix < object_types_.size()); ObjectTypeSlot & slot = object_types_[ix]; slot.store_iface(&meta); return true; } #ifdef MARKED void GCObjectStore::_forward_children_until_fixpoint(DX1Collector * gc, Generation upto, const GCMoveCheckpoint & gray_lo_v) { scope log(XO_DEBUG(config_.debug_flag_)); /** * To-space: * * to_lo = start of to-space * w,W = white objects. An object x is white if x * + all immediate children of x are in to-space * (also implies this GC cycle put it there) * g,G = grey objects. An object x is gray if it's in to-space, * but possibly has >0 black children * _ = free to-space memory * N = nursery space (generation{0}) * T = tenured space (generation{1}) * * wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________... * ^ ^ ^ * to_lo grey_lo(N) free_ptr(N) * * After moving children of one object, * advancing {nursery_grey_lo, nursery_free_ptr} * * wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG______... * ^ ^ ^ * to_lo grey_lo(N) free_ptr(N) * * Invariant: * * objects in [to_lo, gray_lo) are white. * all gray objects are in [gray_lo, free_ptr) * memory starting at free_ptr is free. * * deep_move terminates when gray_lo catches up to free_ptr * * Above is simplified. Complication is that GC (including incremental) may * promote objects from nursery (N) to tenured (T) * * So more accurate before/after picture * * N wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________... * ^ ^ ^ * to_lo(N) grey_lo(N) free_ptr(N) * * T wwwwwwwwwwwwwwgggggggggggg_______________________________... * ^ ^ ^ * to_lo(T) grey_lo(T) free_ptr(N) * * After moving children of one object, * advancing {nursery_grey_lo, nursery_free_ptr} * * N wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG_____... * ^ ^ ^ * to_lo(N) grey_lo(N) free_ptr(N) * * T wwwwwwwwwwwwwwggggggggggggGGGGG_________________________... * ^ ^ ^ * to_lo(T) grey_lo(T) free_ptr(T) * * deep_move terminates when both: * - gray_lo(N) catches up with free_ptr(N) * - gray_lo(T) catches up with free_ptr(T) * **/ std::size_t fixup_work = 0; /* TODO: * - loop here is bad for memory locality * - replace with depth-first traversal */ do { fixup_work = 0; for (Generation g = Generation{0}; g < upto; ++g) { /** object index for this pass **/ size_t i_obj = 0; /* TODO: use AllocIterator here */ while(gray_lo_v[g] < to_space(g)->free_) { AllocHeader * hdr = (AllocHeader *)gray_lo_v[g]; void * src = (hdr + 1); const auto & hdr_cfg = config_.arena_config_.header_; typeseq tseq = typeseq(hdr_cfg.tseq(*hdr)); size_t z = hdr_cfg.size_with_padding(*hdr); log && log("deep_move_gc_owned: fwd to-space children", xtag("g", g), xtag("i_obj", i_obj), xtag("src", src), xtag("tseq", tseq), xtag("tname", TypeRegistry::id2name(tseq)), xtag("z", z)); const AGCObject * iface = this->lookup_type(tseq); assert(iface->_has_null_vptr() == false); auto gc = this->ref(); iface->forward_children(src, gc); gray_lo_v[g] = ((std::byte *)src) + z; ++i_obj; ++fixup_work; } } } while (fixup_work > 0); } /*_forward_children_until_fixpoint*/ #endif } /*namespace mm*/ } /*namespace xo*/ /* end GCObjectStore.cpp */