diff --git a/include/xo/gc/DX1Collector.hpp b/include/xo/gc/DX1Collector.hpp index cf21cf8..86ee992 100644 --- a/include/xo/gc/DX1Collector.hpp +++ b/include/xo/gc/DX1Collector.hpp @@ -49,17 +49,30 @@ namespace xo { * @brief encapsulate state needed while GC is running **/ struct GCRunState { - GCRunState() : gc_upto_{0} {} - explicit GCRunState(generation gc_upto); + enum class Mode { + /** gc not running. X1 available for normal allocation **/ + idle, + /** gc in progress. X1 not available for normal allocation **/ + gc, + /** verify in progress. @ref verify_ok call is on the stack **/ + verify + }; - static GCRunState gc_not_running(); + GCRunState() : gc_upto_{0} {} + GCRunState(Mode mode, generation gc_upto); + + static GCRunState idle(); + static GCRunState verify(); static GCRunState gc_upto(generation g); generation gc_upto() const { return gc_upto_; } - bool is_running() const { return gc_upto_ > 0; } + bool is_running() const { return mode_ == Mode::idle; } + bool is_verify() const { return mode_ == Mode::verify; } private: + /** current collector mode **/ + Mode mode_; /** running gc collecting all generations gi < gc_upto **/ generation gc_upto_; }; @@ -87,6 +100,17 @@ namespace xo { obj * root_ = nullptr; }; + /** @brief info collected during a @ref DX1Collector::verify_ok call + * + **/ + struct VerifyStats { + void clear() { *this = VerifyStats(); } + + std::uint32_t n_ext_ = 0; + std::uint32_t n_from_ = 0; + std::uint32_t n_to_ = 0; + }; + // ----- DX1Collector ----- /** @brief garbage collector 'X1' @@ -111,15 +135,10 @@ namespace xo { template obj ref() { return obj(this); } -#ifdef NOT_YET - /** create instance with default configuration, - * generation size @p gen_z - **/ - static DX1Collector make_std(std::size_t gen_z); -#endif - - std::string_view name() const { return config_.name_; } + // ----- access methods ----- + std::string_view name() const noexcept { return config_.name_; } + GCRunState runstate() const noexcept { return runstate_; } const DArena * get_object_types() const noexcept { return &object_types_; } const RootSet * get_root_set() const noexcept { return &root_set_; } const DArena * get_space(role r, generation g) const noexcept { return space_[r][g]; } @@ -128,6 +147,8 @@ namespace xo { DArena * to_space(generation g) noexcept { return get_space(role::to_space(), g); } DArena * new_space() noexcept { return to_space(generation{0}); } + // ----- basic statistics ----- + /** total reserved memory in bytes, across all {role, generation} **/ size_type reserved_total() const noexcept; /** total size in bytes (same as committed_total()) **/ @@ -139,6 +160,8 @@ namespace xo { /** total allocated memory in bytes, across all {role, generation} **/ size_type allocated_total() const noexcept; + // ----- queries ----- + /** introspection for memory use. * Call @p visitor(info) for each pool owned by this allocator **/ @@ -181,6 +204,9 @@ namespace xo { /** Retreive bookkeeping info for allocation at @p mem. **/ AllocInfo alloc_info(value_type mem) const noexcept; + /** verify that GC state appears consistent **/ + bool verify_ok() noexcept; + // ----- app memory model ----- /** lookup interface from type sequence @@ -194,6 +220,8 @@ namespace xo { **/ bool install_type(const AGCObject & meta) noexcept; + // ------ gc root management ----- + /** add GC root at @p *p_root **/ void add_gc_root_poly(obj * p_root) noexcept; @@ -324,8 +352,16 @@ namespace xo { * no-op if not in gc-space. **/ void * _deep_move_interior(void * from_src, generation upto); - /** common driver for _deep_move_root(), _deep_move_interior() **/ + /** Common driver for _deep_move_root(), _deep_move_interior() **/ void * _deep_move_gc_owned(void * from_src, generation upto); + /** Evacuate object at @p *lhs_data to to-space. + * Replace original with forwarding pointer to new location + **/ + void _forward_inplace_aux(AGCObject * lhs_iface, void ** lhs_data); + /** Verify that pointer {@p iface, @p data} is valid: + * destination either in to-space, or somewhere outside this collector + **/ + void _verify_aux(AGCObject * iface, void * data); public: /** garbage collector configuration **/ @@ -341,6 +377,7 @@ namespace xo { /** gc disabled whenever gc_blocked_ > 0 **/ uint32_t gc_blocked_ = 0; + /** if > 0: need gc for all generations < gc_pending_upto_ **/ generation gc_pending_upto_; @@ -391,6 +428,9 @@ namespace xo { * are reversed each time generation g gets collected. **/ std::array space_[c_n_role]; + + /** counters collected during @ref verify_ok call **/ + VerifyStats verify_stats_; }; } /*namespace mm*/ } /*namespace xo*/ diff --git a/include/xo/gc/MutationLogEntry.hpp b/include/xo/gc/MutationLogEntry.hpp index 7459827..eca086d 100644 --- a/include/xo/gc/MutationLogEntry.hpp +++ b/include/xo/gc/MutationLogEntry.hpp @@ -24,9 +24,16 @@ namespace xo { * - for collector need to traverse data pointer *data **/ class MutationLogEntry { + public: + using AGCObject = xo::mm::AGCObject; + public: MutationLogEntry(void * parent, void ** p_data, obj snap); + void * parent() const { return parent_; } + void ** p_data() const { return p_data_; } + obj snap() const { return snap_; } + private: /** address of object containing logged mutation **/ void * parent_ = nullptr; diff --git a/src/gc/DX1Collector.cpp b/src/gc/DX1Collector.cpp index 96f65a0..03a53f6 100644 --- a/src/gc/DX1Collector.cpp +++ b/src/gc/DX1Collector.cpp @@ -29,20 +29,26 @@ namespace xo { // ----- GCRunState ----- - GCRunState::GCRunState(generation gc_upto) - : gc_upto_{gc_upto} + GCRunState::GCRunState(Mode mode, generation gc_upto) + : mode_{mode}, gc_upto_{gc_upto} {} GCRunState - GCRunState::gc_not_running() + GCRunState::idle() { - return GCRunState(generation(0)); + return GCRunState(Mode::idle, generation::sentinel()); + } + + GCRunState + GCRunState::verify() + { + return GCRunState(Mode::verify, generation::sentinel()); } GCRunState GCRunState::gc_upto(generation g) { - return GCRunState(generation(g + 1)); + return GCRunState(Mode::gc, generation(g + 1)); } // ----- DX1Collector ----- @@ -322,6 +328,78 @@ namespace xo { return (vtable != nullptr); } + AllocInfo + DX1Collector::alloc_info(value_type mem) const noexcept { + for (role ri : role::all()) { + for (generation gj{0}; gj < config_.n_generation_; ++gj) { + const DArena * arena = this->get_space(ri, gj); + + assert(arena); + + if (arena->contains(mem)) { + return arena->alloc_info(mem); + } + } + } + + // deliberately attempt on nursery to-space, to capture error info + return sentinel + return this->get_space(role::to_space(), generation{0})->alloc_info(mem); + } + + bool + DX1Collector::verify_ok() noexcept + { + // 1. visit space pointers + // - verify space_[*] points to space_storage_[*] + // - verify mlog_[*] points to mlog_storage_[*] + // + // 2. visit roots: + // for each root, verify that immediate child pointers are in to-space + // + // 3. scan to-space: + // for each object, verify that immediate children are also in to-space + // + // 4. scan mutation logs: + // verify that entries refer to to-space + + // Each AGCObject impl provides a forward_children() method, + // that calls DX1Collector::forward_inplace(iface, &data) + // + // tactical plan: hijack forward_children. + // Add run state so DX1Collector can recognize forward_inplace() + // calls made for the purpose of checking child pointers. + + GCRunState saved_runstate = runstate_; + { + this->runstate_ = GCRunState::verify(); + this->verify_stats_.clear(); + + // 2. visit roots + for (GCRoot & root_slot : root_set_) { + VerifyStats pre = verify_stats_; + + auto gco = *root_slot.root(); + + if (gco) { + // forward_children is hijacked here to verify + // pointer validity + + gco.forward_children(this->ref()); + } + + VerifyStats post = verify_stats_; + + // assert fail -> root contains ptr to from-space + assert(pre.n_from_ == post.n_from_); + } + } + + // restore run state at end of verify cycle + this->runstate_ = saved_runstate; + + return true; + } + const AGCObject * DX1Collector::lookup_type(typeseq tseq) const noexcept { @@ -389,14 +467,19 @@ namespace xo { //auto t0 = std::chrono::steady_clock::now(); - log && log("step 0a : update run state"); + if (config_.sanitize_flag_) { + log && log("step 0a : verify"); + this->verify_ok(); + } + + log && log("step 0b : update run state"); this->runstate_ = GCRunState::gc_upto(upto); - log && log("step 0a : [STUB] snapshot alloc state"); + log && log("step 0c : [STUB] snapshot alloc state"); - log && log("step 0b : [STUB] scan for object statistics"); + log && log("step 0d : [STUB] scan for object statistics"); - log && log("step 1 : swap from/to roles"); + log && log("step 1 : swap from/to roles (now to-space is empty)"); this->swap_roles(upto); log && log(xtag("from_0", get_space(role::from_space(), generation{0})->lo_), @@ -409,9 +492,13 @@ namespace xo { log && log("step 3a : [STUB] run destructors"); log && log("step 3b : [STUB] keep reachable weak pointers"); - log && log("step 4 : [STUB] cleanup"); + log && log("step 4 : cleanup"); this->cleanup_phase(upto); + if (config_.sanitize_flag_) { + log && log("step 4b : verify"); + this->verify_ok(); + } } void @@ -442,7 +529,7 @@ namespace xo { space_[role::from_space()][g]->clear(); } - this->runstate_ = GCRunState::gc_not_running(); + this->runstate_ = GCRunState::idle(); } void * @@ -684,6 +771,22 @@ namespace xo { void DX1Collector::forward_inplace(AGCObject * lhs_iface, void ** lhs_data) + { + if (runstate_.is_running()) { + // called during collection phase + this->_forward_inplace_aux(lhs_iface, lhs_data); + } else if (runstate_.is_verify()) { + // called during verify_ok + this->_verify_aux(lhs_iface, lhs_data); + } else { + // should be unreachable + assert(false); + } + } + + void + DX1Collector::_forward_inplace_aux(AGCObject * lhs_iface, + void ** lhs_data) { scope log(XO_DEBUG(config_.debug_flag_), xtag("lhs_data", lhs_data), @@ -858,7 +961,30 @@ namespace xo { * e.g. incremental collection + object is tenured */ } - } /*forward_inplace*/ + } /*_forward_inplace*/ + + void + DX1Collector::_verify_aux(AGCObject * iface, void * data) + { + (void)iface; + (void)data; + + generation g = this->generation_of(role::to_space(), data); + + if (g.is_sentinel()) { + g = this->generation_of(role::from_space(), data); + + if (!g.is_sentinel()) { + // verify failure - live pointer still refers to from-space + + ++(verify_stats_.n_from_); + } else { + ++(verify_stats_.n_ext_); + } + } else { + ++(verify_stats_.n_to_); + } + } void * DX1Collector::shallow_move(const AGCObject * iface, void * from_src) @@ -939,24 +1065,6 @@ namespace xo { return false; } - AllocInfo - DX1Collector::alloc_info(value_type mem) const noexcept { - for (role ri : role::all()) { - for (generation gj{0}; gj < config_.n_generation_; ++gj) { - const DArena * arena = this->get_space(ri, gj); - - assert(arena); - - if (arena->contains(mem)) { - return arena->alloc_info(mem); - } - } - } - - // deliberately attempt on nursery to-space, to capture error info + return sentinel - return this->get_space(role::to_space(), generation{0})->alloc_info(mem); - } - // editor bait: write barrier void DX1Collector::assign_member(void * parent, obj * p_lhs, obj rhs)