xo-gc: X1Collector.assign_member() + GCRoots + use ArenaVector

Using ArenaVector for mlog.

include/xo/gc/DX1Collector.hpp

@@ -7,11 +7,13 @@
 
 #include "X1CollectorConfig.hpp"
 #include "GCObject.hpp"
+#include "MutationLogEntry.hpp"
 #include "generation.hpp"
 #include "object_age.hpp"
 #include "role.hpp"
 #include <xo/alloc2/Allocator.hpp>
 #include <xo/arena/DArena.hpp>
+#include <xo/arena/DArenaVector.hpp>
 #include <xo/arena/ArenaConfig.hpp>
 #include <memory>
 #include <array>
@@ -64,9 +66,35 @@ namespace xo {
 
         struct DX1CollectorIterator;
 
+        /** @brief GC root struct
+         *
+         *  A root is traversed much like other gc-owned value:
+         *  a. if root is in GC from-space, move it to to-space.
+         *  b. if root is in GC to-space, skip.
+         *     e.g. root belongs to generation not subject to collection this cycle.
+         *  c. if root it not allocated by GC, still do in-place forward on its
+         *     children.  This is load-bearing for ParserStateMachine, for example.
+         *     Allows non-GC object to refer to a dynamic set of gc-owned objects
+         **/
+        struct GCRoot {
+        public:
+            GCRoot() = default;
+            explicit GCRoot(obj<AGCObject> * x) : root_{x} {}
+
+            obj<AGCObject> * root() { return root_; }
+
+        private:
+            obj<AGCObject> * root_ = nullptr;
+        };
+
         // ----- DX1Collector -----
 
+        /** @brief garbage collector 'X1'
+         **/
         struct DX1Collector {
+        public:
+            using RootSet = DArenaVector<GCRoot>;
+            using MutationLog = DArenaVector<MutationLogEntry>;
             using typeseq = xo::facet::typeseq;
             using size_type = DArena::size_type;
             using value_type = DArena::value_type;
@@ -75,6 +103,7 @@ namespace xo {
             /** hard max typeseq for collector-registered types **/
             static constexpr size_t c_max_typeseq = 4096;
 
+        public:
             /** Create X1 collector instance. **/
             explicit DX1Collector(const X1CollectorConfig & cfg);
 
@@ -92,7 +121,7 @@ namespace xo {
             std::string_view name() const { return config_.name_; }
 
             const DArena * get_object_types() const noexcept { return &object_types_; }
-            const DArena * get_roots() const noexcept { return &roots_; }
+            const RootSet * get_root_set() const noexcept { return &root_set_; }
             const DArena * get_space(role r, generation g) const noexcept { return space_[r][g]; }
             DArena * get_space(role r, generation g) noexcept { return space_[r][g]; }
             DArena * from_space(generation g) noexcept { return get_space(role::from_space(), g); }
@@ -120,6 +149,16 @@ namespace xo {
              **/
             bool contains(role r, const void * addr) const noexcept;
 
+            /** true iff address @p addr allocated from this collector and currently live
+             *  in role @p r (according to current GC state)
+             **/
+            bool contains_allocated(role r, const void * addr) const noexcept;
+
+            /** generation to which pointer @p addr belongs, given role @p r;
+             *  sentinel if not found in this collector
+             **/
+            generation generation_of(role r, const void * addr) const noexcept;
+
             /** return details from last error (will be in gen0 to-space) **/
             AllocError last_error() const noexcept;
 
@@ -224,6 +263,21 @@ namespace xo {
              **/
             bool expand(size_type z) noexcept;
 
+            // ----- mutation -----
+
+            /** Modify a gc-owned pointer @p *p_lhs, within allocation @p parent,
+             *  to point to @p rhs.
+             *
+             *  Motivation: need special handling for cross-generational pointers with
+             *  incremental gc.
+             *
+             *  Require:
+             *  - if parent is owned by this collector, it has it's own allocation
+             *    (alloc header immediately precedes object address @c parent.data_)
+             *  - address @p p_lhs falls within extent of allocation for @c parent.data_
+             **/
+            void assign_member(void * parent, obj<AGCObject> * p_lhs, obj<AGCObject> rhs);
+
             // ----- iteration -----
 
             /** alloc iterator at begin position **/
@@ -242,16 +296,36 @@ namespace xo {
             void clear() noexcept;
 
         private:
+            /** aux init function: initialize @ref object_types_ arena **/
+            void _init_object_types(const X1CollectorConfig & cfg, std::size_t page_z);
+            /** aux init function: initialize @ref roots_ arena **/
+            void _init_gc_roots(const X1CollectorConfig & cfg, std::size_t page_z);
+            /** aux init function: initialize @ref mlog_storage_[][] arenas **/
+            void _init_mlogs(const X1CollectorConfig & cfg, std::size_t page_z);
+            /** aux init function: create mutation log **/
+            MutationLog _make_mlog(uint32_t igen, char tag_char, size_t mlog_z, std::size_t page_z);
+            /** aux init function: initialize @ref space_storage_[][] arenas **/
+            void _init_space(const X1CollectorConfig & cfg);
+
             /** swap from- and to- roles for all generations < @p upto **/
             void swap_roles(generation upto) noexcept;
             /** copy roots + everything reachable from them, to to-space **/
             void copy_roots(generation upto) noexcept;
 
-            /** move subgraph at @p from_src to to-space.
+            /** cleanup after gc **/
-             *
+            void cleanup_phase(generation upto);
+
+            /** move root subgraph at @p from_src to to-space.
+             *  If not in gc-space, visit immediate children and move them.
              *  Require: runstate_.is_running()
              **/
-            void * deep_move(void * from_src, generation upto);
+            void * _deep_move_root(obj<AGCObject> from_src, generation upto);
+            /** move interior subgraph at @p from_src to to-space.
+             *  no-op if not in gc-space.
+             **/
+            void * _deep_move_interior(void * from_src, generation upto);
+            /** common driver for _deep_move_root(), _deep_move_interior() **/
+            void * _deep_move_gc_owned(void * from_src, generation upto);
 
         public:
             /** garbage collector configuration **/
@@ -270,15 +344,42 @@ namespace xo {
             /** if > 0: need gc for all generations < gc_pending_upto_ **/
             generation gc_pending_upto_;
 
-            /** (ab)using arena to get extensible list of root objects.
+            /** using arena to get extensible list of root objects.
              *  For each root store one address (type obj<AGCObject>*)
+             *
+             *  An Object x that supports AGCObject, but doesn't live in gc-space,
+             *  will get special treatment if it appears in root_set_:
+             *  collector will traverse x to forward pointers to gc-owned
+             *  targets. In other contexts collector doesn't look inside
+             *  non-gc-owned objects
+             *
+             *  editor bait: root_v
              **/
-            DArena roots_;
+            RootSet root_set_;
+
+            /** Cross-generational mutations tracked in MutationLogs.
+             *  We need three logs per generation:
+             *  A. one to observe and remember mutations in to-space
+             *     during normal operation (between GC cycles)
+             *  B. during GC: 2nd mlog to hold entries from from-mlog
+             *     that will still be needed post-GC (because ptr direction
+             *     from higher gen to lower gen after cycle).
+             *  C. during GC: 3rd mlog to triage entries for which
+             *     liveness of pointer source isn't yet established.
+             *
+             * NOTE: indexed on generation of pointer *destination*
+             **/
+            std::array<MutationLog, c_max_generation - 1> mlog_storage_[c_n_role + 1];
+
+            /** mlog pointers.  The roles of mlog_storage_[*][g] get permuted
+             *  as each collection cycle proceeds
+             **/
+            std::array<MutationLog *, c_max_generation - 1> mlog_[c_n_role + 1];
 
             /** collector-managed memory here.
              *  - space_[1] is from-space
              *  - space_[0] is to-space
-             *  coordinates with role ingc/role.hpp, see also.
+             *  coordinates with role in gc/role.hpp, see also.
              **/
 
             /** arena objects for collector managed memory

include/xo/gc/MutationLogEntry.hpp

@@ -5,7 +5,7 @@
 
 #pragma once
 
-#include "GCObject.hpp"
+#include <xo/alloc2/GCObject.hpp>
 
 namespace xo {
     namespace mm {

include/xo/gc/X1CollectorConfig.hpp

@@ -26,6 +26,16 @@ namespace xo {
              **/
             X1CollectorConfig with_size(std::size_t gen_z);
 
+            /** copy of this config,
+             *  but with @c debug_flag_ set to @p x
+             **/
+            X1CollectorConfig with_debug_flag(bool x);
+
+            /** copy of this config,
+             *  but with @c sanitize_flag_ set to @p x
+                **/
+            X1CollectorConfig with_sanitize_flag(bool x);
+
             generation age2gen(object_age age) const noexcept {
                 return generation(age % n_survive_threshold_);
             }
@@ -37,13 +47,19 @@ namespace xo {
             std::string name_;
 
             /** Configuration for collector spaces.
-             *  Will have at least {nursery,tenured} x {from,to} spaces.
+             *  Will have (2 x G) of these,
+             *  where G is @ref n_generation_.
              *  Not using name_ member.
              *
              *  REQUIRE:
              *  - arena_config_.store_header_flag_ must be true
              **/
-            ArenaConfig arena_config_;
+            ArenaConfig arena_config_ = ArenaConfig().with_store_header_flag(true);
+
+            /** storage for xgen pointer bookkeeping (aka remembered sets).
+             *  Use 3x this value per generation
+             **/
+            std::size_t mutation_log_z_ = 1024;
 
             /** storage for N object types requires 8*N bytes **/
             std::size_t object_types_z_ = 2*1024*1024;
@@ -85,13 +101,17 @@ namespace xo {
              **/
             uint32_t stats_history_z_ = false;
 
+            /** true to enable sanitize features:
+             *  1. zero out from-space at end of GC cycle
+             **/
+            bool sanitize_flag_ = false;
+
             /** true to enable debug logging **/
             bool debug_flag_ = false;
         };
 
-        
+
     } /*namespace mm*/
 } /*namespace xo*/
 
 /* end X1CollectorConfig.hpp */
-

include/xo/gc/detail/ICollector_DX1Collector.hpp

@@ -49,6 +49,8 @@ namespace xo {
             static void add_gc_root_poly(DX1Collector & d, obj<AGCObject> * p_root);
             static void remove_gc_root_poly(DX1Collector & d, obj<AGCObject> * p_root);
             static void request_gc(DX1Collector & d, generation upto);
+            static void assign_member(DX1Collector & d, void * parent,
+                                      obj<AGCObject> * p_lhs, obj<AGCObject> & rhs);
             static void forward_inplace(DX1Collector & d, AGCObject * lhs_iface, void ** lhs_data);
 
             static int32_t s_typeseq;

include/xo/gc/object_age.hpp

@@ -26,6 +26,18 @@ namespace xo {
 
             std::uint32_t value_;
         };
+
+        inline bool operator==(object_age lhs, object_age rhs) {
+            return lhs.value_ == rhs.value_;
+        }
+
+        inline bool operator<(object_age lhs, object_age rhs) {
+            return lhs.value_ < rhs.value_;
+        }
+
+        inline bool operator>(object_age lhs, object_age rhs) {
+            return lhs.value_ > rhs.value_;
+        }
     }
 }
 

src/gc/CMakeLists.txt

@@ -3,6 +3,9 @@
 set(SELF_LIB xo_gc)
 set(SELF_SRCS
 
+    init_gc.cpp
+    SetupGc.cpp
+
     IAllocator_DX1Collector.cpp
     ICollector_DX1Collector.cpp
     IAllocIterator_DX1CollectorIterator.cpp
@@ -10,6 +13,9 @@ set(SELF_SRCS
     DX1Collector.cpp
     DX1CollectorIterator.cpp
 
+    X1CollectorConfig.cpp
+    MutationLogEntry.cpp
+
     )
 
 xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS})

src/gc/DX1Collector.cpp

@@ -21,27 +21,12 @@
 
 namespace xo {
     using xo::mm::AAllocator;
+    using xo::facet::TypeRegistry;
     using xo::facet::typeseq;
     using xo::facet::with_facet;
 
     namespace mm {
 
-        X1CollectorConfig
-        X1CollectorConfig::with_name(std::string name)
-        {
-            X1CollectorConfig copy = *this;
-            copy.name_ = std::move(name);
-            return copy;
-        }
-
-        X1CollectorConfig
-        X1CollectorConfig::with_size(std::size_t gen_z)
-        {
-            X1CollectorConfig copy = *this;
-            copy.arena_config_ = arena_config_.with_size(gen_z);
-            return copy;
-        }
-
         // ----- GCRunState -----
 
         GCRunState::GCRunState(generation gc_upto)
@@ -72,45 +57,109 @@ namespace xo {
 
             size_t page_z = getpagesize();
 
+            this->_init_object_types(cfg, page_z);
+            this->_init_gc_roots(cfg, page_z);
+            this->_init_mlogs(cfg, page_z);
+            this->_init_space(cfg);
+        }
+
+        void
+        DX1Collector::_init_object_types(const X1CollectorConfig & cfg, std::size_t page_z)
+        {
             /* 1MB reserved address space enough for up to 128k distinct types.
              * In this case don't want to use hugepages since actual #of types
              * likely << .size/8
              */
-            object_types_ = DArena::map(
+            this->object_types_
-                ArenaConfig{
+                = DArena::map(ArenaConfig{.name_ = "x1-object-types",
-                    .name_ = "x1-object-types",
+                                          .size_ = cfg.object_types_z_,
-                    .size_ = cfg.object_types_z_,
+                                          .hugepage_z_ = page_z,
-                    .hugepage_z_ = page_z,
+                                          .store_header_flag_ = false});
-                    .store_header_flag_ = false});
+        }
 
-            roots_ = DArena::map(
+        void
-                ArenaConfig{
+        DX1Collector::_init_gc_roots(const X1CollectorConfig & cfg, std::size_t page_z)
-                    .name_ = "x1-object-roots",
+        {
-                    .size_ = cfg.object_roots_z_,
+            this->root_set_
-                    .hugepage_z_ = page_z,
+                = RootSet::map(ArenaConfig{.name_ = "x1-object-roots",
-                    .store_header_flag_ = false});
+                                          .size_ = cfg.object_roots_z_,
+                                          .hugepage_z_ = page_z,
+                                          .store_header_flag_ = false});
+        }
+
+        void
+        DX1Collector::_init_mlogs(const X1CollectorConfig & cfg, std::size_t page_z)
+        {
+            for (uint32_t igen = 0, ngen = cfg.n_generation_; igen + 1 < ngen; ++igen) {
+                // special case: no use for mutation log for youngest generation,
+                // so don't trouble to allocate one
+
+                if (igen + 1 < c_max_generation) {
+                    this->mlog_storage_[0][igen] = _make_mlog(igen, 'a', cfg.mutation_log_z_, page_z);
+                    this->mlog_storage_[1][igen] = _make_mlog(igen, 'b', cfg.mutation_log_z_, page_z);
+                    this->mlog_storage_[2][igen] = _make_mlog(igen, 'c', cfg.mutation_log_z_, page_z);
+
+                    this->mlog_[0][igen] = &mlog_storage_[0][igen];
+                    this->mlog_[1][igen] = &mlog_storage_[1][igen];
+                    this->mlog_[2][igen] = &mlog_storage_[2][igen];
+                } else {
+                    assert(false);
+                }
+            }
+
+            if (cfg.n_generation_ > 0) {
+                for (uint32_t igen = cfg.n_generation_ - 1; igen + 1 < c_max_generation; ++igen) {
+                    this->mlog_[0][igen] = nullptr;
+                    this->mlog_[1][igen] = nullptr;
+                    this->mlog_[2][igen] = nullptr;
+                }
+            } else {
+                assert(false);
+            }
+        }
+
+        auto
+        DX1Collector::_make_mlog(uint32_t igen, char tag_char, size_t mlog_z, size_t page_z) -> MutationLog
+        {
+            char buf[40];
+            snprintf(buf, sizeof(buf), "x1-mlog-G%u-%c", igen, tag_char);
+
+            return MutationLog::map(ArenaConfig{.name_ = std::string(buf),
+                                                .size_ = mlog_z,
+                                                .hugepage_z_ = page_z,
+                                                .store_header_flag_ = false});
+        }
+
+        void
+        DX1Collector::_init_space(const X1CollectorConfig & cfg)
+        {
+            assert(c_n_role == 2);
 
             for (uint32_t igen = 0, ngen = cfg.n_generation_; igen < ngen; ++igen) {
-                {
+                if (igen < c_max_generation) {
-                    char buf[40];
+                    {
-                    snprintf(buf, sizeof(buf), "x1-space-G%u-a", igen);
+                        char buf[40];
+                        snprintf(buf, sizeof(buf), "x1-space-G%u-a", igen);
 
-                    space_storage_[0][igen] = DArena::map(cfg.arena_config_.with_name(std::string(buf)));
+                        this->space_storage_[0][igen] = DArena::map(cfg.arena_config_.with_name(std::string(buf)));
+                    }
+                    {
+                        char buf[40];
+                        snprintf(buf, sizeof(buf), "x1-space-G%u-b", igen);
+
+                        this->space_storage_[1][igen] = DArena::map(cfg.arena_config_.with_name(std::string(buf)));
+                    }
+
+                    this->space_[role::to_space()][igen] = &space_storage_[0][igen];
+                    this->space_[role::from_space()][igen] = &space_storage_[1][igen];
+                } else {
+                    assert(false);
                 }
-                {
-                    char buf[40];
-                    snprintf(buf, sizeof(buf), "x1-space-G%u-b", igen);
-
-                    space_storage_[1][igen] = DArena::map(cfg.arena_config_.with_name(std::string(buf)));
-                }
-
-                space_[role::to_space()][igen] = &space_storage_[0][igen];
-                space_[role::from_space()][igen] = &space_storage_[1][igen];
             }
 
             for (uint32_t igen = cfg.n_generation_; igen < c_max_generation; ++igen) {
-                space_[role::to_space()][igen] = nullptr;
+                this->space_[role::to_space()][igen] = nullptr;
-                space_[role::from_space()][igen] = nullptr;
+                this->space_[role::from_space()][igen] = nullptr;
             }
         }
 
@@ -118,26 +167,49 @@ namespace xo {
         DX1Collector::visit_pools(const MemorySizeVisitor & visitor) const
         {
             object_types_.visit_pools(visitor);
-            roots_.visit_pools(visitor);
+            root_set_.visit_pools(visitor);
 
-            for (uint32_t i = 0; i < c_n_role; ++i) {
+            for (uint32_t j = 0; j < config_.n_generation_; ++j) {
-                for (uint32_t j = 0; j < config_.n_generation_; ++j) {
+                for (uint32_t i = 0; i < c_n_role; ++i) {
                     space_storage_[i][j].visit_pools(visitor);
                 }
             }
+
+            for (uint32_t j = 1; j < config_.n_generation_; ++j) {
+                for (uint32_t i = 0; i < c_n_role + 1; ++i) {
+                    mlog_storage_[i][j].visit_pools(visitor);
+                }
+            }
         }
 
         bool
         DX1Collector::contains(role r, const void * addr) const noexcept
+        {
+            return !(this->generation_of(r, addr).is_sentinel());
+        }
+
+        bool
+        DX1Collector::contains_allocated(role r, const void * addr) const noexcept
+        {
+            generation g = this->generation_of(r, addr);
+
+            if (g.is_sentinel())
+                return false;
+
+            return this->get_space(r, g)->contains_allocated(addr);
+        }
+
+        generation
+        DX1Collector::generation_of(role r, const void * addr) const noexcept
         {
             for (generation gi{0}; gi < config_.n_generation_; ++gi) {
                 const DArena * arena = get_space(r, gi);
 
                 if (arena->contains(addr))
-                    return true;
+                    return gi;
             }
 
-            return false;
+            return generation::sentinel();
         }
 
         AllocError
@@ -156,7 +228,7 @@ namespace xo {
                                  size_t (DArena::* get_stat_fn)() const) noexcept
             {
                 size_t z1 = (d.object_types_.*get_stat_fn)();
-                size_t z2 = (d.roots_.*get_stat_fn)();
+                size_t z2 = (d.root_set_.store()->*get_stat_fn)();
 
                 size_t z3 = 0;
 
@@ -255,7 +327,11 @@ namespace xo {
         {
             AGCObject * v = reinterpret_cast<AGCObject *>(object_types_.lo_);
 
-            return &(v[tseq.seqno()]);
+            const AGCObject * target = &(v[tseq.seqno()]);
+
+            assert(reinterpret_cast<const std::byte *>(target) < object_types_.limit_);
+
+            return target;
         }
 
         /* editor bait: register_type */
@@ -279,12 +355,7 @@ namespace xo {
         void
         DX1Collector::add_gc_root_poly(obj<AGCObject> * p_root) noexcept
         {
-            std::byte * mem
+            root_set_.push_back(GCRoot(p_root));
-                = roots_.alloc(typeseq::sentinel(),
-                               sizeof(obj<AGCObject>*));
-            assert(mem);
-
-            *(obj<AGCObject> **)mem = p_root;
         }
 
         void
@@ -337,7 +408,10 @@ namespace xo {
             log && log("step 2b : [STUB] copy pinned");
             log && log("step 3a : [STUB] run destructors");
             log && log("step 3b : [STUB] keep reachable weak pointers");
+
             log && log("step 4  : [STUB] cleanup");
+            this->cleanup_phase(upto);
+
         }
 
         void
@@ -352,6 +426,69 @@ namespace xo {
             }
         }
 
+        void
+        DX1Collector::cleanup_phase(generation upto)
+        {
+            scope log(XO_DEBUG(true), xtag("upto", upto));
+
+            // everything live has been copied out of from-space
+            // -> now set to empty
+            //
+            for (generation g = generation{0}; g < upto; ++g) {
+                if (config_.sanitize_flag_) {
+                    space_[role::from_space()][g]->scrub();
+                }
+
+                space_[role::from_space()][g]->clear();
+            }
+
+            this->runstate_ = GCRunState::gc_not_running();
+        }
+
+        void *
+        DX1Collector::_deep_move_root(obj<AGCObject> from_src,
+                                      generation upto)
+        {
+            // NOTE:
+            // Some roots are non-gc-owned nodes.
+            // GC must still visit immediate children of these nodes
+            // to move gc-owned children.
+            // This implements virtual root node feature,
+            // intended to mitigate mutation log churn.
+
+            scope log(XO_DEBUG(config_.debug_flag_));
+
+            if (!from_src)
+                return nullptr;
+
+            bool src_in_from_space = this->contains(role::from_space(), from_src.data());
+
+            if (src_in_from_space) {
+                return _deep_move_gc_owned(from_src.data(), upto);
+            } else {
+                auto self = this->ref<ACollector>();
+                from_src.forward_children(self);
+                return from_src.data();
+            }
+        }
+
+        void *
+        DX1Collector::_deep_move_interior(void * from_src,
+                                          generation upto)
+        {
+            scope log(XO_DEBUG(config_.debug_flag_));
+
+            if (!from_src)
+                return nullptr;
+
+            bool src_in_from_space = this->contains(role::from_space(), from_src);
+
+            if (!src_in_from_space)
+                return from_src;
+
+            return _deep_move_gc_owned(from_src, upto);
+        }
+
         /*
          * rules:
          * - from_src must be in from-space
@@ -364,24 +501,17 @@ namespace xo {
          * EDITOR: gc -> self
          */
         void *
-        DX1Collector::deep_move(void * from_src, generation upto)
+        DX1Collector::_deep_move_gc_owned(void * from_src,
+                                          generation upto)
         {
             scope log(XO_DEBUG(config_.debug_flag_));
 
-            if (!from_src)
-                return nullptr;
-
-            if (!this->contains(role::from_space(), from_src)) {
-                /* presumeably memory not owned by collector
-                 * (e.g. DBoolean {true, false}, DUniqueString {owned by StringTable} etc.)
-                 */
-                return from_src;
-            }
-
             AllocInfo info = this->alloc_info((std::byte *)from_src);
             AllocHeader hdr = info.header();
             typeseq tseq(info.tseq());
 
+            assert(this->contains_allocated(role::from_space(), from_src));
+
             if (is_forwarding_header(hdr)) {
                 /* already forwarded - pickup destination
                  *
@@ -475,6 +605,9 @@ namespace xo {
 
             obj<AAllocator, DX1Collector> alloc(this);
             const AGCObject * iface = lookup_type(tseq);
+
+            assert(iface->_has_null_vptr() == false);
+
             void * to_dest = this->shallow_move(iface, from_src);
 
             std::size_t fixup_work = 0;
@@ -487,23 +620,37 @@ namespace xo {
                 fixup_work = 0;
 
                 for (generation g = generation{0}; g < upto; ++g) {
+                    /** object index for this pass **/
+                    size_t i_obj = 0;
+
                     /* TODO: use AllocIterator here */
                     while(gray_lo_v[g] < to_space(g)->free_) {
                         AllocHeader * hdr = (AllocHeader *)gray_lo_v[g];
                         void * src = (hdr + 1);
 
-                        log && log("fwd children", xtag("src", src));
-
                         const auto & hdr_cfg = config_.arena_config_.header_;
                         typeseq tseq = typeseq(hdr_cfg.tseq(*hdr));
                         size_t z = hdr_cfg.size_with_padding(*hdr);
 
+                        log && log("deep_move_gc_owned: fwd to-space children",
+                                   xtag("g", g),
+                                   xtag("i_obj", i_obj),
+                                   xtag("src", src),
+                                   xtag("tseq", tseq),
+                                   xtag("tname", TypeRegistry::id2name(tseq)),
+                                   xtag("z", z));
+
                         const AGCObject * iface = this->lookup_type(tseq);
+
+                        assert(iface->_has_null_vptr() == false);
+
                         obj<ACollector, DX1Collector> gc(this);
 
                         iface->forward_children(src, gc);
 
                         gray_lo_v[g] = ((std::byte *)src) + z;
+
+                        ++i_obj;
                         ++fixup_work;
                     }
                 }
@@ -512,21 +659,25 @@ namespace xo {
             log && log(xtag("to_dest", to_dest));
 
             return to_dest;
-        }
+        } /*_deep_move_gc_owned*/
 
         void
         DX1Collector::copy_roots(generation upto) noexcept
         {
             scope log(XO_DEBUG(true));
 
-            for (obj<AGCObject> ** p_root = (obj<AGCObject> **)roots_.lo_;
+            for (RootSet::size_type i = 0, n = root_set_.size(); i < n; ++i) {
-                 p_root < (obj<AGCObject> **)roots_.free_; ++p_root)
+                GCRoot & slot = root_set_[i];
-            {
-                log && log("copy root", xtag("**p_root.data.pre", (**p_root).data_));
 
-                (*p_root)->reset_opaque(this->deep_move((*p_root)->data_, upto));
+                log && log("copy root",
+                           xtag("slot.root()", slot.root()),
+                           xtag("slot.root()->data_", slot.root()->data_));
 
-                log && log(xtag("**p_root.data.post", (**p_root).data_));
+                void * root_to = this->_deep_move_root(*slot.root(), upto);
+
+                slot.root()->reset_opaque(root_to);
+
+                log && log(xtag("slot.root()->data_", slot.root()->data_));
             }
         }
 
@@ -590,9 +741,18 @@ namespace xo {
             /* recover allocation size */
             std::size_t alloc_z = some_arena->config_.header_.size_with_padding(alloc_hdr);
 
-            log && log(xtag("some_arena.lo", some_arena->lo_),
+            if (log) {
-                       xtag("p_header", p_header),
+                log(xtag("some_arena.lo", some_arena->lo_),
-                       xtag("alloc_z", alloc_z));
+                    xtag("p_header", p_header),
+                    xtag("alloc_z", alloc_z));
+
+                AllocInfo info = this->alloc_info((std::byte *)object_data);
+                log(xtag("tseq", info.tseq()),
+                    xtag("tname", TypeRegistry::id2name(typeseq(info.tseq()))),
+                    xtag("is_forwarding_tseq", info.is_forwarding_tseq()),
+                    xtag("age", info.age()),
+                    xtag("size", info.size()));
+            }
 
             /* need to be able to fit forwarding pointer
              * in place of forwarded object.
@@ -616,9 +776,9 @@ namespace xo {
                  *                                                   |
                  *                                    (to-space)     |
                  *                                    +---+-+----+   |
-                 *                                    |FWD|G|size|<--/
+                 *                                    |TSQ|G|size|   |
-                 *                                    +---+-+----+
+                 *                                    +---+-+----+   |
-                 *                                    |          |
+                 *                                    |          | <-/
                  *                                    |          |
                  *                                    |          |
                  *                                    +----------+
@@ -638,22 +798,33 @@ namespace xo {
                  *                              |
                  *                              |     (to-space)
                  *                              |     +---+-+----+
-                 *                              \---->|FWD|G|size|
+                 *                              |     |TSQ|G|size|
-                 *                                    +---+-+----+
+                 *                              |     +---+-+----+
-                 *                                    |          |
+                 *                              \---> |          |
                  *                                    |          |
                  *                                    |          |
                  *                                    +----------+
                  */
+
+                if (log) {
+                    log("lhs_data already forwarded", xtag("dest", dest));
+
+                    AllocInfo info = this->alloc_info((std::byte *)dest);
+                    log(xtag("tseq", info.tseq()),
+                        xtag("tname", TypeRegistry::id2name(typeseq(info.tseq()))),
+                        xtag("age", info.age()), xtag("size", info.size()));
+                }
             } else if (this->check_move_policy(alloc_hdr, object_data)) {
                 /* copy object *lhs + replace with forwarding pointer */
 
+                log && log("forward object now");
+
                 /*
                  *   lhs   obj<AGCObject>             (from-space)
                  *    |    +---------+                +---+-+----+
-                 *    \--->| .iface  |                |FWD|G|size| alloc_hdr
+                 *    \--->| .iface  |                |TSQ|G|size| alloc_hdr
                  *         +---------+  object_data   +---+-+----+
-                 *         | .data x----------------->|          |
+                 *         | .data x----------------> |          |
                  *         +---------+                |          |
                  *                                    |          |
                  *                                    +----------+
@@ -673,14 +844,16 @@ namespace xo {
                  *                              |                    |
                  *                              |     (to-space)     |
                  *                              |     +---+-+----+   |
-                 *                              \---->|FWD|G|size|<--/
+                 *                              |     |TSQ|G|size|   |
-                 *                                    +---+-+----+
+                 *                              |     +---+-+----+   |
-                 *                                    |          |
+                 *                              \---> |          | <-/
                  *                                    |          |
                  *                                    |          |
                  *                                    +----------+
                  */
             } else {
+                log && log("object not eligible/required to forward");
+
                 /* object doesn't need to move.
                  * e.g. incremental collection + object is tenured
                  */
@@ -698,6 +871,9 @@ namespace xo {
             void * to_dest = iface->shallow_copy(from_src, alloc);
 
             log && log(xtag("from_src", from_src), xtag("to_dest", to_dest));
+            log && log(xtag("tseq", info.tseq()),
+                       xtag("tname", TypeRegistry::id2name(typeseq(info.tseq()))),
+                       xtag("age", info.age()), xtag("size", info.size()));
 
             if(to_dest == from_src) {
                 assert(false);
@@ -781,6 +957,95 @@ namespace xo {
             return this->get_space(role::to_space(), generation{0})->alloc_info(mem);
         }
 
+        // editor bait: write barrier
+        void
+        DX1Collector::assign_member(void * parent, obj<AGCObject> * p_lhs, obj<AGCObject> rhs)
+        {
+            scope log(XO_DEBUG(config_.debug_flag_),
+                      xtag("parent", parent), xtag("lhs", p_lhs), xtag("rhs", rhs.data()));
+
+            // ++ stats.n_mutation_;
+
+            *p_lhs = rhs;
+
+            if (runstate_.is_running()) {
+                // for removal of all doubt:
+                // don't log mutations during GC cycle
+                return;
+            }
+
+            if (!config_.allow_incremental_gc_) {
+                // only need to log mutations when incremental gc is enabled
+                return;
+            }
+
+            // logging policy depends on:
+            // 1. generation of lhs
+            // 2. generation of rhs
+
+            generation src_g = this->generation_of(role::to_space(), p_lhs);
+
+            if (src_g.is_sentinel()) {
+                // only need mlog entries for gc-owned pointers.
+                // In this case pointer does not originate in gc-owned space
+                return;
+            }
+
+            generation dest_g = this->generation_of(role::to_space(), rhs.data());
+
+            if (dest_g.is_sentinel()) {
+                // similarly, don't need mlog entry to non-gc-owned destination
+                return;
+            }
+
+            if (src_g < dest_g) {
+                // young-to-old pointers don't need to be remembered,
+                // since a GC cycle that collects the old generation is guarnatted
+                // to also collect the young generation.
+                return;
+            }
+
+            if (src_g == dest_g) {
+                // for pointers within the same generation, need to log
+                // if source is older than destination.
+
+                const DArena * arena = this->get_space(role::to_space(), src_g);
+
+                const DArena::header_type * src_hdr = arena->obj2hdr(parent);
+                const DArena::header_type * dest_hdr = arena->obj2hdr(rhs.data());
+
+                assert(src_hdr && dest_hdr);
+
+                if (header2age(*src_hdr) <= header2age(*dest_hdr)) {
+                    // source and destination have the same age;
+                    // therefore are always collected on the same set of GC cycles
+                    // -> no need to remember separately.
+                    return;
+                } else {
+                    // even though {src,dest} belong to the same generation:
+                    // source will be eligible for promotion before destination.
+                    // At that point pointer would become a cross-generational pointer,
+                    // so need to track it now.
+
+                    log && log("xage ptr -> must log");
+                }
+            } else {
+                log && log("xgen ptr -> must log");
+            }
+
+            // control here: we have an older->younger pointer, need to log it
+
+            // mlog keyed by generation in which pointer _destination_ resides:
+            // collection that moves destination generation around needs to also
+            // update pointers such as this one
+            //
+            MutationLog * mlog = this->mlog_[role::to_space()][dest_g];
+
+            mlog->push_back(MutationLogEntry(parent,
+                                             reinterpret_cast<void **>(&(p_lhs->data_)),
+                                             rhs));
+        }
+
         DX1CollectorIterator
         DX1Collector::begin() const noexcept
         {

src/gc/ICollector_DX1Collector.cpp

@@ -81,6 +81,15 @@ namespace xo {
             d.request_gc(upto);
         }
 
+        void
+        ICollector_DX1Collector::assign_member(DX1Collector & d,
+                                               void * parent,
+                                               obj<AGCObject> * p_lhs,
+                                               obj<AGCObject> & rhs)
+        {
+            d.assign_member(parent, p_lhs, rhs);
+        }
+
         void
         ICollector_DX1Collector::forward_inplace(DX1Collector & d,
                                                  AGCObject * lhs_iface,

utest/Collector.test.cpp

@@ -63,10 +63,14 @@ namespace xo {
                                                                    16 /*size_bits*/), };
             X1CollectorConfig cfg = { .arena_config_ = arena_cfg,
                                       .n_generation_ = 2,
-                                      .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0,
+                                      .gc_trigger_v_ = {{64*1024, 1024*1024,
+#ifdef NOPE
+                                                         0, 0,
                                                          0, 0, 0, 0,
                                                          0, 0, 0, 0,
-                                                         0, 0, 0, 0}} };
+                                                         0, 0, 0, 0
+#endif
+                                      }} };
 
             DX1Collector gc = DX1Collector{cfg};
 
@@ -111,10 +115,14 @@ namespace xo {
                                                                    16 /*size_bits*/), };
             X1CollectorConfig cfg = { .arena_config_ = arena_cfg,
                                       .n_generation_ = 2,
-                                      .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0,
+                                      .gc_trigger_v_ = {{64*1024, 1024*1024,
+#ifdef NOPE
+                                                         0, 0,
                                                          0, 0, 0, 0,
                                                          0, 0, 0, 0,
-                                                         0, 0, 0, 0}} };
+                                                         0, 0, 0, 0
+#endif
+                                      }} };
 
             DX1Collector gc = DX1Collector{cfg};
 
@@ -137,10 +145,14 @@ namespace xo {
                                                                    16 /*size-bits*/), };
             X1CollectorConfig cfg = { .arena_config_ = arena_cfg,
                                       .n_generation_ = 2,
-                                      .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0,
+                                      .gc_trigger_v_ = {{64*1024, 1024*1024,
+#ifdef NOPE
+                                                         0, 0,
                                                          0, 0, 0, 0,
                                                          0, 0, 0, 0,
-                                                         0, 0, 0, 0}} };
+                                                         0, 0, 0, 0
+#endif
+                                      }} };
 
             DX1Collector gc = DX1Collector{cfg};
 
@@ -167,10 +179,14 @@ namespace xo {
             /* collector with one generation collapses to a non-generational copying collector */
             X1CollectorConfig cfg = { .arena_config_ = arena_cfg,
                                       .n_generation_ = 1,
-                                      .gc_trigger_v_ = {{64*1024, 0, 0, 0,
+                                      .gc_trigger_v_ = {{64*1024, 0,
+#ifdef NOPE
+                                                         0, 0,
                                                          0, 0, 0, 0,
                                                          0, 0, 0, 0,
-                                                         0, 0, 0, 0}} };
+                                                         0, 0, 0, 0
+#endif
+                                      }} };
 
             DX1Collector x1state = DX1Collector{cfg};
 
@@ -211,10 +227,14 @@ namespace xo {
             /* collector with one generation collapses to a non-generational copying collector */
             X1CollectorConfig cfg = { .arena_config_ = arena_cfg,
                                       .n_generation_ = 1,
-                                      .gc_trigger_v_ = {{64*1024, 0, 0, 0,
+                                      .gc_trigger_v_ = {{64*1024, 0,
+#ifdef NOPE
+                                                         0, 0,
                                                          0, 0, 0, 0,
                                                          0, 0, 0, 0,
-                                                         0, 0, 0, 0}} };
+                                                         0, 0, 0, 0
+#endif
+                                      }} };
 
             /* X1 allocator+collector */
             DX1Collector x1state = DX1Collector{cfg};

utest/DX1CollectorIterator.test.cpp

@@ -53,10 +53,14 @@ namespace xo {
                                                                    16 /*size_bits*/), };
             X1CollectorConfig cfg = { .arena_config_ = arena_cfg,
                                       .n_generation_ = 2,
-                                      .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0,
+                                      .gc_trigger_v_ = {{64*1024, 1024*1024,
+#ifdef NOPE
+                                                         0, 0,
                                                          0, 0, 0, 0,
                                                          0, 0, 0, 0,
-                                                         0, 0, 0, 0}} };
+                                                         0, 0, 0, 0
+#endif
+                                      }} };
 
             DX1Collector gc = DX1Collector{cfg};
 
@@ -96,10 +100,14 @@ namespace xo {
                                                                    16 /*size_bits*/), };
             X1CollectorConfig cfg = { .arena_config_ = arena_cfg,
                                       .n_generation_ = 2,
-                                      .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0,
+                                      .gc_trigger_v_ = {{64*1024, 1024*1024,
+#ifdef NOPE
+                                                         0, 0,
                                                          0, 0, 0, 0,
                                                          0, 0, 0, 0,
-                                                         0, 0, 0, 0}} };
+                                                         0, 0, 0, 0
+#endif
+                                      }} };
 
             DX1Collector gc = DX1Collector{cfg};
             obj<AAllocator, DX1Collector> a1o{&gc};