From 5a5de795c14cd7059014e01b2a083888c852a87d Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Tue, 16 Dec 2025 21:43:51 -0500 Subject: [PATCH] xo-alloc2: add guard-byte spans to AllocInfo --- .../include/xo/alloc2/alloc/AllocHeader.hpp | 112 -------------- .../xo/alloc2/alloc/AllocHeaderConfig.hpp | 144 ++++++++++++++++++ .../include/xo/alloc2/alloc/AllocInfo.hpp | 38 ++++- .../include/xo/alloc2/arena/ArenaConfig.hpp | 14 -- xo-alloc2/src/alloc2/DArena.cpp | 15 +- xo-alloc2/src/alloc2/IAllocator_DArena.cpp | 41 +---- xo-alloc2/utest/Collector.test.cpp | 24 ++- xo-alloc2/utest/arena.test.cpp | 18 ++- 8 files changed, 231 insertions(+), 175 deletions(-) create mode 100644 xo-alloc2/include/xo/alloc2/alloc/AllocHeaderConfig.hpp diff --git a/xo-alloc2/include/xo/alloc2/alloc/AllocHeader.hpp b/xo-alloc2/include/xo/alloc2/alloc/AllocHeader.hpp index 7d6b4ec6..7544f366 100644 --- a/xo-alloc2/include/xo/alloc2/alloc/AllocHeader.hpp +++ b/xo-alloc2/include/xo/alloc2/alloc/AllocHeader.hpp @@ -36,118 +36,6 @@ namespace xo { static_assert(sizeof(AllocHeader) == sizeof(AllocHeader::repr_type)); static_assert(std::is_standard_layout_v); - - /* - * Each allocation is preceded by a 64-bit header. - * Header is split into 3 configurable-width bit fields, - * labelled (from hi to lo bit order) {tseq, age, size}. - * - * 1. tseq. seq# identifying object types; needed for gc. - * 2. gen. age cohort; increases when alloc survives gc. - * 3. size. alloc size. - * - * Arena allocator only uses size. - * X1 collector uses {tseq, gen, size} - * - * alloc header - * - * TTTTTTTTTTTTGGGGGZZZZZZZZZZZZ - * < tseq >< size > - * - * masking - * - * ..432107654321076543210 bit - * - * > < .gen_bits - * 0..............01111111 gen_mask_unshifted - * 0..011111110..........0 gen_mask_shifted - * > < gen_shift - */ - struct AllocHeaderConfig { - using repr_type = AllocHeader; - - AllocHeaderConfig() = default; - AllocHeaderConfig(std::uint8_t t, std::uint8_t a, std::uint8_t z) noexcept - : tseq_bits_{t}, age_bits_{a}, size_bits_{z} {} - - std::uint64_t tseq_mask() const noexcept { - // e.g. - // FF FF FF 00 00 00 00 00 - // with tseq_bits=24, age_bits=8, size_bits=32 - // - return ((1ul << tseq_bits_) - 1) << (age_bits_ + size_bits_); - } - - std::uint64_t age_mask() const noexcept { - // e.g. - // 00 00 00 FF 00 00 00 00 - // with age_bits=8, size_bits=32 - // - return ((1ul << age_bits_) - 1) << size_bits_; - } - - std::uint64_t size_mask() const noexcept { - // e.g. - // 00 00 00 00 FF FF FF FF - // with size_bits=32 - // - return ((1ul << size_bits_) - 1); - } - - /** extract type id from alloc header @p hdr **/ - std::uint32_t tseq(repr_type hdr) const noexcept { - // e.g. - // 0x302010 - // for header - // 30 20 10 -- -- -- -- -- - // with tseq_bits_ = 24, age_bits_ + size_bits_ = 40 - // - return (hdr.repr_ & tseq_mask()) >> (age_bits_ + size_bits_); - } - - /** extract age from alloc header @p hdr **/ - std::uint32_t age(repr_type hdr) const noexcept { - // e.g. - // 0xa0 - // for header - // -- -- -- a0 -- -- -- -- - // with age_bits_ = 8, size_bits_ = 32 - // - return (hdr.repr_ & age_mask()) >> size_bits_; - } - - /** extract size from alloc header @p hdr **/ - std::size_t size(repr_type hdr) const noexcept { - // e.g. - // 0x01020300 - // for header - // -- -- -- -- 01 02 03 00 - // with size_bits_ = 32 - // - return (hdr.repr_ & size_mask()); - } - - /** true iff sentinel tseq, flagging a forwarding pointer **/ - bool is_forwarding_tseq(repr_type hdr) const noexcept { - // e.g. - // 0xFFFFFF - // i.e. header - // FF FF FF -- -- -- -- -- - // with tseq_bits_ = 24, age_bits + size_bits_ = 40 - // - return (hdr.repr_ & tseq_mask()) == tseq_mask(); - } - - bool is_size_enabled() const noexcept { return size_bits_ > 0; } - - /** number of bits for tseq **/ - std::uint8_t tseq_bits_ = 24; - /** number of bits for age **/ - std::uint8_t age_bits_ = 8; - /** number of bits for size **/ - std::uint8_t size_bits_ = 32; - }; - } } diff --git a/xo-alloc2/include/xo/alloc2/alloc/AllocHeaderConfig.hpp b/xo-alloc2/include/xo/alloc2/alloc/AllocHeaderConfig.hpp new file mode 100644 index 00000000..3c2a8f7f --- /dev/null +++ b/xo-alloc2/include/xo/alloc2/alloc/AllocHeaderConfig.hpp @@ -0,0 +1,144 @@ +/** @file AllocHeaderConfig.hpp +* + * @author Roland Conybeare, Dec 2025 + **/ + +#pragma once + +#include "AllocHeader.hpp" +#include + +namespace xo { + namespace mm { + /* + * Each allocation is preceded by a 64-bit header. + * Header is split into 3 configurable-width bit fields, + * labelled (from hi to lo bit order) {tseq, age, size}. + * + * 1. tseq. seq# identifying object types; needed for gc. + * 2. gen. age cohort; increases when alloc survives gc. + * 3. size. alloc size. + * + * Arena allocator only uses size. + * X1 collector uses {tseq, gen, size} + * + * alloc header + * + * TTTTTTTTTTTTGGGGGZZZZZZZZZZZZ + * < tseq >< size > + * + * masking + * + * ..432107654321076543210 bit + * + * > < .gen_bits + * 0..............01111111 gen_mask_unshifted + * 0..011111110..........0 gen_mask_shifted + * > < gen_shift + */ + struct AllocHeaderConfig { + using repr_type = AllocHeader; + using span_type = std::pair; + + AllocHeaderConfig() = default; + AllocHeaderConfig(std::uint32_t gz, + std::uint8_t guard_byte, + std::uint8_t t, + std::uint8_t a, + std::uint8_t z) noexcept : guard_z_{gz}, + guard_byte_{guard_byte}, + tseq_bits_{t}, + age_bits_{a}, + size_bits_{z} {} + + std::uint64_t tseq_mask() const noexcept { + // e.g. + // FF FF FF 00 00 00 00 00 + // with tseq_bits=24, age_bits=8, size_bits=32 + // + return ((1ul << tseq_bits_) - 1) << (age_bits_ + size_bits_); + } + + std::uint64_t age_mask() const noexcept { + // e.g. + // 00 00 00 FF 00 00 00 00 + // with age_bits=8, size_bits=32 + // + return ((1ul << age_bits_) - 1) << size_bits_; + } + + std::uint64_t size_mask() const noexcept { + // e.g. + // 00 00 00 00 FF FF FF FF + // with size_bits=32 + // + return ((1ul << size_bits_) - 1); + } + + /** extract type id from alloc header @p hdr **/ + std::uint32_t tseq(repr_type hdr) const noexcept { + // e.g. + // 0x302010 + // for header + // 30 20 10 -- -- -- -- -- + // with tseq_bits_ = 24, age_bits_ + size_bits_ = 40 + // + return (hdr.repr_ & tseq_mask()) >> (age_bits_ + size_bits_); + } + + /** extract age from alloc header @p hdr **/ + std::uint32_t age(repr_type hdr) const noexcept { + // e.g. + // 0xa0 + // for header + // -- -- -- a0 -- -- -- -- + // with age_bits_ = 8, size_bits_ = 32 + // + return (hdr.repr_ & age_mask()) >> size_bits_; + } + + /** extract size from alloc header @p hdr **/ + std::size_t size(repr_type hdr) const noexcept { + // e.g. + // 0x01020300 + // for header + // -- -- -- -- 01 02 03 00 + // with size_bits_ = 32 + // + return (hdr.repr_ & size_mask()); + } + + /** true iff sentinel tseq, flagging a forwarding pointer **/ + bool is_forwarding_tseq(repr_type hdr) const noexcept { + // e.g. + // 0xFFFFFF + // i.e. header + // FF FF FF -- -- -- -- -- + // with tseq_bits_ = 24, age_bits + size_bits_ = 40 + // + return (hdr.repr_ & tseq_mask()) == tseq_mask(); + } + + bool is_size_enabled() const noexcept { return size_bits_ > 0; } + + /** if non-zero, allocate extra space between allocs, and fill + * with fixed test-pattern contents. Allows for simple + * runtime arena sanitizing checks. + * Will be rounded up to multiple of @ref padding::c_alloc_alignment + **/ + std::uint32_t guard_z_ = 0; + /** if guard_z_ > 0, write at least that many copies + * of this guard byte following each complete allocation + **/ + std::uint8_t guard_byte_ = 0xfd; + /** number of bits for tseq **/ + std::uint8_t tseq_bits_ = 24; + /** number of bits for age **/ + std::uint8_t age_bits_ = 8; + /** number of bits for size **/ + std::uint8_t size_bits_ = 32; + }; + } /*namespace mm*/ +} /*namespace xo*/ + +/* end AllocHeaderConfig.hpp */ diff --git a/xo-alloc2/include/xo/alloc2/alloc/AllocInfo.hpp b/xo-alloc2/include/xo/alloc2/alloc/AllocInfo.hpp index 6e864468..dbc742a2 100644 --- a/xo-alloc2/include/xo/alloc2/alloc/AllocInfo.hpp +++ b/xo-alloc2/include/xo/alloc2/alloc/AllocInfo.hpp @@ -5,7 +5,8 @@ #pragma once -#include "AllocHeader.hpp" +#include "AllocHeaderConfig.hpp" +#include namespace xo { namespace mm { @@ -18,16 +19,34 @@ namespace xo { **/ struct AllocInfo { using size_type = AllocHeader::size_type; + using byte = std::byte; + using span_type = std::pair; - AllocInfo(const AllocHeaderConfig * p_cfg, const AllocHeader * p_hdr) - : p_config_{p_cfg}, p_header_{p_hdr} {} + AllocInfo(const AllocHeaderConfig * p_cfg, + const byte * p_guard_lo, + const AllocHeader * p_hdr, + const byte * p_guard_hi) : p_config_{p_cfg}, + p_guard_lo_{p_guard_lo}, + p_header_{p_hdr}, + p_guard_hi_{p_guard_hi} {} /** error when alloc-header not configured **/ - static AllocInfo error_not_configured(AllocHeaderConfig * p_cfg) { return AllocInfo(p_cfg, nullptr); } + static AllocInfo error_not_configured(AllocHeaderConfig * p_cfg) { + return AllocInfo(p_cfg, nullptr, nullptr, nullptr); + } /** true for non-sentinel AllocInfo instance **/ bool is_valid() const { return (p_config_ != nullptr) && (p_header_ != nullptr); } + /** Guard bytes preceding allocation-header **/ + span_type guard_lo() const noexcept { + if (!p_guard_lo_) + return span_type(nullptr, nullptr); + + return span_type(p_guard_lo_, + p_guard_lo_ + p_config_->guard_z_); + } + /** Type sequence number in garbage collector **/ std::uint32_t tseq() const noexcept { return p_config_->tseq(*p_header_); } /** Allocation age in garbage collector **/ @@ -35,8 +54,19 @@ namespace xo { /** Allocation size (including allocator-supplied padding) **/ size_type size() const noexcept { return p_config_->size(*p_header_); } + /** Guard bytes immediately following allocation **/ + span_type guard_hi() const noexcept { + if (!p_guard_hi_) + return span_type(nullptr, nullptr); + + return span_type(p_guard_hi_, + p_guard_hi_ + p_config_->guard_z_); + } + const AllocHeaderConfig * p_config_ = nullptr; + const byte * p_guard_lo_ = nullptr; const AllocHeader * p_header_ = nullptr; + const byte * p_guard_hi_ = nullptr; }; } /*namespace mm*/ } /*namespace xo*/ diff --git a/xo-alloc2/include/xo/alloc2/arena/ArenaConfig.hpp b/xo-alloc2/include/xo/alloc2/arena/ArenaConfig.hpp index 46311a89..c467f604 100644 --- a/xo-alloc2/include/xo/alloc2/arena/ArenaConfig.hpp +++ b/xo-alloc2/include/xo/alloc2/arena/ArenaConfig.hpp @@ -29,20 +29,6 @@ namespace xo { * (provided you use their full extent :) **/ std::size_t hugepage_z_ = 2 * 1024 * 1024; - /** if non-zero, allocate extra space between allocs, and fill - * with fixed test-pattern contents. Allows for simple - * runtime arena sanitizing checks. - * Will be rounded up to multiple of @ref padding::c_alloc_alignment - **/ - std::size_t guard_z_ = 0; - /** if guard_z_ > 0, write at least that many copies - * of this guard byte following each complete allocation - **/ - std::uint8_t guard_byte_ = 0xfd; - /** if store_header_flag_ is true: mask bits for allocation size. - * remaining bits can be stolen for other purposes - * otherwise ignored - **/ /** true to store header (8 bytes) at the beginning of each allocation. * necessary and sufficient to allows iterating over allocs * present in arena diff --git a/xo-alloc2/src/alloc2/DArena.cpp b/xo-alloc2/src/alloc2/DArena.cpp index 2b3e972e..fe55281e 100644 --- a/xo-alloc2/src/alloc2/DArena.cpp +++ b/xo-alloc2/src/alloc2/DArena.cpp @@ -168,7 +168,8 @@ namespace xo { //retval.checkpoint_ = lo_; /** make sure guard size is aligned **/ - config_.guard_z_ = padding::with_padding(config_.guard_z_); + config_.header_.guard_z_ + = padding::with_padding(config_.header_.guard_z_); } DArena::DArena(DArena && other) { @@ -272,7 +273,17 @@ namespace xo { this->reserved()); } - return AllocInfo(&config_.header_, (AllocHeader *)header_mem); + AllocHeader * header = (AllocHeader *)header_mem; + + const byte * guard_lo + = header_mem - config_.header_.guard_z_; + const byte * guard_hi + = mem + config_.header_.size(*header); + + return AllocInfo(&config_.header_, + guard_lo, + (AllocHeader *)header_mem, + guard_hi); } void diff --git a/xo-alloc2/src/alloc2/IAllocator_DArena.cpp b/xo-alloc2/src/alloc2/IAllocator_DArena.cpp index 39b39b24..af49b257 100644 --- a/xo-alloc2/src/alloc2/IAllocator_DArena.cpp +++ b/xo-alloc2/src/alloc2/IAllocator_DArena.cpp @@ -63,30 +63,6 @@ namespace xo { IAllocator_DArena::alloc_info(DArena & s, value_type mem) noexcept { return s.alloc_info(mem); - - if (!s.config_.store_header_flag_) [[unlikely]] { - ++(s.error_count_); - s.last_error_ = AllocError(error::alloc_info_disabled, - s.error_count_, - 0 /*add_commit_z*/, - s.committed_z_, - reserved(s)); - - return AllocInfo::error_not_configured(&s.config_.header_); - } - - byte * header_mem = mem - sizeof(AllocHeader); - - if (!s.contains(header_mem)) { - ++(s.error_count_); - s.last_error_ = AllocError(error::alloc_info_address, - s.error_count_, - 0 /*add_commit_z*/, - s.committed_z_, - reserved(s)); - } - - return AllocInfo(&s.config_.header_, (AllocHeader*)header_mem); } bool @@ -163,15 +139,14 @@ namespace xo { s.committed_z_ = aligned_target_z; s.limit_ = s.lo_ + s.committed_z_; - if (commit_start == s.lo_) [[unlikely]] - { -/* first expand() for this allocator - start with guard_z_ bytes */ + if (commit_start == s.lo_) [[unlikely]] { + /* first expand() for this allocator - start with guard_z_ bytes */ ::memset(s.free_, - s.config_.guard_byte_, - s.config_.guard_z_); + s.config_.header_.guard_byte_, + s.config_.header_.guard_z_); - s.free_ += s.config_.guard_z_; + s.free_ += s.config_.header_.guard_z_; } assert(s.committed_z_ % s.config_.hugepage_z_ == 0); @@ -403,10 +378,10 @@ namespace xo { if (store_guard) { /* write guard bytes for overrun detection */ ::memset(s.free_, - s.config_.guard_byte_, - s.config_.guard_z_); + s.config_.header_.guard_byte_, + s.config_.header_.guard_z_); - s.free_ += s.config_.guard_z_; + s.free_ += s.config_.header_.guard_z_; } log && log(xtag("self", s.config_.name_), diff --git a/xo-alloc2/utest/Collector.test.cpp b/xo-alloc2/utest/Collector.test.cpp index 93b7faaa..789c149b 100644 --- a/xo-alloc2/utest/Collector.test.cpp +++ b/xo-alloc2/utest/Collector.test.cpp @@ -54,7 +54,11 @@ namespace xo { ArenaConfig arena_cfg = { .name_ = "_test_unused", .size_ = 4*1024*1024, .store_header_flag_ = true, - .header_ = AllocHeaderConfig(0, 0, 16), }; + .header_ = AllocHeaderConfig(0 /*guard_z*/, + 0xfd /*guard_byte*/, + 0 /*tseq_bits*/, + 0 /*age_bits*/, + 16 /*size_bits*/), }; CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 2, .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0, @@ -98,7 +102,11 @@ namespace xo { ArenaConfig arena_cfg = { .name_ = "_test_unused", .size_ = 4*1024*1024, .store_header_flag_ = true, - .header_ = AllocHeaderConfig(0, 0, 16), }; + .header_ = AllocHeaderConfig(0 /*guard_z*/, + 0xfd /*guard_byte*/, + 0 /*tseq_bits*/, + 0 /*age_bits*/, + 16 /*size_bits*/), }; CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 2, .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0, @@ -120,7 +128,11 @@ namespace xo { ArenaConfig arena_cfg = { .name_ = "_test_unused", .size_ = 4*1024*1024, .store_header_flag_ = true, - .header_ = AllocHeaderConfig(0, 0, 16), }; + .header_ = AllocHeaderConfig(0 /*guard_z*/, + 0xfd /*guard_byte*/, + 0 /*tseq-bits*/, + 0 /*age-bits*/, + 16 /*size-bits*/), }; CollectorConfig cfg = { .arena_config_ = arena_cfg, .n_generation_ = 2, .gc_trigger_v_ = {{64*1024, 1024*1024, 0, 0, @@ -142,7 +154,11 @@ namespace xo { ArenaConfig arena_cfg = { .name_ = "_test_unused", .size_ = 4*1024*1024, .store_header_flag_ = true, - .header_ = AllocHeaderConfig(0, 0, 16), }; + .header_ = AllocHeaderConfig(0 /*guard_z*/, + 0xfd /*guard-byte*/, + 0 /*tseq-bits*/, + 0 /*age-bits*/, + 16 /*size-bits*/), }; /* collector with one generation collapses to a non-generational copying collector */ CollectorConfig cfg = { .arena_config_ = arena_cfg, diff --git a/xo-alloc2/utest/arena.test.cpp b/xo-alloc2/utest/arena.test.cpp index 8a013034..34d8fe05 100644 --- a/xo-alloc2/utest/arena.test.cpp +++ b/xo-alloc2/utest/arena.test.cpp @@ -185,7 +185,11 @@ namespace xo { .size_ = 64*1024, .store_header_flag_ = true, /* up to 4GB */ - .header_ = AllocHeaderConfig(0, 0, 32), + .header_ = AllocHeaderConfig(0 /*guard_z*/, + 0xfd /*guard_byte*/, + 0 /*tseq-bits*/, + 0 /*age-bits*/, + 32 /*size-bits*/), .debug_flag_ = false, }; DArena arena = DArena::map(cfg); @@ -222,11 +226,13 @@ namespace xo { /* typed allocator a1o, with object header + guard bytes */ ArenaConfig cfg { .name_ = "testarena", .size_ = 64*1024, - .guard_z_ = 8, - .guard_byte_ = 0xfd, .store_header_flag_ = true, /* up to 4GB */ - .header_ = AllocHeaderConfig(0, 0, 32), + .header_ = AllocHeaderConfig(8 /*guard_z*/, + 0xfd /*guard-byte*/, + 0 /*tseq-bits*/, + 0 /*age-bits*/, + 32 /*size-bits*/), .debug_flag_ = false, }; DArena arena = DArena::map(cfg); @@ -250,7 +256,7 @@ namespace xo { // guard0 header m0 guard1 // - byte * guard0 = m0 - sizeof(header_type) - cfg.guard_z_; + byte * guard0 = m0 - sizeof(header_type) - cfg.header_.guard_z_; header_type* header = (header_type*)(m0 - sizeof(header_type)); size_t pad = padding::with_padding(z0) - z0; byte * guard1 = m0 + z0 + pad; @@ -263,7 +269,7 @@ namespace xo { REQUIRE(a1o.last_error().error_ == error::none); REQUIRE(a1o.last_error().error_seq_ == 0); - REQUIRE(a1o.allocated() == cfg.guard_z_ + sizeof(header_type) + z0 + pad + cfg.guard_z_); + REQUIRE(a1o.allocated() == cfg.header_.guard_z_ + sizeof(header_type) + z0 + pad + cfg.header_.guard_z_); REQUIRE(a1o.allocated() <= a1o.committed()); REQUIRE(a1o.allocated() + a1o.available() == a1o.committed()); REQUIRE(a1o.committed() <= a1o.reserved());