xo-alloc2: streamlining + bugfixes [wip]
This commit is contained in:
parent
8d0af9a0e3
commit
6cf535eaef
11 changed files with 312 additions and 10 deletions
|
|
@ -247,6 +247,7 @@ let
|
|||
pkgs.git
|
||||
pkgs.gh # github cli
|
||||
pkgs.cloc
|
||||
pkgs.bloaty
|
||||
|
||||
pkgs.lcov
|
||||
pkgs.catch2
|
||||
|
|
|
|||
|
|
@ -23,6 +23,10 @@ namespace xo {
|
|||
reserve_exhausted,
|
||||
/** unable to commit (i.e. mprotect failure) **/
|
||||
commit_failed,
|
||||
/** allocation size too big (See @ref ArenaConfig::header_size_mask_) **/
|
||||
header_size_mask,
|
||||
/** sub_alloc not preceded by super alloc (or another sub_alloc) **/
|
||||
orphan_sub_alloc,
|
||||
};
|
||||
|
||||
struct AllocatorError {
|
||||
|
|
@ -67,6 +71,10 @@ namespace xo {
|
|||
///@{
|
||||
/** @brief type used for allocation amounts **/
|
||||
using size_type = std::size_t;
|
||||
/** @brief type used for allocation responses **/
|
||||
using value_type = std::byte *;
|
||||
/** object header, if configured **/
|
||||
using header_type = std::uint64_t;
|
||||
///@}
|
||||
|
||||
/*
|
||||
|
|
@ -121,7 +129,19 @@ namespace xo {
|
|||
**/
|
||||
virtual bool expand(Opaque d, std::size_t z) const noexcept = 0;
|
||||
/** allocate @p z bytes of memory from allocator @p d. **/
|
||||
virtual std::byte * alloc(Opaque d, std::size_t z) const = 0;
|
||||
virtual value_type alloc(Opaque d, size_type z) const = 0;
|
||||
/** like @ref alloc, but follow with one or more consecutive
|
||||
* @ref sub_alloc() calls. This sequence of allocs will share
|
||||
* the initial allocation header.
|
||||
**/
|
||||
virtual value_type super_alloc(Opaque d, size_type z) const = 0;
|
||||
/** follow a preceding @ref super_alloc call with additional
|
||||
* subsidiary allocs that share the same object header.
|
||||
* Must finish sequence with exactly one sub_alloc call
|
||||
* with @p complete_flag set. This sub_alloc call may have
|
||||
* zero @p z
|
||||
**/
|
||||
virtual value_type sub_alloc(Opaque d, size_type z, bool complete_flag) const = 0;
|
||||
/** reset allocator @p d to empty state **/
|
||||
virtual void clear(Opaque d) const = 0;
|
||||
/** destruct allocator @p d **/
|
||||
|
|
|
|||
|
|
@ -26,6 +26,22 @@ namespace xo {
|
|||
* (provided you use their full extent :)
|
||||
**/
|
||||
std::size_t hugepage_z_ = 2 * 1024 * 1024;
|
||||
/** true to store header (8 bytes) at the beginning of each allocation.
|
||||
* necessary and sufficient to allows iterating over allocs
|
||||
* present in arena
|
||||
**/
|
||||
bool store_header_flag_ = false;
|
||||
/** if non-zero, allocate extra space between allocs, and fill
|
||||
* with fixed test-pattern contents. Allows for simple
|
||||
* runtime arena sanitizing checks.
|
||||
* Will be rounded up to multiple of @ref padding::c_alloc_alignment
|
||||
**/
|
||||
std::size_t guard_z_ = 0;
|
||||
/** if store_header_flag_ is true: mask bits for allocation size.
|
||||
* remaining bits can be stolen for other purposes
|
||||
* otherwise ignored
|
||||
**/
|
||||
std::uint64_t header_size_mask_ = 0xffffffff;
|
||||
/** true to enable debug logging **/
|
||||
bool debug_flag_ = false;
|
||||
|
||||
|
|
@ -59,6 +75,8 @@ namespace xo {
|
|||
using size_type = std::size_t;
|
||||
/** @brief a contiguous memory range **/
|
||||
using range_type = std::pair<std::byte*,std::byte*>;
|
||||
/** @brief type for allocation header (if enabled) **/
|
||||
using header_type = std::uint64_t;
|
||||
|
||||
///@}
|
||||
|
||||
|
|
@ -103,6 +121,11 @@ namespace xo {
|
|||
**/
|
||||
size_type committed_z_ = 0;
|
||||
|
||||
/** if config_.store_header_flag_:
|
||||
* Pointer to header for last allocation.
|
||||
**/
|
||||
header_type * last_header_ = nullptr;
|
||||
|
||||
/** free pointer.
|
||||
* Memory in range [@ref lo_, @ref free_) current in use
|
||||
**/
|
||||
|
|
|
|||
|
|
@ -41,7 +41,9 @@ namespace xo {
|
|||
[[noreturn]] AllocatorError last_error(Copaque) const noexcept override { _fatal(); }
|
||||
|
||||
[[noreturn]] bool expand(Opaque, std::size_t) const noexcept override { _fatal(); }
|
||||
[[noreturn]] std::byte * alloc(Opaque, std::size_t) const override { _fatal(); }
|
||||
[[noreturn]] value_type alloc(Opaque, std::size_t) const override { _fatal(); }
|
||||
[[noreturn]] value_type super_alloc(Opaque, std::size_t) const override { _fatal(); }
|
||||
[[noreturn]] value_type sub_alloc(Opaque, std::size_t, bool) const override { _fatal(); }
|
||||
[[noreturn]] void clear(Opaque) const override { _fatal(); }
|
||||
[[noreturn]] void destruct_data(Opaque) const override { _fatal(); }
|
||||
|
||||
|
|
|
|||
|
|
@ -29,6 +29,14 @@ namespace xo {
|
|||
*/
|
||||
struct IAllocator_DArena {
|
||||
using size_type = std::size_t;
|
||||
using value_type = std::byte *;
|
||||
|
||||
enum class alloc_mode : uint8_t {
|
||||
standard,
|
||||
super,
|
||||
sub,
|
||||
sub_complete,
|
||||
};
|
||||
|
||||
static std::string_view name(const DArena &) noexcept;
|
||||
static size_type reserved(const DArena &) noexcept;
|
||||
|
|
@ -43,11 +51,34 @@ namespace xo {
|
|||
* to size at least @p z
|
||||
* In practice will round up to a multiple of @ref page_z_.
|
||||
**/
|
||||
static bool expand(DArena & d, std::size_t z) noexcept;
|
||||
static bool expand(DArena & d, size_type z) noexcept;
|
||||
|
||||
static std::byte * alloc(DArena &, std::size_t z);
|
||||
static value_type alloc(DArena &, size_type z);
|
||||
/** when store_header_flag enabled:
|
||||
* like alloc(), but combine memory consumed by this alloc
|
||||
* plus following consecutive sub_alloc()'s into a single header.
|
||||
* otherwise equivalent to alloc()
|
||||
**/
|
||||
static value_type super_alloc(DArena &, size_type z);
|
||||
/** when store_header_flag enabled:
|
||||
* follow preceding super_alloc() by one or more sub_allocs().
|
||||
* accumulate total allocated size (including padding) into
|
||||
* single header. All sub_allocs() except the last must set
|
||||
* @p complete_flag to false. The last sub_alloc() must set
|
||||
* @p complete_flag to true.
|
||||
**/
|
||||
static value_type sub_alloc(DArena &, size_type z, bool complete_flag);
|
||||
static void clear(DArena &);
|
||||
static void destruct_data(DArena &);
|
||||
|
||||
private:
|
||||
/** alloc driver. shared by alloc(), super_alloc(), sub_alloc() **/
|
||||
static value_type _alloc(DArena &,
|
||||
size_type z,
|
||||
alloc_mode mode,
|
||||
bool store_header_flag,
|
||||
bool remember_header_flag);
|
||||
|
||||
};
|
||||
|
||||
// template <>
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ namespace xo {
|
|||
struct IAllocator_Xfer : public AAllocator {
|
||||
// parallel interface to AAllocator, with specific data type
|
||||
using Impl = IAllocator_DRepr;
|
||||
using size_type = std::size_t;
|
||||
using size_type = AAllocator::size_type;
|
||||
using value_type = AAllocator::value_type;
|
||||
|
||||
static const DRepr & _dcast(Copaque d) { return *(const DRepr *)d; }
|
||||
static DRepr & _dcast(Opaque d) { return *(DRepr *)d; }
|
||||
|
|
@ -43,8 +44,15 @@ namespace xo {
|
|||
|
||||
bool expand(Opaque d,
|
||||
std::size_t z) const noexcept override { return I::expand(_dcast(d), z); }
|
||||
std::byte * alloc(Opaque d,
|
||||
value_type alloc(Opaque d,
|
||||
std::size_t z) const override { return I::alloc(_dcast(d), z); }
|
||||
value_type super_alloc(Opaque d,
|
||||
std::size_t z) const override { return I::super_alloc(_dcast(d), z); }
|
||||
value_type sub_alloc(Opaque d,
|
||||
std::size_t z,
|
||||
bool complete_flag) const override {
|
||||
return I::sub_alloc(_dcast(d), z, complete_flag);
|
||||
}
|
||||
void clear(Opaque d) const override { return I::clear(_dcast(d)); }
|
||||
void destruct_data(Opaque d) const override { return I::destruct_data(_dcast(d)); }
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ namespace xo {
|
|||
public:
|
||||
using ObjectType = Object;
|
||||
using size_type = std::size_t;
|
||||
using value_type = std::byte *;
|
||||
|
||||
RAllocator() {}
|
||||
RAllocator(Object::DataPtr data) : Object{std::move(data)} {}
|
||||
|
|
@ -33,7 +34,11 @@ namespace xo {
|
|||
AllocatorError last_error() const { return O::iface()->last_error(O::data()); }
|
||||
|
||||
bool expand(size_type z) { return O::iface()->expand(O::data(), z); }
|
||||
std::byte * alloc(size_type z) { return O::iface()->alloc(O::data(), z); }
|
||||
value_type alloc(size_type z) { return O::iface()->alloc(O::data(), z); }
|
||||
value_type super_alloc(size_type z) { return O::iface()->super_alloc(O::data(), z); }
|
||||
value_type sub_alloc(size_type z,
|
||||
bool complete_flag) { return O::iface()->sub_alloc(O::data(),
|
||||
z, complete_flag); }
|
||||
|
||||
static bool _valid;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ namespace xo {
|
|||
**/
|
||||
static inline
|
||||
std::size_t with_padding(std::size_t z,
|
||||
std::size_t align)
|
||||
std::size_t align = c_alloc_alignment)
|
||||
{
|
||||
return z + alloc_padding(z, align);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -166,6 +166,9 @@ namespace xo {
|
|||
last_error_{}
|
||||
{
|
||||
//retval.checkpoint_ = lo_;
|
||||
|
||||
/** make sure guard size is aligned **/
|
||||
config_.guard_z_ = padding::with_padding(config_.guard_z_);
|
||||
}
|
||||
|
||||
DArena::DArena(DArena && other) {
|
||||
|
|
|
|||
|
|
@ -145,23 +145,194 @@ namespace xo {
|
|||
std::byte *
|
||||
IAllocator_DArena::alloc(DArena & s,
|
||||
std::size_t req_z)
|
||||
{
|
||||
/* - primary allocation path:
|
||||
* exactly 1 header per alloc() call.
|
||||
* - store_header_flag follows configuration
|
||||
*/
|
||||
|
||||
return _alloc(s, req_z,
|
||||
alloc_mode::standard);
|
||||
}
|
||||
|
||||
std::byte *
|
||||
IAllocator_DArena::super_alloc(DArena & s,
|
||||
std::size_t req_z)
|
||||
{
|
||||
/* - (uncommon) pattern for parent alloc immediately followed by
|
||||
* zero-or-more susidiary allocs, all sharing a single header.
|
||||
* - collapses into alloc() behavior when
|
||||
* ArenaConfig.store_header_flag_ disabled
|
||||
*/
|
||||
|
||||
bool remember_header_flag = s.config_.store_header_flag_;
|
||||
|
||||
return _alloc(s, req_z,
|
||||
alloc_mode::super);
|
||||
}
|
||||
|
||||
std::byte *
|
||||
IAllocator_DArena::sub_alloc(DArena & s,
|
||||
std::size_t req_z,
|
||||
bool complete_flag)
|
||||
{
|
||||
/* - (uncommon) pattern for subsidiary allocs:
|
||||
* that piggyback onto preceding super_alloc()
|
||||
* - collapses into alloc() behavior when
|
||||
* ArenaConfig.store_header_flag_ disabled
|
||||
*/
|
||||
|
||||
return _alloc(s, req_z,
|
||||
(complete_flag
|
||||
? alloc_mode::sub_complete
|
||||
: alloc_mode::sub_incomplete));
|
||||
|
||||
#ifdef OBSOLETE
|
||||
if ((req_z == 0) && complete_flag) [[unlikely]] {
|
||||
/** use zero req_z with complete_flag to clear s.last_header_ **/
|
||||
|
||||
if (s.config_.store_header_flag_) {
|
||||
if (!s.last_header_) [[unlikely]] {
|
||||
++(s.error_count_);
|
||||
s.last_error_ = AllocatorError(error::orphan_sub_alloc,
|
||||
s.error_count_,
|
||||
0 /*add_commit_z*/, s.committed_z_, reserved(s));
|
||||
} else {
|
||||
s.last_header_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
byte * free0 = s.free_;
|
||||
byte * mem = _alloc(s, req_z,
|
||||
complete_flag ? alloc_mode::sub_complete : alloc_mode::sub,
|
||||
false /*!store_header_flag*/,
|
||||
false /*!remember_header_flag*/);
|
||||
|
||||
if (!mem) [[unlikely]] {
|
||||
/* error already captured */
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
byte * free1 = s.free_;
|
||||
/* used: accounting for padding applied to req_z */
|
||||
size_t z0 = (free1 - free0);
|
||||
|
||||
assert(z0 > 0);
|
||||
|
||||
if (s.config_.store_header_flag_) {
|
||||
if (!s.last_header_) [[unlikely]] {
|
||||
++(s.error_count_);
|
||||
s.last_error_ = AllocatorError(error::orphan_sub_alloc,
|
||||
s.error_count_,
|
||||
0 /*add_commit_z*/, s.committed_z_, reserved(s));
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
/* s.last_header_ holds aggregate size of preceding super_alloc
|
||||
* (+ any sub-alloc's).
|
||||
*
|
||||
* Accumulate allocation size
|
||||
*/
|
||||
uint64_t header = *s.last_header_;
|
||||
|
||||
if ((header & s.config_.header_size_mask_ & z0) != z0) [[unlikely]] {
|
||||
/* cumulative alloc size doesn't fit in configured header_size_mask bits */
|
||||
++(s.error_count_);
|
||||
s.last_error_ = AllocatorError(error::header_size_mask,
|
||||
s.error_count_,
|
||||
0 /*add_commit_z*/, s.committed_z_, reserved(s));
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
*s.last_header_ = ((header & ~s.config_.header_size_mask_) | z0);
|
||||
|
||||
if (complete_flag) {
|
||||
s.last_header_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
return mem;
|
||||
#endif
|
||||
}
|
||||
|
||||
std::byte *
|
||||
IAllocator_DArena::_alloc(DArena & s,
|
||||
std::size_t req_z,
|
||||
bool store_header_flag,
|
||||
bool remember_header_flag)
|
||||
{
|
||||
scope log(XO_DEBUG(s.config_.debug_flag_));
|
||||
|
||||
assert(padding::is_aligned((size_t)s.free_));
|
||||
/* remember_header_flag -implies-> store_header_flag */
|
||||
assert(store_header_flag || !remember_header_flag);
|
||||
|
||||
/*
|
||||
* free_(pre)
|
||||
* v
|
||||
*
|
||||
* <-------------z1--------------->
|
||||
* < guard >< hz >< req_z >< dz >< guard >
|
||||
*
|
||||
* used <== +++++++++0000zzzz@@@@@@@@@@@@@@@@@ppppppp+++++++++ ==> unallocated
|
||||
*
|
||||
* ^ ^ ^
|
||||
* header mem |
|
||||
* ^ |
|
||||
* last_header_ free_(post)
|
||||
*
|
||||
* [+] guard after each allocation, for simple sanitize checks
|
||||
* [0] unused header bits (avail to application)
|
||||
* [z] record allocation size
|
||||
* [@] new allocated memory
|
||||
* [p] padding (to uintptr_t alignment)
|
||||
*/
|
||||
|
||||
/* non-zero if header feature enabled */
|
||||
size_t hz = 0;
|
||||
/* dz: pad req_z to alignment size (multiple of 8 bytes, probably) */
|
||||
size_t dz = padding::alloc_padding(req_z);
|
||||
size_t z1 = req_z + dz;
|
||||
size_t z0 = req_z + dz;
|
||||
/* if non-zero: will store padded alloc size at the beginning of each allocation
|
||||
* reminder: important to store padded size for correct arena iteration
|
||||
*/
|
||||
uint64_t header = req_z + dz;
|
||||
|
||||
if (store_header_flag) {
|
||||
if ((s.config_.header_size_mask_ & z0) == z0) [[likely]] {
|
||||
hz = sizeof(header);
|
||||
} else {
|
||||
/* req_z doesn't fit in configured header_size_mask bits */
|
||||
++(s.error_count_);
|
||||
s.last_error_ = AllocatorError(error::header_size_mask,
|
||||
s.error_count_,
|
||||
0 /*add_commit_z*/, s.committed_z_, reserved(s));
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
size_t z1 = hz + z0;
|
||||
|
||||
assert(padding::is_aligned(z1));
|
||||
|
||||
if (expand(s, allocated(s) + z1)) [[likely]] {
|
||||
byte * mem = s.free_;
|
||||
if (store_header_flag) {
|
||||
(*(uint64_t *)s.free_) = header;
|
||||
|
||||
if (remember_header_flag) {
|
||||
s.last_header_ = (uint64_t *)s.free_;
|
||||
}
|
||||
}
|
||||
|
||||
byte * mem = s.free_ + hz;
|
||||
|
||||
s.free_ += z1;
|
||||
|
||||
log && log(xtag("self", s.config_.name_),
|
||||
xtag("hz", hz),
|
||||
xtag("z0", req_z),
|
||||
xtag("+pad", dz),
|
||||
xtag("z1", z1),
|
||||
|
|
|
|||
|
|
@ -174,6 +174,44 @@ namespace xo {
|
|||
REQUIRE(a1o.committed() <= a1o.reserved());
|
||||
}
|
||||
|
||||
TEST_CASE("allocator-alloc-2", "[alloc2][Allocator]")
|
||||
{
|
||||
using header_type = AAllocator::header_type;
|
||||
|
||||
/* typed allocator a1o, with object header */
|
||||
ArenaConfig cfg { .name_ = "testarena",
|
||||
.size_ = 64*1024,
|
||||
.store_header_flag_ = true,
|
||||
/* up to 4GB */
|
||||
.header_size_mask_ = 0xffffffff,
|
||||
.debug_flag_ = false,
|
||||
};
|
||||
DArena arena = DArena::map(cfg);
|
||||
obj<AAllocator, DArena> a1o{&arena};
|
||||
|
||||
REQUIRE(a1o.reserved() >= cfg.size_);
|
||||
REQUIRE(a1o.committed() == 0);
|
||||
REQUIRE(a1o.available() == 0);
|
||||
REQUIRE(a1o.allocated() == 0);
|
||||
|
||||
size_t z0 = 1;
|
||||
byte * m0 = a1o.alloc(1);
|
||||
|
||||
REQUIRE(m0);
|
||||
|
||||
header_type* header = (header_type*)(m0 - sizeof(header_type));
|
||||
|
||||
REQUIRE(a1o.contains(header));
|
||||
REQUIRE(((*header) & cfg.header_size_mask_) == padding::with_padding(z0));
|
||||
REQUIRE(a1o.last_error().error_ == error::none);
|
||||
REQUIRE(a1o.last_error().error_seq_ == 0);
|
||||
REQUIRE(a1o.allocated() >= z0);
|
||||
REQUIRE(a1o.allocated() < sizeof(AAllocator::header_type) + z0 + padding::c_alloc_alignment );
|
||||
REQUIRE(a1o.allocated() <= a1o.committed());
|
||||
REQUIRE(a1o.allocated() + a1o.available() == a1o.committed());
|
||||
REQUIRE(a1o.committed() <= a1o.reserved());
|
||||
}
|
||||
|
||||
TEST_CASE("allocator-fail-1", "[alloc2][AAllocator]")
|
||||
{
|
||||
/* typed allocator a1o */
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue