+ xo-alloc + xo-object + xo-alloc docs + GC utests
This commit is contained in:
parent
8970f51dbd
commit
5f46b51f12
32 changed files with 2903 additions and 82 deletions
13
src/alloc/AllocPolicy.cpp
Normal file
13
src/alloc/AllocPolicy.cpp
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/* AllocPolicy.cpp
|
||||
*
|
||||
* author: Roland Conybeare, Jul 2025
|
||||
*/
|
||||
|
||||
#include "AllocPolicy.hpp"
|
||||
|
||||
/* note: inline/.hpp definition not allowed for operator delete */
|
||||
void operator delete(void * ptr) noexcept {
|
||||
xo::xo.free(ptr);
|
||||
}
|
||||
|
||||
/* end AllocPolicy.cpp */
|
||||
|
|
@ -1,29 +1,33 @@
|
|||
/* file LinearAlloc.cpp
|
||||
/* file ArenaAlloc.cpp
|
||||
*
|
||||
* author: Roland Conybeare
|
||||
*/
|
||||
|
||||
#include "LinearAlloc.hpp"
|
||||
#include "ArenaAlloc.hpp"
|
||||
#include "xo/indentlog/scope.hpp"
|
||||
#include "xo/indentlog/print/tag.hpp"
|
||||
#include <cassert>
|
||||
|
||||
namespace xo {
|
||||
namespace gc {
|
||||
LinearAlloc::LinearAlloc(std::size_t rz, std::size_t z)
|
||||
ArenaAlloc::ArenaAlloc(const std::string & name, std::size_t rz, std::size_t z, bool debug_flag)
|
||||
{
|
||||
this->lo_ = (new std::uint8_t [rz + z]);
|
||||
this->name_ = name;
|
||||
this->lo_ = (new std::byte [rz + z]);
|
||||
this->checkpoint_ = lo_;
|
||||
this->free_ptr_ = lo_;
|
||||
this->limit_ = lo_ + z;
|
||||
this->redline_z_ = rz;
|
||||
this->hi_ = limit_ + rz;
|
||||
this->debug_flag_ = debug_flag;
|
||||
|
||||
if (!lo_) {
|
||||
throw std::runtime_error(tostr("LinearAlloc: allocation failed",
|
||||
throw std::runtime_error(tostr("ArenaAlloc: allocation failed",
|
||||
xtag("size", rz + z)));
|
||||
}
|
||||
}
|
||||
|
||||
LinearAlloc::~LinearAlloc()
|
||||
ArenaAlloc::~ArenaAlloc()
|
||||
{
|
||||
delete [] this->lo_;
|
||||
|
||||
|
|
@ -33,17 +37,19 @@ namespace xo {
|
|||
this->checkpoint_ = nullptr;
|
||||
this->free_ptr_ = nullptr;
|
||||
this->limit_ = nullptr;
|
||||
this->redline_z_ = 0;
|
||||
this->hi_ = nullptr;
|
||||
this->debug_flag_ = false;
|
||||
}
|
||||
|
||||
up<LinearAlloc>
|
||||
LinearAlloc::make(std::size_t rz, std::size_t z)
|
||||
up<ArenaAlloc>
|
||||
ArenaAlloc::make(const std::string & name, std::size_t rz, std::size_t z, bool debug_flag)
|
||||
{
|
||||
return up<LinearAlloc>(new LinearAlloc(rz, z));
|
||||
return up<ArenaAlloc>(new ArenaAlloc(name, rz, z, debug_flag));
|
||||
}
|
||||
|
||||
void
|
||||
LinearAlloc::set_free_ptr(std::uint8_t * x)
|
||||
ArenaAlloc::set_free_ptr(std::byte * x)
|
||||
{
|
||||
assert(lo_ <= x);
|
||||
assert(x < limit_);
|
||||
|
|
@ -57,70 +63,79 @@ namespace xo {
|
|||
}
|
||||
|
||||
std::size_t
|
||||
LinearAlloc::size() const {
|
||||
ArenaAlloc::size() const {
|
||||
return limit_ - lo_;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
LinearAlloc::available() const {
|
||||
ArenaAlloc::available() const {
|
||||
return limit_ - free_ptr_;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
LinearAlloc::allocated() const {
|
||||
ArenaAlloc::allocated() const {
|
||||
return free_ptr_ - lo_;
|
||||
}
|
||||
|
||||
bool
|
||||
LinearAlloc::is_before_checkpoint(const std::uint8_t * x) const {
|
||||
ArenaAlloc::contains(const void * x) const {
|
||||
return (lo_ <= x) && (x < hi_);
|
||||
}
|
||||
|
||||
bool
|
||||
ArenaAlloc::is_before_checkpoint(const void * x) const {
|
||||
return (lo_ <= x) && (x < checkpoint_);
|
||||
}
|
||||
|
||||
std::size_t
|
||||
LinearAlloc::before_checkpoint() const
|
||||
ArenaAlloc::before_checkpoint() const
|
||||
{
|
||||
return checkpoint_ - lo_;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
LinearAlloc::after_checkpoint() const
|
||||
ArenaAlloc::after_checkpoint() const
|
||||
{
|
||||
return free_ptr_ - checkpoint_;
|
||||
}
|
||||
|
||||
void
|
||||
LinearAlloc::clear()
|
||||
ArenaAlloc::clear()
|
||||
{
|
||||
this->checkpoint_ = lo_;
|
||||
this->free_ptr_ = lo_;
|
||||
this->limit_ = lo_;
|
||||
this->limit_ = hi_ - redline_z_;
|
||||
}
|
||||
|
||||
void
|
||||
LinearAlloc::checkpoint()
|
||||
ArenaAlloc::checkpoint()
|
||||
{
|
||||
this->checkpoint_ = this->free_ptr_;
|
||||
}
|
||||
|
||||
std::uint8_t *
|
||||
LinearAlloc::alloc(std::size_t z)
|
||||
std::byte *
|
||||
ArenaAlloc::alloc(std::size_t z0)
|
||||
{
|
||||
scope log(XO_DEBUG(debug_flag_));
|
||||
|
||||
/* word size for alignment */
|
||||
constexpr uint32_t c_bpw = sizeof(void*);
|
||||
constexpr uint32_t c_bpw = sizeof(std::uintptr_t);
|
||||
|
||||
std::uintptr_t free_u64 = reinterpret_cast<std::uintptr_t>(free_ptr_);
|
||||
|
||||
assert(free_u64 % c_bpw == 0ul);
|
||||
|
||||
/* round up to multiple of c_bpw */
|
||||
std::uint32_t dz = (c_bpw - (z % c_bpw));
|
||||
z += dz;
|
||||
std::uint32_t dz = alloc_padding(z0);
|
||||
|
||||
assert(z % c_bpw == 0ul);
|
||||
std::size_t z1 = z0 + dz;
|
||||
|
||||
std::uint8_t * retval = this->free_ptr_;
|
||||
assert(z1 % c_bpw == 0ul);
|
||||
|
||||
this->free_ptr_ += z;
|
||||
std::byte * retval = this->free_ptr_;
|
||||
|
||||
this->free_ptr_ += z1;
|
||||
|
||||
log && log(xtag("self", name_), xtag("z0", z0), xtag("+pad", dz), xtag("z1", z1));
|
||||
|
||||
if (free_ptr_ > limit_) {
|
||||
return nullptr;
|
||||
|
|
@ -128,8 +143,14 @@ namespace xo {
|
|||
|
||||
return retval;
|
||||
}
|
||||
|
||||
void
|
||||
ArenaAlloc::release_redline_memory() {
|
||||
this->limit_ = this->hi_;
|
||||
}
|
||||
|
||||
} /*namespace gc*/
|
||||
} /*namespace xo*/
|
||||
|
||||
|
||||
/* end LinearAlloc.cpp */
|
||||
/* end ArenaAlloc.cpp */
|
||||
|
|
@ -2,7 +2,12 @@
|
|||
|
||||
set(SELF_LIB xo_alloc)
|
||||
set(SELF_SRCS
|
||||
LinearAlloc.cpp
|
||||
IAlloc.cpp
|
||||
ArenaAlloc.cpp
|
||||
ListAlloc.cpp
|
||||
GC.cpp
|
||||
Object.cpp
|
||||
Forwarding1.cpp
|
||||
)
|
||||
|
||||
xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS})
|
||||
|
|
|
|||
45
src/alloc/Forwarding1.cpp
Normal file
45
src/alloc/Forwarding1.cpp
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
/* file Forwarding1.cpp
|
||||
*
|
||||
* author: Roland Conybeare, Aug 2025
|
||||
*/
|
||||
|
||||
#include "Forwarding1.hpp"
|
||||
#include <cstddef>
|
||||
#include <cassert>
|
||||
|
||||
namespace xo {
|
||||
namespace obj {
|
||||
Forwarding1::Forwarding1(gp<Object> dest)
|
||||
: dest_{dest}
|
||||
{}
|
||||
|
||||
Object *
|
||||
Forwarding1::_offset_destination(Object * src) const
|
||||
{
|
||||
intptr_t offset = src - static_cast<const Object *>(this);
|
||||
|
||||
return dest_.ptr() + offset;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
Forwarding1::_shallow_size() const {
|
||||
assert(false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
Object *
|
||||
Forwarding1::_shallow_copy() const {
|
||||
assert(false);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
Forwarding1::_forward_children() {
|
||||
assert(false);
|
||||
return 0;
|
||||
}
|
||||
|
||||
} /*namespace obj*/
|
||||
} /*namespace xo*/
|
||||
|
||||
/* end Forwarding1.cpp */
|
||||
492
src/alloc/GC.cpp
Normal file
492
src/alloc/GC.cpp
Normal file
|
|
@ -0,0 +1,492 @@
|
|||
/* GC.cpp
|
||||
*
|
||||
* author: Roland Conybeare, Jul 2025
|
||||
*/
|
||||
|
||||
#include "GC.hpp"
|
||||
#include "Object.hpp"
|
||||
#include "xo/indentlog/scope.hpp"
|
||||
#include <cassert>
|
||||
#include <cstddef>
|
||||
|
||||
namespace xo {
|
||||
namespace gc {
|
||||
void
|
||||
PerGenerationStatistics::include_gc(std::size_t alloc_z,
|
||||
std::size_t before_z,
|
||||
std::size_t after_z,
|
||||
std::size_t promote_z)
|
||||
{
|
||||
this->update_snapshot(after_z);
|
||||
|
||||
new_alloc_z_ += alloc_z;
|
||||
scanned_z_ += before_z;
|
||||
survive_z_ += after_z;
|
||||
promote_z_ += promote_z;
|
||||
}
|
||||
|
||||
void
|
||||
PerGenerationStatistics::update_snapshot(std::size_t after_z)
|
||||
{
|
||||
used_z_ = after_z;
|
||||
}
|
||||
|
||||
void
|
||||
PerGenerationStatistics::display(std::ostream & os) const
|
||||
{
|
||||
os << "<PerGenerationStatistics"
|
||||
<< xtag("used", used_z_)
|
||||
<< xtag("n_gc", n_gc_)
|
||||
<< xtag("new_alloc_z", new_alloc_z_)
|
||||
<< xtag("scanned_z", scanned_z_)
|
||||
<< xtag("survive_z", survive_z_)
|
||||
<< xtag("promote_z", promote_z_)
|
||||
<< ">";
|
||||
}
|
||||
|
||||
void
|
||||
GcStatistics::include_gc(generation upto,
|
||||
std::size_t alloc_z,
|
||||
std::size_t before_z,
|
||||
std::size_t after_z,
|
||||
std::size_t promote_z)
|
||||
{
|
||||
gen_v_[static_cast<std::size_t>(upto)].include_gc(alloc_z, before_z, after_z, promote_z);
|
||||
}
|
||||
|
||||
void
|
||||
GcStatistics::update_snapshot(generation upto,
|
||||
std::size_t after_z)
|
||||
{
|
||||
gen_v_[static_cast<std::size_t>(upto)].update_snapshot(after_z);
|
||||
}
|
||||
|
||||
void
|
||||
GcStatistics::display(std::ostream & os) const
|
||||
{
|
||||
os << "<GcStatistics"
|
||||
<< xtag("gen_v", gen_v_)
|
||||
<< xtag("total_allocated", total_allocated_)
|
||||
// << xtag("per_type_stats", per_type_stats_)
|
||||
<< ">";
|
||||
}
|
||||
|
||||
GC::GC(const Config & config)
|
||||
: config_{config}
|
||||
{
|
||||
enum { NurseryFrom, NurseryTo, TenuredFrom, TenuredTo };
|
||||
|
||||
std::size_t nursery_size = config.initial_nursery_z_;
|
||||
std::size_t tenured_size = config.initial_tenured_z_;
|
||||
|
||||
nursery_[role2int(role::from_space)]
|
||||
= ListAlloc::make("NA", nursery_size, 2 * nursery_size, config.debug_flag_);
|
||||
nursery_[role2int(role::to_space) ]
|
||||
= ListAlloc::make("NB", nursery_size, 2 * nursery_size, config.debug_flag_);
|
||||
|
||||
tenured_[role2int(role::from_space)]
|
||||
= ListAlloc::make("TA", tenured_size, 2 * tenured_size, config.debug_flag_);
|
||||
tenured_[role2int(role::to_space) ]
|
||||
= ListAlloc::make("TB", tenured_size, 2 * tenured_size, config.debug_flag_);
|
||||
|
||||
this->checkpoint();
|
||||
}
|
||||
|
||||
up<GC>
|
||||
GC::make(const Config & config)
|
||||
{
|
||||
GC * gc = new GC(config);
|
||||
|
||||
return up<GC>{gc};
|
||||
}
|
||||
|
||||
std::size_t
|
||||
GC::size() const
|
||||
{
|
||||
return nursery_[role2int(role::to_space)]->size() + tenured_[role2int(role::to_space)]->size();
|
||||
}
|
||||
|
||||
std::size_t
|
||||
GC::allocated() const
|
||||
{
|
||||
return (nursery_[role2int(role::to_space)]->allocated()
|
||||
+ tenured_[role2int(role::to_space)]->allocated());
|
||||
}
|
||||
|
||||
std::size_t
|
||||
GC::available() const
|
||||
{
|
||||
return nursery_[role2int(role::to_space)]->available();
|
||||
}
|
||||
|
||||
bool
|
||||
GC::fromspace_contains(const void * x) const
|
||||
{
|
||||
return (nursery_[role2int(role::from_space)]->contains(x)
|
||||
|| tenured_[role2int(role::from_space)]->contains(x));
|
||||
}
|
||||
|
||||
bool
|
||||
GC::contains(const void * x) const
|
||||
{
|
||||
return (nursery_[role2int(role::to_space)]->contains(x)
|
||||
|| tenured_[role2int(role::to_space)]->contains(x));
|
||||
}
|
||||
|
||||
bool
|
||||
GC::is_before_checkpoint(const void * x) const
|
||||
{
|
||||
return nursery_[role2int(role::to_space)]->is_before_checkpoint(x);
|
||||
}
|
||||
|
||||
std::size_t
|
||||
GC::before_checkpoint() const
|
||||
{
|
||||
return nursery_[role2int(role::to_space)]->before_checkpoint();
|
||||
}
|
||||
|
||||
std::size_t
|
||||
GC::after_checkpoint() const
|
||||
{
|
||||
return nursery_[role2int(role::to_space)]->after_checkpoint();
|
||||
}
|
||||
|
||||
generation
|
||||
GC::fromspace_generation_of(const void * x) const
|
||||
{
|
||||
if (tenured_[role2int(role::from_space)]->contains(x))
|
||||
return generation::tenured;
|
||||
|
||||
return generation::nursery;
|
||||
}
|
||||
|
||||
generation
|
||||
GC::generation_of(const void * x) const
|
||||
{
|
||||
if (tenured_[role2int(role::to_space)]->contains(x))
|
||||
return generation::tenured;
|
||||
|
||||
return generation::nursery;
|
||||
}
|
||||
|
||||
std::byte *
|
||||
GC::free_ptr(generation gen)
|
||||
{
|
||||
switch(gen) {
|
||||
case generation::nursery:
|
||||
return nursery_[role2int(role::to_space)]->free_ptr();
|
||||
case generation::tenured:
|
||||
return tenured_[role2int(role::to_space)]->free_ptr();
|
||||
case generation::N:
|
||||
assert(false);
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void
|
||||
GC::clear()
|
||||
{
|
||||
nursery_[role2int(role::from_space)]->clear();
|
||||
nursery_[role2int(role::to_space) ]->clear();
|
||||
|
||||
tenured_[role2int(role::from_space)]->clear();
|
||||
tenured_[role2int(role::to_space) ]->clear();
|
||||
}
|
||||
|
||||
void
|
||||
GC::add_gc_root(Object ** addr)
|
||||
{
|
||||
gc_root_v_.push_back(addr);
|
||||
}
|
||||
|
||||
void
|
||||
GC::checkpoint()
|
||||
{
|
||||
nursery_[role2int(role::to_space) ]->checkpoint();
|
||||
}
|
||||
|
||||
std::byte *
|
||||
GC::alloc(std::size_t z)
|
||||
{
|
||||
std::byte * x = nursery_[role2int(role::to_space)]->alloc(z);
|
||||
|
||||
if (!x) {
|
||||
this->request_gc(generation::nursery);
|
||||
|
||||
if (incr_gc_pending_ || full_gc_pending_)
|
||||
nursery_[role2int(role::to_space)]->release_redline_memory();
|
||||
|
||||
/* try (just once) more, maybe request fits in redline space */
|
||||
x = nursery_[role2int(role::to_space)]->alloc(z);
|
||||
|
||||
assert(x);
|
||||
}
|
||||
|
||||
return x;
|
||||
}
|
||||
|
||||
std::byte *
|
||||
GC::alloc_gc_copy(std::size_t z, const void * src)
|
||||
{
|
||||
scope log(XO_DEBUG(config_.debug_flag_), xtag("z", z), xtag("+pad", IAlloc::alloc_padding(z)));
|
||||
|
||||
generation g = this->fromspace_generation_of(src);
|
||||
|
||||
std::byte * retval = nullptr;
|
||||
|
||||
if (g == generation::tenured)
|
||||
{
|
||||
log && log("tenured");
|
||||
|
||||
retval = tenured_[role2int(role::to_space)]->alloc(z);
|
||||
} else if (nursery_[role2int(role::from_space)]->is_before_checkpoint(src))
|
||||
{
|
||||
log && log("promote");
|
||||
|
||||
/* nursery object has survived 2nd collection cycle
|
||||
* -> promote into tenured generation
|
||||
*/
|
||||
retval = tenured_[role2int(role::to_space)]->alloc(z);
|
||||
|
||||
this->gc_statistics_.total_promoted_ += IAlloc::with_padding(z);
|
||||
} else {
|
||||
log && log("nursery");
|
||||
|
||||
retval = nursery_[role2int(role::to_space)]->alloc(z);
|
||||
|
||||
if (!retval) {
|
||||
/* nursery space exhausted */
|
||||
|
||||
this->request_gc(generation::nursery);
|
||||
|
||||
nursery_[role2int(role::to_space)]->release_redline_memory();
|
||||
|
||||
retval = nursery_[role2int(role::to_space)]->alloc(z);
|
||||
}
|
||||
}
|
||||
|
||||
assert(retval);
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
void
|
||||
GC::release_redline_memory()
|
||||
{
|
||||
// not supported feature for GC
|
||||
}
|
||||
|
||||
void
|
||||
GC::swap_nursery()
|
||||
{
|
||||
up<ListAlloc> tmp = std::move(nursery_[role2int(role::to_space)]);
|
||||
nursery_[role2int(role::to_space)] = std::move(nursery_[role2int(role::from_space)]);
|
||||
nursery_[role2int(role::from_space)] = std::move(tmp);
|
||||
}
|
||||
|
||||
void
|
||||
GC::swap_tenured()
|
||||
{
|
||||
up<ListAlloc> tmp = std::move(tenured_[role2int(role::to_space)]);
|
||||
tenured_[role2int(role::to_space)] = std::move(tenured_[role2int(role::from_space)]);
|
||||
tenured_[role2int(role::from_space)] = std::move(tmp);
|
||||
}
|
||||
|
||||
void
|
||||
GC::swap_spaces(generation target)
|
||||
{
|
||||
// will be copying into storage currently labelled FromSpace
|
||||
|
||||
/* gc will copy some to-be-determined amount in [0..promote_z]
|
||||
from nursery->tenured generation.
|
||||
*/
|
||||
std::size_t promote_z = nursery_[role2int(role::to_space)]->before_checkpoint();
|
||||
if (target == generation::tenured) {
|
||||
/* gc on tenured generation may need this much space */
|
||||
std::size_t tenured_z = (tenured_[role2int(role::to_space)]->allocated()
|
||||
+ promote_z
|
||||
+ full_gc_threshold_);
|
||||
|
||||
tenured_[role2int(role::from_space)]->reset(tenured_z);
|
||||
|
||||
this->swap_tenured();
|
||||
} else {
|
||||
if (tenured_[role2int(role::to_space)]->available() < promote_z) {
|
||||
tenured_[role2int(role::to_space)]->expand(promote_z);
|
||||
}
|
||||
}
|
||||
|
||||
nursery_[role2int(role::from_space)]->reset(nursery_[role2int(role::to_space)]->allocated()
|
||||
- promote_z
|
||||
+ incr_gc_threshold_);
|
||||
this->swap_nursery();
|
||||
} /*swap_spaces*/
|
||||
|
||||
void
|
||||
GC::copy_object(Object ** pp_object, generation upto, ObjectStatistics * object_stats)
|
||||
{
|
||||
void * object_address = *pp_object;
|
||||
|
||||
if (nursery_[role2int(role::to_space)]->contains(object_address)
|
||||
|| ((upto == generation::tenured)
|
||||
&& tenured_[role2int(role::to_space)]->contains(object_address)))
|
||||
{
|
||||
/* global is already in to-space */
|
||||
;
|
||||
} else if((upto == generation::nursery) && tenured_[role2int(role::to_space)]->contains(object_address))
|
||||
{
|
||||
/* skip tenured objects when incremental collection */
|
||||
;
|
||||
} else {
|
||||
*pp_object = Object::_deep_move(*pp_object, this, object_stats);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
GC::copy_globals(generation upto)
|
||||
{
|
||||
for (Object ** pp_root : gc_root_v_) {
|
||||
this->copy_object(pp_root, upto, &gc_statistics_.per_type_stats_);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
GC::cleanup_phase(generation upto)
|
||||
{
|
||||
scope log(XO_DEBUG(config_.debug_flag_));
|
||||
|
||||
std::size_t N_allocated = nursery_[role2int(role::from_space)]->after_checkpoint();
|
||||
std::size_t T_allocated = tenured_[role2int(role::from_space)]->after_checkpoint();
|
||||
|
||||
std::size_t N_before_gc = nursery_[role2int(role::from_space)]->allocated();
|
||||
std::size_t T_before_gc = tenured_[role2int(role::from_space)]->allocated();
|
||||
|
||||
std::size_t N_after_gc = nursery_[role2int(role::to_space)]->allocated();
|
||||
std::size_t T_after_gc = tenured_[role2int(role::to_space)]->allocated();
|
||||
//std::byte * N_free_ptr = nursery_[role2int(role::to_space)]->free_ptr();
|
||||
|
||||
std::size_t promote_z = gc_statistics_.total_promoted_ - gc_statistics_.total_promoted_sab_;
|
||||
|
||||
this->nursery_[role2int(role::from_space)]->reset(0);
|
||||
this->tenured_[role2int(role::from_space)]->reset(0);
|
||||
|
||||
/* objects currenty in to-space nursery have survived one collection */
|
||||
this->nursery_[role2int(role::to_space)]->checkpoint();
|
||||
|
||||
// nursery_[role2int(role::to_space)]->set_redline(nursery_[role2int(role::to_space)]->allocated() + incr_gc_threshold_)
|
||||
|
||||
if (upto == generation::tenured)
|
||||
this->tenured_[role2int(role::to_space)]->checkpoint();
|
||||
|
||||
if (log) {
|
||||
log(xtag("N_allocated", N_allocated));
|
||||
log(xtag("N_before_gc", N_before_gc));
|
||||
log(xtag("N_after_gc", N_after_gc));
|
||||
log(xtag("T_allocated", T_allocated));
|
||||
log(xtag("T_before_gc", T_before_gc));
|
||||
log(xtag("T_after_gc", T_after_gc));
|
||||
}
|
||||
|
||||
this->incr_gc_pending_ = false;
|
||||
this->gc_statistics_.include_gc(generation::nursery, N_allocated, N_before_gc, N_after_gc, promote_z);
|
||||
|
||||
if (upto == generation::tenured) {
|
||||
this->full_gc_pending_ = false;
|
||||
this->gc_statistics_.include_gc(generation::tenured, T_allocated, T_before_gc, T_after_gc, 0);
|
||||
} else {
|
||||
// still want to update tenured stats for current alloc size
|
||||
this->gc_statistics_.update_snapshot(generation::tenured, T_after_gc);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
GC::execute_gc(generation target)
|
||||
{
|
||||
scope log(XO_DEBUG(config_.debug_flag_));
|
||||
|
||||
bool full_move = (target == generation::tenured);
|
||||
|
||||
// TODO: RAII version in case of exceptions
|
||||
this->runstate_ = GCRunstate(true /*in_progress*/, full_move);
|
||||
|
||||
log && log("step 0: snapshot alloc stats");
|
||||
|
||||
/* new allocation since last GC */
|
||||
std::size_t new_alloc = this->after_checkpoint();
|
||||
|
||||
++(gc_statistics_.gen_v_[static_cast<std::size_t>(target)].n_gc_);
|
||||
gc_statistics_.total_allocated_ += new_alloc;
|
||||
gc_statistics_.total_promoted_sab_ = gc_statistics_.total_promoted_;
|
||||
|
||||
log && log(xtag("new_alloc", new_alloc));
|
||||
|
||||
log && log("step 1: swap to/from roles");
|
||||
|
||||
this->swap_spaces(target);
|
||||
|
||||
log && log("step 2a: copy globals");
|
||||
|
||||
this->copy_globals(target);
|
||||
|
||||
log && log("step 2b: TODO: copy pinned");
|
||||
|
||||
log && log("step 3: TODO: forward mutation log");
|
||||
|
||||
log && log("step 4: TODO: notify destructor log");
|
||||
|
||||
log && log("step 5: TODO: keep reachable weak pointers");
|
||||
|
||||
log && log("step 6: cleanup");
|
||||
|
||||
this->cleanup_phase(target);
|
||||
|
||||
this->runstate_ = GCRunstate();
|
||||
|
||||
log && log("statistics:");
|
||||
log && log(gc_statistics_);
|
||||
}
|
||||
|
||||
void
|
||||
GC::request_gc(generation target)
|
||||
{
|
||||
if (!runstate_.in_progress() && (gc_enabled_ == 0)) {
|
||||
if (!config_.allow_incremental_gc_)
|
||||
target = generation::tenured;
|
||||
|
||||
if ((target == generation::nursery)
|
||||
&& (tenured_[role2int(role::to_space)]->after_checkpoint() > full_gc_threshold_))
|
||||
{
|
||||
/** full collection when >= @ref full_gc_threshold_ bytes added to tenured
|
||||
* generation, since last full collection
|
||||
**/
|
||||
target = generation::tenured;
|
||||
}
|
||||
|
||||
this->execute_gc(target);
|
||||
} else {
|
||||
this->incr_gc_pending_ = true;
|
||||
if (target == generation::tenured)
|
||||
this->full_gc_pending_ = true;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
GC::disable_gc() {
|
||||
--gc_enabled_;
|
||||
}
|
||||
|
||||
void
|
||||
GC::enable_gc() {
|
||||
++gc_enabled_;
|
||||
|
||||
if (gc_enabled_ == 0) {
|
||||
/* unblock gc */
|
||||
if (incr_gc_pending_)
|
||||
this->request_gc(full_gc_pending_ ? generation::tenured : generation::nursery);
|
||||
}
|
||||
}
|
||||
} /*namespace gc*/
|
||||
} /*namespace xo*/
|
||||
|
||||
/* end GC.cpp */
|
||||
54
src/alloc/IAlloc.cpp
Normal file
54
src/alloc/IAlloc.cpp
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
/* @file IAlloc.cpp
|
||||
*
|
||||
* author: Roland Conybeare, Aug 2025
|
||||
*/
|
||||
|
||||
#include "IAlloc.hpp"
|
||||
#include <cassert>
|
||||
#include <cstddef>
|
||||
|
||||
namespace xo {
|
||||
namespace gc {
|
||||
|
||||
std::uint32_t
|
||||
IAlloc::alloc_padding(std::size_t z)
|
||||
{
|
||||
/* word size for alignment */
|
||||
constexpr uint32_t c_bpw = sizeof(std::uintptr_t);
|
||||
|
||||
/* round up to multiple of c_bpw, but map 0 -> 0
|
||||
* (table assuming c_bpw==8)
|
||||
*
|
||||
* z%c_bpw dz
|
||||
* ------------
|
||||
* 0 0
|
||||
* 1 7
|
||||
* 2 6
|
||||
* .. ..
|
||||
* 7 1
|
||||
*/
|
||||
std::uint32_t dz = (c_bpw - (z % c_bpw)) % c_bpw;
|
||||
z += dz;
|
||||
|
||||
assert(z % c_bpw == 0ul);
|
||||
|
||||
return dz;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
IAlloc::with_padding(std::size_t z)
|
||||
{
|
||||
return z + alloc_padding(z);
|
||||
}
|
||||
|
||||
std::byte *
|
||||
IAlloc::alloc_gc_copy(std::size_t /*z*/, const void * /*src*/)
|
||||
{
|
||||
assert(false);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
} /*namespace gc*/
|
||||
} /*namespace xo*/
|
||||
|
||||
/* end IAlloc.cpp */
|
||||
318
src/alloc/ListAlloc.cpp
Normal file
318
src/alloc/ListAlloc.cpp
Normal file
|
|
@ -0,0 +1,318 @@
|
|||
/* file ListAlloc.cpp
|
||||
*
|
||||
* author: Roland Conybeare, Jul 2025
|
||||
*/
|
||||
|
||||
#include "ListAlloc.hpp"
|
||||
#include "ArenaAlloc.hpp"
|
||||
#include <cassert>
|
||||
#include <cstddef>
|
||||
|
||||
namespace xo {
|
||||
namespace gc {
|
||||
ListAlloc::ListAlloc(std::unique_ptr<ArenaAlloc> hd,
|
||||
ArenaAlloc * marked,
|
||||
std::size_t cz, std::size_t nz, std::size_t tz,
|
||||
bool use_redline,
|
||||
bool debug_flag)
|
||||
: start_z_{cz},
|
||||
hd_{std::move(hd)},
|
||||
marked_{marked},
|
||||
full_l_{},
|
||||
current_z_{cz},
|
||||
next_z_{nz},
|
||||
total_z_{tz},
|
||||
use_redline_{use_redline},
|
||||
debug_flag_{debug_flag}
|
||||
{}
|
||||
|
||||
ListAlloc::~ListAlloc()
|
||||
{
|
||||
this->clear();
|
||||
}
|
||||
|
||||
up<ListAlloc>
|
||||
ListAlloc::make(const std::string & name, std::size_t cz, std::size_t nz, bool debug_flag)
|
||||
{
|
||||
std::unique_ptr<ArenaAlloc> hd{ArenaAlloc::make(name, 0, cz, debug_flag)};
|
||||
|
||||
if (!hd)
|
||||
return nullptr;
|
||||
|
||||
ArenaAlloc * marked = nullptr;
|
||||
|
||||
up<ListAlloc> retval{new ListAlloc(std::move(hd),
|
||||
marked,
|
||||
cz, nz, cz,
|
||||
false /*!use_redline*/,
|
||||
debug_flag)};
|
||||
|
||||
return retval;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
ListAlloc::size() const {
|
||||
return total_z_;
|
||||
}
|
||||
|
||||
std::byte *
|
||||
ListAlloc::free_ptr() const {
|
||||
return hd_->free_ptr();
|
||||
}
|
||||
|
||||
std::size_t
|
||||
ListAlloc::available() const {
|
||||
if (hd_) {
|
||||
/* can only allocate from @ref hd_,
|
||||
* so even if there were available space in @ref full_l_,
|
||||
* it's not accessible to ListAlloc.
|
||||
*/
|
||||
|
||||
return hd_->available();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
ListAlloc::allocated() const {
|
||||
std::size_t total = 0;
|
||||
|
||||
if (hd_)
|
||||
total += hd_->allocated();
|
||||
|
||||
for (const auto & alloc : full_l_)
|
||||
total += alloc->allocated();
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
bool
|
||||
ListAlloc::contains(const void * x) const {
|
||||
if (hd_ && hd_->contains(x))
|
||||
return true;
|
||||
|
||||
for (const auto & alloc : full_l_) {
|
||||
if (alloc->contains(x))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
ListAlloc::is_before_checkpoint(const void * x) const {
|
||||
if (!marked_)
|
||||
return false;
|
||||
|
||||
if ((marked_ == hd_.get()) && hd_->contains(x))
|
||||
return hd_->is_before_checkpoint(x);
|
||||
|
||||
/*
|
||||
* 1. allocs in full_l_ appear in youngest-to-oldest order
|
||||
* 2. allocators that appear before marked_ in full_l_ count as 'after checkpoint'
|
||||
* 3. allocators that appear after marked_ in full_l_ count as 'before checkpoint'
|
||||
*/
|
||||
|
||||
bool younger_than_marked = true;
|
||||
|
||||
for (const auto & alloc : full_l_) {
|
||||
if (younger_than_marked) {
|
||||
if (alloc.get() == marked_) {
|
||||
/* nothing else to test on this iteration,
|
||||
* already checked .marked_ specifically
|
||||
*/
|
||||
younger_than_marked = false;
|
||||
} else {
|
||||
/* after checkpoint */
|
||||
if (alloc->contains(x))
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
if (alloc->contains(x))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
ListAlloc::before_checkpoint() const
|
||||
{
|
||||
if (marked_) {
|
||||
if (full_l_.empty()) {
|
||||
assert(marked_ == hd_.get());
|
||||
|
||||
return marked_->before_checkpoint();
|
||||
}
|
||||
} else {
|
||||
/* count everything allocated */
|
||||
return this->allocated();
|
||||
}
|
||||
|
||||
std::size_t z = 0;
|
||||
|
||||
/* control here: .marked & .full_l non-empty. */
|
||||
if (hd_.get() == marked_) {
|
||||
z += hd_->before_checkpoint();
|
||||
|
||||
/* anything in .full_l older than marked .hd */
|
||||
for (const auto & alloc : full_l_) {
|
||||
z += alloc->allocated();
|
||||
}
|
||||
|
||||
return z;
|
||||
} else {
|
||||
/* messiest case: .marked is true,
|
||||
* and not the youngest arena
|
||||
*/
|
||||
bool younger_than_marked = true;
|
||||
|
||||
for (const auto & alloc : full_l_) {
|
||||
if (younger_than_marked) {
|
||||
if (alloc.get() == marked_) {
|
||||
younger_than_marked = false;
|
||||
z += marked_->before_checkpoint();
|
||||
} else {
|
||||
;
|
||||
}
|
||||
} else {
|
||||
z += alloc->allocated();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return z;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
ListAlloc::after_checkpoint() const
|
||||
{
|
||||
if (!marked_)
|
||||
return 0;
|
||||
|
||||
if (full_l_.empty()) {
|
||||
assert(marked_ == hd_.get());
|
||||
|
||||
return marked_->after_checkpoint();
|
||||
}
|
||||
|
||||
bool younger_than_marked = true;
|
||||
|
||||
std::size_t z = 0;
|
||||
|
||||
for (const auto & alloc : full_l_) {
|
||||
if (younger_than_marked) {
|
||||
if (alloc.get() == marked_) {
|
||||
younger_than_marked = false;
|
||||
z += marked_->after_checkpoint();
|
||||
break;
|
||||
} else {
|
||||
z += alloc->allocated();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return z;
|
||||
}
|
||||
|
||||
void
|
||||
ListAlloc::clear() {
|
||||
// general hygiene
|
||||
start_z_ = 0;
|
||||
hd_.reset();
|
||||
marked_ = nullptr;
|
||||
full_l_.clear();
|
||||
current_z_ = 0;
|
||||
next_z_ = 0;
|
||||
total_z_ = 0;
|
||||
use_redline_ = false;
|
||||
}
|
||||
|
||||
bool
|
||||
ListAlloc::reset(std::size_t z)
|
||||
{
|
||||
// warning: hd_->size() does not include redline memory
|
||||
hd_->release_redline_memory();
|
||||
|
||||
bool recycle_head_bucket = hd_ && (z <= hd_->size());
|
||||
|
||||
this->full_l_.clear();
|
||||
this->marked_ = nullptr;
|
||||
this->redlined_flag_ = false;
|
||||
|
||||
if (recycle_head_bucket) {
|
||||
this->hd_->clear();
|
||||
this->total_z_ = hd_->size();
|
||||
|
||||
return true;
|
||||
} else {
|
||||
this->hd_.reset(nullptr);
|
||||
this->total_z_ = 0;
|
||||
|
||||
return this->expand(z);
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
ListAlloc::expand(std::size_t z)
|
||||
{
|
||||
std::size_t cz = current_z_;
|
||||
std::size_t nz = next_z_;
|
||||
std::size_t tz;
|
||||
|
||||
do {
|
||||
tz = cz + nz;
|
||||
cz = nz;
|
||||
nz = tz;
|
||||
} while (cz < z);
|
||||
|
||||
std::string name = hd_->name() + "+exp";
|
||||
|
||||
std::unique_ptr<ArenaAlloc> new_alloc = ArenaAlloc::make(name, 0, cz, debug_flag_);
|
||||
|
||||
if (!new_alloc)
|
||||
return false;
|
||||
|
||||
this->current_z_ = cz;
|
||||
this->next_z_ = nz;
|
||||
this->total_z_ += cz;
|
||||
|
||||
this->hd_ = std::move(new_alloc);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
ListAlloc::checkpoint() {
|
||||
hd_->checkpoint();
|
||||
|
||||
this->marked_ = hd_.get();
|
||||
}
|
||||
|
||||
std::byte *
|
||||
ListAlloc::alloc(std::size_t z) {
|
||||
std::byte * retval = hd_->alloc(z);
|
||||
|
||||
if (retval)
|
||||
return retval;
|
||||
|
||||
if (this->expand(z))
|
||||
return hd_->alloc(z);
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void
|
||||
ListAlloc::release_redline_memory()
|
||||
{
|
||||
if (use_redline_)
|
||||
redlined_flag_ = true;
|
||||
|
||||
this->hd_->release_redline_memory();
|
||||
}
|
||||
} /*namespace gc*/
|
||||
} /*namespace xo*/
|
||||
|
||||
/* end ListAlloc.cpp */
|
||||
196
src/alloc/Object.cpp
Normal file
196
src/alloc/Object.cpp
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
/* Object.cpp
|
||||
*
|
||||
* author: Roalnd Conybeare, Jul 2025
|
||||
*/
|
||||
|
||||
#include "Object.hpp"
|
||||
#include "GC.hpp"
|
||||
#include "Forwarding1.hpp"
|
||||
|
||||
using xo::obj::Forwarding1;
|
||||
|
||||
void *
|
||||
operator new (std::size_t z, const xo::Cpof & cpof)
|
||||
{
|
||||
using xo::gc::GC;
|
||||
|
||||
GC * gc = reinterpret_cast<GC *>(xo::Object::mm);
|
||||
|
||||
return gc->alloc_gc_copy(z, cpof.src_);
|
||||
}
|
||||
|
||||
namespace xo {
|
||||
gc::IAlloc *
|
||||
Object::mm = nullptr;
|
||||
|
||||
Object *
|
||||
Object::_forward(Object * src, gc::GC * gc)
|
||||
{
|
||||
if (!src)
|
||||
return src;
|
||||
|
||||
if (src->_is_forwarded())
|
||||
return src->_offset_destination(src);
|
||||
|
||||
bool full_move = gc->runstate().full_move();
|
||||
|
||||
if (!full_move && (gc->generation_of(src) == gc::generation::tenured)) {
|
||||
/* don't move tenured objects during incremental collection */
|
||||
return src;
|
||||
}
|
||||
|
||||
Object::_shallow_move(src, gc);
|
||||
|
||||
/* *src is now a forwarding pointer to copy in to-space */
|
||||
|
||||
return src->_offset_destination(src);
|
||||
}
|
||||
|
||||
Object *
|
||||
Object::_deep_move(Object * from_src, gc::GC * gc, gc::ObjectStatistics * /*stats*/)
|
||||
{
|
||||
using gc::generation;
|
||||
|
||||
if (!from_src)
|
||||
return nullptr;
|
||||
|
||||
Object * retval = from_src->_destination();
|
||||
|
||||
if (retval)
|
||||
return retval;
|
||||
|
||||
bool full_move = gc->runstate().full_move();
|
||||
|
||||
if (!full_move && gc->generation_of(from_src) == generation::tenured) {
|
||||
/** incremental collection does not move already-tenured objects **/
|
||||
return from_src;
|
||||
}
|
||||
|
||||
/**
|
||||
* To-space:
|
||||
*
|
||||
* to_lo = start of to-space
|
||||
* w,W = white objects. An object x is white if x + all immediate children of x are in to-space
|
||||
* (also implies this GC cycle put it there)
|
||||
* g,G = grey objects. An object x is gray if it's in to-space,
|
||||
* but possibly has >0 black children
|
||||
* _ = free to-space memory
|
||||
* N = nursery space
|
||||
* T = tenured space
|
||||
*
|
||||
* wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________...
|
||||
* ^ ^ ^
|
||||
* to_lo grey_lo(N) free_ptr(N)
|
||||
*
|
||||
* After moving children of one object, advancing {nursery_grey_lo, nursery_free_ptr}
|
||||
*
|
||||
* wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG______...
|
||||
* ^ ^ ^
|
||||
* to_lo grey_lo(N) free_ptr(N)
|
||||
*
|
||||
* Invariant:
|
||||
*
|
||||
* objects in [to_lo, gray_lo) are white.
|
||||
* all gray objects are in [gray_lo, free_ptr)
|
||||
* memory starting at free_ptr is free.
|
||||
*
|
||||
* deep_move terminates when gray_lo catches up to free_ptr
|
||||
*
|
||||
* Above is simplified. Complication is that GC (including incremental) may
|
||||
* promote objects from nursery (N) to tenured (T)
|
||||
*
|
||||
* So more accurate before/after picture
|
||||
*
|
||||
* N wwwwwwwwwwwwwwwwwwwggggggggggggggggggggg_________________...
|
||||
* ^ ^ ^
|
||||
* to_lo(N) grey_lo(N) free_ptr(N)
|
||||
*
|
||||
* T wwwwwwwwwwwwwwgggggggggggg_______________________________...
|
||||
* ^ ^ ^
|
||||
* to_lo(T) grey_lo(T) free_ptr(N)
|
||||
*
|
||||
* After moving children of one object, advancing {nursery_grey_lo, nursery_free_ptr}
|
||||
*
|
||||
* N wwwwwwwwwwwwwwwwwwwWWWWgggggggggggggggggGGGGGGGGGGG_____...
|
||||
* ^ ^ ^
|
||||
* to_lo(N) grey_lo(N) free_ptr(N)
|
||||
*
|
||||
* T wwwwwwwwwwwwwwggggggggggggGGGGG_________________________...
|
||||
* ^ ^ ^
|
||||
* to_lo(T) grey_lo(T) free_ptr(T)
|
||||
*
|
||||
* deep_move terminates when both:
|
||||
* - gray_lo(N) catches up with free_ptr(N)
|
||||
* - gray_lo(T) catches up with free_ptr(T)
|
||||
*
|
||||
**/
|
||||
|
||||
std::array<std::byte *, gen2int(generation::N)> gray_lo_v
|
||||
= { gc->free_ptr(generation::nursery), gc->free_ptr(generation::tenured) };
|
||||
|
||||
Object * to_src = Object::_shallow_move(from_src, gc);
|
||||
|
||||
std::size_t fixup_work = 0;
|
||||
do {
|
||||
fixup_work = 0;
|
||||
|
||||
auto fixup_generation = [gc, &gray_lo_v](generation gen) {
|
||||
std::size_t work = 0;
|
||||
while(gray_lo_v[gen2int(gen)] < gc->free_ptr(gen)) {
|
||||
Object * x = reinterpret_cast<Object *>(gray_lo_v[gen2int(gen)]);
|
||||
|
||||
// update per-class stats here
|
||||
|
||||
std::size_t xz = x->_forward_children();
|
||||
|
||||
// must pad xz to multiple of word size,
|
||||
// to match behavior of LinearAlloc::alloc()
|
||||
//
|
||||
xz += gc::IAlloc::alloc_padding(xz);
|
||||
|
||||
gray_lo_v[gen2int(gen)] += xz;
|
||||
++work;
|
||||
}
|
||||
|
||||
return work;
|
||||
};
|
||||
|
||||
fixup_work += fixup_generation(generation::nursery);
|
||||
fixup_work += fixup_generation(generation::tenured);
|
||||
} while (fixup_work > 0);
|
||||
|
||||
return to_src;
|
||||
} /*deep_move*/
|
||||
|
||||
Object *
|
||||
Object::_shallow_move(Object * src, gc::GC * gc)
|
||||
{
|
||||
/* filter for source objects that are owned by GC.
|
||||
* Care required though -- during GC from/to spaces have been swapped already
|
||||
*/
|
||||
if (gc->fromspace_contains(src))
|
||||
{
|
||||
Object * dest = src->_shallow_copy();
|
||||
|
||||
if (dest != src)
|
||||
src->_forward_to(dest);
|
||||
|
||||
return dest;
|
||||
} else {
|
||||
return src;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
Object::_forward_to(Object * dest)
|
||||
{
|
||||
char * mem = reinterpret_cast<char *>(this);
|
||||
|
||||
Forwarding1 * fwd = new (mem) Forwarding1(dest);
|
||||
|
||||
(void)fwd;
|
||||
}
|
||||
|
||||
} /*namespace xo*/
|
||||
|
||||
/* end Object.cpp*/
|
||||
Loading…
Add table
Add a link
Reference in a new issue