diff --git a/CMakeLists.txt b/CMakeLists.txt index eebf3aff..0e9de5c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# alloc/CMakeLists.txt +# xo-alloc/CMakeLists.txt cmake_minimum_required(VERSION 3.10) diff --git a/include/xo/alloc/ArenaAlloc.hpp b/include/xo/alloc/ArenaAlloc.hpp index 8e162a95..63de43e8 100644 --- a/include/xo/alloc/ArenaAlloc.hpp +++ b/include/xo/alloc/ArenaAlloc.hpp @@ -24,6 +24,8 @@ namespace xo { **/ class ArenaAlloc : public IAlloc { public: + ArenaAlloc(const ArenaAlloc &) = delete; + ArenaAlloc(ArenaAlloc &&) = delete; ~ArenaAlloc(); /** create allocator with capacity @p z, @@ -38,10 +40,14 @@ namespace xo { void capture_object_statistics(capture_phase phase, ObjectStatistics * p_dest) const; + /** expand available (i.e. committed) space to size @p z **/ + bool expand(std::size_t z); + // inherited from IAlloc... virtual const std::string & name() const final override; virtual std::size_t size() const final override; + virtual std::size_t committed() const final override; virtual std::size_t available() const final override; virtual std::size_t allocated() const final override; virtual bool contains(const void * x) const final override; @@ -54,6 +60,9 @@ namespace xo { virtual void checkpoint() final override; virtual std::byte * alloc(std::size_t z) final override; + ArenaAlloc & operator=(const ArenaAlloc &) = delete; + ArenaAlloc & operator=(ArenaAlloc &&) = delete; + private: ArenaAlloc(const std::string & name, std::size_t z, bool debug_flag); @@ -67,8 +76,15 @@ namespace xo { /** optional instance name, for diagnostics **/ std::string name_; + /** size of a VM page **/ + std::size_t page_z_; + /** allocator owns memory in range [@ref lo_, @ref hi_) **/ std::byte * lo_ = nullptr; + /** prefix of this size is actually committed. + * Remainder uses uncommitted virtual address space + **/ + std::size_t committed_z_ = 0; /** checkpoint (for GC support); divides objects into * older (addresses below checkpoint) * and younger (addresses above checkpoint) @@ -76,9 +92,9 @@ namespace xo { std::byte * checkpoint_; /** free pointer. memory in range [@ref free_, @ref limit_) available **/ std::byte * free_ptr_ = nullptr; - /** soft limit: end of released memory **/ + /** soft limit: end of committed virtual memory **/ std::byte * limit_ = nullptr; - /** hard limit: end of allocated memory **/ + /** hard limit: end of reserved virtual memory **/ std::byte * hi_ = nullptr; /** true to enable detailed debug logging **/ bool debug_flag_ = false; diff --git a/include/xo/alloc/GC.hpp b/include/xo/alloc/GC.hpp index 4560f3ac..c5432935 100644 --- a/include/xo/alloc/GC.hpp +++ b/include/xo/alloc/GC.hpp @@ -164,7 +164,8 @@ namespace xo { * since one role is always held empty between collections. **/ virtual std::size_t size() const final override; - + /** for committed count both to-space and from-space **/ + virtual std::size_t committed() const final override; virtual std::size_t allocated() const final override; virtual std::size_t available() const final override; /** only tests to-space **/ @@ -194,10 +195,6 @@ namespace xo { virtual std::byte * alloc(std::size_t z) final override; virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override; -#ifdef REDLINE_MEMORY - virtual void release_redline_memory() final override; -#endif - private: ListAlloc * nursery_to() const { return nursery(role::to_space); } ListAlloc * nursery_from() const { return nursery(role::from_space); } @@ -208,6 +205,8 @@ namespace xo { ListAlloc * nursery(role r) const { return nursery_[role2int(r)].get(); } ListAlloc * tenured(role r) const { return tenured_[role2int(r)].get(); } + MutationLog * mutation_log(role r) const { return mutation_log_[role2int(r)].get(); } + /** begin GC now **/ void execute_gc(generation g); /** cleanup phase. aux function for @ref execute_gc **/ diff --git a/include/xo/alloc/IAlloc.hpp b/include/xo/alloc/IAlloc.hpp index b8270f53..0b6791f0 100644 --- a/include/xo/alloc/IAlloc.hpp +++ b/include/xo/alloc/IAlloc.hpp @@ -31,10 +31,12 @@ namespace xo { /** optional name for this allocator; labelling for diagnostics **/ virtual const std::string & name() const = 0; - /** allocator size in bytes (up to soft limit). + /** allocator size in bytes (up to reserved limit) * Includes unallocated mmeory **/ virtual std::size_t size() const = 0; + /** committed size in bytes **/ + virtual std::size_t committed() const = 0; /** number of unallocated bytes available (up to soft limit) * from this allocator **/ diff --git a/include/xo/alloc/ListAlloc.hpp b/include/xo/alloc/ListAlloc.hpp index ac2f9894..7b592b4e 100644 --- a/include/xo/alloc/ListAlloc.hpp +++ b/include/xo/alloc/ListAlloc.hpp @@ -55,6 +55,7 @@ namespace xo { virtual const std::string & name() const final override; virtual std::size_t size() const final override; + virtual std::size_t committed() const final override; virtual std::size_t available() const final override; virtual std::size_t allocated() const final override; virtual bool contains(const void * x) const final override; diff --git a/src/alloc/ArenaAlloc.cpp b/src/alloc/ArenaAlloc.cpp index e978f566..49a2d16d 100644 --- a/src/alloc/ArenaAlloc.cpp +++ b/src/alloc/ArenaAlloc.cpp @@ -8,6 +8,7 @@ #include "ObjectStatistics.hpp" #include "xo/indentlog/scope.hpp" #include "xo/indentlog/print/tag.hpp" +#include #include namespace xo { @@ -15,27 +16,52 @@ namespace xo { ArenaAlloc::ArenaAlloc(const std::string & name, std::size_t z, bool debug_flag) { + scope log(XO_DEBUG(debug_flag), xtag("name", name)); + this->name_ = name; - this->lo_ = (new std::byte [z]); - this->checkpoint_ = lo_; - this->free_ptr_ = lo_; - this->limit_ = lo_ + z; - this->hi_ = limit_; - this->debug_flag_ = debug_flag; + this->page_z_ = getpagesize(); + + // reserve virtual memory + + void * base = mmap(nullptr, z, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + // could use this as fallback.. + //base = (new std::byte [z]); + + if (base == MAP_FAILED) { + throw std::runtime_error(tostr("ArenaAlloc: uncommitted allocation failed", + xtag("size", z))); + } + + this->lo_ = reinterpret_cast(base); + this->committed_z_ = 0; + this->checkpoint_ = lo_; + this->free_ptr_ = lo_; + this->limit_ = lo_ + z; + this->hi_ = limit_; + this->debug_flag_ = debug_flag; if (!lo_) { throw std::runtime_error(tostr("ArenaAlloc: allocation failed", xtag("size", z))); } + + log && log(xtag("lo", (void*)lo_), xtag("page_z", page_z_)); } ArenaAlloc::~ArenaAlloc() { - delete [] this->lo_; // hygiene.. + if (lo_) { + munmap(lo_, hi_ - lo_); + } + // could use this as fallback if not using uncommitted technique + //delete [] this->lo_; + this->lo_ = nullptr; + this->committed_z_ = 0; this->checkpoint_ = nullptr; this->free_ptr_ = nullptr; this->limit_ = nullptr; @@ -51,6 +77,46 @@ namespace xo { z, debug_flag)); } + namespace { + /* alignment better be a power of 2 */ + std::size_t + align_lub(std::size_t x, std::size_t align) + { + /* e.g: + * align = 4096, x%align = 100 -> dx = 3996 + * align = 4096, x%align = 0 -> dx = 0 + */ + std::size_t dx = (align - (x % align)) % align; + + return x + dx; + } + } + + bool + ArenaAlloc::expand(size_t offset_z) { + scope log(XO_DEBUG(debug_flag_), xtag("offset_z", offset_z), xtag("committed_z", committed_z_)); + + if (offset_z <= committed_z_) + return true; + + std::size_t align_offset_z = align_lub(offset_z, page_z_); + std::byte * commit_start = lo_ + committed_z_; + std::size_t new_commit_z = align_offset_z - committed_z_; + + log && log(xtag("align_offset_z", align_offset_z), + xtag("new_commit_z", new_commit_z)); + + if (mprotect(commit_start, new_commit_z, PROT_READ | PROT_WRITE) != 0) { + throw std::runtime_error(tostr("ArenaAlloc::expand: commit failure", + xtag("committed_z", committed_z_), + xtag("new_commit_z", new_commit_z))); + } + + this->committed_z_ = align_offset_z; + + return true; + } + void ArenaAlloc::set_free_ptr(std::byte * x) { @@ -59,7 +125,7 @@ namespace xo { if (lo_ <= x && x < limit_) { this->free_ptr_ = x; - if (this->checkpoint_ > free_ptr_) + if (checkpoint_ > free_ptr_) this->checkpoint_ = free_ptr_; } else { throw std::runtime_error(tostr("LinearAllog::set_free_ptr(x): expected lo <= x < limit", @@ -128,6 +194,11 @@ namespace xo { return limit_ - lo_; } + std::size_t + ArenaAlloc::committed() const { + return committed_z_; + } + std::size_t ArenaAlloc::available() const { return limit_ - free_ptr_; @@ -197,13 +268,15 @@ namespace xo { assert(z1 % c_bpw == 0ul); + this->expand(this->allocated() + z1); + std::byte * retval = this->free_ptr_; - log && log(xtag("self", name_), xtag("z0", z0), xtag("+pad", dz), xtag("z1", z1), xtag("avail", this->available())); - - if (free_ptr_ + z1 > limit_) { - return nullptr; - } + log && log(xtag("self", name_), + xtag("z0", z0), + xtag("+pad", dz), + xtag("z1", z1), + xtag("avail", this->available())); this->free_ptr_ += z1; diff --git a/src/alloc/GC.cpp b/src/alloc/GC.cpp index 2990adf6..b72ac721 100644 --- a/src/alloc/GC.cpp +++ b/src/alloc/GC.cpp @@ -115,7 +115,16 @@ namespace xo { std::size_t GC::size() const { - return nursery_[role2int(role::to_space)]->size() + tenured_[role2int(role::to_space)]->size(); + return nursery_to()->size() + tenured_to()->size(); + } + + std::size_t + GC::committed() const + { + return (nursery_to()->committed() + + nursery_from()->committed() + + tenured_to()->committed() + + tenured_from()->committed()); } std::size_t @@ -182,6 +191,12 @@ namespace xo { return retval; } + std::size_t + GC::nursery_to_committed() const + { + return nursery_to()->committed(); + } + generation_result GC::fromspace_generation_of(const void * x) const { diff --git a/src/alloc/ListAlloc.cpp b/src/alloc/ListAlloc.cpp index 6e37a325..29cdb477 100644 --- a/src/alloc/ListAlloc.cpp +++ b/src/alloc/ListAlloc.cpp @@ -75,6 +75,17 @@ namespace xo { return total_z_; } + std::size_t + ListAlloc::committed() const { + std::size_t z = 0; + if (hd_) + z += hd_->committed(); + for (const auto & a : full_l_) + z += a->committed(); + + return z; + } + std::byte * ListAlloc::free_ptr() const { return hd_->free_ptr();