xo-alloc: reserve virtual memory, commit pages on demand

CMakeLists.txt

@@ -1,4 +1,4 @@
-# alloc/CMakeLists.txt
+# xo-alloc/CMakeLists.txt
 
 cmake_minimum_required(VERSION 3.10)
 

include/xo/alloc/ArenaAlloc.hpp

@@ -24,6 +24,8 @@ namespace xo {
          **/
         class ArenaAlloc : public IAlloc {
         public:
+            ArenaAlloc(const ArenaAlloc &) = delete;
+            ArenaAlloc(ArenaAlloc &&) = delete;
             ~ArenaAlloc();
 
             /** create allocator with capacity @p z,
@@ -38,10 +40,14 @@ namespace xo {
             void capture_object_statistics(capture_phase phase,
                                            ObjectStatistics * p_dest) const;
 
+            /** expand available (i.e. committed) space to size @p z **/
+            bool expand(std::size_t z);
+
             // inherited from IAlloc...
 
             virtual const std::string & name() const final override;
             virtual std::size_t size() const final override;
+            virtual std::size_t committed() const final override;
             virtual std::size_t available() const final override;
             virtual std::size_t allocated() const final override;
             virtual bool contains(const void * x) const final override;
@@ -54,6 +60,9 @@ namespace xo {
             virtual void checkpoint() final override;
             virtual std::byte * alloc(std::size_t z) final override;
 
+            ArenaAlloc & operator=(const ArenaAlloc &) = delete;
+            ArenaAlloc & operator=(ArenaAlloc &&) = delete;
+
         private:
             ArenaAlloc(const std::string & name,
                        std::size_t z, bool debug_flag);
@@ -67,8 +76,15 @@ namespace xo {
             /** optional instance name, for diagnostics **/
             std::string name_;
 
+            /** size of a VM page **/
+            std::size_t page_z_;
+
             /** allocator owns memory in range [@ref lo_, @ref hi_) **/
             std::byte * lo_ = nullptr;
+            /** prefix of this size is actually committed.
+             *  Remainder uses uncommitted virtual address space
+             **/
+            std::size_t committed_z_ = 0;
             /** checkpoint (for GC support); divides objects into
              *  older (addresses below checkpoint)
              *  and younger (addresses above checkpoint)
@@ -76,9 +92,9 @@ namespace xo {
             std::byte * checkpoint_;
             /** free pointer. memory in range [@ref free_, @ref limit_) available **/
             std::byte * free_ptr_ = nullptr;
-            /** soft limit: end of released memory **/
+            /** soft limit: end of committed virtual memory **/
             std::byte * limit_ = nullptr;
-            /** hard limit: end of allocated memory **/
+            /** hard limit: end of reserved virtual memory **/
             std::byte * hi_ = nullptr;
             /** true to enable detailed debug logging **/
             bool debug_flag_ = false;

include/xo/alloc/GC.hpp

@@ -164,7 +164,8 @@ namespace xo {
              *  since one role is always held empty between collections.
              **/
             virtual std::size_t size() const final override;
-
+            /** for committed count both to-space and from-space **/
+            virtual std::size_t committed() const final override;
             virtual std::size_t allocated() const final override;
             virtual std::size_t available() const final override;
             /** only tests to-space **/
@@ -194,10 +195,6 @@ namespace xo {
             virtual std::byte * alloc(std::size_t z) final override;
             virtual std::byte * alloc_gc_copy(std::size_t z, const void * src) final override;
 
-#ifdef REDLINE_MEMORY
-            virtual void release_redline_memory() final override;
-#endif
-
         private:
             ListAlloc * nursery_to() const { return nursery(role::to_space); }
             ListAlloc * nursery_from() const { return nursery(role::from_space); }
@@ -208,6 +205,8 @@ namespace xo {
             ListAlloc * nursery(role r) const { return nursery_[role2int(r)].get(); }
             ListAlloc * tenured(role r) const { return tenured_[role2int(r)].get(); }
 
+            MutationLog * mutation_log(role r) const { return mutation_log_[role2int(r)].get(); }
+
             /** begin GC now **/
             void execute_gc(generation g);
             /** cleanup phase.  aux function for @ref execute_gc **/

include/xo/alloc/IAlloc.hpp

@@ -31,10 +31,12 @@ namespace xo {
 
             /** optional name for this allocator; labelling for diagnostics **/
             virtual const std::string & name() const = 0;
-            /** allocator size in bytes (up to soft limit).
+            /** allocator size in bytes (up to reserved limit)
              *  Includes unallocated mmeory
              **/
             virtual std::size_t size() const = 0;
+            /** committed size in bytes **/
+            virtual std::size_t committed() const = 0;
             /** number of unallocated bytes available (up to soft limit)
              *  from this allocator
              **/

include/xo/alloc/ListAlloc.hpp

@@ -55,6 +55,7 @@ namespace xo {
 
             virtual const std::string & name() const final override;
             virtual std::size_t size() const final override;
+            virtual std::size_t committed() const final override;
             virtual std::size_t available() const final override;
             virtual std::size_t allocated() const final override;
             virtual bool contains(const void * x) const final override;

src/alloc/ArenaAlloc.cpp

@@ -8,6 +8,7 @@
 #include "ObjectStatistics.hpp"
 #include "xo/indentlog/scope.hpp"
 #include "xo/indentlog/print/tag.hpp"
+#include <sys/mman.h>
 #include <cassert>
 
 namespace xo {
@@ -15,27 +16,52 @@ namespace xo {
         ArenaAlloc::ArenaAlloc(const std::string & name,
                                std::size_t z, bool debug_flag)
         {
+            scope log(XO_DEBUG(debug_flag), xtag("name", name));
+
             this->name_       = name;
-            this->lo_         = (new std::byte [z]);
+            this->page_z_     = getpagesize();
-            this->checkpoint_ = lo_;
+
-            this->free_ptr_   = lo_;
+            // reserve virtual memory
-            this->limit_      = lo_ + z;
+
-            this->hi_         = limit_;
+            void * base = mmap(nullptr, z, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-            this->debug_flag_ = debug_flag;
+
+            // could use this as fallback..
+            //base         = (new std::byte [z]);
+
+            if (base == MAP_FAILED) {
+                throw std::runtime_error(tostr("ArenaAlloc: uncommitted allocation failed",
+                                               xtag("size", z)));
+            }
+
+            this->lo_          = reinterpret_cast<std::byte *>(base);
+            this->committed_z_ = 0;
+            this->checkpoint_  = lo_;
+            this->free_ptr_    = lo_;
+            this->limit_       = lo_ + z;
+            this->hi_          = limit_;
+            this->debug_flag_  = debug_flag;
 
             if (!lo_) {
                 throw std::runtime_error(tostr("ArenaAlloc: allocation failed",
                                                xtag("size", z)));
             }
+
+            log && log(xtag("lo", (void*)lo_), xtag("page_z", page_z_));
         }
 
         ArenaAlloc::~ArenaAlloc()
         {
-            delete [] this->lo_;
 
             // hygiene..
 
+            if (lo_) {
+                munmap(lo_, hi_ - lo_);
+            }
+            // could use this as fallback if not using uncommitted technique
+            //delete [] this->lo_;
+
             this->lo_ = nullptr;
+            this->committed_z_ = 0;
             this->checkpoint_ = nullptr;
             this->free_ptr_ = nullptr;
             this->limit_ = nullptr;
@@ -51,6 +77,46 @@ namespace xo {
                                                  z, debug_flag));
         }
 
+        namespace {
+            /* alignment better be a power of 2 */
+            std::size_t
+            align_lub(std::size_t x, std::size_t align)
+            {
+                /* e.g:
+                 *   align = 4096, x%align = 100 -> dx = 3996
+                 *   align = 4096, x%align = 0   -> dx = 0
+                 */
+                std::size_t dx = (align - (x % align)) % align;
+
+                return x + dx;
+            }
+        }
+
+        bool
+        ArenaAlloc::expand(size_t offset_z) {
+            scope log(XO_DEBUG(debug_flag_), xtag("offset_z", offset_z), xtag("committed_z", committed_z_));
+
+            if (offset_z <= committed_z_)
+                return true;
+
+            std::size_t align_offset_z = align_lub(offset_z, page_z_);
+            std::byte * commit_start = lo_ + committed_z_;
+            std::size_t new_commit_z = align_offset_z - committed_z_;
+
+            log && log(xtag("align_offset_z", align_offset_z),
+                       xtag("new_commit_z", new_commit_z));
+
+            if (mprotect(commit_start, new_commit_z, PROT_READ | PROT_WRITE) != 0) {
+                throw std::runtime_error(tostr("ArenaAlloc::expand: commit failure",
+                                               xtag("committed_z", committed_z_),
+                                               xtag("new_commit_z", new_commit_z)));
+            }
+
+            this->committed_z_ = align_offset_z;
+
+            return true;
+        }
+
         void
         ArenaAlloc::set_free_ptr(std::byte * x)
         {
@@ -59,7 +125,7 @@ namespace xo {
 
             if (lo_ <= x && x < limit_) {
                 this->free_ptr_ = x;
-                if (this->checkpoint_ > free_ptr_)
+                if (checkpoint_ > free_ptr_)
                     this->checkpoint_ = free_ptr_;
             } else {
                 throw std::runtime_error(tostr("LinearAllog::set_free_ptr(x): expected lo <= x < limit",
@@ -128,6 +194,11 @@ namespace xo {
             return limit_ - lo_;
         }
 
+        std::size_t
+        ArenaAlloc::committed() const {
+            return committed_z_;
+        }
+
         std::size_t
         ArenaAlloc::available() const {
             return limit_ - free_ptr_;
@@ -197,13 +268,15 @@ namespace xo {
 
             assert(z1 % c_bpw == 0ul);
 
+            this->expand(this->allocated() + z1);
+
             std::byte * retval = this->free_ptr_;
 
-            log && log(xtag("self", name_), xtag("z0", z0), xtag("+pad", dz), xtag("z1", z1), xtag("avail", this->available()));
+            log && log(xtag("self", name_),
-
+                       xtag("z0", z0),
-            if (free_ptr_ + z1 > limit_) {
+                       xtag("+pad", dz),
-                return nullptr;
+                       xtag("z1", z1),
-            }
+                       xtag("avail", this->available()));
 
             this->free_ptr_ += z1;
 

src/alloc/GC.cpp

@@ -115,7 +115,16 @@ namespace xo {
         std::size_t
         GC::size() const
         {
-            return nursery_[role2int(role::to_space)]->size() + tenured_[role2int(role::to_space)]->size();
+            return nursery_to()->size() + tenured_to()->size();
+        }
+
+        std::size_t
+        GC::committed() const
+        {
+            return (nursery_to()->committed()
+                    + nursery_from()->committed()
+                    + tenured_to()->committed()
+                    + tenured_from()->committed());
         }
 
         std::size_t
@@ -182,6 +191,12 @@ namespace xo {
             return retval;
         }
 
+        std::size_t
+        GC::nursery_to_committed() const
+        {
+            return nursery_to()->committed();
+        }
+
         generation_result
         GC::fromspace_generation_of(const void * x) const
         {

src/alloc/ListAlloc.cpp

@@ -75,6 +75,17 @@ namespace xo {
             return total_z_;
         }
 
+        std::size_t
+        ListAlloc::committed() const {
+            std::size_t z = 0;
+            if (hd_)
+                z += hd_->committed();
+            for (const auto & a : full_l_)
+                z += a->committed();
+
+            return z;
+        }
+
         std::byte *
         ListAlloc::free_ptr() const {
             return hd_->free_ptr();