roland/xo-arena
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-tokenizer: example tokenrepl restored to wokring order

Browse source 
Now with CBufferedInput in Tokenizer
This commit is contained in:
Roland Conybeare 2026-01-11 19:10:42 -05:00
commit 94e02f0eeb
4 changed files with 27 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

6
include/xo/arena/DCircularBuffer.hpp
View file

@ -39,8 +39,8 @@ namespace xo {
using size_type = std::size_t; using size_type = std::size_t;
using byte = std::byte; using byte = std::byte;
/** a contiguous addres range **/ /** a contiguous addres range **/
using span_type = span<byte>; using span_type = span<char>;
using const_span_type = span<const byte>; using const_span_type = span<const char>;
///@} ///@}
@ -162,7 +162,7 @@ namespace xo {
* *
* Require: @p hi < @ref reserved_range_.hi * Require: @p hi < @ref reserved_range_.hi
**/ **/
bool _expand_to(byte * hi); bool _expand_to(char * hi);
/** shrink occupied rnage to the smallest contiguous range that contains both: /** shrink occupied rnage to the smallest contiguous range that contains both:
* all of .input_range_, and all pinned ranges in .pinned_spans_ * all of .input_range_, and all pinned ranges in .pinned_spans_

8
include/xo/arena/span.hpp
View file

@ -88,6 +88,14 @@ namespace xo {
return span(lo, hi); return span(lo, hi);
} }
/** @brief create span from raw memory **/
static span from_memory(span<std::byte> span_memory) {
CharT * lo = (CharT *)span_memory.lo();
CharT * hi = (CharT *)span_memory.hi();
return span(lo, hi);
}
/** @brief concatenate two contiguous spans */ /** @brief concatenate two contiguous spans */
static span concat(const span & span1, const span & span2) { static span concat(const span & span1, const span & span2) {
if (span1.is_null()) if (span1.is_null())

19
src/arena/DCircularBuffer.cpp
View file

@ -48,17 +48,19 @@ namespace xo {
log && log(xtag("page_z", page_z), log && log(xtag("page_z", page_z),
xtag("align_z", align_z)); xtag("align_z", align_z));
auto span = mmap_util::map_aligned_range(config.max_capacity_, auto mapped_span
align_z, = span<char>::from_memory(mmap_util::map_aligned_range
enable_hugepage_flag, (config.max_capacity_,
config.debug_flag_); align_z,
enable_hugepage_flag,
config.debug_flag_));
if (!span.lo()) { if (!mapped_span.lo()) {
throw std::runtime_error(tostr("DCircularBuffer: reserve address range failed", throw std::runtime_error(tostr("DCircularBuffer: reserve address range failed",
xtag("size", config.max_capacity_))); xtag("size", config.max_capacity_)));
} }
return DCircularBuffer(config, page_z, align_z, span); return DCircularBuffer(config, page_z, align_z, mapped_span);
} }
DCircularBuffer::DCircularBuffer(const CircularBufferConfig & config, DCircularBuffer::DCircularBuffer(const CircularBufferConfig & config,
@ -302,7 +304,8 @@ namespace xo {
} }
bool bool
DCircularBuffer::_expand_to(byte * hi) { DCircularBuffer::_expand_to(char * hi)
{
scope log(XO_DEBUG(config_.debug_flag_)); scope log(XO_DEBUG(config_.debug_flag_));
if (hi < mapped_range_.hi()) { if (hi < mapped_range_.hi()) {
@ -312,7 +315,7 @@ namespace xo {
size_t add_z = hi - mapped_range_.hi(); size_t add_z = hi - mapped_range_.hi();
size_t add_commit_z = padding::with_padding(add_z, buffer_align_z_); size_t add_commit_z = padding::with_padding(add_z, buffer_align_z_);
byte * commit_start = mapped_range_.hi(); char * commit_start = mapped_range_.hi();
if (::mprotect(commit_start, if (::mprotect(commit_start,
add_commit_z, add_commit_z,

10
utest/DCircularBuffer.test.cpp
View file

@ -34,18 +34,16 @@ namespace xo {
REQUIRE(buf.occupied_range().size() == 0); REQUIRE(buf.occupied_range().size() == 0);
REQUIRE(buf.input_range().size() == 0); REQUIRE(buf.input_range().size() == 0);
std::string_view s0 = "abcdefghijk"; auto s0 = DCircularBuffer::const_span_type::from_cstr("abcdefghijk");
/* return value is unaccepted suffix of input */ /* return value is unaccepted suffix of input */
REQUIRE(buf.append(DCircularBuffer::span_type((byte *)s0.begin(), REQUIRE(buf.append(s0).empty());
(byte *)s0.end())).empty());
REQUIRE(buf.verify_ok(verify_policy::log_only())); REQUIRE(buf.verify_ok(verify_policy::log_only()));
REQUIRE(buf.mapped_range().size() == getpagesize()); REQUIRE(buf.mapped_range().size() == getpagesize());
REQUIRE(buf.occupied_range().size() == s0.size()); REQUIRE(buf.occupied_range().size() == s0.size());
REQUIRE(buf.input_range().size() == s0.size()); REQUIRE(buf.input_range().size() == s0.size());
std::string_view s1 = "lmnopq"; auto s1 = DCircularBuffer::const_span_type::from_cstr("lmnopq");
REQUIRE(buf.append(DCircularBuffer::span_type((byte *)s1.begin(), REQUIRE(buf.append(s1).empty());
(byte *)s1.end())).empty());
REQUIRE(buf.mapped_range().size() == getpagesize()); REQUIRE(buf.mapped_range().size() == getpagesize());
REQUIRE(buf.occupied_range().size() == s0.size() + s1.size()); REQUIRE(buf.occupied_range().size() == s0.size() + s1.size());