diff --git a/xo-arena/include/xo/arena/DCircularBuffer.hpp b/xo-arena/include/xo/arena/DCircularBuffer.hpp index 0cb007ad..30a38960 100644 --- a/xo-arena/include/xo/arena/DCircularBuffer.hpp +++ b/xo-arena/include/xo/arena/DCircularBuffer.hpp @@ -39,8 +39,8 @@ namespace xo { using size_type = std::size_t; using byte = std::byte; /** a contiguous addres range **/ - using span_type = span; - using const_span_type = span; + using span_type = span; + using const_span_type = span; ///@} @@ -162,7 +162,7 @@ namespace xo { * * Require: @p hi < @ref reserved_range_.hi **/ - bool _expand_to(byte * hi); + bool _expand_to(char * hi); /** shrink occupied rnage to the smallest contiguous range that contains both: * all of .input_range_, and all pinned ranges in .pinned_spans_ diff --git a/xo-arena/include/xo/arena/span.hpp b/xo-arena/include/xo/arena/span.hpp index c038fce5..b9192662 100644 --- a/xo-arena/include/xo/arena/span.hpp +++ b/xo-arena/include/xo/arena/span.hpp @@ -88,6 +88,14 @@ namespace xo { return span(lo, hi); } + /** @brief create span from raw memory **/ + static span from_memory(span span_memory) { + CharT * lo = (CharT *)span_memory.lo(); + CharT * hi = (CharT *)span_memory.hi(); + + return span(lo, hi); + } + /** @brief concatenate two contiguous spans */ static span concat(const span & span1, const span & span2) { if (span1.is_null()) diff --git a/xo-arena/src/arena/DCircularBuffer.cpp b/xo-arena/src/arena/DCircularBuffer.cpp index 5ad7e021..e00aaaf7 100644 --- a/xo-arena/src/arena/DCircularBuffer.cpp +++ b/xo-arena/src/arena/DCircularBuffer.cpp @@ -48,17 +48,19 @@ namespace xo { log && log(xtag("page_z", page_z), xtag("align_z", align_z)); - auto span = mmap_util::map_aligned_range(config.max_capacity_, - align_z, - enable_hugepage_flag, - config.debug_flag_); + auto mapped_span + = span::from_memory(mmap_util::map_aligned_range + (config.max_capacity_, + align_z, + enable_hugepage_flag, + config.debug_flag_)); - if (!span.lo()) { + if (!mapped_span.lo()) { throw std::runtime_error(tostr("DCircularBuffer: reserve address range failed", xtag("size", config.max_capacity_))); } - return DCircularBuffer(config, page_z, align_z, span); + return DCircularBuffer(config, page_z, align_z, mapped_span); } DCircularBuffer::DCircularBuffer(const CircularBufferConfig & config, @@ -302,7 +304,8 @@ namespace xo { } bool - DCircularBuffer::_expand_to(byte * hi) { + DCircularBuffer::_expand_to(char * hi) + { scope log(XO_DEBUG(config_.debug_flag_)); if (hi < mapped_range_.hi()) { @@ -312,7 +315,7 @@ namespace xo { size_t add_z = hi - mapped_range_.hi(); size_t add_commit_z = padding::with_padding(add_z, buffer_align_z_); - byte * commit_start = mapped_range_.hi(); + char * commit_start = mapped_range_.hi(); if (::mprotect(commit_start, add_commit_z, diff --git a/xo-arena/utest/DCircularBuffer.test.cpp b/xo-arena/utest/DCircularBuffer.test.cpp index be869478..2d8df93d 100644 --- a/xo-arena/utest/DCircularBuffer.test.cpp +++ b/xo-arena/utest/DCircularBuffer.test.cpp @@ -34,18 +34,16 @@ namespace xo { REQUIRE(buf.occupied_range().size() == 0); REQUIRE(buf.input_range().size() == 0); - std::string_view s0 = "abcdefghijk"; + auto s0 = DCircularBuffer::const_span_type::from_cstr("abcdefghijk"); /* return value is unaccepted suffix of input */ - REQUIRE(buf.append(DCircularBuffer::span_type((byte *)s0.begin(), - (byte *)s0.end())).empty()); + REQUIRE(buf.append(s0).empty()); REQUIRE(buf.verify_ok(verify_policy::log_only())); REQUIRE(buf.mapped_range().size() == getpagesize()); REQUIRE(buf.occupied_range().size() == s0.size()); REQUIRE(buf.input_range().size() == s0.size()); - std::string_view s1 = "lmnopq"; - REQUIRE(buf.append(DCircularBuffer::span_type((byte *)s1.begin(), - (byte *)s1.end())).empty()); + auto s1 = DCircularBuffer::const_span_type::from_cstr("lmnopq"); + REQUIRE(buf.append(s1).empty()); REQUIRE(buf.mapped_range().size() == getpagesize()); REQUIRE(buf.occupied_range().size() == s0.size() + s1.size()); diff --git a/xo-tokenizer2/example/tokenrepl/tokenrepl.cpp b/xo-tokenizer2/example/tokenrepl/tokenrepl.cpp index 0852f028..1cf02244 100644 --- a/xo-tokenizer2/example/tokenrepl/tokenrepl.cpp +++ b/xo-tokenizer2/example/tokenrepl/tokenrepl.cpp @@ -51,6 +51,8 @@ main() { using xo::scm::operator<<; using xo::mm::CircularBufferConfig; using xo::mm::span; + using xo::scope; + using xo::xtag; using replxx::Replxx; using namespace std; @@ -65,10 +67,13 @@ main() { rx.set_max_history_size(1000); rx.history_load("repl_history.txt"); + constexpr bool c_debug_flag = true; + scope log(XO_DEBUG(c_debug_flag)); + Tokenizer tkz(CircularBufferConfig{.name_ = "tokenrepl-input", .max_capacity_ = 4*1024, .max_captured_span_ = 128}, - true /*debug_flag*/); + c_debug_flag); const char * input_cstr = nullptr;; @@ -84,9 +89,17 @@ main() { if (input_cstr && *input_cstr) { auto [error, input] = tkz.buffer_input_line(input_cstr, false /*!eof*/); + if (log) { + log(xtag("msg", "buffered input line")); + log(xtag("input", input)); + } + + while (!input.empty()) { auto [tk, consumed, error] = tkz.scan(input); + log && log(xtag("consumed", consumed), xtag("tk", tk)); + if (tk.is_valid()) { cout << tk << endl; } else if (error.is_error()) { diff --git a/xo-tokenizer2/src/tokenizer2/Tokenizer.cpp b/xo-tokenizer2/src/tokenizer2/Tokenizer.cpp index 888a0c43..4fa98a97 100644 --- a/xo-tokenizer2/src/tokenizer2/Tokenizer.cpp +++ b/xo-tokenizer2/src/tokenizer2/Tokenizer.cpp @@ -622,15 +622,9 @@ namespace xo { auto buf_input_0 = input_buffer_.input_range().hi(); auto remainder = input_buffer_.append - (DCircularBuffer::const_span_type - ((const byte *)input_cstr, - (const byte *)input_cstr + strlen(input_cstr))); - - const char * newline_cstr = "\n"; + (DCircularBuffer::const_span_type::from_cstr(input_cstr)); auto remainder2 = input_buffer_.append - (DCircularBuffer::const_span_type - ((const byte *)newline_cstr, - (const byte *)newline_cstr + strlen(newline_cstr))); + (DCircularBuffer::const_span_type::from_cstr("\n")); if (!remainder.empty() || !remainder2.empty()) { throw std::runtime_error(tostr("Tokenizer::buffer_line: line too long!", @@ -639,8 +633,8 @@ namespace xo { auto buf_input_1 = input_buffer_.input_range().hi(); - span_type input = span_type((const char *)buf_input_0, - (const char *)buf_input_1); + span_type input = span_type(buf_input_0, + buf_input_1); return this->input_state_.capture_current_line(input, eof_flag); }