roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-interpreter2 .. xo-arena. memory pool introspection

Browse source
This commit is contained in:
Roland Conybeare 2026-02-03 01:05:36 -05:00
commit 3f5bd39ed8
2 changed files with 6 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

8
include/xo/tokenizer2/Tokenizer.hpp
View file

@ -61,7 +61,7 @@ namespace xo {
using error_type = TokenizerError;
using DCircularBuffer = xo::mm::DCircularBuffer;
using CircularBufferConfig = xo::mm::CircularBufferConfig;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using MemorySizeVisitor = xo::mm::MemorySizeVisitor;
using span_type = xo::mm::span<const CharT>;
//using input_state_type = TkInputState;
using result_type = scan_result;
@ -91,10 +91,8 @@ namespace xo {
const TkInputState & input_state() const { return input_state_; }
#pragma GCC diagnostic pop
/** number of distinct memory pools owned by tokenizer **/
std::size_t _n_store() const noexcept;
/** memory consumption for i'th memory pool **/
MemorySizeInfo _store_info(std::size_t i) const noexcept;
/** visit tokenizer-owned memory pools; invoke visitor(info) for each one **/
void visit_pools(const MemorySizeVisitor & visitor) const;
///@}

12
src/tokenizer2/Tokenizer.cpp
View file

@ -22,16 +22,10 @@ namespace xo {
this->input_state_.discard_current_line();
}
std::size_t
Tokenizer::_n_store() const noexcept
void
Tokenizer::visit_pools(const MemorySizeVisitor & visitor) const
{
return input_buffer_._n_store();
}
MemorySizeInfo
Tokenizer::_store_info(std::size_t i) const noexcept
{
return input_buffer_._store_info(i);
input_buffer_.visit_pools(visitor);
}
bool