roland/xo-tokenizer2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-interpreter2: scaffold repl + alloc measurement frameowkr

Browse source
This commit is contained in:
Roland Conybeare 2026-02-02 21:55:34 -05:00
commit 415a382442
2 changed files with 19 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
include/xo/tokenizer2/Tokenizer.hpp
View file

@ -61,6 +61,7 @@ namespace xo {
using error_type = TokenizerError;
using DCircularBuffer = xo::mm::DCircularBuffer;
using CircularBufferConfig = xo::mm::CircularBufferConfig;
using MemorySizeInfo = xo::mm::MemorySizeInfo;
using span_type = xo::mm::span<const CharT>;
//using input_state_type = TkInputState;
using result_type = scan_result;
@ -90,6 +91,11 @@ namespace xo {
const TkInputState & input_state() const { return input_state_; }
#pragma GCC diagnostic pop
/** number of distinct memory pools owned by tokenizer **/
std::size_t _n_store() const noexcept;
/** memory consumption for i'th memory pool **/
MemorySizeInfo _store_info(std::size_t i) const noexcept;
///@}
/** @defgroup tokenizer-general-methods tokenizer methods **/

13
src/tokenizer2/Tokenizer.cpp
View file

@ -6,6 +6,7 @@
#include "Tokenizer.hpp"
namespace xo {
using xo::mm::MemorySizeInfo;
using std::byte;
namespace scm {
@ -21,6 +22,18 @@ namespace xo {
this->input_state_.discard_current_line();
}
std::size_t
Tokenizer::_n_store() const noexcept
{
return input_buffer_._n_store();
}
MemorySizeInfo
Tokenizer::_store_info(std::size_t i) const noexcept
{
return input_buffer_._store_info(i);
}
bool
Tokenizer::is_1char_punctuation(CharT ch)
{