roland/xo-expression2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

xo-reader xo-expression: nested lambdas working properly + docs

Browse source
This commit is contained in:
Roland Conybeare 2025-07-06 14:13:44 -05:00
commit 699ee5d38e
41 changed files with 736 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

2
xo-cmake/cmake/xo_macros/xo_cxx.cmake
View file

@ -1363,7 +1363,7 @@ macro(xo_external_target_dependency target pkg pkgtarget)
#target_link_libraries(${target} ${pkgtarget})
endmacro()
# dependency on external (non-xo) target
# dependency on external (non-XO) target
#
macro(xo_external_dependency target pkg)
xo_external_target_dependency(${target} ${pkg} ${target})

7
xo-expression/include/xo/expression/Apply.hpp
View file

@ -98,12 +98,7 @@ namespace xo {
return xform_fn(this);
}
virtual void attach_envs(bp<Environment> p) override {
fn_->attach_envs(p);
for (const auto & arg : argv_)
arg->attach_envs(p);
}
virtual void attach_envs(bp<Environment> p) override;
virtual void display(std::ostream & os) const override;

8
xo-expression/include/xo/expression/Environment.hpp
View file

@ -33,7 +33,15 @@ namespace xo {
* returns llvm::Value representing code that produces a value for vname
**/
virtual bp<Expression> lookup_var(const std::string & vname) const = 0;
virtual void print(std::ostream & os) const = 0;
};
inline std::ostream &
operator<< (std::ostream & os, const Environment & x) {
x.print(os);
return os;
}
} /*namespace ast*/
} /*namespace xo*/

9
xo-expression/include/xo/expression/GlobalEnv.hpp
View file

@ -17,10 +17,7 @@ namespace xo {
static rp<GlobalEnv> make() { return new GlobalEnv(); }
bp<Expression> require_global(const std::string & vname,
bp<Expression> expr) {
global_map_[vname] = expr.get();
return expr;
} /*require_global*/
bp<Expression> expr);
// ----- Environment -----
@ -44,8 +41,10 @@ namespace xo {
return ix->second;
}
virtual void print(std::ostream & os) const override;
private:
GlobalEnv() = default;
GlobalEnv();
private:
/* for assignable globals, need to allocate memory

21
xo-expression/include/xo/expression/Lambda.hpp
View file

@ -26,10 +26,21 @@ namespace xo {
* @p name Name for this lambda -- must be unique
* @p argv Formal parameters, in left-to-right order
* @p body Expression for body of this function
* @p parent_env Environment for enclosing lexical scope
**/
static rp<Lambda> make(const std::string & name,
const std::vector<rp<Variable>> & argv,
const rp<Expression> & body);
const rp<Expression> & body,
const rp<Environment> & parent_env);
/**
* @p name Name for this lambda -- must be unique
* @p env Environment with {name,type} for each formal parameter
* @p body Expression for body of function
**/
static rp<Lambda> make_from_env(const std::string & name,
const rp<LocalEnv> & env,
const rp<Expression> & body);
/** downcast from Expression **/
static bp<Lambda> from(bp<Expression> x) {
@ -174,16 +185,18 @@ namespace xo {
inline rp<Lambda>
make_lambda(const std::string & name,
const std::vector<rp<Variable>> & argv,
const rp<Expression> & body)
const rp<Expression> & body,
const rp<Environment> & parent_env)
{
return Lambda::make(name, argv, body);
return Lambda::make(name, argv, body, parent_env);
}
class LambdaAccess : public Lambda {
public:
static rp<LambdaAccess> make(const std::string & name,
const std::vector<rp<Variable>> & argv,
const rp<Expression> & body);
const rp<Expression> & body,
const rp<Environment> & parent_env);
static rp<LambdaAccess> make_empty();
/** assign body + compute derived members

27
xo-expression/include/xo/expression/LocalEnv.hpp
View file

@ -25,10 +25,13 @@ namespace xo {
using TypeDescr = xo::reflect::TypeDescr;
public:
static rp<LocalEnv> make_empty();
/** named ctor idiom. Create instance with local variables per @p argv **/
static rp<LocalEnv> make(const std::vector<rp<Variable>> & argv) {
return new LocalEnv(argv);
}
static rp<LocalEnv> make(const std::vector<rp<Variable>> & argv,
const rp<Environment> & parent_env);
/** Create instance with single local variable @ap argv1 **/
static rp<LocalEnv> make1(const rp<Variable> & arg1,
const rp<Environment> & parent_env);
Lambda * origin() const { return origin_; }
const std::vector<rp<Variable>> & argv() const { return argv_; }
@ -48,10 +51,14 @@ namespace xo {
}
/** single-assign this environment's parent **/
void assign_parent(bp<Environment> p) {
assert(parent_env_.get() == nullptr);
parent_env_ = p.get();
}
void assign_parent(bp<Environment> p);
/** create/replace local variable @p target.
* Narrow use case: intended for when LocalEnv represents a top-level session environment
**/
void upsert_local(bp<Variable> target);
bp<Variable> lookup_local(const std::string & vname) const;
// ----- Environment -----
@ -71,9 +78,10 @@ namespace xo {
return parent_env_->lookup_var(target);
}
virtual void print(std::ostream & os) const override;
private:
LocalEnv(const std::vector<rp<Variable>> & argv)
: origin_{nullptr}, argv_(argv) {}
LocalEnv(const std::vector<rp<Variable>> & argv, const rp<Environment> & parent_env);
private:
/** Lambnda for which this environment created.
@ -93,6 +101,7 @@ namespace xo {
**/
rp<Environment> parent_env_;
};
} /*namespace ast*/
} /*namespace xo*/

5
xo-expression/include/xo/expression/Variable.hpp
View file

@ -16,6 +16,11 @@ namespace xo {
**/
class Variable : public Expression {
public:
/** Generate unique symbol-name beginning with @p prefix.
* Relies on static counter
**/
static std::string gensym(const std::string & prefix);
/** create expression representing a variable
* identified by @p name, that can take on values
* described by @p var_type.

8
xo-expression/src/expression/Apply.cpp
View file

@ -58,6 +58,14 @@ namespace xo {
{lhs, rhs});
}
void
Apply::attach_envs(bp<Environment> p) {
fn_->attach_envs(p);
for (const auto & arg : argv_)
arg->attach_envs(p);
}
void
Apply::display(std::ostream & os) const {
os << "<Apply"

1
xo-expression/src/expression/CMakeLists.txt
View file

@ -11,6 +11,7 @@ set(SELF_SRCS
Variable.cpp
IfExpr.cpp
Sequence.cpp
GlobalEnv.cpp
LocalEnv.cpp
ConvertExpr.cpp
Primitive.cpp

26
xo-expression/src/expression/GlobalEnv.cpp Normal file
View file

@ -0,0 +1,26 @@
/* file GlobalEnv.cpp
*
* author: Roland Conybeare, Jul 2025
*/
#include "GlobalEnv.hpp"
#include "Expression.hpp"
namespace xo {
namespace ast {
GlobalEnv::GlobalEnv() = default;
bp<Expression>
GlobalEnv::require_global(const std::string & vname,
bp<Expression> expr)
{
this->global_map_[vname] = expr.get();
return expr;
} /*require_global*/
void
GlobalEnv::print(std::ostream & os) const {
os << "<globalenv" << xtag("size", global_map_.size()) << ">";
}
} /*namespace ast*/
} /*namespace xo*/

28
xo-expression/src/expression/Lambda.cpp
View file

@ -66,15 +66,11 @@ namespace xo {
}
rp<Lambda>
Lambda::make(const std::string & name,
const std::vector<rp<Variable>> & argv,
const rp<Expression> & body)
Lambda::make_from_env(const std::string & name,
const rp<LocalEnv> & env,
const rp<Expression> & body)
{
using xo::reflect::FunctionTdx;
rp<LocalEnv> env = LocalEnv::make(argv);
TypeDescr lambda_td = assemble_lambda_td(argv, body);
TypeDescr lambda_td = assemble_lambda_td(env->argv(), body);
rp<Lambda> retval
= new Lambda(name,
@ -86,6 +82,17 @@ namespace xo {
env->assign_origin(retval.get());
return retval;
}
rp<Lambda>
Lambda::make(const std::string & name,
const std::vector<rp<Variable>> & argv,
const rp<Expression> & body,
const rp<Environment> & parent_env)
{
rp<LocalEnv> env = LocalEnv::make(argv, parent_env);
return make_from_env(name, env, body);
} /*make*/
std::set<std::string>
@ -315,10 +322,11 @@ namespace xo {
rp<LambdaAccess>
LambdaAccess::make(const std::string & name,
const std::vector<rp<Variable>> & argv,
const rp<Expression> & body)
const rp<Expression> & body,
const rp<Environment> & parent_env)
{
TypeDescr lambda_td = assemble_lambda_td(argv, body);
rp<LocalEnv> env = LocalEnv::make(argv);
rp<LocalEnv> env = LocalEnv::make(argv, parent_env);
rp<LambdaAccess> retval
= new LambdaAccess(name,

85
xo-expression/src/expression/LocalEnv.cpp
View file

@ -4,12 +4,43 @@
*/
#include "LocalEnv.hpp"
#include "xo/indentlog/print/vector.hpp"
namespace xo {
namespace ast {
binding_path
LocalEnv::lookup_local_binding(const std::string & vname) const
rp<LocalEnv>
LocalEnv::make_empty() {
return new LocalEnv(std::vector<rp<Variable>>(), nullptr);
}
rp<LocalEnv>
LocalEnv::make(const std::vector<rp<Variable>> & argv,
const rp<Environment> & parent_env)
{
return new LocalEnv(argv, parent_env);
}
rp<LocalEnv>
LocalEnv::make1(const rp<Variable> & arg1,
const rp<Environment> & parent_env)
{
std::vector<rp<Variable>> argv = { arg1 };
return make(argv, parent_env);
}
LocalEnv::LocalEnv(const std::vector<rp<Variable>> & argv,
const rp<Environment> & parent_env)
: origin_{nullptr},
argv_(argv),
parent_env_{parent_env}
{
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag), xtag("this", (void*)this), xtag("argv", argv_));
}
binding_path
LocalEnv::lookup_local_binding(const std::string & vname) const {
int j_slot = 0;
for (const auto & arg : argv_) {
if (arg->name() == vname)
@ -21,11 +52,9 @@ namespace xo {
} /*lookup_local_binding*/
binding_path
LocalEnv::lookup_binding(const std::string & vname) const
{
LocalEnv::lookup_binding(const std::string & vname) const {
{
auto local = this->lookup_local_binding(vname);
if (local.i_link_ == 0)
return local;
}
@ -37,6 +66,52 @@ namespace xo {
else
return { free.i_link_ + 1, free.j_slot_ };
} /*lookup_binding*/
bp<Variable>
LocalEnv::lookup_local(const std::string & vname) const {
for (const auto & var : this->argv_) {
if (var->name() == vname)
return var;
}
return bp<Variable>::from_native(nullptr);
}
void
LocalEnv::assign_parent(bp<Environment> p) {
if ((parent_env_.get() != nullptr) && (parent_env_.get() != p.get())) {
throw std::runtime_error(tostr("LocalEnv::assign_parent(P2): already have established parent P1",
xtag("P1", parent_env_),
xtag("P2", p)));
assert(false);
}
parent_env_ = p.promote();
}
void
LocalEnv::upsert_local(bp<Variable> target) {
for (auto & var : this->argv_) {
if (var->name() == target->name()) {
/* replace existing variable. May change its type */
var = target.promote();
return;
}
}
/* control here: target not already present in this frame -> append */
this->argv_.push_back(target.promote());
}
void
LocalEnv::print(std::ostream& os) const {
os << "<localenv"
<< xtag("this", (void*)this)
<< xtag("argv", argv_)
<< ">";
}
} /*namespace ast*/
} /*namespace xo*/

12
xo-expression/src/expression/Variable.cpp
View file

@ -5,6 +5,18 @@
namespace xo {
namespace ast {
std::string
Variable::gensym(const std::string & prefix) {
static std::size_t s_counter = 0;
++s_counter;
char buf[32];
snprintf(buf, sizeof(buf), "%ld", s_counter);
return prefix + std::string(buf);
}
void
Variable::attach_envs(bp<Environment> e) {
/** e makes accessible all enclosing lexical scopes **/

2
xo-indentlog/include/xo/indentlog/log_state.hpp
View file

@ -142,7 +142,7 @@ namespace xo {
std::ostream ss_;
}; /*state_impl*/
constexpr uint32_t c_default_buf_size = 1024;
constexpr uint32_t c_default_buf_size = 1024*1024;
template <typename CharT, typename Traits>
state_impl<CharT, Traits>::state_impl()

2
xo-reader/docs/CMakeLists.txt
View file

@ -3,5 +3,5 @@
xo_doxygen_collect_deps()
xo_docdir_doxygen_config()
xo_docdir_sphinx_config(
index.rst
index.rst example.rst install.rst implementation.rst
)

2
xo-reader/docs/example.rst
View file

@ -29,7 +29,7 @@ See ``xo-reader/examples`` for built examples
for (auto rem = input; !rem.empty();) {
// res: (parsed-expr, used)
auto [expr, rem2] = rdr.read_expr(rem, eof);
auto [expr, rem2, _] = rdr.read_expr(rem, eof);
if (expr) {
cout << expr << endl;

137
xo-reader/docs/implementation.rst Normal file
View file

@ -0,0 +1,137 @@
.. _implementation:
.. toctree::
:maxdepth: 2
Components
==========
Library dependency tower for *xo-reader*:
.. ditaa::
+------------------------------------------+
| xo_reader |
+-------------------------+----------------+
| xo_expression | |
+-------------------------+ |
| xo_reflect | xo_tokenizer |
+-----------+-------------+ |
| | xo_refcnt | |
| xo_subsys +-------------+----------------+
| | xo_indentlog |
+-----------+------------------------------+
Install instructions :doc:`here<install>`
Abstraction tower for *xo-reader* components:
.. ditaa::
:--scale: 0.85
+--------------------------------+
| reader |
+--------------------------------+
| parser |
+----------------+---------------+
| exprstatestack | envframestack |
+----------------+---------------+
| exprstate | envframe |
+----------------+---------------+
``exprstate`` provides an abstract api.
We use runtime polymorphism to represent concrete parsing states.
Different expression types inherit from ``exprstate`` to encapsulate
parsing for each expression type.
.. uml::
:caption: exprstate
:scale: 99%
:align: center
class exprstate {
}
class define_xs {
}
exprstate <|-- define_xs
class lambda_xs {
}
exprstate <|-- lambda_xs
class exprseq_xs {
}
exprstate <|-- exprseq_xs
class let1_xs {
}
exprstate <|-- let1_xs
class paren_xs {
}
exprstate <|-- paren_xs
class sequence_xs {
}
exprstate <|-- sequence_xs
There are also classes for nested state machines:
.. uml::
:caption: exprstate
:scale: 99%
:align: center
class exprstate {
}
class progress_xs {
}
exprstate <|-- progress_xs
class expect_symbol_xs {
}
exprstate <|-- expect_symbol_xs
class expect_type_xs {
}
exprstate <|-- expect_type_xs
class expect_expr_xs {
}
exprstate <|-- expect_expr_xs
class expect_formal_xs {
}
exprstate <|-- expect_formal_xs
class expect_formal_arglist_xs {
}
exprstate <|-- expect_formal_arglist_xs
Putting these in context:
.. list-table:: Schematika Parsing States
:widths: 15 30
:header-rows: 1
* - exprstate class
- target syntax
* - define_xs
- ``def foo : f64 = 1;``, ``def sq = lambda (x : i64) { x * x; }``
* - progress_xs
- possibly-incomplete arithmetic expressions
``(a + b) * 7``..

2
xo-reader/docs/index.rst
View file

@ -9,4 +9,6 @@ xo-reader provides a parser for the Schematika language.
:maxdepth: 2
:caption: xo-reader contents
install
example
implementation

146
xo-reader/docs/install.rst Normal file
View file

@ -0,0 +1,146 @@
.. _install:
.. toctree::
:maxdepth: 2
Source
======
Source code lives on github `here`_
.. _here: https://github.com/rconybea/xo-reader
To clone from git:
.. code-block:: bash
git clone https://github.com/rconybea/xo-reader
Tested with gcc 13.3
Install
=======
One-step Install
----------------
Install along with the rest of *XO* from `xo-umbrella2 source`_
.. _xo-umbrella2 source: https://github.com/rconybea/xo-umbrella2
Minimal Dependencies
--------------------
``xo-reader`` uses several supporting libraries from the *XO* project:
- `xo-expression source`_ (Schematika AST representation)
- `xo-tokenizer source`_ (Schematika lexer)
- `xo-reflect source`_ (reflection library)
- `xo-refcnt source`_ (reference-counting library)
- `xo-indentlog source`_ (structured logging)
- `xo-subsys source`_ (utility library)
- `xo-cmake source`_ (shared cmake macros)
.. _xo-expression source: https://github.com/rconybea/xo-expression
.. _xo-tokenizer source: https://github.com/rconybea/xo-tokenizer
.. _xo-reflect source: https://github.com/rconybea/xo-reflect
.. _xo-refcnt source: https://github.com/rconybea/refcnt
.. _xo-indentlog source: https://github.com/rconybea/indentlog
.. _xo-subsys source: https://github.com/rconybea/subsys
.. _xo-cmake source: https://github.com/rconybea/xo-cmake
Installing from source
----------------------
Install scripts for XO libraries depend on helper scripts installed from `xo-cmake`.
Preamble:
.. code-block:: bash
mkdir -p ~/proj/xo
cd ~/proj/xo
git clone https://github.com/rconybea/xo-cmake
PREFIX=/usr/local # ..or desired installation prefix
# want PREFIX/bin in PATH to use xo-cmake helpers
PATH=$PREFIX/bin:$PATH
Install `xo-cmake`:
.. code-block:: bash
cmake -B xo-cmake/.build -S xo-cmake
cmake --install xo-cmake/.build
Install dependencies in topological order:
.. code-block:: bash
xo-build --clone --configure --build --install xo-indentlog
xo-build --clone --configure --build --install xo-subsys
xo-build --clone --configure --build --install xo-refcnt
xo-build --clone --configure --build --install xo-reflect
xo-build --clone --configure --build --install xo-expression
xo-build --clone --configure --build --install xo-tokenizer
xo-build --clone --configure --build --install xo-reader
Directories under ``PREFIX`` will then contain:
.. code-block::
PREFIX
+- bin
| +- xo-build
| +- xo-cmake-config
| \- xo-cmake-lcov-harness
+- include
| \- xo
| +- cxxutil/
| +- expression/
| +- indentlog/
| +- reader/
| +- refcnt/
| +- reflect/
| +- subsys/
| \- tokenizer/
+- lib
| +- cmake
| | +- indentlog/
| | +- refcnt/
| | +- reflect/
| | +- subsys/
| | +- xo_expression/
| | +- xo_reader/
| | \- xo_tokenizer/
| +- lib*.so
+- share
+- cmake
| \- xo_macros
| +- code-coverage.cmake
| +- xo-project-macros.cmake
| \- xo_cxx.cmake
+- etc
| \- xo
| \- subsystem-list
\- xo-macros
+- Doxyfile.in
+- gen-ccov.in
\- xo-bootstrap-macros.cmake
CMake Support
-------------
To use built-in cmake support, when using ``xo-reader`` from another project:
Make sure ``PREFIX/lib/cmake`` is searched by cmake (if necessary, include it in ``CMAKE_PREFIX_PATH``)
Add to your ``CMakeLists.txt``:
.. code-block:: cmake
FindPackage(xo_reader CONFIG REQUIRED)
target_link_libraries(mytarget INTERFACE xo_reader)

2
xo-reader/examples/exprreplxx/exprreplxx.cpp
View file

@ -35,6 +35,8 @@ bool replxx_getline(bool interactive, std::size_t parser_stack_size, replxx::Rep
rx.history_add(input);
input.push_back('\n');
return retval;
}

22
xo-reader/include/xo/reader/envframe.hpp
View file

@ -5,11 +5,15 @@
#pragma once
xxx;
#include "xo/expression/Variable.hpp"
#include "xo/expression/LocalEnv.hpp"
#include <vector>
namespace xo {
namespace scm {
#ifdef OBSOLETE
/** @class envframe
* @brief names/types of formal paremeters introduced by a function
*
@ -17,26 +21,31 @@ namespace xo {
**/
class envframe {
public:
using Variable = xo::ast::Variable;
using Environment = xo::ast::Environment;
using LocalEnv = xo::ast::LocalEnv;
using Variable = xo::ast::Variable;
public:
envframe() = default;
envframe(const std::vector<rp<Variable>> & argl) : argl_(argl) {}
envframe(const std::vector<rp<Variable>> & argl, const rp<Environment> & parent_env);
const std::vector<rp<Variable>> & argl() const { return argl_; }
const std::vector<rp<Variable>> & argl() const { return env_->argv(); }
const rp<LocalEnv> & local_env() const { return env_; }
/** lookup variable by @p name. If found, return it.
* Otherwise return nullptr
**/
rp<Variable> lookup(const std::string & name) const;
bp<Variable> lookup(const std::string & name) const;
/** establish (replacing if already exists) binding for variable @p var **/
/** establish (replacing if already exists) binding for variable @p var.
* Replacement intended for use in interactive sessions
**/
void upsert(bp<Variable> var);
void print (std::ostream & os) const;
private:
std::vector<rp<Variable>> argl_;
rp<LocalEnv> env_;
};
inline std::ostream &
@ -44,6 +53,7 @@ namespace xo {
x.print(os);
return os;
}
#endif
} /*namespace scm*/
} /*namespace xo*/

21
xo-reader/include/xo/reader/envframestack.hpp
View file

@ -5,7 +5,7 @@
#pragma once
#include "envframe.hpp"
#include "xo/expression/LocalEnv.hpp"
namespace xo {
namespace scm {
@ -14,6 +14,7 @@ namespace xo {
**/
class envframestack {
public:
using LocalEnv = xo::ast::LocalEnv;
using Variable = xo::ast::Variable;
public:
@ -26,40 +27,40 @@ namespace xo {
* Visit frames in fifo order, report first match;
* nullptr if no matches.
**/
rp<Variable> lookup(const std::string & x) const;
bp<Variable> lookup(const std::string & x) const;
/** update/replace binding for variable @p target.
* New binding may have a different type.
**/
void upsert(bp<Variable> target);
envframe & top_envframe();
void push_envframe(envframe x);
void pop_envframe();
bp<LocalEnv> top_envframe();
void push_envframe(const rp<LocalEnv> & x);
rp<LocalEnv> pop_envframe();
/** relative to top-of-stack.
* 0 -> top (last in), z-1 -> bottom (first in)
**/
envframe & operator[](std::size_t i) {
bp<LocalEnv> operator[](std::size_t i) {
std::size_t z = stack_.size();
assert(i < z);
return stack_[z - i - 1];
return stack_[z - i - 1].get();
}
const envframe & operator[](std::size_t i) const {
bp<LocalEnv> operator[](std::size_t i) const {
std::size_t z = stack_.size();
assert(i < z);
return stack_[z - i - 1];
return stack_[z - i - 1].get();
}
void print (std::ostream & os) const;
private:
std::vector<envframe> stack_;
std::vector<rp<LocalEnv>> stack_;
};
inline std::ostream &

13
xo-reader/include/xo/reader/lambda_xs.hpp
View file

@ -6,6 +6,7 @@
#pragma once
#include "exprstate.hpp"
#include "xo/expression/LocalEnv.hpp"
//#include <cstdint>
namespace xo {
@ -46,6 +47,10 @@ namespace xo {
*
**/
class lambda_xs : public exprstate {
public:
using Environment = xo::ast::Environment;
using LocalEnv = xo::ast::LocalEnv;
public:
lambda_xs();
@ -71,11 +76,15 @@ namespace xo {
/** parsing state-machine state **/
lambdastatetype lmxs_type_ = lambdastatetype::lm_0;
/** formal parameter list **/
std::vector<rp<Variable>> argl_;
/** lambda environment (for formal parameters) **/
rp<LocalEnv> local_env_;
/** body expression **/
rp<Expression> body_;
/** parent environment **/
rp<Environment> parent_env_;
};
} /*namespace scm*/
} /*namespace xo*/

11
xo-reader/include/xo/reader/let1_xs.hpp
View file

@ -6,14 +6,17 @@
#pragma once
#include "exprstate.hpp"
#include "xo/expression/LocalEnv.hpp"
namespace xo {
namespace scm {
class let1_xs : public exprstate {
public:
using LocalEnv = xo::ast::LocalEnv;
public:
/** given local definition equivalent to
* def lhs_name = rhs
* def lhs_name = rhs;
* rest...
* parse sequence of incoming expressions rest... (until '}')
*
@ -27,21 +30,27 @@ namespace xo {
virtual void on_expr(bp<Expression> expr,
parserstatemachine * p_psm) override;
virtual void on_expr_with_semicolon(bp<Expression> expr,
parserstatemachine * p_psm) override;
virtual void on_rightbrace_token(const token_type & tk,
parserstatemachine * p_psm) override;
private:
let1_xs(std::string lhs_name,
rp<LocalEnv> local_env,
rp<Expression> rhs);
/** named ctor idiom **/
static std::unique_ptr<let1_xs> make(std::string lhs_name,
rp<LocalEnv> local_env,
rp<Expression> rhs);
private:
/** name for new local variable **/
std::string lhs_name_;
/** environment. contains just @ref lhs_name_ **/
rp<LocalEnv> local_env_;
/** set initial value for @ref lhs_name_ from value of this expression **/
rp<Expression> rhs_;

9
xo-reader/include/xo/reader/parserstatemachine.hpp
View file

@ -21,6 +21,7 @@ namespace xo {
public:
using Expression = xo::ast::Expression;
using Variable = xo::ast::Variable;
using LocalEnv = xo::ast::LocalEnv;
using token_type = token<char>;
public:
@ -38,15 +39,17 @@ namespace xo {
/** lookup variable name in lexical context represented by
* this psm. nullptr if not found
**/
rp<Variable> lookup_var(const std::string & x) const;
bp<Variable> lookup_var(const std::string & x) const;
/** update/replace binding for variable @p x in lexical context
* represented by this psm. Always acts on innermost frame.
**/
void upsert_var(bp<Variable> x);
void push_envframe(envframe x);
void pop_envframe();
bp<LocalEnv> top_envframe() const;
void push_envframe(const rp<LocalEnv> & x);
rp<LocalEnv> pop_envframe();
size_t env_stack_size() const { return p_env_stack_->size(); }
// ----- parsing outputs -----

5
xo-reader/include/xo/reader/progress_xs.hpp
View file

@ -51,6 +51,9 @@ namespace xo {
* Handles an expression that produces a value, for example appearing on the
* right-hand side of a definition.
*
* Deals with infix operators, handles operator precedence.
* Also generates argument-type-specific arithmetic expressions,
* for example using ``Apply::make_add2_f64()`` when adding floating-point numbers
**/
class progress_xs : public exprstate {
public:
@ -71,6 +74,8 @@ namespace xo {
virtual void on_expr(bp<Expression> expr,
parserstatemachine * p_psm) override;
virtual void on_expr_with_semicolon(bp<Expression> expr,
parserstatemachine * p_psm) override;
virtual void on_symbol_token(const token_type & tk,
parserstatemachine * p_psm) override;
virtual void on_typedescr(TypeDescr td,

3
xo-reader/src/reader/CMakeLists.txt
View file

@ -19,8 +19,7 @@ set(SELF_SRCS
expect_type_xs.cpp
lambda_xs.cpp
let1_xs.cpp
envframestack.cpp
envframe.cpp)
envframestack.cpp)
xo_add_shared_library4(${SELF_LIB} ${PROJECT_NAME}Targets ${PROJECT_VERSION} 1 ${SELF_SRCS})
xo_dependency(${SELF_LIB} xo_expression)

34
xo-reader/src/reader/define_xs.cpp
View file

@ -93,6 +93,11 @@ namespace xo {
define_xs::on_expr_with_semicolon(bp<Expression> expr,
parserstatemachine * p_psm)
{
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
log && log(xtag("defxs_type", defxs_type_));
this->on_expr(expr, p_psm);
/* semicolon is allowed to terminate def expr */
this->on_semicolon_token(token_type::semicolon(), p_psm);
@ -190,10 +195,31 @@ namespace xo {
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
/* remember variable binding in lexical context,
* so we can refer to it later
*/
p_psm->upsert_var(def_expr->lhs_variable());
// if this is a genuine top-level define (i.e. nesting level = 0),
// then we need to upsert so we can refer to rhs later.
//
// In other contexts (e.g. body-of-lambda) will be rewriting
// {
// def y = foo(x,x);
// bar(y,y);
// }
// into something like
// {
// (lambda (y123) bar(y123,y123))(foo(x,x));
// }
//
// This works in the body of lambda, because we don't evaluate anything
// until lambda definition is complete.
//
// For interactive top-level defs we want to evaluate as we go,
// so need incremental bindings.
if (p_psm->env_stack_size() == 1) {
/* remember variable binding in lexical context,
* so we can refer to it later
*/
p_psm->upsert_var(def_expr->lhs_variable());
}
p_psm->top_exprstate().on_expr(def_expr, p_psm);
} else {

31
xo-reader/src/reader/envframe.cpp
View file

@ -10,37 +10,30 @@ namespace xo {
using xo::ast::Variable;
namespace scm {
rp<Variable>
envframe::lookup(const std::string & x) const {
for (const auto & var : argl_) {
if (x == var->name())
return var;
}
#ifdef OBSOLETE
envframe::envframe(const std::vector<rp<Variable>> & argl,
const rp<Environment>& parent_env)
{
this->env_ = LocalEnv::make(argl, parent_env);
}
return nullptr;
bp<Variable>
envframe::lookup(const std::string & target) const {
return env_->lookup_local(target);
}
void
envframe::upsert(bp<Variable> target) {
for (auto & var : this->argl_) {
if (var->name() == target->name()) {
/* replace existing variable -- may change type */
var = target.promote();
return;
}
}
/* here: target not already present in this frame, append it */
this->argl_.push_back(target.promote());
env_->upsert_local(target);
}
void
envframe::print(std::ostream & os) const {
os << "<envframe"
<< xtag("argl", argl_)
<< xtag("argv", env_->argv())
<< ">";
}
#endif
} /*namespace scm */
} /*namespace xo*/

26
xo-reader/src/reader/envframestack.cpp
View file

@ -6,10 +6,11 @@
#include "envframestack.hpp"
namespace xo {
using xo::ast::LocalEnv;
using xo::ast::Variable;
namespace scm {
envframe &
bp<LocalEnv>
envframestack::top_envframe() {
std::size_t z = stack_.size();
@ -18,11 +19,12 @@ namespace xo {
("parser::top_exprstate: unexpected empty stack");
}
return stack_[z-1];
return stack_[z-1].get();
}
void
envframestack::push_envframe(envframe frame) {
envframestack::push_envframe(const rp<LocalEnv> & frame)
{
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag),
xtag("frame", frame));
@ -31,10 +33,10 @@ namespace xo {
stack_.resize(z+1);
stack_[z] = std::move(frame);
stack_[z] = frame;
}
void
rp<LocalEnv>
envframestack::pop_envframe() {
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
@ -44,26 +46,28 @@ namespace xo {
if (z > 0) {
//std::unique_ptr<exprstate> top = std::move(stack_[z-1]);
rp<LocalEnv> retval = stack_.at(z-1);
stack_.resize(z-1);
//return top;
return retval;
} else {
//return nullptr;
return nullptr;
}
}
rp<Variable>
bp<Variable>
envframestack::lookup(const std::string & x) const {
for (std::size_t i = 0, z = this->size(); i < z; ++i) {
const auto & frame = (*this)[i];
auto retval = frame.lookup(x);
auto retval = frame->lookup_local(x);
if (retval)
return retval;
}
return nullptr;
return bp<Variable>::from_native(nullptr);
}
void
@ -71,7 +75,7 @@ namespace xo {
/* upsert should always happen in the innermost lexical context.
* We are providing new variable binding (perhaps shadowing an existing binding)
*/
this->top_envframe().upsert(x);
this->top_envframe()->upsert_local(x);
}
void

4
xo-reader/src/reader/expect_expr_xs.cpp
View file

@ -139,7 +139,7 @@ namespace xo {
* and {(2), (3)} (symbol is function call)
*/
rp<Variable> var = p_psm->lookup_var(tk.text());
bp<Variable> var = p_psm->lookup_var(tk.text());
if (!var) {
throw std::runtime_error
@ -157,7 +157,7 @@ namespace xo {
* def y = foo(pi2);
* ^
*/
progress_xs::start(var, p_psm);
progress_xs::start(var.promote(), p_psm);
#ifdef NOT_YET
p_stack->push_exprstate(exprstate(exprstatetype::expr_progress,

4
xo-reader/src/reader/expect_symbol_xs.cpp
View file

@ -33,10 +33,12 @@ namespace xo {
log && log(xtag("tk", tk));
assert(&p_psm->top_exprstate() == this);
/* have to do pop first, before sending symbol to
* the o.g. symbol-requester
*/
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
std::unique_ptr<exprstate> self{p_psm->pop_exprstate()};
p_psm->on_symbol(tk.text());
}

4
xo-reader/src/reader/exprseq_xs.cpp
View file

@ -63,10 +63,10 @@ namespace xo {
* a + b; // rhs expression
* Variable must have been defined!
*/
rp<Variable> var = p_psm->lookup_var(tk.text());
bp<Variable> var = p_psm->lookup_var(tk.text());
if (var.get()) {
progress_xs::start(var, p_psm);
progress_xs::start(var.promote(), p_psm);
} else {
this->unknown_variable_error(c_self_name, tk);
}

1
xo-reader/src/reader/exprstate.cpp
View file

@ -289,6 +289,7 @@ namespace xo {
log && log(xtag("tk", tk));
log && log(xtag("state", *this));
log && log(xtag("psm", *p_psm));
log && log(xtag("proofoflogging", true));
switch (tk.tk_type()) {

13
xo-reader/src/reader/lambda_xs.cpp
View file

@ -9,6 +9,7 @@
namespace xo {
using xo::ast::Lambda;
using xo::ast::LocalEnv;
namespace scm {
const char *
@ -60,9 +61,10 @@ namespace xo {
{
if (lmxs_type_ == lambdastatetype::lm_1) {
this->lmxs_type_ = lambdastatetype::lm_2;
this->argl_ = argl;
this->parent_env_ = p_psm->top_envframe().promote();
this->local_env_ = LocalEnv::make(argl, parent_env_);
p_psm->push_envframe(envframe(argl));
p_psm->push_envframe(local_env_);
expect_expr_xs::start(p_psm);
} else {
@ -99,12 +101,13 @@ namespace xo {
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
std::string name = "fixmename";
rp<Lambda> lm = Lambda::make(name, argl_, body_);
std::string name = Variable::gensym("lambda");
/* top env frame recorded arguments to this lambda */
p_psm->pop_envframe();
rp<Lambda> lm = Lambda::make_from_env(name, local_env_, body_);
p_psm->top_exprstate().on_expr(lm, p_psm);
p_psm->top_exprstate().on_semicolon_token(tk, p_psm);

70
xo-reader/src/reader/let1_xs.cpp
View file

@ -17,20 +17,18 @@ namespace xo {
using Apply = xo::ast::Apply;
using Lambda = xo::ast::Lambda;
using LambdaAccess = xo::ast::LambdaAccess;
using Environment = xo::ast::Environment;
using LocalEnv = xo::ast::LocalEnv;
using Variable = xo::ast::Variable;
namespace {
std::string gensym() {
return "genanotherxx";
}
}
namespace scm {
std::unique_ptr<let1_xs>
let1_xs::make(std::string lhs_name,
rp<LocalEnv> local_env,
rp<Expression> rhs)
{
return std::make_unique<let1_xs>(let1_xs(std::move(lhs_name),
std::move(local_env),
std::move(rhs)));
}
@ -39,7 +37,16 @@ namespace xo {
const rp<Expression> & rhs,
parserstatemachine * p_psm)
{
p_psm->push_exprstate(let1_xs::make(lhs_name, rhs));
rp<Environment> parent_env = p_psm->top_envframe().promote();
rp<Variable> var1 = Variable::make(lhs_name, rhs->valuetype());
rp<LocalEnv> let_env = LocalEnv::make1(var1, parent_env);
p_psm->push_envframe(let_env);
// TODO: stash let_env in let1_xs, then pick up directly in .on_rightbrace_token()
// still have to push here so vars can find it
//
p_psm->push_exprstate(let1_xs::make(lhs_name, let_env, rhs));
expect_expr_xs::start(true /*allow_defs*/,
true /*cxl_on_rightbrace*/,
@ -47,9 +54,11 @@ namespace xo {
}
let1_xs::let1_xs(std::string lhs_name,
rp<LocalEnv> local_env,
rp<Expression> rhs)
: exprstate(),
: exprstate(exprstatetype::let1expr),
lhs_name_{std::move(lhs_name)},
local_env_{std::move(local_env)},
rhs_{std::move(rhs)}
{}
@ -63,7 +72,8 @@ namespace xo {
bp<DefineExpr> def_expr = DefineExpr::from(expr);
if (def_expr) {
/** nested_start: control returns via
/** starting a nested let here:
* control returns via
* .on_expr(x)
* with x something like:
* Apply(Lambda(gensym(),
@ -85,6 +95,32 @@ namespace xo {
}
}
void
let1_xs::on_expr_with_semicolon(bp<Expression> expr,
parserstatemachine * p_psm)
{
/* same as on_expr(), since we only use let1_xs inside a block { .. }
* This means final ';' is unnecessary
*/
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
bp<DefineExpr> def_expr = DefineExpr::from(expr);
if (def_expr) {
let1_xs::start(def_expr->lhs_name(),
def_expr->rhs(),
p_psm);
} else {
this->expr_v_.push_back(expr.promote());
expect_expr_xs::start(true /*allow_defs*/,
true /*cxl_on_rightbvrace*/,
p_psm);
}
}
void
let1_xs::on_rightbrace_token(const token_type & tk,
parserstatemachine * p_psm)
@ -93,13 +129,19 @@ namespace xo {
auto expr = Sequence::make(this->expr_v_);
std::string argname = gensym();
/* top env frame was established by let1_xs::start();
* now unwind it
*/
p_psm->pop_envframe();
std::string lambda_name = Variable::gensym("let1");
rp<Environment> parent_env = p_psm->top_envframe().promote();
rp<Expression> lambda
= Lambda::make(this->lhs_name_,
{Variable::make(argname,
this->rhs_->valuetype())},
expr);
= Lambda::make_from_env(lambda_name,
local_env_,
expr);
rp<Expression> result
= Apply::make(lambda, {this->rhs_});

11
xo-reader/src/reader/parser.cpp
View file

@ -10,11 +10,15 @@
#include "xo/expression/DefineExpr.hpp"
#include "xo/expression/Constant.hpp"
#include "xo/expression/ConvertExpr.hpp"
//#include "xo/expression/GlobalEnv.hpp"
#include "xo/expression/LocalEnv.hpp"
//#include <regex>
#include <stdexcept>
namespace xo {
using xo::ast::Expression;
//using xo::ast::GlobalEnv;
using xo::ast::LocalEnv;
//using xo::ast::DefineExpr;
//using xo::ast::ConvertExpr;
//using xo::ast::Constant;
@ -28,14 +32,15 @@ namespace xo {
: xs_stack_{}, env_stack_{}
{
/* top-level environment. initially empty */
envframe toplevel_env;
rp<LocalEnv> toplevel_env = LocalEnv::make_empty();
this->env_stack_.push_envframe(toplevel_env);
}
bool
parser::has_incomplete_expr() const {
return !xs_stack_.empty();
/* (don't count toplevel exprseq) */
return xs_stack_.size() > 1;
}
void
@ -73,6 +78,8 @@ namespace xo {
/* stack_ is non-empty */
log && log(xtag("top", xs_stack_.top_exprstate()));
rp<Expression> retval;
parserstatemachine psm(&xs_stack_, &env_stack_, &retval);

18
xo-reader/src/reader/parserstatemachine.cpp
View file

@ -7,10 +7,11 @@
#include "exprstatestack.hpp"
namespace xo {
using xo::ast::LocalEnv;
using xo::ast::Variable;
namespace scm {
rp<Variable>
bp<Variable>
parserstatemachine::lookup_var(const std::string & x) const {
return p_env_stack_->lookup(x);
}
@ -35,22 +36,27 @@ namespace xo {
p_stack_->push_exprstate(std::move(x));
}
bp<LocalEnv>
parserstatemachine::top_envframe() const {
return p_env_stack_->top_envframe();
}
void
parserstatemachine::push_envframe(envframe x) {
parserstatemachine::push_envframe(const rp<LocalEnv> & x) {
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
log && log(xtag("frame", x));
p_env_stack_->push_envframe(std::move(x));
p_env_stack_->push_envframe(x);
}
void
rp<LocalEnv>
parserstatemachine::pop_envframe() {
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
p_env_stack_->pop_envframe();
return p_env_stack_->pop_envframe();
}
void
@ -75,6 +81,8 @@ namespace xo {
log && log(xtag("x", x),
xtag("psm", *this));
assert(!this->p_stack_->empty());
this->p_stack_
->top_exprstate().on_expr_with_semicolon(x, this);
}

31
xo-reader/src/reader/progress_xs.cpp
View file

@ -187,6 +187,35 @@ namespace xo {
this->rhs_ = expr.promote();
}
void
progress_xs::on_expr_with_semicolon(bp<Expression> expr,
parserstatemachine * p_psm)
{
constexpr bool c_debug_flag = true;
scope log(XO_DEBUG(c_debug_flag));
log && log(xtag("lhs", lhs_), xtag("op", op_type_), xtag("expr", expr));
constexpr const char * c_self_name = "progress_xs::on_expr_with_semicolon";
if (op_type_ == optype::invalid) {
throw std::runtime_error(tostr(c_self_name,
": consecutive unseparated exprs not legal"));
}
this->rhs_ = expr.promote();
// FORBIDDEN, because .on_semicolon_token() destroys *this before returning
// this->on_semicolon_token(token_type::semicolon(), p_psm);
// INSTEAD, spell out the body
rp<Expression> expr2 = this->assemble_expr();
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
p_psm->on_expr_with_semicolon(expr2);
}
void
progress_xs::on_symbol_token(const token_type & /*tk*/,
parserstatemachine * /*p_psm*/)
@ -225,6 +254,8 @@ namespace xo {
rp<Expression> expr = this->assemble_expr();
log && log(xtag("assembled-expr", expr));
std::unique_ptr<exprstate> self = p_psm->pop_exprstate();
p_psm->on_expr_with_semicolon(expr);

4
xo-reader/utest/reader.test.cpp
View file

@ -15,11 +15,11 @@ namespace xo {
std::vector<test_case> s_testcase_v = {
{"def foo : f64 = 3.14159265;"},
{"def foo : f64 = (3.14159265);"},
//{"def foo : f64 = 2.0 * 3.14159265;"},
{"def foo = 2.0 * 3.141569265;"},
{"def foo = lambda (x : f64) 3.1415965;"},
{"def foo = lambda (x : f64, y : f64) 3.1415965;"},
{"def foo = lambda (x : f64) x;"},
{"def foo = lambda (x : f64) { def y = x * x; y; }"},
//{"def foo = lambda (x : f64) { def y = x * x; y; }"},
};
}

4
xo-tokenizer/docs/install.rst
View file

@ -21,7 +21,7 @@ Tested with gcc 13.3
Install
=======
``xo-tokenizer`` uses supporting library ``xo-tokenizer`` and cmake macros ``xo-cmake``.
``xo-tokenizer`` uses supporting library ``xo-indentlog`` and cmake macros ``xo-cmake``.
These are on github:
- `xo-tokenizer source`_ (Schematika tokenizer)
@ -29,7 +29,7 @@ These are on github:
- `xo-cmake source`_ (shared cmake macros)
.. _xo-tokenizer source: https://github.com/rconybea/xo-tokenizer
.. _xo-indentlog source: https://github.com/rconybea/xo-indentlog
.. _xo-indentlog source: https://github.com/rconybea/indentlog
.. _xo-cmake source: https://github.com/rconybea/xo-cmake
Installing from source