From 3cc8e73ab4e07f66034b4c6fd4f6be2b8d18d7f1 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 14 Jun 2024 15:00:57 -0400 Subject: [PATCH 01/33] xo-pyjit: initial commit --- .gitignore | 6 ++++ CMakeLists.txt | 28 +++++++++++++++ cmake/xo-bootstrap-macros.cmake | 35 +++++++++++++++++++ cmake/xo_pyjitConfig.cmake.in | 7 ++++ include/README.md | 1 + src/pyjit/CMakeLists.txt | 9 +++++ src/pyjit/pyjit.cpp | 60 +++++++++++++++++++++++++++++++++ src/pyjit/pyjit.hpp.in | 25 ++++++++++++++ 8 files changed, 171 insertions(+) create mode 100644 .gitignore create mode 100644 CMakeLists.txt create mode 100644 cmake/xo-bootstrap-macros.cmake create mode 100644 cmake/xo_pyjitConfig.cmake.in create mode 100644 include/README.md create mode 100644 src/pyjit/CMakeLists.txt create mode 100644 src/pyjit/pyjit.cpp create mode 100644 src/pyjit/pyjit.hpp.in diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..13c0afb7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# clangd working space (see emacs+lsp) +.cache +# typical cmake build directory (source-tree-nephew) +.build* +# symlink to builddir/compile_commands.json; should be set manually in dev sandbox +compile_commands.json diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..bacd5c53 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,28 @@ +# xo-pyjit/CMakeLists.txt + +cmake_minimum_required(VERSION 3.10) + +project(xo_pyjit VERSION 0.1) + +include(GNUInstallDirs) +include(cmake/xo-bootstrap-macros.cmake) + +xo_cxx_toplevel_options3() + +# ---------------------------------------------------------------- +# c++ settings (usually temporary) + +set(PROJECT_CXX_FLAGS "") +add_definitions(${PROJECT_CXX_FLAGS}) + +# ---------------------------------------------------------------- + +add_subdirectory(src/pyjit) +#add_subdirectory(utest) + +# ---------------------------------------------------------------- +# provide find_package() support + +xo_export_cmake_config(${PROJECT_NAME} ${PROJECT_VERSION} ${PROJECT_NAME}Targets) + +# end CMakeLists.txt diff --git a/cmake/xo-bootstrap-macros.cmake b/cmake/xo-bootstrap-macros.cmake new file mode 100644 index 00000000..aba31169 --- /dev/null +++ b/cmake/xo-bootstrap-macros.cmake @@ -0,0 +1,35 @@ +# ---------------------------------------------------------------- +# for example: +# $ PREFIX=/usr/local # for example +# $ cmake -DCMAKE_MODULE_PATH=prefix -DCMAKE_INSTALL_PREFIX=$PREFIX -B .build +# +# will get +# CMAKE_MODULE_PATH +# from xo-cmake-config --cmake-module-path +# +# and expect .cmake macros in +# CMAKE_MODULE_PATH/xo_macros/xo_cxx.cmake +# ---------------------------------------------------------------- + +find_program(XO_CMAKE_CONFIG_EXECUTABLE NAMES xo-cmake-config REQUIRED) + +if ("${XO_CMAKE_CONFIG_EXECUTABLE}" STREQUAL "XO_CMAKE_CONFIG_EXECUTABLE-NOT_FOUND") + message(FATAL "could not find xo-cmake-config executable") +endif() + +message(STATUS "XO_CMAKE_CONFIG_EXECUTABLE=${XO_CMAKE_CONFIG_EXECUTABLE}") + +if (NOT XO_SUBMODULE_BUILD) + if (("${CMAKE_MODULE_PATH}" STREQUAL "") OR ("${CMAKE_MODULE_PATH}" STREQUAL prefix)) + # default to typical install location for xo-project-macros + execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --cmake-module-path OUTPUT_VARIABLE CMAKE_MODULE_PATH) + message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + endif() +endif() + +# needs to have been installed somewhere on CMAKE_MODULE_PATH, +# (e.g. from xo-cmake with the same value for CMAKE_INSTALL_PREFIX) +# +include(xo_macros/xo_cxx) + +xo_cxx_bootstrap_message() diff --git a/cmake/xo_pyjitConfig.cmake.in b/cmake/xo_pyjitConfig.cmake.in new file mode 100644 index 00000000..5d7de08a --- /dev/null +++ b/cmake/xo_pyjitConfig.cmake.in @@ -0,0 +1,7 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(xo_jit) +find_dependency(xo_pyexpression) +include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") +check_required_components("@PROJECT_NAME@") diff --git a/include/README.md b/include/README.md new file mode 100644 index 00000000..9db0605f --- /dev/null +++ b/include/README.md @@ -0,0 +1 @@ +placeholder for future xo-pymatrix header files diff --git a/src/pyjit/CMakeLists.txt b/src/pyjit/CMakeLists.txt new file mode 100644 index 00000000..d6d84521 --- /dev/null +++ b/src/pyjit/CMakeLists.txt @@ -0,0 +1,9 @@ +# xo-pyjit/src/pyjit/CMakeLists.txt + +set(SELF_LIB xo_pyjit) +set(SELF_SRCS pyjit.cpp) + +xo_pybind11_library(${SELF_LIB} ${PROJECT_NAME}Targets ${SELF_SRCS}) +xo_pybind11_dependency(${SELF_LIB} xo_jit) +xo_pybind11_dependency(${SELF_LIB} xo_pyexpression) +xo_dependency(${SELF_LIB} refcnt) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp new file mode 100644 index 00000000..a8a532ec --- /dev/null +++ b/src/pyjit/pyjit.cpp @@ -0,0 +1,60 @@ +/* @file pyjit.cpp */ + +#include "pyjit.hpp" +#include "xo/pyexpression/pyexpression.hpp" +#include "xo/jit/Jit.hpp" +#include "xo/pyutil/pyutil.hpp" + +namespace xo { + namespace jit { + using xo::ast::Expression; + using xo::ref::rp; + using xo::ref::unowned_ptr; + namespace py = pybind11; + + PYBIND11_MODULE(XO_PYJIT_MODULE_NAME(), m) { + // e.g. for xo::ast::Expression + XO_PYEXPRESSION_IMPORT_MODULE(); // py::module_::import("pyexpression"); + + m.doc() = "pybind11 plugin for xo-jit"; + + py::class_>(m, "Jit") + .def_static("make", &Jit::make, + py::doc("create Jit instance. Not threadsafe," + " but does not share resources with any other Jit instance")) + + .def("codegen", + [](Jit & jit, const rp & expr) { + return jit.codegen(expr.borrow()); + }, + py::arg("x"), + py::doc("generate llvm (IR) code for Expression x"), + /* we're assuming llvm-generated code lives for as long as the Jit + * instance that created it. + * + * RC 14jun2024 - I think this is true modulo use of llvm resource trackers. + */ + py::return_value_policy::reference_internal) + ; + + py::class_>(m, "llvm_Value") + .def("print", + [](llvm::Value & x) { + std::string buf; + llvm::raw_string_ostream ss(buf); + x.print(ss); + return buf; + }) + .def("__repr__", + &Jit::display_string) + ; + + } + + + } /*namespace jit*/ +} /*namespace xo*/ + + +/* end pyjit.cpp */ diff --git a/src/pyjit/pyjit.hpp.in b/src/pyjit/pyjit.hpp.in new file mode 100644 index 00000000..8abdc8e3 --- /dev/null +++ b/src/pyjit/pyjit.hpp.in @@ -0,0 +1,25 @@ +/* @file pyjit.hpp + * + * automatically generated from src/xo_pyjit/pyjit.hpp.in + * see src/xo_pyjit/CMakeLists.txt + */ + +/* python requires module name = library name + * example: + * PYBIND11_MODULE(XO_PYJIT_MODULE_NAME(), m) { ... } + */ +#define XO_PYJIT_MODULE_NAME() @SELF_LIB@ + +/* example: + * py::module_::import(XO_PYJIT_MODULE_NAME_STR) + */ +#define XO_PYJIT_MODULE_NAME_STR "@SELF_LIB@" + +/* example: + * XO_PYJIT_IMPORT_MODULE() + * replaces + * py::module_::import("pyjit") + */ +#define XO_PYJIT_IMPORT_MODULE() py::module_::import("@SELF_LIB@") + +/* end pyjit.hpp */ From 5b8b08145b250749d5d50cf004747edf14687e6b Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 14 Jun 2024 15:05:47 -0400 Subject: [PATCH 02/33] + README.md --- README.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 00000000..9e55c217 --- /dev/null +++ b/README.md @@ -0,0 +1,73 @@ +# python bindings for llvm JIT for EGAD (xo-pyjit) + +## Getting Started + +### Build + install dependencies + +- [github/Rconybea/xo-jit](https://github.com/Rconybea/xo-jit) +- [github/Rconybea/xo-pyexpression](https://github.com/Rconybea/xo-pyexpression) + +### build + isntall + +``` +$ cd xo-pyjit +$ PREFIX=/usr/local # or preferred install location +$ cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -S . -B .build +$ cmake --build .build -j +$ cmake --install .build +``` +(also see .github/workflows/main.yml) + +## Examples + +Assumes `xo-pyjit` installed to `~/local2/lib`, +i.e. built with `PREFIX=~/local2`. +``` +PYTHONPATH=~/local2/lib:$PYTHONPATH python +>>> from xo_pyexpression import * +>>> from xo_pyjit import * +>>> x=make_constant(3.14159) +>>> jit=Jit.make() +>>> code=jit.codegen(x) +>>> x.print +double 3.141600e+00 +``` + +## Development + +### use from build tree + +Limited utility: requires that supporting libraries (e.g. `xo_pyexpression`) appear in PYTHONPATH +``` +$ cd xo-pyjit/.build/src/pyjit +$ python +>>> import xo_pyjit +``` + +### build for unit test coverage +``` +$ cd xo-pyexpression +$ cmake -DCMAKE_BUILD_TYPE=coverage -DENABLE_TESTING=on -S . -B .build-ccov +$ cmake --build .build-ccov -j +``` + +### LSP (language server) support + +LSP looks for compile commands in the root of the source tree; +while Cmake creates them in the root of its build directory. + +``` +$ cd xo-pyexpression +$ ln -s .build/compile_commands.json # supply compile commands to LSP +``` + +### display cmake variables + +- `-L` list variables +- `-A` include 'advanced' variables +- `-H` include help text + +``` +$ cd xo-pyjit/.build +$ cmake -LAH +``` From d315d4212b781e6b0ab4e2aed6a0363b35ab05dc Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 14 Jun 2024 15:06:15 -0400 Subject: [PATCH 03/33] .gitignore: + ignore .projectile --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 13c0afb7..e6766880 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +# emacs configuration for workspace +.projectile # clangd working space (see emacs+lsp) .cache # typical cmake build directory (source-tree-nephew) From f8b73140557fbadb6fae00134119761bd47e6ed2 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 14 Jun 2024 15:42:24 -0400 Subject: [PATCH 04/33] xo-pyjit: README: expand + use xo-build for XO deps --- README.md | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 9e55c217..f78c49fd 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,45 @@ ## Getting Started -### Build + install dependencies +### Build + install `xo-cmake` dependency -- [github/Rconybea/xo-jit](https://github.com/Rconybea/xo-jit) -- [github/Rconybea/xo-pyexpression](https://github.com/Rconybea/xo-pyexpression) +- [github/Rconybea/xo-cmake](https://github.com/Rconybea/xo-cmake) -### build + isntall +Installs a few cmake ingredients, along with a build assistant `xo-build` for XO projects such as this one. + +### build + install other necessary XO dependencies +``` +$ xo-build --clone --configure --build --install xo-indentlog +$ xo-build --clone --configure --build --install xo-refnct +$ xo-build --clone --configure --build --install xo-subsys +$ xo-build --clone --configure --build --install xo-reflect +$ xo-build --clone --configure --build --install xo-expression +$ xo-build --clone --configure --build --install xo-jit +$ xo-build --clone --configure --build --install xo-pyutil +$ xo-build --clone --configure --build --install xo-pyexpression +``` +### copy `xo-pyjit` repository locally +``` +$ xo-build --clone xo-pyjit +``` + +or equivalently +``` +$ git clone git@github.com:Rconybea/xo-pyjit.git +``` + +### build + install xo-pyjit +``` +$ xo-build --configure --build --install xo-pyjit +``` + +or equivalently: ``` -$ cd xo-pyjit $ PREFIX=/usr/local # or preferred install location -$ cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -S . -B .build -$ cmake --build .build -j -$ cmake --install .build +$ cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -S xo-pyjit -B xo-pyjit/.build +$ cmake --build xo-pyjit/.build -j +$ cmake --install xo-pyjit/.build ``` (also see .github/workflows/main.yml) From 94d03915854ee6ca1493f5c22f8300580cdd6bfd Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 14 Jun 2024 16:12:22 -0400 Subject: [PATCH 05/33] xo-pyjit: minor -- drop printing to procrastinate on compiler nit --- README.md | 2 ++ src/pyjit/pyjit.cpp | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f78c49fd..1f46c9a5 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ $ xo-build --clone --configure --build --install xo-jit $ xo-build --clone --configure --build --install xo-pyutil $ xo-build --clone --configure --build --install xo-pyexpression ``` +note: can use `xo-build -n` to dry-run here + ### copy `xo-pyjit` repository locally ``` $ xo-build --clone xo-pyjit diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index a8a532ec..db202824 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -46,8 +46,8 @@ namespace xo { x.print(ss); return buf; }) - .def("__repr__", - &Jit::display_string) +// .def("__repr__", +// &Jit::display_string) ; } From dafa567eaa8ce10fce989fb8d4869c97cbff9c9e Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 15 Jun 2024 15:15:07 -0400 Subject: [PATCH 06/33] xo-pyjit: invoke pybind11 stl support --- src/pyjit/pyjit.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index db202824..5f233fff 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -4,6 +4,7 @@ #include "xo/pyexpression/pyexpression.hpp" #include "xo/jit/Jit.hpp" #include "xo/pyutil/pyutil.hpp" +#include namespace xo { namespace jit { From 7402746dbd0f1e224d7a537f6b2675c1147da361 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 15 Jun 2024 15:15:22 -0400 Subject: [PATCH 07/33] xo-pyjit: expose Jit instrumentation getters --- src/pyjit/pyjit.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 5f233fff..2bd2afaf 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -24,6 +24,12 @@ namespace xo { py::doc("create Jit instance. Not threadsafe," " but does not share resources with any other Jit instance")) + .def_property_readonly("target_triple", &Jit::target_triple, + py::doc("string describing target host for code generation")) + .def("get_function_name_v", &Jit::get_function_name_v, + py::doc("get vector of function names defined in jit module")) + .def("dump_execution_session", &Jit::dump_execution_session, + py::doc("write to console with state of all jit-owned dynamic libraries")) .def("codegen", [](Jit & jit, const rp & expr) { return jit.codegen(expr.borrow()); From 01e0b8ffbafe55f9ceb632304885614ea158dfdf Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 15 Jun 2024 15:15:45 -0400 Subject: [PATCH 08/33] xo-pyjit: + XferFn for symbol lookup [wip] --- src/pyjit/pyjit.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 2bd2afaf..14bfc9f4 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -7,6 +7,16 @@ #include namespace xo { + struct XferFn : public ref::Refcount { + using fptr_type = double (*) (double); + + explicit XferFn(fptr_type fptr) : fptr_{fptr} {} + + double operator() (double x) { return (*fptr_)(x); } + + fptr_type fptr_; + }; + namespace jit { using xo::ast::Expression; using xo::ref::rp; @@ -42,6 +52,10 @@ namespace xo { * RC 14jun2024 - I think this is true modulo use of llvm resource trackers. */ py::return_value_policy::reference_internal) + py::class_>(m, "XferFn") + .def("__call__", + [](XferFn & self, double x) { return self(x); } + ) ; py::class_ Date: Sat, 15 Jun 2024 15:16:00 -0400 Subject: [PATCH 09/33] xo-pyjit: + lookup_dbl_dbl_fn [wip -- not actually working yet] --- src/pyjit/pyjit.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 14bfc9f4..c69558a1 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -49,9 +49,19 @@ namespace xo { /* we're assuming llvm-generated code lives for as long as the Jit * instance that created it. * - * RC 14jun2024 - I think this is true modulo use of llvm resource trackers. + * RC 14jun2024 - I think this is true, modulo use of llvm resource trackers. */ py::return_value_policy::reference_internal) + .def("lookup_dbl_dbl_fn", + [](Jit & jit, const std::string & symbol) { + auto llvm_addr = jit.lookup_symbol(symbol); + + auto fn_addr = llvm_addr.toPtr(); + + return new XferFn(fn_addr); + }) + ; + py::class_>(m, "XferFn") .def("__call__", [](XferFn & self, double x) { return self(x); } From 595a3a2d5b3ff3b645a547b8fe473e00ca2a1e59 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 15 Jun 2024 17:14:43 -0400 Subject: [PATCH 10/33] xo-pyjit: + Jit.machgen_current_module --- src/pyjit/pyjit.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index c69558a1..69defaec 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -52,6 +52,9 @@ namespace xo { * RC 14jun2024 - I think this is true, modulo use of llvm resource trackers. */ py::return_value_policy::reference_internal) + .def("machgen_current_module", &Jit::machgen_current_module, + py::doc("Make current module available for execution via the jit.\n" + "Adds all functions generated since last call to this method.")) .def("lookup_dbl_dbl_fn", [](Jit & jit, const std::string & symbol) { auto llvm_addr = jit.lookup_symbol(symbol); From e22841148c9849296a4ab68646775ded2e781919 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sat, 15 Jun 2024 17:14:59 -0400 Subject: [PATCH 11/33] xo-pyjit: README: spelled out working example. --- README.md | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1f46c9a5..c739cd31 100644 --- a/README.md +++ b/README.md @@ -52,13 +52,66 @@ Assumes `xo-pyjit` installed to `~/local2/lib`, i.e. built with `PREFIX=~/local2`. ``` PYTHONPATH=~/local2/lib:$PYTHONPATH python ->>> from xo_pyexpression import * >>> from xo_pyjit import * ->>> x=make_constant(3.14159) +>>> from xo_pyexpression import * +``` + +create a jit from within python +``` >>> jit=Jit.make() ->>> code=jit.codegen(x) ->>> x.print -double 3.141600e+00 +>>> jit.dump_execution_sesion() +JITDylib "
" (ES: 0x0000000000446ee0, State = Open) +Link order: [ ("
", MatchAllSymbols) ] +Symbol table: +``` + +build an AST from within python +``` +>>> x=make_var('x') # "x" a variable (context not yet known) +>>> f1=make_sin_pm() # "sin()" +>>> c1=make_apply(f1,x) # "sin(x)" +>>> f2=make_cos_pm() # "cos()" +>>> c2=make_apply(f2,c1) # "cos(sin(x))" +>>> lm=make_lambda('foo', ['x'], c2) # "def foo(x): cos(sin(x))" +>>> lm + :argv "[ :argv \"[]\">]">> +``` + +generate llvm IR for our AST +``` +>>> code=jit.codegen(lm) +>>> print(code.print()) +define double @foo(double %x) { +entry: + %calltmp = call double @sin(double %x) + %calltmp1 = call double @cos(double %calltmp) + ret double %calltmp1 +} +``` + +generate machine code for our AST, lookup compiled function so we can invoke it directly +``` +>>> jit.machgen_current_module() +>>> jit.dump_execution_session() +JITDylib "
" (ES: 0x0000000000446ee0, State = Open) +Link order: [ ("
", MatchAllSymbols) ] +Symbol table: + "foo": [Callable] Never-Searched (Materializer 0x646fe0, xojit) +>>> fn=jit.lookup_dbl_dbl_fn('foo') + +>>> jit.dump_execution_session() +JITDylib "
" (ES: 0x0000000000446ee0, State = Open) +Link order: [ ("
", MatchAllSymbols) ] +Symbol table: + "cos": 0x7ffff7926670 [Data] Ready + "foo": 0x7fffee2b6000 [Callable] Ready + "sin": 0x7ffff7925e50 [Data] Ready +``` + +invoke just-compiled code! +``` +>>> fn(22) +0.999960827417674 ``` ## Development From a4af2a79d01c5847dea9cbd291747fc51e6f1de1 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 16 Jun 2024 11:49:50 -0400 Subject: [PATCH 12/33] xo-pyjit: refactor: Jit -> MachPipeline --- README.md | 14 +++++++------- src/pyjit/pyjit.cpp | 23 ++++++++++++----------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c739cd31..e9f08673 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,8 @@ PYTHONPATH=~/local2/lib:$PYTHONPATH python create a jit from within python ``` ->>> jit=Jit.make() ->>> jit.dump_execution_sesion() +>>> mp=MachPipeline.make() +>>> mp.dump_execution_sesion() JITDylib "
" (ES: 0x0000000000446ee0, State = Open) Link order: [ ("
", MatchAllSymbols) ] Symbol table: @@ -79,7 +79,7 @@ build an AST from within python generate llvm IR for our AST ``` ->>> code=jit.codegen(lm) +>>> code=mp.codegen(lm) >>> print(code.print()) define double @foo(double %x) { entry: @@ -91,15 +91,15 @@ entry: generate machine code for our AST, lookup compiled function so we can invoke it directly ``` ->>> jit.machgen_current_module() ->>> jit.dump_execution_session() +>>> mp.machgen_current_module() +>>> mp.dump_execution_session() JITDylib "
" (ES: 0x0000000000446ee0, State = Open) Link order: [ ("
", MatchAllSymbols) ] Symbol table: "foo": [Callable] Never-Searched (Materializer 0x646fe0, xojit) ->>> fn=jit.lookup_dbl_dbl_fn('foo') +>>> fn=mp.lookup_dbl_dbl_fn('foo') ->>> jit.dump_execution_session() +>>> mp.dump_execution_session() JITDylib "
" (ES: 0x0000000000446ee0, State = Open) Link order: [ ("
", MatchAllSymbols) ] Symbol table: diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 69defaec..624f8882 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -2,7 +2,7 @@ #include "pyjit.hpp" #include "xo/pyexpression/pyexpression.hpp" -#include "xo/jit/Jit.hpp" +#include "xo/jit/MachPipeline.hpp" #include "xo/pyutil/pyutil.hpp" #include @@ -29,19 +29,20 @@ namespace xo { m.doc() = "pybind11 plugin for xo-jit"; - py::class_>(m, "Jit") - .def_static("make", &Jit::make, - py::doc("create Jit instance. Not threadsafe," - " but does not share resources with any other Jit instance")) + py::class_>(m, "MachPipeline") + .def_static("make", &MachPipeline::make, + py::doc("Create machine pipeline for in-process code generation" + " and execution. Not threadsafe.\n" + "Does not share resources with any other instance")) - .def_property_readonly("target_triple", &Jit::target_triple, + .def_property_readonly("target_triple", &MachPipeline::target_triple, py::doc("string describing target host for code generation")) - .def("get_function_name_v", &Jit::get_function_name_v, + .def("get_function_name_v", &MachPipeline::get_function_name_v, py::doc("get vector of function names defined in jit module")) - .def("dump_execution_session", &Jit::dump_execution_session, + .def("dump_execution_session", &MachPipeline::dump_execution_session, py::doc("write to console with state of all jit-owned dynamic libraries")) .def("codegen", - [](Jit & jit, const rp & expr) { + [](MachPipeline & jit, const rp & expr) { return jit.codegen(expr.borrow()); }, py::arg("x"), @@ -52,11 +53,11 @@ namespace xo { * RC 14jun2024 - I think this is true, modulo use of llvm resource trackers. */ py::return_value_policy::reference_internal) - .def("machgen_current_module", &Jit::machgen_current_module, + .def("machgen_current_module", &MachPipeline::machgen_current_module, py::doc("Make current module available for execution via the jit.\n" "Adds all functions generated since last call to this method.")) .def("lookup_dbl_dbl_fn", - [](Jit & jit, const std::string & symbol) { + [](MachPipeline & jit, const std::string & symbol) { auto llvm_addr = jit.lookup_symbol(symbol); auto fn_addr = llvm_addr.toPtr(); From 93819cbb6c6b2093d6c2ced0ef7ff94fa2930df6 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 17 Jun 2024 17:16:07 -0400 Subject: [PATCH 13/33] xo-pyjit: README: bring example up-to-date --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e9f08673..342b3308 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,9 @@ build an AST from within python ``` >>> x=make_var('x') # "x" a variable (context not yet known) >>> f1=make_sin_pm() # "sin()" ->>> c1=make_apply(f1,x) # "sin(x)" +>>> c1=make_apply(f1,[x]) # "sin(x)" >>> f2=make_cos_pm() # "cos()" ->>> c2=make_apply(f2,c1) # "cos(sin(x))" +>>> c2=make_apply(f2,[c1]) # "cos(sin(x))" >>> lm=make_lambda('foo', ['x'], c2) # "def foo(x): cos(sin(x))" >>> lm :argv "[ :argv \"[]\">]">> @@ -97,7 +97,7 @@ JITDylib "
" (ES: 0x0000000000446ee0, State = Open) Link order: [ ("
", MatchAllSymbols) ] Symbol table: "foo": [Callable] Never-Searched (Materializer 0x646fe0, xojit) ->>> fn=mp.lookup_dbl_dbl_fn('foo') +>>> fn=mp.lookup_dbl2dbl_fn('foo') >>> mp.dump_execution_session() JITDylib "
" (ES: 0x0000000000446ee0, State = Open) From 5ea46cf22e46c50b1d1c3c5e17d872eb02660c88 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 17 Jun 2024 17:17:13 -0400 Subject: [PATCH 14/33] xo-pyjit: 1-arg and 2-arg double->double + double^2->double fns --- src/pyjit/pyjit.cpp | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 624f8882..2c76f219 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -7,15 +7,25 @@ #include namespace xo { - struct XferFn : public ref::Refcount { + struct XferDbl2DblFn : public ref::Refcount { using fptr_type = double (*) (double); - explicit XferFn(fptr_type fptr) : fptr_{fptr} {} + explicit XferDbl2DblFn(fptr_type fptr) : fptr_{fptr} {} double operator() (double x) { return (*fptr_)(x); } fptr_type fptr_; - }; + }; /*XferDbl2DblFn*/ + + struct XferDblDbl2DblFn : public ref::Refcount { + using fptr_type = double (*) (double, double); + + explicit XferDblDbl2DblFn(fptr_type fptr) : fptr_{fptr} {} + + double operator() (double x, double y) { return (*fptr_)(x, y); } + + fptr_type fptr_; + }; /*XferDblDbl2DblFn*/ namespace jit { using xo::ast::Expression; @@ -56,19 +66,36 @@ namespace xo { .def("machgen_current_module", &MachPipeline::machgen_current_module, py::doc("Make current module available for execution via the jit.\n" "Adds all functions generated since last call to this method.")) - .def("lookup_dbl_dbl_fn", + /* double -> double */ + .def("lookup_dbl2dbl_fn", [](MachPipeline & jit, const std::string & symbol) { auto llvm_addr = jit.lookup_symbol(symbol); auto fn_addr = llvm_addr.toPtr(); - return new XferFn(fn_addr); + return new XferDbl2DblFn(fn_addr); + }) + + /* (double x double) -> double */ + .def("lookup_dbldbl2dbl_fn", + [](MachPipeline & jit, const std::string & symbol) { + auto llvm_addr = jit.lookup_symbol(symbol); + + auto fn_addr = llvm_addr.toPtr(); + + return new XferDblDbl2DblFn(fn_addr); }) ; - py::class_>(m, "XferFn") + + py::class_>(m, "XferDbl2DblFn") .def("__call__", - [](XferFn & self, double x) { return self(x); } + [](XferDbl2DblFn & self, double x) { return self(x); } + ) + ; + py::class_>(m, "XferDblDbl2DblFn") + .def("__call__", + [](XferDblDbl2DblFn & self, double x, double y) { return self(x, y); } ) ; From 4acf521609810ca6241e07be112b20617b7a07eb Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 17 Jun 2024 17:17:46 -0400 Subject: [PATCH 15/33] xo-pyjit: use pycaller to streamline function pointer handling --- src/pyjit/pyjit.cpp | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 2c76f219..b446a2c9 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -3,6 +3,7 @@ #include "pyjit.hpp" #include "xo/pyexpression/pyexpression.hpp" #include "xo/jit/MachPipeline.hpp" +#include "xo/pyutil/pycaller.hpp" #include "xo/pyutil/pyutil.hpp" #include @@ -29,7 +30,10 @@ namespace xo { namespace jit { using xo::ast::Expression; + using xo::pyutil::pycaller_base; + using xo::pyutil::pycaller; using xo::ref::rp; + //using xo::ref::Refcount; using xo::ref::unowned_ptr; namespace py = pybind11; @@ -39,6 +43,8 @@ namespace xo { m.doc() = "pybind11 plugin for xo-jit"; + pycaller::declare_once(m); + pycaller::declare_once(m); py::class_>(m, "MachPipeline") .def_static("make", &MachPipeline::make, py::doc("Create machine pipeline for in-process code generation" @@ -85,6 +91,30 @@ namespace xo { return new XferDblDbl2DblFn(fn_addr); }) + + .def("lookup_fn", + [](MachPipeline & jit, const std::string & prototype, const std::string & symbol) -> pycaller_base* { + auto llvm_addr = jit.lookup_symbol(symbol); + + /* note: llvm_addr.toPtr<..> always succeeds, + * event if pointer refers to an object of incompatible type + * + * note: return value policy is for python to own the wrapper + */ + + if((prototype == "double(double,double)") || (prototype == "double(*)(double,double)")) { + auto fn_addr = llvm_addr.toPtr(); + + return new pycaller(fn_addr); + //return new XferDblDbl2DblFn(fn_addr); + } else if ((prototype == "double(double)") || (prototype == "double(*)(double)")) { + auto fn_addr = llvm_addr.toPtr(); + + return new pycaller(fn_addr); + } else { + throw std::runtime_error(tostr("MachPipeline.lookup_fn: unknown function prototype", + xtag("p", prototype))); + }}) ; From 80df3e013cefd21b9786ae351519abaec7e5db64 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 17 Jun 2024 17:18:09 -0400 Subject: [PATCH 16/33] xo-pyjit: + MachPipeline.dump_current_module() --- src/pyjit/pyjit.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index b446a2c9..ad454e6d 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -45,6 +45,7 @@ namespace xo { pycaller::declare_once(m); pycaller::declare_once(m); + py::class_>(m, "MachPipeline") .def_static("make", &MachPipeline::make, py::doc("Create machine pipeline for in-process code generation" @@ -72,6 +73,9 @@ namespace xo { .def("machgen_current_module", &MachPipeline::machgen_current_module, py::doc("Make current module available for execution via the jit.\n" "Adds all functions generated since last call to this method.")) + .def("dump_current_module", &MachPipeline::dump_current_module, + py::doc("Dump contents of current module to console")) + /* double -> double */ .def("lookup_dbl2dbl_fn", [](MachPipeline & jit, const std::string & symbol) { From c8d5633d812d856c3c1bc4abfa0b49daa8792f46 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 17 Jun 2024 17:18:24 -0400 Subject: [PATCH 17/33] xo-pyjit: README: + links --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 342b3308..ff4c99f4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # python bindings for llvm JIT for EGAD (xo-pyjit) +## Links + +- [cheatsheet for pyobject<->c++ conversion](https://github.com/pybind/pybind11/issues/1201) + ## Getting Started ### Build + install `xo-cmake` dependency From 444ea0f4b5390d3a04aab61732ea0226fd95863d Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 17 Jun 2024 18:35:27 -0400 Subject: [PATCH 18/33] xo-pyjit: obsolete XferDbl2DblFn etc; prefer pycaller<> --- src/pyjit/pyjit.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index ad454e6d..6a935202 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -8,6 +8,7 @@ #include namespace xo { +#ifdef OBSOLETE struct XferDbl2DblFn : public ref::Refcount { using fptr_type = double (*) (double); @@ -27,6 +28,7 @@ namespace xo { fptr_type fptr_; }; /*XferDblDbl2DblFn*/ +#endif namespace jit { using xo::ast::Expression; @@ -76,6 +78,7 @@ namespace xo { .def("dump_current_module", &MachPipeline::dump_current_module, py::doc("Dump contents of current module to console")) +#ifdef OBSOLETE /* double -> double */ .def("lookup_dbl2dbl_fn", [](MachPipeline & jit, const std::string & symbol) { @@ -95,6 +98,7 @@ namespace xo { return new XferDblDbl2DblFn(fn_addr); }) +#endif .def("lookup_fn", [](MachPipeline & jit, const std::string & prototype, const std::string & symbol) -> pycaller_base* { @@ -122,6 +126,7 @@ namespace xo { ; +#ifdef OBSOLETE py::class_>(m, "XferDbl2DblFn") .def("__call__", [](XferDbl2DblFn & self, double x) { return self(x); } @@ -132,6 +137,7 @@ namespace xo { [](XferDblDbl2DblFn & self, double x, double y) { return self(x, y); } ) ; +#endif py::class_>(m, "llvm_Value") From c6da0fb58b4ae0b7dc63f1e98be3200cda526baf Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Mon, 17 Jun 2024 19:35:02 -0400 Subject: [PATCH 19/33] xo-pyjit: + pycaller_store, streamlining c++ function signatures --- README.md | 4 +- src/pyjit/pyjit.cpp | 105 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 99 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ff4c99f4..6119b705 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,9 @@ build an AST from within python ``` >>> x=make_var('x') # "x" a variable (context not yet known) >>> f1=make_sin_pm() # "sin()" ->>> c1=make_apply(f1,[x]) # "sin(x)" +>>> c1=make_apply(f1,[x]) # "sin(x)" >>> f2=make_cos_pm() # "cos()" ->>> c2=make_apply(f2,[c1]) # "cos(sin(x))" +>>> c2=make_apply(f2,[c1]) # "cos(sin(x))" >>> lm=make_lambda('foo', ['x'], c2) # "def foo(x): cos(sin(x))" >>> lm :argv "[ :argv \"[]\">]">> diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 6a935202..1b53fdfb 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -39,14 +39,85 @@ namespace xo { using xo::ref::unowned_ptr; namespace py = pybind11; + /** storage for pycaller glue functions for different function signatures. + * each pycaller instance embodies captures a canonical (architecture-dependent) + * calling sequence for a C/C++ function with that signature. + **/ + struct pycaller_store { + public: + /** singleton instance **/ + static pycaller_store * instance() { return &s_instance; } + + /** establish caller for signature @p prototype_str. + * This needs to be called at most once for each distinct signature. + * + * Although it takes module as argument, the module being used + * doesn't (shoudn't ??) matter + **/ + template + pycaller_base::factory_function_type + require_prototype(py::module & m, + const std::string & prototype_str) + { + using caller_type = pycaller; + + caller_type::declare_once(m); + + /* factory function takes function pointer of type + * Retval(*)(Args...) + * and returns new instance of caller_type for that function + */ + + auto ix = pycaller_map_.find(prototype_str); + + auto retval = &caller_type::make; + + if(ix == pycaller_map_.end()) + pycaller_map_[prototype_str] = retval; + + return retval; + } + + /** lookup caller for signature @p prototype_str **/ + pycaller_base::factory_function_type + lookup_prototype(const std::string & prototype_str) const + { + auto ix = pycaller_map_.find(prototype_str); + + if (ix == pycaller_map_.end()) + return nullptr; + else + return ix->second; + } + + private: + static pycaller_store s_instance; + + /** map prototype string to pycaller factory for that prototype. + * For example + * "double(double)" -> pycaller() + **/ + std::unordered_map pycaller_map_; + + }; /*pycaller_store*/ + + pycaller_store + pycaller_store::s_instance; + PYBIND11_MODULE(XO_PYJIT_MODULE_NAME(), m) { // e.g. for xo::ast::Expression XO_PYEXPRESSION_IMPORT_MODULE(); // py::module_::import("pyexpression"); m.doc() = "pybind11 plugin for xo-jit"; - pycaller::declare_once(m); - pycaller::declare_once(m); + pycaller_store::instance() + ->require_prototype(m, "double(double)"); + pycaller_store::instance() + ->require_prototype(m, "double(double,double)"); + + //pycaller::declare_once(m); + //pycaller::declare_once(m); py::class_>(m, "MachPipeline") .def_static("make", &MachPipeline::make, @@ -104,25 +175,43 @@ namespace xo { [](MachPipeline & jit, const std::string & prototype, const std::string & symbol) -> pycaller_base* { auto llvm_addr = jit.lookup_symbol(symbol); + /* llvm doesn't know the actual function signature, + * so any function type will appear to succeed here. + * We cast to particular function type within the pycaller<..> template + */ + auto fn_addr = llvm_addr.toPtr(); + /* note: llvm_addr.toPtr<..> always succeeds, * event if pointer refers to an object of incompatible type * * note: return value policy is for python to own the wrapper + * + * note: pycaller signatures need to have been introduced in advance + * (in practice determined at compile time, + * since they encode a function-signature-specific calling sequence) + * by calling pycaller_store::instance()->require_prototype(prototype); */ + auto factory = pycaller_store::instance()->lookup_prototype(prototype); + + if (!factory) { + throw std::runtime_error(tostr("MachPipeline.lookup_fn: unknown function prototype", + xtag("p", prototype))); + } + + return (*factory)(fn_addr); + +#ifdef OBSOLETE if((prototype == "double(double,double)") || (prototype == "double(*)(double,double)")) { - auto fn_addr = llvm_addr.toPtr(); - return new pycaller(fn_addr); - //return new XferDblDbl2DblFn(fn_addr); } else if ((prototype == "double(double)") || (prototype == "double(*)(double)")) { - auto fn_addr = llvm_addr.toPtr(); - return new pycaller(fn_addr); } else { throw std::runtime_error(tostr("MachPipeline.lookup_fn: unknown function prototype", xtag("p", prototype))); - }}) + } +#endif + }) ; From 4132c7fba3778d8391fb509d3640249f3fa9b820 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 19 Jun 2024 11:02:18 -0400 Subject: [PATCH 20/33] xo-pyjit: except (instead of abort) on failed prototype lookup --- src/pyjit/pyjit.cpp | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 1b53fdfb..1a3903c4 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -112,9 +112,9 @@ namespace xo { m.doc() = "pybind11 plugin for xo-jit"; pycaller_store::instance() - ->require_prototype(m, "double(double)"); + ->require_prototype(m, "double (*)(double)"); pycaller_store::instance() - ->require_prototype(m, "double(double,double)"); + ->require_prototype(m, "double (*)(double,double)"); //pycaller::declare_once(m); //pycaller::declare_once(m); @@ -179,28 +179,33 @@ namespace xo { * so any function type will appear to succeed here. * We cast to particular function type within the pycaller<..> template */ - auto fn_addr = llvm_addr.toPtr(); + if (llvm_addr) { + auto fn_addr = llvm_addr.get().toPtr(); - /* note: llvm_addr.toPtr<..> always succeeds, - * event if pointer refers to an object of incompatible type - * - * note: return value policy is for python to own the wrapper - * - * note: pycaller signatures need to have been introduced in advance - * (in practice determined at compile time, - * since they encode a function-signature-specific calling sequence) - * by calling pycaller_store::instance()->require_prototype(prototype); - */ + /* note: llvm_addr.toPtr<..> always succeeds, + * event if pointer refers to an object of incompatible type + * + * note: return value policy is for python to own the wrapper + * + * note: pycaller signatures need to have been introduced in advance + * (in practice determined at compile time, + * since they encode a function-signature-specific calling sequence) + * by calling pycaller_store::instance()->require_prototype(prototype); + */ - auto factory = pycaller_store::instance()->lookup_prototype(prototype); + auto factory = pycaller_store::instance()->lookup_prototype(prototype); - if (!factory) { - throw std::runtime_error(tostr("MachPipeline.lookup_fn: unknown function prototype", - xtag("p", prototype))); + if (!factory) { + throw std::runtime_error(tostr("MachPipeline.lookup_fn: unknown function prototype", + xtag("p", prototype))); + } + + return (*factory)(fn_addr); + } else { + throw std::runtime_error(tostr("MachPipeline.lookup_fn: lookup on symbol S failed", + xtag("S", symbol))); } - return (*factory)(fn_addr); - #ifdef OBSOLETE if((prototype == "double(double,double)") || (prototype == "double(*)(double,double)")) { return new pycaller(fn_addr); From 6edb6abd44ede4249670e7fdcff1ad0eb6b036ff Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 19 Jun 2024 11:03:21 -0400 Subject: [PATCH 21/33] xo-pyjit: doc: update README example --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6119b705..a749ee43 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Assumes `xo-pyjit` installed to `~/local2/lib`, i.e. built with `PREFIX=~/local2`. ``` PYTHONPATH=~/local2/lib:$PYTHONPATH python +>>> from xo_pyreflect import * >>> from xo_pyjit import * >>> from xo_pyexpression import * ``` @@ -71,12 +72,13 @@ Symbol table: build an AST from within python ``` ->>> x=make_var('x') # "x" a variable (context not yet known) +>>> f64_t=TypeDescr.lookup_by_name('double') +>>> x=make_var('x',f64_t) # "x" a variable (context not yet known) >>> f1=make_sin_pm() # "sin()" >>> c1=make_apply(f1,[x]) # "sin(x)" >>> f2=make_cos_pm() # "cos()" >>> c2=make_apply(f2,[c1]) # "cos(sin(x))" ->>> lm=make_lambda('foo', ['x'], c2) # "def foo(x): cos(sin(x))" +>>> lm=make_lambda('foo', [x], c2) # "def foo(x): cos(sin(x))" >>> lm :argv "[ :argv \"[]\">]">> ``` From faed5f59c6bafbeea5e46feb2ee97484bdc1b9ae Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 19 Jun 2024 18:29:15 -0400 Subject: [PATCH 22/33] xo-pyjit: remove OBSOLETE code --- src/pyjit/pyjit.cpp | 54 --------------------------------------------- 1 file changed, 54 deletions(-) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 1a3903c4..c1706a2c 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -8,28 +8,6 @@ #include namespace xo { -#ifdef OBSOLETE - struct XferDbl2DblFn : public ref::Refcount { - using fptr_type = double (*) (double); - - explicit XferDbl2DblFn(fptr_type fptr) : fptr_{fptr} {} - - double operator() (double x) { return (*fptr_)(x); } - - fptr_type fptr_; - }; /*XferDbl2DblFn*/ - - struct XferDblDbl2DblFn : public ref::Refcount { - using fptr_type = double (*) (double, double); - - explicit XferDblDbl2DblFn(fptr_type fptr) : fptr_{fptr} {} - - double operator() (double x, double y) { return (*fptr_)(x, y); } - - fptr_type fptr_; - }; /*XferDblDbl2DblFn*/ -#endif - namespace jit { using xo::ast::Expression; using xo::pyutil::pycaller_base; @@ -149,27 +127,6 @@ namespace xo { .def("dump_current_module", &MachPipeline::dump_current_module, py::doc("Dump contents of current module to console")) -#ifdef OBSOLETE - /* double -> double */ - .def("lookup_dbl2dbl_fn", - [](MachPipeline & jit, const std::string & symbol) { - auto llvm_addr = jit.lookup_symbol(symbol); - - auto fn_addr = llvm_addr.toPtr(); - - return new XferDbl2DblFn(fn_addr); - }) - - /* (double x double) -> double */ - .def("lookup_dbldbl2dbl_fn", - [](MachPipeline & jit, const std::string & symbol) { - auto llvm_addr = jit.lookup_symbol(symbol); - - auto fn_addr = llvm_addr.toPtr(); - - return new XferDblDbl2DblFn(fn_addr); - }) -#endif .def("lookup_fn", [](MachPipeline & jit, const std::string & prototype, const std::string & symbol) -> pycaller_base* { @@ -205,17 +162,6 @@ namespace xo { throw std::runtime_error(tostr("MachPipeline.lookup_fn: lookup on symbol S failed", xtag("S", symbol))); } - -#ifdef OBSOLETE - if((prototype == "double(double,double)") || (prototype == "double(*)(double,double)")) { - return new pycaller(fn_addr); - } else if ((prototype == "double(double)") || (prototype == "double(*)(double)")) { - return new pycaller(fn_addr); - } else { - throw std::runtime_error(tostr("MachPipeline.lookup_fn: unknown function prototype", - xtag("p", prototype))); - } -#endif }) ; From 48efe6b31935cc75fdc6029e76ad358b2cafffe5 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 19 Jun 2024 18:29:43 -0400 Subject: [PATCH 23/33] xo-pyjit: + llvm_version() --- src/pyjit/pyjit.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index c1706a2c..69e4a4c3 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -5,6 +5,7 @@ #include "xo/jit/MachPipeline.hpp" #include "xo/pyutil/pycaller.hpp" #include "xo/pyutil/pyutil.hpp" +#include #include namespace xo { @@ -97,6 +98,9 @@ namespace xo { //pycaller::declare_once(m); //pycaller::declare_once(m); + m.def("llvm_version", []() { return LLVM_VERSION_STRING; }, + py::doc("llvm_version() reports compile-time llvm version string (via [llvm-config.h])")); + py::class_>(m, "MachPipeline") .def_static("make", &MachPipeline::make, py::doc("Create machine pipeline for in-process code generation" From 7e5aca41e7e4f50b6c53df9f28a3852db622cdd3 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 19 Jun 2024 18:33:51 -0400 Subject: [PATCH 24/33] xo-pyjit: experiment: + install mul_i32/mul_f64 intrinsics here --- src/pyjit/pyjit.cpp | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 69e4a4c3..0409c016 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -3,6 +3,8 @@ #include "pyjit.hpp" #include "xo/pyexpression/pyexpression.hpp" #include "xo/jit/MachPipeline.hpp" +#include "xo/jit/intrinsics.hpp" +#include "xo/expression/Primitive.hpp" #include "xo/pyutil/pycaller.hpp" #include "xo/pyutil/pyutil.hpp" #include @@ -11,6 +13,7 @@ namespace xo { namespace jit { using xo::ast::Expression; + using xo::ast::make_primitive; using xo::pyutil::pycaller_base; using xo::pyutil::pycaller; using xo::ref::rp; @@ -32,15 +35,25 @@ namespace xo { * * Although it takes module as argument, the module being used * doesn't (shoudn't ??) matter + * + * note: pybind11 requires [const char *] pycaller_id_str + * + * Example: + * pycaller_store::instance() + * ->require_prototype*(m, "pycaller_i32_i32", "int (*)(int)") + * + * @p pycaller_id_str python pycaller class name; must be unique + * @p prototype_str prototype string for @ref lookup_prototype; must be unique **/ template pycaller_base::factory_function_type require_prototype(py::module & m, - const std::string & prototype_str) + const char * pycaller_id_str, + const char * prototype_str) { using caller_type = pycaller; - caller_type::declare_once(m); + caller_type::declare_once(m, pycaller_id_str); /* factory function takes function pointer of type * Retval(*)(Args...) @@ -90,10 +103,15 @@ namespace xo { m.doc() = "pybind11 plugin for xo-jit"; + /* reminder: prototype_str must be valid python class name */ pycaller_store::instance() - ->require_prototype(m, "double (*)(double)"); + ->require_prototype(m, "pycaller_i32_i32", "int (*)(int)"); pycaller_store::instance() - ->require_prototype(m, "double (*)(double,double)"); + ->require_prototype(m, "pycaller_i32_i32_i32", "int (*)(int, int)"); + pycaller_store::instance() + ->require_prototype(m, "pycaller_f64_f64", "double (*)(double)"); + pycaller_store::instance() + ->require_prototype(m, "pycaller_f64_f64_f64", "double (*)(double, double)"); //pycaller::declare_once(m); //pycaller::declare_once(m); @@ -101,6 +119,22 @@ namespace xo { m.def("llvm_version", []() { return LLVM_VERSION_STRING; }, py::doc("llvm_version() reports compile-time llvm version string (via [llvm-config.h])")); + m.def("make_mul_i32_pm", + []() + { + return make_primitive + ("mul_i32", ::mul_i32, true /*explicit_symbol_def*/); + }, + py::doc("create primitive for 32-bit signed integer multiplication")); + + m.def("make_mul_f64_pm", + []() + { + return make_primitive + ("mul_f64", ::mul_f64, true /*explicit_symbol_def*/); + }, + py::doc("create primitive for 64-bit floating point multiplication")); + py::class_>(m, "MachPipeline") .def_static("make", &MachPipeline::make, py::doc("Create machine pipeline for in-process code generation" From b7c4e8f93c2fc4b22185198c2197278ac385602d Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 19 Jun 2024 18:34:39 -0400 Subject: [PATCH 25/33] xo-pyjit: + MachPipeline::mangle --- src/pyjit/pyjit.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 0409c016..e905c43c 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -165,6 +165,10 @@ namespace xo { .def("dump_current_module", &MachPipeline::dump_current_module, py::doc("Dump contents of current module to console")) + .def("mangle", &MachPipeline::mangle, + py::arg("symbol"), + py::doc("mangle(symbol) reports mangled version of symbol.\n" + "throws exception if mangling fails")) .def("lookup_fn", [](MachPipeline & jit, const std::string & prototype, const std::string & symbol) -> pycaller_base* { From 8f57d15eac72fbb5f265efcd0050207d1b15c38b Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Wed, 19 Jun 2024 18:34:52 -0400 Subject: [PATCH 26/33] xo-pyjit: cosmetic: drop OBSOLETE code --- src/pyjit/pyjit.cpp | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index e905c43c..959f0931 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -195,7 +195,7 @@ namespace xo { auto factory = pycaller_store::instance()->lookup_prototype(prototype); if (!factory) { - throw std::runtime_error(tostr("MachPipeline.lookup_fn: unknown function prototype", + throw std::runtime_error(tostr("MachPipeline.lookup_fn: unknown function prototype p", xtag("p", prototype))); } @@ -204,23 +204,14 @@ namespace xo { throw std::runtime_error(tostr("MachPipeline.lookup_fn: lookup on symbol S failed", xtag("S", symbol))); } - }) + }, + py::arg("prototype"), py::arg("symbol"), + py::doc("lookup_fn(proto,sym) fetches function associated with sym in jit,\n" + "and wraps it as a callable python function.\n" + "proto *must* match (with exact spelling) pycaller registered at compile time with pycaller_store::instance,\n" + "for example 'int (*)(int, int)'")) ; - -#ifdef OBSOLETE - py::class_>(m, "XferDbl2DblFn") - .def("__call__", - [](XferDbl2DblFn & self, double x) { return self(x); } - ) - ; - py::class_>(m, "XferDblDbl2DblFn") - .def("__call__", - [](XferDblDbl2DblFn & self, double x, double y) { return self(x, y); } - ) - ; -#endif - py::class_>(m, "llvm_Value") .def("print", From bdd51539f447178ff07579c92e3e34a26b3ba2be Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 21 Jun 2024 14:09:30 -0400 Subject: [PATCH 27/33] xo-pyjit: supply intrinsic to Primitive defns --- src/pyjit/pyjit.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 959f0931..184fb309 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -14,6 +14,7 @@ namespace xo { namespace jit { using xo::ast::Expression; using xo::ast::make_primitive; + using xo::ast::llvmintrinsic; using xo::pyutil::pycaller_base; using xo::pyutil::pycaller; using xo::ref::rp; @@ -123,7 +124,7 @@ namespace xo { []() { return make_primitive - ("mul_i32", ::mul_i32, true /*explicit_symbol_def*/); + ("mul_i32", ::mul_i32, true /*explicit_symbol_def*/, llvmintrinsic::i_mul); }, py::doc("create primitive for 32-bit signed integer multiplication")); @@ -131,7 +132,7 @@ namespace xo { []() { return make_primitive - ("mul_f64", ::mul_f64, true /*explicit_symbol_def*/); + ("mul_f64", ::mul_f64, true /*explicit_symbol_def*/, llvmintrinsic::fp_mul); }, py::doc("create primitive for 64-bit floating point multiplication")); From aae67dd79462591eede58c360ee4743f0dbd234e Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 21 Jun 2024 14:09:53 -0400 Subject: [PATCH 28/33] xo-pyjit: doc: fix atavism in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a749ee43..61a2d0c5 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ JITDylib "
" (ES: 0x0000000000446ee0, State = Open) Link order: [ ("
", MatchAllSymbols) ] Symbol table: "foo": [Callable] Never-Searched (Materializer 0x646fe0, xojit) ->>> fn=mp.lookup_dbl2dbl_fn('foo') +>>> fn=mp.lookup_fn('double (*)(double, double)', 'foo') >>> mp.dump_execution_session() JITDylib "
" (ES: 0x0000000000446ee0, State = Open) From 6f4f06f1b1f69734438d201da9abfb1660b31c37 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 5 Jul 2024 13:52:03 -0400 Subject: [PATCH 29/33] xo-pyjit: + reflect fyunction type so can declare vars w/ that type --- src/pyjit/pyjit.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 184fb309..04cfd968 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -17,6 +17,7 @@ namespace xo { using xo::ast::llvmintrinsic; using xo::pyutil::pycaller_base; using xo::pyutil::pycaller; + using xo::reflect::Reflect; using xo::ref::rp; //using xo::ref::Refcount; using xo::ref::unowned_ptr; @@ -54,6 +55,11 @@ namespace xo { { using caller_type = pycaller; + /* we want native function type reflected; + * need this so we can declare function-valued variables + */ + Reflect::require(); + caller_type::declare_once(m, pycaller_id_str); /* factory function takes function pointer of type From a0d748ef9f506597132a9859fb37221d707dbb64 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Fri, 5 Jul 2024 13:52:31 -0400 Subject: [PATCH 30/33] xo-pyjit: fix: MachPipeline::codegen_toplevel() instead of codegen() --- src/pyjit/pyjit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 04cfd968..527cfe1a 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -156,7 +156,7 @@ namespace xo { py::doc("write to console with state of all jit-owned dynamic libraries")) .def("codegen", [](MachPipeline & jit, const rp & expr) { - return jit.codegen(expr.borrow()); + return jit.codegen_toplevel(expr.borrow()); }, py::arg("x"), py::doc("generate llvm (IR) code for Expression x"), From 99b56a354087d7fdbdaf516392b48cde0af2be8f Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 15 Sep 2024 11:14:20 -0500 Subject: [PATCH 31/33] xo-pyjit: drop xo-pyexpression dep --- src/pyjit/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyjit/CMakeLists.txt b/src/pyjit/CMakeLists.txt index d6d84521..31d04f60 100644 --- a/src/pyjit/CMakeLists.txt +++ b/src/pyjit/CMakeLists.txt @@ -5,5 +5,5 @@ set(SELF_SRCS pyjit.cpp) xo_pybind11_library(${SELF_LIB} ${PROJECT_NAME}Targets ${SELF_SRCS}) xo_pybind11_dependency(${SELF_LIB} xo_jit) -xo_pybind11_dependency(${SELF_LIB} xo_pyexpression) +#xo_pybind11_dependency(${SELF_LIB} xo_pyexpression) xo_dependency(${SELF_LIB} refcnt) From 88852c1498e0c8daabf67c3e13df4760bae47ae3 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Sun, 15 Sep 2024 11:14:36 -0500 Subject: [PATCH 32/33] xo-pyjit: jit.codegen() -> jit.codegen_toplevel() --- src/pyjit/pyjit.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 184fb309..195a908d 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -150,7 +150,7 @@ namespace xo { py::doc("write to console with state of all jit-owned dynamic libraries")) .def("codegen", [](MachPipeline & jit, const rp & expr) { - return jit.codegen(expr.borrow()); + return jit.codegen_toplevel(expr.borrow()); }, py::arg("x"), py::doc("generate llvm (IR) code for Expression x"), From ee20e799063f8924150525328d81df6c29065707 Mon Sep 17 00:00:00 2001 From: Roland Conybeare Date: Thu, 8 May 2025 23:58:16 -0500 Subject: [PATCH 33/33] xo-pyjit: build + dep fixes --- src/pyjit/CMakeLists.txt | 2 +- src/pyjit/pyjit.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pyjit/CMakeLists.txt b/src/pyjit/CMakeLists.txt index 31d04f60..b9c1a2b5 100644 --- a/src/pyjit/CMakeLists.txt +++ b/src/pyjit/CMakeLists.txt @@ -5,5 +5,5 @@ set(SELF_SRCS pyjit.cpp) xo_pybind11_library(${SELF_LIB} ${PROJECT_NAME}Targets ${SELF_SRCS}) xo_pybind11_dependency(${SELF_LIB} xo_jit) -#xo_pybind11_dependency(${SELF_LIB} xo_pyexpression) +xo_pybind11_header_dependency(${SELF_LIB} xo_pyexpression) xo_dependency(${SELF_LIB} refcnt) diff --git a/src/pyjit/pyjit.cpp b/src/pyjit/pyjit.cpp index 527cfe1a..c83ef36a 100644 --- a/src/pyjit/pyjit.cpp +++ b/src/pyjit/pyjit.cpp @@ -18,7 +18,7 @@ namespace xo { using xo::pyutil::pycaller_base; using xo::pyutil::pycaller; using xo::reflect::Reflect; - using xo::ref::rp; + using xo::rp; //using xo::ref::Refcount; using xo::ref::unowned_ptr; namespace py = pybind11;