roland/xo-umbrella2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add 'xo-pyjit/' from commit 'ee20e79906'

Browse source 
git-subtree-dir: xo-pyjit
git-subtree-mainline: 56380e9aed
git-subtree-split: ee20e79906
This commit is contained in:
Roland Conybeare 2025-05-11 14:38:02 -05:00
commit 662a8a5b40
9 changed files with 515 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

8
xo-pyjit/.gitignore vendored Normal file
View file

@ -0,0 +1,8 @@
# emacs configuration for workspace
.projectile
# clangd working space (see emacs+lsp)
.cache
# typical cmake build directory (source-tree-nephew)
.build*
# symlink to builddir/compile_commands.json; should be set manually in dev sandbox
compile_commands.json

28
xo-pyjit/CMakeLists.txt Normal file
View file

@ -0,0 +1,28 @@
# xo-pyjit/CMakeLists.txt
cmake_minimum_required(VERSION 3.10)
project(xo_pyjit VERSION 0.1)
include(GNUInstallDirs)
include(cmake/xo-bootstrap-macros.cmake)
xo_cxx_toplevel_options3()
# ----------------------------------------------------------------
# c++ settings (usually temporary)
set(PROJECT_CXX_FLAGS "")
add_definitions(${PROJECT_CXX_FLAGS})
# ----------------------------------------------------------------
add_subdirectory(src/pyjit)
#add_subdirectory(utest)
# ----------------------------------------------------------------
# provide find_package() support
xo_export_cmake_config(${PROJECT_NAME} ${PROJECT_VERSION} ${PROJECT_NAME}Targets)
# end CMakeLists.txt

160
xo-pyjit/README.md Normal file
View file

@ -0,0 +1,160 @@
# python bindings for llvm JIT for EGAD (xo-pyjit)
## Links
- [cheatsheet for pyobject<->c++ conversion](https://github.com/pybind/pybind11/issues/1201)
## Getting Started
### Build + install `xo-cmake` dependency
- [github/Rconybea/xo-cmake](https://github.com/Rconybea/xo-cmake)
Installs a few cmake ingredients, along with a build assistant `xo-build` for XO projects such as this one.
### build + install other necessary XO dependencies
```
$ xo-build --clone --configure --build --install xo-indentlog
$ xo-build --clone --configure --build --install xo-refnct
$ xo-build --clone --configure --build --install xo-subsys
$ xo-build --clone --configure --build --install xo-reflect
$ xo-build --clone --configure --build --install xo-expression
$ xo-build --clone --configure --build --install xo-jit
$ xo-build --clone --configure --build --install xo-pyutil
$ xo-build --clone --configure --build --install xo-pyexpression
```
note: can use `xo-build -n` to dry-run here
### copy `xo-pyjit` repository locally
```
$ xo-build --clone xo-pyjit
```
or equivalently
```
$ git clone git@github.com:Rconybea/xo-pyjit.git
```
### build + install xo-pyjit
```
$ xo-build --configure --build --install xo-pyjit
```
or equivalently:
```
$ PREFIX=/usr/local # or preferred install location
$ cmake -DCMAKE_INSTALL_PREFIX=$PREFIX -S xo-pyjit -B xo-pyjit/.build
$ cmake --build xo-pyjit/.build -j
$ cmake --install xo-pyjit/.build
```
(also see .github/workflows/main.yml)
## Examples
Assumes `xo-pyjit` installed to `~/local2/lib`,
i.e. built with `PREFIX=~/local2`.
```
PYTHONPATH=~/local2/lib:$PYTHONPATH python
>>> from xo_pyreflect import *
>>> from xo_pyjit import *
>>> from xo_pyexpression import *
```
create a jit from within python
```
>>> mp=MachPipeline.make()
>>> mp.dump_execution_sesion()
JITDylib "<main>" (ES: 0x0000000000446ee0, State = Open)
Link order: [ ("<main>", MatchAllSymbols) ]
Symbol table:
```
build an AST from within python
```
>>> f64_t=TypeDescr.lookup_by_name('double')
>>> x=make_var('x',f64_t) # "x" a variable (context not yet known)
>>> f1=make_sin_pm() # "sin()"
>>> c1=make_apply(f1,[x]) # "sin(x)"
>>> f2=make_cos_pm() # "cos()"
>>> c2=make_apply(f2,[c1]) # "cos(sin(x))"
>>> lm=make_lambda('foo', [x], c2) # "def foo(x): cos(sin(x))"
>>> lm
<Lambda :name foo :argv [x] :body <Apply :fn <Primitive :name cos :type "double (*)(double)" :value 1> :argv "[<Apply :fn <Primitive :name sin :type \"double (*)(double)\" :value 1> :argv \"[<Variable :name x>]\">]">>
```
generate llvm IR for our AST
```
>>> code=mp.codegen(lm)
>>> print(code.print())
define double @foo(double %x) {
entry:
%calltmp = call double @sin(double %x)
%calltmp1 = call double @cos(double %calltmp)
ret double %calltmp1
}
```
generate machine code for our AST, lookup compiled function so we can invoke it directly
```
>>> mp.machgen_current_module()
>>> mp.dump_execution_session()
JITDylib "<main>" (ES: 0x0000000000446ee0, State = Open)
Link order: [ ("<main>", MatchAllSymbols) ]
Symbol table:
"foo": <not resolved> [Callable] Never-Searched (Materializer 0x646fe0, xojit)
>>> fn=mp.lookup_fn('double (*)(double, double)', 'foo')
>>> mp.dump_execution_session()
JITDylib "<main>" (ES: 0x0000000000446ee0, State = Open)
Link order: [ ("<main>", MatchAllSymbols) ]
Symbol table:
"cos": 0x7ffff7926670 [Data] Ready
"foo": 0x7fffee2b6000 [Callable] Ready
"sin": 0x7ffff7925e50 [Data] Ready
```
invoke just-compiled code!
```
>>> fn(22)
0.999960827417674
```
## Development
### use from build tree
Limited utility: requires that supporting libraries (e.g. `xo_pyexpression`) appear in PYTHONPATH
```
$ cd xo-pyjit/.build/src/pyjit
$ python
>>> import xo_pyjit
```
### build for unit test coverage
```
$ cd xo-pyexpression
$ cmake -DCMAKE_BUILD_TYPE=coverage -DENABLE_TESTING=on -S . -B .build-ccov
$ cmake --build .build-ccov -j
```
### LSP (language server) support
LSP looks for compile commands in the root of the source tree;
while Cmake creates them in the root of its build directory.
```
$ cd xo-pyexpression
$ ln -s .build/compile_commands.json # supply compile commands to LSP
```
### display cmake variables
- `-L` list variables
- `-A` include 'advanced' variables
- `-H` include help text
```
$ cd xo-pyjit/.build
$ cmake -LAH
```

35
xo-pyjit/cmake/xo-bootstrap-macros.cmake Normal file
View file

@ -0,0 +1,35 @@
# ----------------------------------------------------------------
# for example:
# $ PREFIX=/usr/local # for example
# $ cmake -DCMAKE_MODULE_PATH=prefix -DCMAKE_INSTALL_PREFIX=$PREFIX -B .build
#
# will get
# CMAKE_MODULE_PATH
# from xo-cmake-config --cmake-module-path
#
# and expect .cmake macros in
# CMAKE_MODULE_PATH/xo_macros/xo_cxx.cmake
# ----------------------------------------------------------------
find_program(XO_CMAKE_CONFIG_EXECUTABLE NAMES xo-cmake-config REQUIRED)
if ("${XO_CMAKE_CONFIG_EXECUTABLE}" STREQUAL "XO_CMAKE_CONFIG_EXECUTABLE-NOT_FOUND")
message(FATAL "could not find xo-cmake-config executable")
endif()
message(STATUS "XO_CMAKE_CONFIG_EXECUTABLE=${XO_CMAKE_CONFIG_EXECUTABLE}")
if (NOT XO_SUBMODULE_BUILD)
if (("${CMAKE_MODULE_PATH}" STREQUAL "") OR ("${CMAKE_MODULE_PATH}" STREQUAL prefix))
# default to typical install location for xo-project-macros
execute_process(COMMAND ${XO_CMAKE_CONFIG_EXECUTABLE} --cmake-module-path OUTPUT_VARIABLE CMAKE_MODULE_PATH)
message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}")
endif()
endif()
# needs to have been installed somewhere on CMAKE_MODULE_PATH,
# (e.g. from xo-cmake with the same value for CMAKE_INSTALL_PREFIX)
#
include(xo_macros/xo_cxx)
xo_cxx_bootstrap_message()

7
xo-pyjit/cmake/xo_pyjitConfig.cmake.in Normal file
View file

@ -0,0 +1,7 @@
@PACKAGE_INIT@
include(CMakeFindDependencyMacro)
find_dependency(xo_jit)
find_dependency(xo_pyexpression)
include("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
check_required_components("@PROJECT_NAME@")

1
xo-pyjit/include/README.md Normal file
View file

@ -0,0 +1 @@
placeholder for future xo-pymatrix header files

9
xo-pyjit/src/pyjit/CMakeLists.txt Normal file
View file

@ -0,0 +1,9 @@
# xo-pyjit/src/pyjit/CMakeLists.txt
set(SELF_LIB xo_pyjit)
set(SELF_SRCS pyjit.cpp)
xo_pybind11_library(${SELF_LIB} ${PROJECT_NAME}Targets ${SELF_SRCS})
xo_pybind11_dependency(${SELF_LIB} xo_jit)
xo_pybind11_header_dependency(${SELF_LIB} xo_pyexpression)
xo_dependency(${SELF_LIB} refcnt)

242
xo-pyjit/src/pyjit/pyjit.cpp Normal file
View file

@ -0,0 +1,242 @@
/* @file pyjit.cpp */
#include "pyjit.hpp"
#include "xo/pyexpression/pyexpression.hpp"
#include "xo/jit/MachPipeline.hpp"
#include "xo/jit/intrinsics.hpp"
#include "xo/expression/Primitive.hpp"
#include "xo/pyutil/pycaller.hpp"
#include "xo/pyutil/pyutil.hpp"
#include <llvm/Config/llvm-config.h>
#include <pybind11/stl.h>
namespace xo {
namespace jit {
using xo::ast::Expression;
using xo::ast::make_primitive;
using xo::ast::llvmintrinsic;
using xo::pyutil::pycaller_base;
using xo::pyutil::pycaller;
using xo::reflect::Reflect;
using xo::rp;
//using xo::ref::Refcount;
using xo::ref::unowned_ptr;
namespace py = pybind11;
/** storage for pycaller glue functions for different function signatures.
* each pycaller instance embodies captures a canonical (architecture-dependent)
* calling sequence for a C/C++ function with that signature.
**/
struct pycaller_store {
public:
/** singleton instance **/
static pycaller_store * instance() { return &s_instance; }
/** establish caller for signature @p prototype_str.
* This needs to be called at most once for each distinct signature.
*
* Although it takes module as argument, the module being used
* doesn't (shoudn't ??) matter
*
* note: pybind11 requires [const char *] pycaller_id_str
*
* Example:
* pycaller_store::instance()
* ->require_prototype<int, int>*(m, "pycaller_i32_i32", "int (*)(int)")
*
* @p pycaller_id_str python pycaller class name; must be unique
* @p prototype_str prototype string for @ref lookup_prototype; must be unique
**/
template <typename Retval, typename... Args>
pycaller_base::factory_function_type
require_prototype(py::module & m,
const char * pycaller_id_str,
const char * prototype_str)
{
using caller_type = pycaller<Retval, Args...>;
/* we want native function type reflected;
* need this so we can declare function-valued variables
*/
Reflect::require<typename caller_type::function_type>();
caller_type::declare_once(m, pycaller_id_str);
/* factory function takes function pointer of type
* Retval(*)(Args...)
* and returns new instance of caller_type for that function
*/
auto ix = pycaller_map_.find(prototype_str);
auto retval = &caller_type::make;
if(ix == pycaller_map_.end())
pycaller_map_[prototype_str] = retval;
return retval;
}
/** lookup caller for signature @p prototype_str **/
pycaller_base::factory_function_type
lookup_prototype(const std::string & prototype_str) const
{
auto ix = pycaller_map_.find(prototype_str);
if (ix == pycaller_map_.end())
return nullptr;
else
return ix->second;
}
private:
static pycaller_store s_instance;
/** map prototype string to pycaller factory for that prototype.
* For example
* "double(double)" -> pycaller<double,double>()
**/
std::unordered_map<std::string,
pycaller_base::factory_function_type> pycaller_map_;
}; /*pycaller_store*/
pycaller_store
pycaller_store::s_instance;
PYBIND11_MODULE(XO_PYJIT_MODULE_NAME(), m) {
// e.g. for xo::ast::Expression
XO_PYEXPRESSION_IMPORT_MODULE(); // py::module_::import("pyexpression");
m.doc() = "pybind11 plugin for xo-jit";
/* reminder: prototype_str must be valid python class name */
pycaller_store::instance()
->require_prototype<int, int>(m, "pycaller_i32_i32", "int (*)(int)");
pycaller_store::instance()
->require_prototype<int, int, int>(m, "pycaller_i32_i32_i32", "int (*)(int, int)");
pycaller_store::instance()
->require_prototype<double, double>(m, "pycaller_f64_f64", "double (*)(double)");
pycaller_store::instance()
->require_prototype<double, double, double>(m, "pycaller_f64_f64_f64", "double (*)(double, double)");
//pycaller<double, double>::declare_once(m);
//pycaller<double, double, double>::declare_once(m);
m.def("llvm_version", []() { return LLVM_VERSION_STRING; },
py::doc("llvm_version() reports compile-time llvm version string (via [llvm-config.h])"));
m.def("make_mul_i32_pm",
[]()
{
return make_primitive<int32_t (*)(int32_t, int32_t)>
("mul_i32", ::mul_i32, true /*explicit_symbol_def*/, llvmintrinsic::i_mul);
},
py::doc("create primitive for 32-bit signed integer multiplication"));
m.def("make_mul_f64_pm",
[]()
{
return make_primitive<double (*)(double, double)>
("mul_f64", ::mul_f64, true /*explicit_symbol_def*/, llvmintrinsic::fp_mul);
},
py::doc("create primitive for 64-bit floating point multiplication"));
py::class_<MachPipeline, rp<MachPipeline>>(m, "MachPipeline")
.def_static("make", &MachPipeline::make,
py::doc("Create machine pipeline for in-process code generation"
" and execution. Not threadsafe.\n"
"Does not share resources with any other instance"))
.def_property_readonly("target_triple", &MachPipeline::target_triple,
py::doc("string describing target host for code generation"))
.def("get_function_name_v", &MachPipeline::get_function_name_v,
py::doc("get vector of function names defined in jit module"))
.def("dump_execution_session", &MachPipeline::dump_execution_session,
py::doc("write to console with state of all jit-owned dynamic libraries"))
.def("codegen",
[](MachPipeline & jit, const rp<Expression> & expr) {
return jit.codegen_toplevel(expr.borrow());
},
py::arg("x"),
py::doc("generate llvm (IR) code for Expression x"),
/* we're assuming llvm-generated code lives for as long as the Jit
* instance that created it.
*
* RC 14jun2024 - I think this is true, modulo use of llvm resource trackers.
*/
py::return_value_policy::reference_internal)
.def("machgen_current_module", &MachPipeline::machgen_current_module,
py::doc("Make current module available for execution via the jit.\n"
"Adds all functions generated since last call to this method."))
.def("dump_current_module", &MachPipeline::dump_current_module,
py::doc("Dump contents of current module to console"))
.def("mangle", &MachPipeline::mangle,
py::arg("symbol"),
py::doc("mangle(symbol) reports mangled version of symbol.\n"
"throws exception if mangling fails"))
.def("lookup_fn",
[](MachPipeline & jit, const std::string & prototype, const std::string & symbol) -> pycaller_base* {
auto llvm_addr = jit.lookup_symbol(symbol);
/* llvm doesn't know the actual function signature,
* so any function type will appear to succeed here.
* We cast to particular function type within the pycaller<..> template
*/
if (llvm_addr) {
auto fn_addr = llvm_addr.get().toPtr<void(*)()>();
/* note: llvm_addr.toPtr<..> always succeeds,
* event if pointer refers to an object of incompatible type
*
* note: return value policy is for python to own the wrapper
*
* note: pycaller signatures need to have been introduced in advance
* (in practice determined at compile time,
* since they encode a function-signature-specific calling sequence)
* by calling pycaller_store::instance()->require_prototype<Retval, Args...>(prototype);
*/
auto factory = pycaller_store::instance()->lookup_prototype(prototype);
if (!factory) {
throw std::runtime_error(tostr("MachPipeline.lookup_fn: unknown function prototype p",
xtag("p", prototype)));
}
return (*factory)(fn_addr);
} else {
throw std::runtime_error(tostr("MachPipeline.lookup_fn: lookup on symbol S failed",
xtag("S", symbol)));
}
},
py::arg("prototype"), py::arg("symbol"),
py::doc("lookup_fn(proto,sym) fetches function associated with sym in jit,\n"
"and wraps it as a callable python function.\n"
"proto *must* match (with exact spelling) pycaller registered at compile time with pycaller_store::instance,\n"
"for example 'int (*)(int, int)'"))
;
py::class_<llvm::Value,
unowned_ptr<llvm::Value>>(m, "llvm_Value")
.def("print",
[](llvm::Value & x) {
std::string buf;
llvm::raw_string_ostream ss(buf);
x.print(ss);
return buf;
})
// .def("__repr__",
// &Jit::display_string)
;
}
} /*namespace jit*/
} /*namespace xo*/
/* end pyjit.cpp */

25
xo-pyjit/src/pyjit/pyjit.hpp.in Normal file
View file

@ -0,0 +1,25 @@
/* @file pyjit.hpp
*
* automatically generated from src/xo_pyjit/pyjit.hpp.in
* see src/xo_pyjit/CMakeLists.txt
*/
/* python requires module name = library name
* example:
* PYBIND11_MODULE(XO_PYJIT_MODULE_NAME(), m) { ... }
*/
#define XO_PYJIT_MODULE_NAME() @SELF_LIB@
/* example:
* py::module_::import(XO_PYJIT_MODULE_NAME_STR)
*/
#define XO_PYJIT_MODULE_NAME_STR "@SELF_LIB@"
/* example:
* XO_PYJIT_IMPORT_MODULE()
* replaces
* py::module_::import("pyjit")
*/
#define XO_PYJIT_IMPORT_MODULE() py::module_::import("@SELF_LIB@")
/* end pyjit.hpp */