From dacdeb2cd77ef203dfcb90afddba75c99ba19b1c Mon Sep 17 00:00:00 2001
From: Roland Conybeare <rconybeare@gmail.com>
Date: Sun, 11 May 2025 01:38:24 -0500
Subject: [PATCH] + xo-tokenizer

---
 .github/workflows/main-stock-ubuntu.yml |  3 +++
 CMakeLists.txt                          |  2 +-
 default.nix                             |  5 +++--
 pkgs/xo-tokenizer.nix                   | 22 ++++++++++++++++++++++
 pkgs/xo-userenv.nix                     |  6 ++++--
 5 files changed, 33 insertions(+), 5 deletions(-)
 create mode 100644 pkgs/xo-tokenizer.nix

diff --git a/.github/workflows/main-stock-ubuntu.yml b/.github/workflows/main-stock-ubuntu.yml
index dcecd246..ec67fb5b 100644
--- a/.github/workflows/main-stock-ubuntu.yml
+++ b/.github/workflows/main-stock-ubuntu.yml
@@ -110,6 +110,9 @@ jobs:
           echo "::group::xo-pyexpression build"
           cmake --build .build -- xo-pyexpression
 
+          echo "::group::xo-tokenizer build"
+          cmake --build .build -- xo-tokenizer
+
           ##############
 
           echo "::group::xo build (catch all)"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 55119a3d..e4373c31 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -81,7 +81,7 @@ add_subdirectory(xo-pyunit)
 #
 add_subdirectory(xo-expression)
 add_subdirectory(xo-pyexpression)
-#add_subdirectory(xo-tokenizer)
+add_subdirectory(xo-tokenizer)
 #add_subdirectory(xo-reader)
 #add_subdirectory(xo-jit)
 #add_subdirectory(xo-pyjit)
diff --git a/default.nix b/default.nix
index b608a335..8a716640 100644
--- a/default.nix
+++ b/default.nix
@@ -53,8 +53,9 @@ let
           xo-pyunit         = self.callPackage pkgs/xo-pyunit.nix         {};
 
           xo-expression     = self.callPackage pkgs/xo-expression.nix     {};
-#          xo-pyexpression   = self.callPackage pkgs/xo-pyexpression.nix   {};
-#          xo-tokenizer      = self.callPackage pkgs/xo-tokenizer.nix      {};
+          xo-pyexpression   = self.callPackage pkgs/xo-pyexpression.nix   {};
+          xo-tokenizer      = self.callPackage pkgs/xo-tokenizer.nix      {};
+
 #          xo-reader         = self.callPackage pkgs/xo-reader.nix         {};
 #          xo-jit            = self.callPackage pkgs/xo-jit.nix            { stdenv = customStdenv;
 #                                                                            clang = llvmPackages.clang;
diff --git a/pkgs/xo-tokenizer.nix b/pkgs/xo-tokenizer.nix
new file mode 100644
index 00000000..ff02f2d8
--- /dev/null
+++ b/pkgs/xo-tokenizer.nix
@@ -0,0 +1,22 @@
+{
+  # dependencies
+  stdenv, cmake, catch2,
+
+  xo-cmake, xo-indentlog
+} :
+
+stdenv.mkDerivation (finalattrs:
+  {
+    name = "xo-tokenizer";
+    version = "1.0";
+
+    src = ../xo-tokenizer;
+
+    cmakeFlags = ["-DCMAKE_MODULE_PATH=${xo-cmake}/share/cmake"];
+    doCheck = true;
+    propagatedBuildInputs = [ ];
+    nativeBuildInputs = [ cmake catch2
+                          xo-cmake
+                          xo-indentlog
+                        ];
+  })
diff --git a/pkgs/xo-userenv.nix b/pkgs/xo-userenv.nix
index bdddd92c..020102af 100644
--- a/pkgs/xo-userenv.nix
+++ b/pkgs/xo-userenv.nix
@@ -46,10 +46,11 @@
 #  xo-pyreactor,
 #  xo-simulator,
 #  xo-pysimulator, xo-distribution, xo-pydistribution, xo-process, xo-pyprocess, xo-statistics, xo-kalmanfilter,
-#  xo-pykalmanfilter, xo-websock, xo-pywebsock, xo-tokenizer,
+#  xo-pykalmanfilter, xo-websock, xo-pywebsock,
 
   xo-expression,
   xo-pyexpression,
+  xo-tokenizer,
 # xo-reader,
 #  xo-jit,
 #  xo-pyjit
@@ -78,7 +79,8 @@ buildEnv {
 
             xo-expression
             xo-pyexpression
-#            xo-tokenizer
+            xo-tokenizer
+
 #            xo-reader
 #            xo-jit
 #            xo-pyjit