Python Built-in Functions via libpython-clj2
Table of Contents
1. Examples
(quote
(require '[python.builtins :as python])
(require '[python.builtins :as p])
(doc python/list)
(doc python/dict)
(doc python/set)
(doc python/tuple)
(doc python/abs)
(python/abs -2)
(python/all [])
(python/any [])
(python/ascii 'a)
(python/bin 4)
(python/bool true)
(python/bytes 8)
(python/callable 'a)
(python/chr 16)
(python/compile "main.py" "main.py" "single")
(python/complex 2)
(python/copyright)
(python/credits)
(python/delattr (python/dict {"a" "b"}) "a")
(python/delattr {"a" 1} "a")
(python/dict {"a" "b"})
(python/dir)
(python/divmod)
(python/enumerate)
(python/eval)
(python/hex)
(python/id)
(python/input)
(python/int)
(python/isinstance)
(python/len)
(python/license)
(python/list)
(python/locals)
(python/map)
(python/max)
(python/memoryview)
(python/min)
(python/next)
(python/object)
(python/object)
(python/open)
(python/ord)
(python/pow)
(python/print)
(python/property)
(python/quit)
(python/range)
(python/repr)
(python/reversed)
(python/round)
(python/set)
(python/setattr)
(python/slice)
(python/sorted)
(python/super)
(python/tuple)
(python/type)
(python/vars)
(python/zip)
)
2. Modern ML Library Integration
2.1. PyTorch Integration
(require '[libpython-clj2.require :refer [require-python]])
(require '[libpython-clj2.python :as py :refer [py. py.. py.-]])
;; Import PyTorch
(require-python '[torch :as torch])
(require-python '[torch.nn :as nn])
(require-python '[torch.optim :as optim])
;; Create tensors
(def x (torch/tensor [[1.0 2.0] [3.0 4.0]]))
(def y (torch/tensor [[5.0 6.0] [7.0 8.0]]))
;; Matrix operations
(torch/matmul x y)
(torch/add x y)
;; GPU support (if available)
(when (torch/cuda.is_available)
(def x-gpu (py. x "cuda")))
;; Simple neural network
(def model
(py/call-attr nn "Sequential"
(nn/Linear 784 256)
(nn/ReLU)
(nn/Linear 256 10)))
;; Forward pass
(def input (torch/randn [1 784]))
(def output (py. model "__call__" input))
2.2. Hugging Face Transformers
correct
(require-python '[transformers :as hf])
;; Load a pre-trained model and tokenizer
(def tokenizer (hf/AutoTokenizer.from_pretrained "bert-base-uncased"))
(def model (hf/AutoModel.from_pretrained "bert-base-uncased"))
;; Tokenize text
(def inputs (py. tokenizer "__call__"
"Hello, Clojure meets Python!"
:return_tensors "pt"))
;; Get embeddings
(def outputs (py. model "__call__" :py/kwargs inputs))
(def last-hidden (py.- outputs last_hidden_state))
;; Text generation with GPT-2
(def generator (hf/pipeline "text-generation" :model "gpt2"))
(generator "Clojure is a functional programming language that")
;; Sentiment analysis
(def classifier (hf/pipeline "sentiment-analysis"))
(classifier "libpython-clj2 makes Python integration seamless!")
2.3. LangChain Integration
corrected
LangChain did not exist at this note's 2022-01-01 date -- the langchain-ai/langchain GitHub repo was created 2022-10-17 and its first PyPI release (0.0.1) shipped 2022-10-25, nine-plus months later. No period-accurate substitute library offered the same prompt/chain/vectorstore abstraction in early 2022; flagging the anachronism rather than fabricating a stand-in.
(require-python '[langchain.llms :as llms])
(require-python '[langchain.chains :as chains])
(require-python '[langchain.prompts :as prompts])
(require-python '[langchain.text_splitter :as splitter])
(require-python '[langchain.embeddings :as embeddings])
(require-python '[langchain.vectorstores :as vectorstores])
;; Initialize an LLM
(def llm (llms/OpenAI :temperature 0.7))
;; Create a prompt template
(def prompt
(prompts/PromptTemplate
:input_variables ["topic"]
:template "Write a brief explanation of {topic} for a Clojure developer."))
;; Create a chain
(def chain (chains/LLMChain :llm llm :prompt prompt))
;; Run the chain
(py. chain "run" "functional programming patterns")
;; Document loading and RAG setup
(def text-splitter (splitter/RecursiveCharacterTextSplitter
:chunk_size 1000
:chunk_overlap 200))
(def embeddings-model (embeddings/OpenAIEmbeddings))
;; Create vector store from documents
(defn create-knowledge-base [documents]
(let [chunks (py. text-splitter "split_documents" documents)]
(vectorstores/FAISS.from_documents chunks embeddings-model)))
3. Clojure Data Science Stack Integration
3.1. dtype-next Integration
(require '[tech.v3.datatype :as dtype])
(require '[tech.v3.tensor :as dtt])
(require '[libpython-clj2.python :as py])
;; Convert between Clojure and NumPy arrays
(require-python '[numpy :as np])
;; Create a dtype-next tensor
(def clj-tensor (dtt/->tensor [[1.0 2.0 3.0]
[4.0 5.0 6.0]]))
;; Convert to NumPy array (zero-copy when possible)
(def np-array (py/->py-array clj-tensor))
;; Perform NumPy operations
(def result (np/dot np-array (np/transpose np-array)))
;; Convert back to Clojure
(def clj-result (dtype/->array-buffer result))
;; Direct interop with PyTorch tensors
(def torch-tensor (torch/from_numpy np-array))
(def clj-from-torch (dtype/->array-buffer (py. torch-tensor "numpy")))
3.2. tech.ml.dataset Integration
(require '[tech.v3.dataset :as ds])
(require-python '[pandas :as pd])
(require-python '[sklearn.model_selection :as model-sel])
(require-python '[sklearn.ensemble :as ensemble])
;; Load data with tech.ml.dataset
(def dataset (ds/->dataset "data.csv"))
;; Convert to Pandas DataFrame
(defn dataset->pandas [ds]
(pd/DataFrame (into {} (ds/columns ds))))
(def df (dataset->pandas dataset))
;; Use scikit-learn for ML
(def X (py/get-attr df "drop" "target" :axis 1))
(def y (py/get-attr df "target"))
;; Train-test split
(def [X-train X-test y-train y-test]
(model-sel/train_test_split X y :test_size 0.2 :random_state 42))
;; Train a model
(def rf (ensemble/RandomForestClassifier :n_estimators 100))
(py. rf "fit" X-train y-train)
;; Predictions
(def predictions (py. rf "predict" X-test))
;; Convert predictions back to Clojure
(def clj-predictions (vec (py/->jvm predictions)))
3.3. Hybrid Data Pipeline Example
(require '[tech.v3.dataset :as ds])
(require '[tech.v3.dataset.column :as col])
(require-python '[pandas :as pd])
(require-python '[torch.utils.data :as torch-data])
;; Define a data processing pipeline that uses both ecosystems
(defn process-pipeline [input-path]
;; Step 1: Load with tech.ml.dataset (fast Clojure CSV parsing)
(let [raw-ds (ds/->dataset input-path)
;; Step 2: Clojure data transformations
cleaned-ds (-> raw-ds
(ds/filter-column :value #(> % 0))
(ds/add-column :log-value
(map #(Math/log %) (raw-ds :value))))
;; Step 3: Convert to Pandas for Python-specific ops
df (dataset->pandas cleaned-ds)
;; Step 4: Use Pandas rolling window
df-with-rolling (py/set-attr! df "rolling_mean"
(py. (py. df "rolling" 7) "mean"))]
;; Step 5: Return as tech.ml.dataset
(pandas->dataset df-with-rolling)))
(defn pandas->dataset [df]
(ds/->dataset (py. df "to_dict" "list")))
4. Performance Considerations
4.1. Memory Management
(require '[libpython-clj2.python :as py])
;; Explicit garbage collection coordination
(defn with-python-gc [f]
(try
(f)
(finally
(py/run-simple-string "import gc; gc.collect()"))))
;; Release large Python objects explicitly
(defmacro with-python-resources [bindings & body]
`(let ~bindings
(try
~@body
(finally
~@(for [[name _] (partition 2 bindings)]
`(py/del ~name))))))
;; Example usage
(with-python-resources [large-tensor (torch/randn [10000 10000])
result (torch/matmul large-tensor large-tensor)]
(py. result "mean"))
4.2. Zero-Copy Data Transfer
;; Zero-copy between NumPy and dtype-next when possible
(require '[tech.v3.datatype.ffi :as ffi])
;; NumPy arrays backed by native memory can be shared
(def np-array (np/zeros [1000000] :dtype "float64"))
;; Access the underlying buffer without copying
(def native-buffer (py/->jvm np-array :as-buffer))
;; Modify in-place from Clojure
(dtype/set-value! native-buffer 0 42.0)
;; Changes visible in Python
(np/sum np-array) ;; Reflects the modification
4.3. Async and Parallel Processing
(require '[clojure.core.async :as async])
;; Python GIL considerations - use separate processes for true parallelism
(require-python '[multiprocessing :as mp])
(require-python '[concurrent.futures :as futures])
;; For I/O-bound Python operations, use ThreadPoolExecutor
(defn parallel-fetch [urls]
(let [executor (futures/ThreadPoolExecutor :max_workers 10)]
(try
(let [future-to-url
(into {} (for [url urls]
[(py. executor "submit" fetch-url url) url]))
results (for [future (futures/as_completed future-to-url)]
(py. future "result"))]
(doall results))
(finally
(py. executor "shutdown")))))
;; For CPU-bound work, leverage Clojure's parallelism with smaller Python calls
(defn parallel-inference [items model]
(->> items
(partition-all 32) ;; Batch for efficiency
(pmap (fn [batch]
(py. model "predict" (py/->py-list batch))))
(mapcat identity)))
4.4. Caching and Memoization
;; Cache expensive Python model loading
(def model-cache (atom {}))
(defn get-or-load-model [model-name]
(if-let [model (@model-cache model-name)]
model
(let [model (hf/AutoModel.from_pretrained model-name)]
(swap! model-cache assoc model-name model)
model)))
;; Memoize tokenization
(def memoized-tokenize
(memoize
(fn [tokenizer text]
(py/->jvm (py. tokenizer "__call__" text :return_tensors "pt")))))
5. Best Practices
5.1. Project Setup
corrected
all three pinned versions postdate this note's 2022-01-01 date. Clojars pom timestamps show clj-python/libpython-clj 2.025 published 2023-09-05 and techascent/tech.ml.dataset 7.000 published 2023-08-22 -- both over a year and a half later. org.bytedeco/numpy "1.26.0-1.5.10" was never published at all (Maven Central has no such artifact); the nearest real version, 1.26.3-1.5.10, shipped 2024-01-26. Corrected to the latest versions actually available before the note's date: libpython-clj 2.006 (2021-12-28), tech.ml.dataset 6.042 (2021-12-18), org.bytedeco/numpy 1.21.1-1.5.6 (2021-07-30).
Add to deps.edn:
{:deps {clj-python/libpython-clj {:mvn/version "2.006"}
techascent/tech.ml.dataset {:mvn/version "6.042"}
org.bytedeco/numpy {:mvn/version "1.21.1-1.5.6"}}}
5.2. Python Environment Management
corrected
LIBPYTHON_CLJ_PYTHON_EXECUTABLE is not a real libpython-clj environment variable -- a grep of the clj-python/libpython-clj source tree finds no such var (the only one read via System/getenv is PYTHONHOME, which does a different job). Removed the fabricated line; py/initialize! shown above is the only config path. Also corrected the example library path from Python 3.11 (CPython 3.11.0 did not ship until October 2022) to 3.10, the stable release current at the note's 2022-01-01 date (released October 2021).
;; Specify Python executable at startup
(require '[libpython-clj2.python :as py])
(py/initialize! :python-executable "/path/to/venv/bin/python"
:library-path "/path/to/libpython3.10.so")
;; Install Python packages programmatically
(require-python '[pip :as pip])
(pip/main ["install" "torch" "transformers" "langchain"])
5.3. Error Handling
(defmacro with-python-error-handling [& body]
`(try
~@body
(catch Exception e#
(let [py-error# (py/python-exception-message e#)]
(throw (ex-info "Python error"
{:python-error py-error#
:clojure-error (.getMessage e#)}))))))
;; Usage
(with-python-error-handling
(py. model "forward" invalid-input))