pyroxide
Pyroxide: A bounded background task engine for Python powered by Rust.
Exposes a thread-safe background execution engine for offloading operations from the main Python interpreter. Supports Python callable tasks, sandboxed WebAssembly execution, and dynamically compiled shared library (dylib) plugins.
Exports: - task: Decorator to submit Python functions to the background execution pool. - TaskHandle: Object returned by task submission to query status and await results. - register_wasm / wasm_task: WebAssembly sandbox registration and execution. - compile_rust / dylib_task: Dynamic shared library compilation and execution.
1""" 2Pyroxide: A bounded background task engine for Python powered by Rust. 3 4Exposes a thread-safe background execution engine for offloading operations from 5the main Python interpreter. Supports Python callable tasks, sandboxed WebAssembly 6execution, and dynamically compiled shared library (dylib) plugins. 7 8Exports: 9 - task: Decorator to submit Python functions to the background execution pool. 10 - TaskHandle: Object returned by task submission to query status and await results. 11 - register_wasm / wasm_task: WebAssembly sandbox registration and execution. 12 - compile_rust / dylib_task: Dynamic shared library compilation and execution. 13""" 14 15from . import config 16from ._pyroxide import ( # noqa: F401 17 ForkSafetyError, 18 get_status, 19 register_dylib, 20 submit_task, 21) 22from ._pyroxide import ( 23 shutdown_engine as _shutdown_engine, 24) 25from .config import is_free_threaded, scoped, set_queue_timeout, set_wasm_limits, stats 26from .decorators import task 27from .plugins import ( 28 CompilerNotFoundError, 29 compile_c, 30 compile_rust, 31 compile_zig, 32 dylib_task, 33 load_dylib, 34 unregister_dylib, 35) 36from .stubs import generate_stubs 37from .types import TaskHandle 38from .wasm import ( 39 compile_c_wasm, 40 compile_rust_wasm, 41 compile_wasm, 42 compile_wat_wasm, 43 compile_zig_wasm, 44 load_wasm, 45 register_wasm, 46 register_wasm_wat, 47 wasm_task, 48) 49from .workflows import TaskGroup, group 50 51__version__ = "1.0.0rc1" 52 53 54def shutdown(wait: bool = True, cancel_pending: bool = False) -> None: 55 """Stop accepting work and shut down Pyroxide's workers. 56 57 Shutdown is irreversible for the current process. Accepted work drains by 58 default; ``cancel_pending=True`` cancels work that has not started. 59 A Pyroxide worker task must use ``wait=False`` to avoid waiting for itself. 60 """ 61 if type(wait) is not bool or type(cancel_pending) is not bool: 62 raise TypeError("wait and cancel_pending must be bool values") 63 _shutdown_engine(wait=wait, cancel_pending=cancel_pending) 64 if wait: 65 from .types import _cleanup_waker 66 67 _cleanup_waker() 68 69 70__all__ = [ 71 "task", 72 "TaskHandle", 73 "register_wasm", 74 "register_wasm_wat", 75 "wasm_task", 76 "load_wasm", 77 "compile_wasm", 78 "compile_wat_wasm", 79 "compile_c_wasm", 80 "compile_rust_wasm", 81 "compile_zig_wasm", 82 "compile_rust", 83 "dylib_task", 84 "load_dylib", 85 "unregister_dylib", 86 "compile_c", 87 "compile_zig", 88 "group", 89 "TaskGroup", 90 "shutdown", 91 "ForkSafetyError", 92 "generate_stubs", 93 "set_wasm_limits", 94 "set_queue_timeout", 95 "scoped", 96 "is_free_threaded", 97 "stats", 98 "config", 99 "CompilerNotFoundError", 100]
def
task(func_or_none=None, *, isolated: bool = False):
37def task(func_or_none=None, *, isolated: bool = False): 38 """ 39 Decorator to offload a Python function to the Rust background worker pool. 40 41 The decorated function is executed on a background OS thread. On a 42 free-threaded CPython build, in-process Python work can execute across CPU 43 cores. On regular CPython, ``isolated=True`` uses a separate interpreter 44 process and serialized IPC; large serialized frames may use shared memory. 45 46 Args: 47 func_or_none: The Python callable to execute. 48 isolated: Set to True to run the task in an isolated worker process. 49 """ 50 51 def decorator(func: Callable[[P], R]) -> Callable[[P], TaskHandle]: 52 @functools.wraps(func) 53 def wrapper(payload: P) -> TaskHandle: 54 import os 55 56 from .config import _get_scoped_queue_timeout_ms 57 58 if os.environ.get("PYROXIDE_WORKER") == "1": 59 return cast(TaskHandle, func(payload)) 60 61 target_callable: Any = func 62 if isolated: 63 target_callable = ( 64 _FunctionalIsolatedCallable(func) 65 if _registered_original(func) 66 else wrapper 67 ) 68 queue_time = _get_scoped_queue_timeout_ms() 69 task_id = submit_task( 70 target_callable, payload, isolated=isolated, queue_timeout_ms=queue_time 71 ) 72 return TaskHandle(task_id) 73 74 def batch(payloads: list) -> list[TaskHandle]: 75 import os 76 77 from ._pyroxide import submit_batch 78 from .config import _get_scoped_queue_timeout_ms 79 80 if os.environ.get("PYROXIDE_WORKER") == "1": 81 return [cast(TaskHandle, func(p)) for p in payloads] 82 83 target_callable: Any = func 84 if isolated: 85 target_callable = ( 86 _FunctionalIsolatedCallable(func) 87 if _registered_original(func) 88 else wrapper 89 ) 90 queue_time = _get_scoped_queue_timeout_ms() 91 task_ids = submit_batch( 92 target_callable, 93 payloads, 94 isolated=isolated, 95 queue_timeout_ms=queue_time, 96 ) 97 return [TaskHandle(tid) for tid in task_ids] 98 99 setattr(wrapper, "batch", batch) 100 return wrapper 101 102 if func_or_none is None: 103 return decorator 104 else: 105 return decorator(func_or_none)
Decorator to offload a Python function to the Rust background worker pool.
The decorated function is executed on a background OS thread. On a
free-threaded CPython build, in-process Python work can execute across CPU
cores. On regular CPython, isolated=True uses a separate interpreter
process and serialized IPC; large serialized frames may use shared memory.
Args: func_or_none: The Python callable to execute. isolated: Set to True to run the task in an isolated worker process.
class
TaskHandle:
24class TaskHandle: 25 def __init__(self, task_id: int) -> None: 26 self.task_id: int = task_id 27 self._consumed: bool = False 28 29 def __repr__(self) -> str: 30 return f"<TaskHandle id={self.task_id}>" 31 32 @property 33 def status(self) -> str: 34 """Queries the current status from the Rust Slab.""" 35 return get_status(self.task_id) 36 37 def cancel(self) -> bool: 38 """ 39 Attempts to prevent or terminate task execution. 40 41 Pending tasks can be cancelled. Running isolated tasks can be 42 terminated. Running in-process Python, native, and WASM tasks cannot 43 be interrupted safely, so cancellation returns False and their real 44 result remains available. 45 """ 46 from ._pyroxide import cancel_task 47 48 return cancel_task(self.task_id) 49 50 def wait( 51 self, poll_interval_ms: int = 10, timeout_sec: Optional[float] = None 52 ) -> str: 53 """ 54 Blocks the Python runtime until the background Rust worker completes the task. 55 Uses native Rust condvar signal to sleep with 0% CPU usage. 56 """ 57 if timeout_sec is not None: 58 if timeout_sec < 0: 59 raise ValueError("timeout_sec must be non-negative") 60 timeout_ms: Optional[int] = int(timeout_sec * 1000) 61 else: 62 timeout_ms = None 63 current_status: str = wait_status(self.task_id, timeout_ms) 64 65 if current_status == "Cancelled": 66 raise RuntimeError("Task cancelled") 67 68 if timeout_sec is not None and current_status not in ("Completed", "Failed"): 69 raise TimeoutError(f"Task {self.task_id} timed out.") 70 71 return current_status 72 73 def result(self, timeout_sec: Optional[float] = None, consume: bool = True) -> Any: 74 """ 75 Blocks until the task is complete, then returns the result. 76 If the task failed, raises the exception encountered. 77 78 Args: 79 timeout_sec: Maximum time in seconds to wait. 80 consume: If True, automatically evicts the task from the Rust Slab once retrieved. 81 """ 82 self.wait(timeout_sec=timeout_sec) 83 from ._pyroxide import free_task, get_result 84 85 res = get_result(self.task_id) 86 if consume: 87 free_task(self.task_id) 88 self._consumed = True 89 return res 90 91 async def result_async( 92 self, timeout_sec: Optional[float] = None, consume: bool = True 93 ) -> Any: 94 """ 95 Asynchronously awaits the task result, yielding control to the event loop. 96 97 A task may have only one active asynchronous waiter. A second concurrent 98 call raises ``RuntimeError``. 99 """ 100 with _pending_futures_lock: 101 if self.task_id in _async_waiters: 102 raise RuntimeError(f"Task {self.task_id} is already being awaited") 103 _async_waiters.add(self.task_id) 104 105 fut: Optional[asyncio.Future] = None 106 try: 107 if sys.platform == "win32": 108 current_status = self.status 109 if current_status in ("Completed", "Failed", "Cancelled"): 110 return self.result(timeout_sec=0, consume=consume) 111 loop = asyncio.get_running_loop() 112 await loop.run_in_executor(None, self.wait, 10, timeout_sec) 113 return self.result(timeout_sec=0, consume=consume) 114 115 loop = asyncio.get_running_loop() 116 ensure_waker_registered(loop) 117 118 fut = loop.create_future() 119 with _pending_futures_lock: 120 _pending_futures[self.task_id] = fut 121 122 current_status = self.status 123 if current_status in ("Completed", "Failed", "Cancelled"): 124 return self.result(timeout_sec=0, consume=consume) 125 126 try: 127 if timeout_sec is not None: 128 await asyncio.wait_for(fut, timeout=timeout_sec) 129 else: 130 await fut 131 except asyncio.TimeoutError: 132 raise TimeoutError(f"Task {self.task_id} timed out.") 133 134 return self.result(timeout_sec=0, consume=consume) 135 finally: 136 with _pending_futures_lock: 137 if fut is not None and _pending_futures.get(self.task_id) is fut: 138 _pending_futures.pop(self.task_id, None) 139 _async_waiters.discard(self.task_id) 140 141 def close(self) -> None: 142 """ 143 Explicitly releases and frees the task memory in the Rust Slab. 144 """ 145 if getattr(self, "_consumed", False): 146 return 147 try: 148 current_status = self.status 149 if current_status in ("Completed", "Failed", "Cancelled"): 150 from ._pyroxide import free_task 151 152 free_task(self.task_id) 153 else: 154 from ._pyroxide import set_autofree 155 156 set_autofree(self.task_id) 157 self._consumed = True 158 except Exception: 159 pass 160 161 def __enter__(self) -> "TaskHandle": 162 return self 163 164 def __exit__(self, exc_type, exc_val, exc_tb) -> None: 165 self.close() 166 167 def __del__(self) -> None: 168 """ 169 Garbage collection destructor. 170 Automatically frees the task memory in the Rust Slab when the Python handle is deleted/dropped. 171 """ 172 self.close()
status: str
32 @property 33 def status(self) -> str: 34 """Queries the current status from the Rust Slab.""" 35 return get_status(self.task_id)
Queries the current status from the Rust Slab.
def
cancel(self) -> bool:
37 def cancel(self) -> bool: 38 """ 39 Attempts to prevent or terminate task execution. 40 41 Pending tasks can be cancelled. Running isolated tasks can be 42 terminated. Running in-process Python, native, and WASM tasks cannot 43 be interrupted safely, so cancellation returns False and their real 44 result remains available. 45 """ 46 from ._pyroxide import cancel_task 47 48 return cancel_task(self.task_id)
Attempts to prevent or terminate task execution.
Pending tasks can be cancelled. Running isolated tasks can be terminated. Running in-process Python, native, and WASM tasks cannot be interrupted safely, so cancellation returns False and their real result remains available.
def
wait( self, poll_interval_ms: int = 10, timeout_sec: Optional[float] = None) -> str:
50 def wait( 51 self, poll_interval_ms: int = 10, timeout_sec: Optional[float] = None 52 ) -> str: 53 """ 54 Blocks the Python runtime until the background Rust worker completes the task. 55 Uses native Rust condvar signal to sleep with 0% CPU usage. 56 """ 57 if timeout_sec is not None: 58 if timeout_sec < 0: 59 raise ValueError("timeout_sec must be non-negative") 60 timeout_ms: Optional[int] = int(timeout_sec * 1000) 61 else: 62 timeout_ms = None 63 current_status: str = wait_status(self.task_id, timeout_ms) 64 65 if current_status == "Cancelled": 66 raise RuntimeError("Task cancelled") 67 68 if timeout_sec is not None and current_status not in ("Completed", "Failed"): 69 raise TimeoutError(f"Task {self.task_id} timed out.") 70 71 return current_status
Blocks the Python runtime until the background Rust worker completes the task. Uses native Rust condvar signal to sleep with 0% CPU usage.
def
result(self, timeout_sec: Optional[float] = None, consume: bool = True) -> Any:
73 def result(self, timeout_sec: Optional[float] = None, consume: bool = True) -> Any: 74 """ 75 Blocks until the task is complete, then returns the result. 76 If the task failed, raises the exception encountered. 77 78 Args: 79 timeout_sec: Maximum time in seconds to wait. 80 consume: If True, automatically evicts the task from the Rust Slab once retrieved. 81 """ 82 self.wait(timeout_sec=timeout_sec) 83 from ._pyroxide import free_task, get_result 84 85 res = get_result(self.task_id) 86 if consume: 87 free_task(self.task_id) 88 self._consumed = True 89 return res
Blocks until the task is complete, then returns the result. If the task failed, raises the exception encountered.
Args: timeout_sec: Maximum time in seconds to wait. consume: If True, automatically evicts the task from the Rust Slab once retrieved.
async def
result_async(self, timeout_sec: Optional[float] = None, consume: bool = True) -> Any:
91 async def result_async( 92 self, timeout_sec: Optional[float] = None, consume: bool = True 93 ) -> Any: 94 """ 95 Asynchronously awaits the task result, yielding control to the event loop. 96 97 A task may have only one active asynchronous waiter. A second concurrent 98 call raises ``RuntimeError``. 99 """ 100 with _pending_futures_lock: 101 if self.task_id in _async_waiters: 102 raise RuntimeError(f"Task {self.task_id} is already being awaited") 103 _async_waiters.add(self.task_id) 104 105 fut: Optional[asyncio.Future] = None 106 try: 107 if sys.platform == "win32": 108 current_status = self.status 109 if current_status in ("Completed", "Failed", "Cancelled"): 110 return self.result(timeout_sec=0, consume=consume) 111 loop = asyncio.get_running_loop() 112 await loop.run_in_executor(None, self.wait, 10, timeout_sec) 113 return self.result(timeout_sec=0, consume=consume) 114 115 loop = asyncio.get_running_loop() 116 ensure_waker_registered(loop) 117 118 fut = loop.create_future() 119 with _pending_futures_lock: 120 _pending_futures[self.task_id] = fut 121 122 current_status = self.status 123 if current_status in ("Completed", "Failed", "Cancelled"): 124 return self.result(timeout_sec=0, consume=consume) 125 126 try: 127 if timeout_sec is not None: 128 await asyncio.wait_for(fut, timeout=timeout_sec) 129 else: 130 await fut 131 except asyncio.TimeoutError: 132 raise TimeoutError(f"Task {self.task_id} timed out.") 133 134 return self.result(timeout_sec=0, consume=consume) 135 finally: 136 with _pending_futures_lock: 137 if fut is not None and _pending_futures.get(self.task_id) is fut: 138 _pending_futures.pop(self.task_id, None) 139 _async_waiters.discard(self.task_id)
Asynchronously awaits the task result, yielding control to the event loop.
A task may have only one active asynchronous waiter. A second concurrent
call raises RuntimeError.
def
close(self) -> None:
141 def close(self) -> None: 142 """ 143 Explicitly releases and frees the task memory in the Rust Slab. 144 """ 145 if getattr(self, "_consumed", False): 146 return 147 try: 148 current_status = self.status 149 if current_status in ("Completed", "Failed", "Cancelled"): 150 from ._pyroxide import free_task 151 152 free_task(self.task_id) 153 else: 154 from ._pyroxide import set_autofree 155 156 set_autofree(self.task_id) 157 self._consumed = True 158 except Exception: 159 pass
Explicitly releases and frees the task memory in the Rust Slab.
def
register_wasm(module_name: str, wasm_bytes: bytes):
10def register_wasm(module_name: str, wasm_bytes: bytes): 11 """ 12 Registers a pre-compiled WebAssembly module in the global registry. 13 """ 14 register_wasm_module(module_name, wasm_bytes)
Registers a pre-compiled WebAssembly module in the global registry.
def
register_wasm_wat(module_name: str, wat_str: str):
17def register_wasm_wat(module_name: str, wat_str: str): 18 """ 19 Registers a WebAssembly module from WAT text format. 20 """ 21 from ._pyroxide import register_wasm_wat as reg_wat 22 23 reg_wat(module_name, wat_str)
Registers a WebAssembly module from WAT text format.
def
wasm_task(module_name: str, func_name: str = 'run', *, isolated: bool = False):
26def wasm_task(module_name: str, func_name: str = "run", *, isolated: bool = False): 27 """ 28 Decorator to submit string or byte payloads to be processed by a registered WASM module. 29 """ 30 31 def decorator(func): 32 @functools.wraps(func) 33 def wrapper(payload) -> TaskHandle: 34 from .config import ( 35 _get_scoped_queue_timeout_ms, 36 _get_scoped_wasm_memory_limit_bytes, 37 _get_scoped_wasm_timeout_ms, 38 ) 39 40 wasm_mem = _get_scoped_wasm_memory_limit_bytes() 41 wasm_time = _get_scoped_wasm_timeout_ms() 42 queue_time = _get_scoped_queue_timeout_ms() 43 task_id = submit_wasm_task( 44 module_name, 45 func_name, 46 payload, 47 isolated=isolated, 48 wasm_memory_limit_bytes=wasm_mem, 49 wasm_timeout_ms=wasm_time, 50 queue_timeout_ms=queue_time, 51 ) 52 return TaskHandle(task_id) 53 54 return wrapper 55 56 return decorator
Decorator to submit string or byte payloads to be processed by a registered WASM module.
def
load_wasm( module_name: str, *, generate_stubs: bool = False, isolated: bool = False) -> pyroxide.wasm.WasmProxy:
113def load_wasm( 114 module_name: str, 115 *, 116 generate_stubs: bool = False, 117 isolated: bool = False, 118) -> WasmProxy: 119 """ 120 Loads a registered WebAssembly (WASM) module and returns an object-oriented proxy 121 allowing direct invocation of any exported WASM function on the background worker pool. 122 """ 123 proxy_class_name = f"{module_name.capitalize()}WasmProxy" 124 ProxyClass = type(proxy_class_name, (WasmProxy,), {}) 125 proxy = ProxyClass(module_name, isolated=isolated) 126 if generate_stubs: 127 from pyroxide.stubs import generate_stubs as run_gen 128 129 run_gen(module_name, library_type="wasm") 130 return proxy
Loads a registered WebAssembly (WASM) module and returns an object-oriented proxy allowing direct invocation of any exported WASM function on the background worker pool.
def
compile_wasm(module_name: str, source_code: str, lang: str = 'wat') -> str:
245def compile_wasm(module_name: str, source_code: str, lang: str = "wat") -> str: 246 """ 247 Compiles and registers source code (WAT, C, Rust, or Zig) to sandboxed WebAssembly (WASM) on-the-fly. 248 """ 249 lang_lower = lang.lower() 250 if lang_lower in ("wat", "wasm"): 251 return compile_wat_wasm(module_name, source_code) 252 elif lang_lower == "c": 253 return compile_c_wasm(module_name, source_code) 254 elif lang_lower == "rust": 255 return compile_rust_wasm(module_name, source_code) 256 elif lang_lower == "zig": 257 return compile_zig_wasm(module_name, source_code) 258 else: 259 raise ValueError( 260 f"Unsupported WASM compilation language '{lang}'. Supported: 'wat', 'c', 'rust', 'zig'" 261 )
Compiles and registers source code (WAT, C, Rust, or Zig) to sandboxed WebAssembly (WASM) on-the-fly.
def
compile_wat_wasm(module_name: str, wat_code: str) -> str:
35def compile_wat_wasm(module_name: str, wat_code: str) -> str: 36 """ 37 Registers a WebAssembly module from WAT text format string on-the-fly. 38 """ 39 register_wasm_wat(module_name, wat_code) 40 return module_name
Registers a WebAssembly module from WAT text format string on-the-fly.
def
compile_c_wasm(module_name: str, source_code: str) -> str:
43def compile_c_wasm(module_name: str, source_code: str) -> str: 44 """ 45 Compiles C source code on-the-fly into WebAssembly bytecode (WASM/WASI) and registers it for sandboxed execution. 46 """ 47 stripped = source_code.strip() 48 if stripped.startswith("(module"): 49 register_wasm_wat(module_name, source_code) 50 return module_name 51 52 _check_compilation_enabled() 53 cc = os.environ.get("CC", "clang" if sys.platform == "darwin" else "gcc") 54 _verify_compiler(cc) 55 56 with _wasm_compilation_guard(): 57 temp_dir = tempfile.mkdtemp(prefix=f"pyroxide_c_wasm_{module_name}_") 58 try: 59 src_path = os.path.join(temp_dir, f"{module_name}.c") 60 out_path = os.path.join(temp_dir, f"{module_name}.wasm") 61 with open(src_path, "w") as f: 62 f.write(source_code) 63 64 cmd = [ 65 cc, 66 "--target=wasm32-wasi", 67 "-O3", 68 "-nostdlib", 69 "-Wl,--no-entry", 70 "-Wl,--export-all", 71 "-o", 72 out_path, 73 src_path, 74 ] 75 res = subprocess.run( 76 cmd, 77 capture_output=True, 78 text=True, 79 timeout=_compiler_timeout_seconds(), 80 ) 81 if res.returncode != 0 or not os.path.exists(out_path): 82 cmd_fb = [ 83 cc, 84 "--target=wasm32", 85 "-O3", 86 "-nostdlib", 87 "-Wl,--no-entry", 88 "-Wl,--export-all", 89 "-o", 90 out_path, 91 src_path, 92 ] 93 res_fb = subprocess.run( 94 cmd_fb, 95 capture_output=True, 96 text=True, 97 timeout=_compiler_timeout_seconds(), 98 ) 99 if res_fb.returncode != 0 or not os.path.exists(out_path): 100 raise RuntimeError( 101 f"C to WASM compilation failed:\n{res.stderr}\n{res_fb.stderr}" 102 ) 103 104 with open(out_path, "rb") as f: 105 wasm_bytes = f.read() 106 107 register_wasm(module_name, wasm_bytes) 108 return module_name 109 finally: 110 shutil.rmtree(temp_dir, ignore_errors=True)
Compiles C source code on-the-fly into WebAssembly bytecode (WASM/WASI) and registers it for sandboxed execution.
def
compile_rust_wasm(module_name: str, source_code: str) -> str:
113def compile_rust_wasm(module_name: str, source_code: str) -> str: 114 """ 115 Compiles Rust source code on-the-fly into WebAssembly bytecode (WASM/WASI) and registers it for sandboxed execution. 116 """ 117 stripped = source_code.strip() 118 if stripped.startswith("(module"): 119 register_wasm_wat(module_name, source_code) 120 return module_name 121 122 _check_compilation_enabled() 123 _verify_compiler("cargo") 124 125 with _wasm_compilation_guard(): 126 temp_dir = tempfile.mkdtemp(prefix=f"pyroxide_rust_wasm_{module_name}_") 127 try: 128 subprocess.run( 129 ["cargo", "init", "--lib", "--name", module_name], 130 cwd=temp_dir, 131 check=True, 132 stdout=subprocess.DEVNULL, 133 stderr=subprocess.DEVNULL, 134 timeout=_compiler_timeout_seconds(), 135 ) 136 137 cargo_toml_path = os.path.join(temp_dir, "Cargo.toml") 138 with open(cargo_toml_path, "r") as f: 139 cargo_content = f.read() 140 141 cargo_content = cargo_content.replace( 142 'edition = "2024"', 'edition = "2021"' 143 ) 144 cargo_content += '\n[lib]\ncrate-type = ["cdylib"]\n' 145 with open(cargo_toml_path, "w") as f: 146 f.write(cargo_content) 147 148 lib_rs_path = os.path.join(temp_dir, "src", "lib.rs") 149 with open(lib_rs_path, "w") as f: 150 f.write(source_code) 151 152 target = "wasm32-wasip1" 153 res = subprocess.run( 154 ["cargo", "build", "--target", target, "--release"], 155 cwd=temp_dir, 156 capture_output=True, 157 text=True, 158 timeout=_compiler_timeout_seconds(), 159 ) 160 out_wasm = os.path.join( 161 temp_dir, "target", target, "release", f"{module_name}.wasm" 162 ) 163 164 if res.returncode != 0 or not os.path.exists(out_wasm): 165 fallback_target = "wasm32-unknown-unknown" 166 res_fb = subprocess.run( 167 ["cargo", "build", "--target", fallback_target, "--release"], 168 cwd=temp_dir, 169 capture_output=True, 170 text=True, 171 timeout=_compiler_timeout_seconds(), 172 ) 173 out_wasm = os.path.join( 174 temp_dir, 175 "target", 176 fallback_target, 177 "release", 178 f"{module_name}.wasm", 179 ) 180 if res_fb.returncode != 0 or not os.path.exists(out_wasm): 181 raise RuntimeError( 182 f"Rust to WASM compilation failed:\n{res.stderr}\n{res_fb.stderr}" 183 ) 184 185 with open(out_wasm, "rb") as f: 186 wasm_bytes = f.read() 187 188 register_wasm(module_name, wasm_bytes) 189 return module_name 190 finally: 191 shutil.rmtree(temp_dir, ignore_errors=True)
Compiles Rust source code on-the-fly into WebAssembly bytecode (WASM/WASI) and registers it for sandboxed execution.
def
compile_zig_wasm(module_name: str, source_code: str) -> str:
194def compile_zig_wasm(module_name: str, source_code: str) -> str: 195 """ 196 Compiles Zig source code on-the-fly into WebAssembly bytecode (WASM/WASI) and registers it for sandboxed execution. 197 """ 198 stripped = source_code.strip() 199 if stripped.startswith("(module"): 200 register_wasm_wat(module_name, source_code) 201 return module_name 202 203 _check_compilation_enabled() 204 _verify_compiler("zig") 205 206 with _wasm_compilation_guard(): 207 temp_dir = tempfile.mkdtemp(prefix=f"pyroxide_zig_wasm_{module_name}_") 208 try: 209 src_path = os.path.join(temp_dir, f"{module_name}.zig") 210 out_path = os.path.join(temp_dir, f"{module_name}.wasm") 211 with open(src_path, "w") as f: 212 f.write(source_code) 213 214 cmd = [ 215 "zig", 216 "build-exe", 217 "-target", 218 "wasm32-wasi", 219 "-O", 220 "ReleaseFast", 221 f"-femit-bin={out_path}", 222 src_path, 223 ] 224 res = subprocess.run( 225 cmd, 226 cwd=temp_dir, 227 capture_output=True, 228 text=True, 229 timeout=_compiler_timeout_seconds(), 230 ) 231 if res.returncode != 0 or not os.path.exists(out_path): 232 raise RuntimeError( 233 f"Zig to WASM compilation failed:\n{res.stderr}\n{res.stdout}" 234 ) 235 236 with open(out_path, "rb") as f: 237 wasm_bytes = f.read() 238 239 register_wasm(module_name, wasm_bytes) 240 return module_name 241 finally: 242 shutil.rmtree(temp_dir, ignore_errors=True)
Compiles Zig source code on-the-fly into WebAssembly bytecode (WASM/WASI) and registers it for sandboxed execution.
def
compile_rust( name: str, source_code: str, dependencies: Optional[Dict[str, str]] = None) -> str:
160def compile_rust( 161 name: str, source_code: str, dependencies: Optional[Dict[str, str]] = None 162) -> str: 163 """ 164 Compiles Rust source code on-the-fly into a dynamic shared library (.so / .dylib / .dll), 165 and registers it with the Pyroxide background broker for GIL-free execution. 166 167 The compilation is handled automatically by invoking ``cargo build --release`` inside 168 a temporary directory. The user does not need to install or configure anything beyond 169 having a working Rust toolchain (``rustc`` + ``cargo``). 170 171 Args: 172 name: Unique name for the dylib. Used to reference it in ``@dylib_task``. 173 source_code: Raw Rust source code string. Must export two C-compatible symbols: 174 175 - ``pyroxide_plugin_run(ptr, len, out_len) -> *mut u8`` 176 - ``pyroxide_plugin_free(ptr, len)`` 177 dependencies: Optional dict of Cargo dependencies, e.g. ``{"serde": "1.0"}``. 178 179 Returns: 180 Absolute path to the compiled shared library file. 181 182 Raises: 183 RuntimeError: If the Cargo compilation fails. 184 FileNotFoundError: If the compiled library binary is not found after build. 185 186 Example: 187 >>> compile_rust("my_lib", RUST_SOURCE_CODE) 188 >>> @dylib_task("my_lib") 189 ... def process(payload): pass 190 >>> handle = process("hello") 191 >>> print(handle.result()) 192 """ 193 _check_compilation_enabled() 194 _verify_compiler("cargo") 195 196 cache_dir = _cache_dir() 197 lock_path = os.path.join(cache_dir, "compile.lock") 198 lock = CrossProcessLock(lock_path) 199 _acquire_compilation_locks(lock) 200 201 temp_dir = tempfile.mkdtemp(prefix=f"pyroxide_dylib_{name}_") 202 try: 203 # Run cargo init 204 subprocess.run( 205 ["cargo", "init", "--lib", "--name", name], 206 cwd=temp_dir, 207 check=True, 208 stdout=subprocess.DEVNULL, 209 stderr=subprocess.DEVNULL, 210 timeout=_compiler_timeout_seconds(), 211 ) 212 213 cargo_toml_path = os.path.join(temp_dir, "Cargo.toml") 214 with open(cargo_toml_path, "r") as f: 215 cargo_content = f.read() 216 217 # Force Edition 2021 to prevent newer Rust 2024 edition strict compiler errors 218 cargo_content = cargo_content.replace('edition = "2024"', 'edition = "2021"') 219 220 # Add cdylib configuration 221 cargo_content += '\n[lib]\ncrate-type = ["cdylib"]\n' 222 223 # Add dependencies 224 if dependencies: 225 cargo_content += "\n[dependencies]\n" 226 for dep, ver in dependencies.items(): 227 cargo_content += f'{dep} = "{ver}"\n' 228 229 with open(cargo_toml_path, "w") as f: 230 f.write(cargo_content) 231 232 # Write Rust source code to src/lib.rs 233 lib_rs_path = os.path.join(temp_dir, "src", "lib.rs") 234 with open(lib_rs_path, "w") as f: 235 f.write(source_code) 236 237 # Run cargo build in release mode 238 res = subprocess.run( 239 ["cargo", "build", "--release"], 240 cwd=temp_dir, 241 capture_output=True, 242 text=True, 243 timeout=_compiler_timeout_seconds(), 244 ) 245 if res.returncode != 0: 246 raise RuntimeError(f"Cargo build failed:\n{res.stderr}\n{res.stdout}") 247 248 # Find compiled library 249 lib_ext = "dylib" if sys.platform == "darwin" else "so" 250 if sys.platform == "win32": 251 lib_ext = "dll" 252 253 lib_name = f"lib{name}.{lib_ext}" 254 if sys.platform == "win32": 255 lib_name = f"{name}.{lib_ext}" 256 257 compiled_path = os.path.join(temp_dir, "target", "release", lib_name) 258 if not os.path.exists(compiled_path): 259 raise FileNotFoundError(f"Compiled library not found at: {compiled_path}") 260 261 # Copy to persistent cache directory 262 dest_path = _publish_library(compiled_path, cache_dir, lib_name) 263 264 # Register dylib with the Rust core engine 265 register_dylib(name, dest_path) 266 return dest_path 267 268 except Exception as e: 269 raise RuntimeError(f"Failed to compile dylib '{name}' via Cargo: {e}") from e 270 finally: 271 shutil.rmtree(temp_dir, ignore_errors=True) 272 lock.release() 273 _compile_lock.release()
Compiles Rust source code on-the-fly into a dynamic shared library (.so / .dylib / .dll), and registers it with the Pyroxide background broker for GIL-free execution.
The compilation is handled automatically by invoking cargo build --release inside
a temporary directory. The user does not need to install or configure anything beyond
having a working Rust toolchain (rustc + cargo).
Args:
name: Unique name for the dylib. Used to reference it in @dylib_task.
source_code: Raw Rust source code string. Must export two C-compatible symbols:
- ``pyroxide_plugin_run(ptr, len, out_len) -> *mut u8``
- ``pyroxide_plugin_free(ptr, len)``
dependencies: Optional dict of Cargo dependencies, e.g. ``{"serde": "1.0"}``.
Returns: Absolute path to the compiled shared library file.
Raises: RuntimeError: If the Cargo compilation fails. FileNotFoundError: If the compiled library binary is not found after build.
Example:
compile_rust("my_lib", RUST_SOURCE_CODE) @dylib_task("my_lib") ... def process(payload): pass handle = process("hello") print(handle.result())
def
dylib_task( dylib_name: str, symbol_name: str = 'pyroxide_plugin_run', ffi_sig: Optional[tuple] = None, *, isolated: bool = False):
11def dylib_task( 12 dylib_name: str, 13 symbol_name: str = "pyroxide_plugin_run", 14 ffi_sig: Optional[tuple] = None, 15 *, 16 isolated: bool = False, 17): 18 """ 19 Decorator that routes task payloads to a registered dynamic shared library (dylib) 20 for GIL-free execution on the background Rust worker pool. 21 22 The dylib must have been previously compiled and registered via ``compile_rust()``. 23 24 Args: 25 dylib_name: The name of the dylib as registered with ``compile_rust()``. 26 symbol_name: The function symbol to load from the dylib. Defaults to "pyroxide_plugin_run". 27 ffi_sig: Optional FFI signature tuple, e.g. (['i32', 'i32'], 'i32') 28 isolated: Set to True to run in an isolated worker process for crash isolation. 29 """ 30 31 def decorator(func: Callable[[Any], Any]) -> Callable[[Any], TaskHandle]: 32 def wrapper(payload: Any) -> TaskHandle: 33 from .config import _get_scoped_queue_timeout_ms 34 35 queue_time = _get_scoped_queue_timeout_ms() 36 task_id = submit_dylib_task( 37 dylib_name, 38 symbol_name, 39 payload, 40 ffi_sig=ffi_sig, 41 isolated=isolated, 42 queue_timeout_ms=queue_time, 43 ) 44 return TaskHandle(task_id) 45 46 def batch(payloads: list) -> list[TaskHandle]: 47 from .config import _get_scoped_queue_timeout_ms 48 49 queue_time = _get_scoped_queue_timeout_ms() 50 task_ids = submit_dylib_batch( 51 dylib_name, 52 symbol_name, 53 payloads, 54 ffi_sig=ffi_sig, 55 isolated=isolated, 56 queue_timeout_ms=queue_time, 57 ) 58 return [TaskHandle(task_id) for task_id in task_ids] 59 60 setattr(wrapper, "batch", batch) 61 return wrapper 62 63 return decorator
Decorator that routes task payloads to a registered dynamic shared library (dylib) for GIL-free execution on the background Rust worker pool.
The dylib must have been previously compiled and registered via compile_rust().
Args:
dylib_name: The name of the dylib as registered with compile_rust().
symbol_name: The function symbol to load from the dylib. Defaults to "pyroxide_plugin_run".
ffi_sig: Optional FFI signature tuple, e.g. (['i32', 'i32'], 'i32')
isolated: Set to True to run in an isolated worker process for crash isolation.
def
load_dylib( lib_name: str, *, signatures: Optional[dict] = None, generate_stubs: bool = False, isolated: bool = False, free_fn_name: Optional[str] = None) -> pyroxide.plugins.DylibProxy:
67def load_dylib( 68 lib_name: str, 69 *, 70 signatures: Optional[dict] = None, 71 generate_stubs: bool = False, 72 isolated: bool = False, 73 free_fn_name: Optional[str] = None, 74) -> DylibProxy: 75 """ 76 Loads a registered dynamic shared library (dylib) and returns an object-oriented proxy 77 allowing direct invocation of any C-ABI exported symbol on the background worker pool. 78 """ 79 # 0. Auto-register if not already registered 80 try: 81 from pyroxide._pyroxide import get_dylib_exports, get_dylib_path 82 83 get_dylib_exports(lib_name) 84 if free_fn_name is not None: 85 try: 86 reg_path = get_dylib_path(lib_name) 87 if reg_path: 88 clean_path = reg_path.split(";")[0] 89 register_dylib(lib_name, clean_path, free_fn_name=free_fn_name) 90 except Exception: 91 pass 92 except ValueError: 93 try: 94 register_dylib(lib_name, lib_name, free_fn_name=free_fn_name) 95 except Exception: 96 pass 97 98 # 1. Auto-discover signatures if none are provided 99 if signatures is None: 100 from pyroxide._pyroxide import get_dylib_metadata 101 102 metadata_str = get_dylib_metadata(lib_name) 103 if metadata_str: 104 signatures = {} 105 for entry in metadata_str.split(";"): 106 if not entry: 107 continue 108 func_parts = entry.split(":") 109 if len(func_parts) == 2: 110 func_name, sig_part = func_parts 111 sig_parts = sig_part.split("|") 112 if len(sig_parts) == 2: 113 args_part, ret_type = sig_parts 114 args = [a for a in args_part.split(",") if a] 115 signatures[func_name] = {"args": args, "ret": ret_type} 116 117 # 2. Create the proxy 118 proxy_class_name = f"{lib_name.capitalize()}DylibProxy" 119 ProxyClass = type(proxy_class_name, (DylibProxy,), {}) 120 proxy = ProxyClass(lib_name, signatures=signatures, isolated=isolated) 121 122 # 3. Generate stubs if requested 123 if generate_stubs: 124 from pyroxide.stubs import generate_stubs as run_gen 125 126 run_gen(lib_name, library_type="dylib") 127 128 return proxy
Loads a registered dynamic shared library (dylib) and returns an object-oriented proxy allowing direct invocation of any C-ABI exported symbol on the background worker pool.
def
unregister_dylib(name: str) -> None:
131def unregister_dylib(name: str) -> None: 132 """ 133 Unregisters a dynamic shared library from the Pyroxide registries. 134 """ 135 from pyroxide._pyroxide import unregister_dylib as _raw_unregister 136 137 _raw_unregister(name)
Unregisters a dynamic shared library from the Pyroxide registries.
def
compile_c(name: str, source_code: str) -> str:
276def compile_c(name: str, source_code: str) -> str: 277 """ 278 Compiles C source code on-the-fly into a dynamic shared library (.so / .dylib / .dll), 279 and registers it with the Pyroxide background broker for GIL-free execution. 280 281 Args: 282 name: Unique name for the library. Used to reference it in @dylib_task. 283 source_code: Raw C source code string. Must export two functions: 284 - ``pyroxide_plugin_run(ptr, len, out_len) -> uint8_t*`` 285 - ``pyroxide_plugin_free(ptr, len)`` 286 """ 287 _check_compilation_enabled() 288 cc = os.environ.get("CC", "clang" if sys.platform == "darwin" else "gcc") 289 _verify_compiler(cc) 290 291 cache_dir = _cache_dir() 292 lock_path = os.path.join(cache_dir, "compile.lock") 293 lock = CrossProcessLock(lock_path) 294 _acquire_compilation_locks(lock) 295 296 temp_dir = tempfile.mkdtemp(prefix=f"pyroxide_c_{name}_") 297 try: 298 src_path = os.path.join(temp_dir, f"{name}.c") 299 with open(src_path, "w") as f: 300 f.write(source_code) 301 lib_ext = "dylib" if sys.platform == "darwin" else "so" 302 if sys.platform == "win32": 303 lib_ext = "dll" 304 305 lib_name = f"lib{name}.{lib_ext}" 306 if sys.platform == "win32": 307 lib_name = f"{name}.{lib_ext}" 308 compiled_path = os.path.join(temp_dir, lib_name) 309 310 cmd = [cc, "-shared", "-o", compiled_path, "-fPIC", src_path] 311 res = subprocess.run( 312 cmd, 313 capture_output=True, 314 text=True, 315 timeout=_compiler_timeout_seconds(), 316 ) 317 if res.returncode != 0: 318 raise RuntimeError(f"C compilation failed:\n{res.stderr}\n{res.stdout}") 319 320 if not os.path.exists(compiled_path): 321 raise FileNotFoundError(f"Compiled C library not found at: {compiled_path}") 322 323 # Copy to persistent cache directory 324 dest_path = _publish_library(compiled_path, cache_dir, lib_name) 325 326 register_dylib(name, dest_path) 327 return dest_path 328 329 except Exception as e: 330 raise RuntimeError(f"Failed to compile C library '{name}': {e}") from e 331 finally: 332 shutil.rmtree(temp_dir, ignore_errors=True) 333 lock.release() 334 _compile_lock.release()
Compiles C source code on-the-fly into a dynamic shared library (.so / .dylib / .dll), and registers it with the Pyroxide background broker for GIL-free execution.
Args:
name: Unique name for the library. Used to reference it in @dylib_task.
source_code: Raw C source code string. Must export two functions:
- pyroxide_plugin_run(ptr, len, out_len) -> uint8_t*
- pyroxide_plugin_free(ptr, len)
def
compile_zig(name: str, source_code: str) -> str:
337def compile_zig(name: str, source_code: str) -> str: 338 """ 339 Compiles Zig source code on-the-fly into a dynamic shared library (.so / .dylib / .dll), 340 and registers it with the Pyroxide background broker for GIL-free execution. 341 342 Args: 343 name: Unique name for the library. Used to reference it in @dylib_task. 344 source_code: Raw Zig source code string. Must export two functions: 345 - ``pyroxide_plugin_run(ptr, len, out_len) -> [*]u8`` 346 - ``pyroxide_plugin_free(ptr, len)`` 347 """ 348 _check_compilation_enabled() 349 _verify_compiler("zig") 350 351 cache_dir = _cache_dir() 352 lock_path = os.path.join(cache_dir, "compile.lock") 353 lock = CrossProcessLock(lock_path) 354 _acquire_compilation_locks(lock) 355 356 temp_dir = tempfile.mkdtemp(prefix=f"pyroxide_zig_{name}_") 357 try: 358 src_path = os.path.join(temp_dir, f"{name}.zig") 359 with open(src_path, "w") as f: 360 f.write(source_code) 361 362 # Compiles dynamic library. Zig build-lib generates output in cwd 363 cmd = ["zig", "build-lib", "-dynamic", "-O", "ReleaseFast", src_path] 364 res = subprocess.run( 365 cmd, 366 cwd=temp_dir, 367 capture_output=True, 368 text=True, 369 timeout=_compiler_timeout_seconds(), 370 ) 371 if res.returncode != 0: 372 raise RuntimeError(f"Zig compilation failed:\n{res.stderr}\n{res.stdout}") 373 374 lib_ext = "dylib" if sys.platform == "darwin" else "so" 375 if sys.platform == "win32": 376 lib_ext = "dll" 377 378 lib_name = f"lib{name}.{lib_ext}" 379 if sys.platform == "win32": 380 lib_name = f"{name}.{lib_ext}" 381 382 compiled_path = os.path.join(temp_dir, lib_name) 383 if not os.path.exists(compiled_path): 384 raise FileNotFoundError( 385 f"Compiled Zig library not found at: {compiled_path}" 386 ) 387 388 # Copy to persistent cache directory 389 dest_path = _publish_library(compiled_path, cache_dir, lib_name) 390 391 register_dylib(name, dest_path) 392 return dest_path 393 394 except Exception as e: 395 raise RuntimeError(f"Failed to compile Zig library '{name}': {e}") from e 396 finally: 397 shutil.rmtree(temp_dir, ignore_errors=True) 398 lock.release() 399 _compile_lock.release()
Compiles Zig source code on-the-fly into a dynamic shared library (.so / .dylib / .dll), and registers it with the Pyroxide background broker for GIL-free execution.
Args:
name: Unique name for the library. Used to reference it in @dylib_task.
source_code: Raw Zig source code string. Must export two functions:
- pyroxide_plugin_run(ptr, len, out_len) -> [*]u8
- pyroxide_plugin_free(ptr, len)
118def group(handles: Iterable[TaskHandle]) -> TaskGroup: 119 """Wraps multiple task handles into a parallel TaskGroup.""" 120 return TaskGroup(handles)
Wraps multiple task handles into a parallel TaskGroup.
class
TaskGroup:
21class TaskGroup: 22 """A collection of tasks that run in parallel and can be managed as a unit.""" 23 24 def __init__(self, handles: Iterable[TaskHandle]): 25 self.handles = list(handles) 26 27 def __repr__(self) -> str: 28 return f"<TaskGroup handles={self.handles} status={self.status}>" 29 30 @property 31 def status(self) -> str: 32 statuses = [h.status for h in self.handles] 33 if "Failed" in statuses: 34 return "Failed" 35 if "Cancelled" in statuses: 36 return "Cancelled" 37 if all(s == "Completed" for s in statuses): 38 return "Completed" 39 return "Running" 40 41 def wait(self): 42 """Blocks until all tasks in the group are completed.""" 43 for h in self.handles: 44 h.wait() 45 46 def result(self, consume: bool = True) -> List: 47 """Waits for all tasks and returns their results in order.""" 48 return [h.result(consume=consume) for h in self.handles] 49 50 def cancel(self) -> bool: 51 """Cancels all tasks in the group. Returns True if all were successfully cancelled.""" 52 results = [h.cancel() for h in self.handles] 53 return all(results) 54 55 async def __aenter__(self): 56 """Enters the asynchronous context manager.""" 57 return self 58 59 async def __aexit__(self, exc_type, exc_val, exc_tb): 60 """ 61 Exits the asynchronous context manager. 62 If an exception occurred, cancels all tasks. Otherwise, waits for all tasks to complete. 63 """ 64 if exc_type is not None: 65 self.cancel() 66 67 exceptions = [] 68 if exc_val is not None: 69 exceptions.append(exc_val) 70 71 tasks = [ 72 asyncio.create_task(h.result_async(consume=False)) for h in self.handles 73 ] 74 if tasks: 75 try: 76 while tasks: 77 done, pending = await asyncio.wait( 78 tasks, return_when=asyncio.FIRST_EXCEPTION 79 ) 80 for t in done: 81 try: 82 await t 83 except Exception as e: 84 self.cancel() 85 if ( 86 exc_val is not None 87 and isinstance(e, RuntimeError) 88 and "task cancelled" in str(e).lower() 89 ): 90 continue 91 exceptions.append(e) 92 tasks = list(pending) 93 except Exception as e: 94 self.cancel() 95 exceptions.append(e) 96 97 if exceptions: 98 # Sibling task cancellation errors should be filtered out to reduce noise if there is another root exception 99 has_real_exception = any( 100 not (isinstance(e, RuntimeError) and "cancelled" in str(e).lower()) 101 for e in exceptions 102 ) 103 if has_real_exception: 104 exceptions = [ 105 e 106 for e in exceptions 107 if not ( 108 isinstance(e, RuntimeError) and "cancelled" in str(e).lower() 109 ) 110 ] 111 112 if len(exceptions) == 1 and exceptions[0] == exc_val: 113 return False 114 115 raise ExceptionGroup("TaskGroup errors", exceptions)
A collection of tasks that run in parallel and can be managed as a unit.
TaskGroup(handles: Iterable[TaskHandle])
def
wait(self):
41 def wait(self): 42 """Blocks until all tasks in the group are completed.""" 43 for h in self.handles: 44 h.wait()
Blocks until all tasks in the group are completed.
def
shutdown(wait: bool = True, cancel_pending: bool = False) -> None:
55def shutdown(wait: bool = True, cancel_pending: bool = False) -> None: 56 """Stop accepting work and shut down Pyroxide's workers. 57 58 Shutdown is irreversible for the current process. Accepted work drains by 59 default; ``cancel_pending=True`` cancels work that has not started. 60 A Pyroxide worker task must use ``wait=False`` to avoid waiting for itself. 61 """ 62 if type(wait) is not bool or type(cancel_pending) is not bool: 63 raise TypeError("wait and cancel_pending must be bool values") 64 _shutdown_engine(wait=wait, cancel_pending=cancel_pending) 65 if wait: 66 from .types import _cleanup_waker 67 68 _cleanup_waker()
Stop accepting work and shut down Pyroxide's workers.
Shutdown is irreversible for the current process. Accepted work drains by
default; cancel_pending=True cancels work that has not started.
A Pyroxide worker task must use wait=False to avoid waiting for itself.
class
ForkSafetyError(builtins.RuntimeError):
Unspecified run-time error.
def
generate_stubs(name: str, library_type: str, out_path: Optional[str] = None) -> str:
10def generate_stubs(name: str, library_type: str, out_path: Optional[str] = None) -> str: 11 """ 12 Generates a Python PEP 484 type stub (.pyi) file for a registered dynamic library or WASM module, 13 providing full IDE autocompletion and hover documentation for all dynamic exports. 14 15 Args: 16 name: The registered name of the dynamic library or WASM module. 17 library_type: Either 'dylib' or 'wasm'. 18 out_path: Optional output file path. Defaults to '{name}_proxy.pyi' in the current directory. 19 20 Returns: 21 The absolute path to the generated type stub file. 22 """ 23 sigs = {} 24 if library_type.lower() == "wasm": 25 from ._pyroxide import get_wasm_exports 26 27 exports = get_wasm_exports(name) 28 class_name = f"{name.capitalize()}WasmProxy" 29 factory_name = f"load_wasm_{name}" 30 elif library_type.lower() == "dylib": 31 from ._pyroxide import get_dylib_exports, get_dylib_metadata 32 33 exports = get_dylib_exports(name) 34 class_name = f"{name.capitalize()}DylibProxy" 35 factory_name = f"load_dylib_{name}" 36 37 # Discover FFI signatures from metadata 38 metadata_str = get_dylib_metadata(name) 39 if metadata_str: 40 for entry in metadata_str.split(";"): 41 if not entry: 42 continue 43 func_parts = entry.split(":") 44 if len(func_parts) == 2: 45 func_name, sig_part = func_parts 46 sig_parts = sig_part.split("|") 47 if len(sig_parts) == 2: 48 args_part, ret_type = sig_parts 49 args = [a for a in args_part.split(",") if a] 50 sigs[func_name] = {"args": args, "ret": ret_type} 51 else: 52 raise ValueError("library_type must be either 'wasm' or 'dylib'") 53 54 # Print compile-time warnings for potential memory leaks if pyroxide_plugin_free is missing for raw tasks 55 if library_type.lower() == "dylib" and "pyroxide_plugin_free" not in exports: 56 raw_funcs = [ 57 s 58 for s in exports 59 if s not in ("pyroxide_metadata", "pyroxide_plugin_free") and s not in sigs 60 ] 61 if raw_funcs: 62 import sys 63 64 print( 65 f"⚠️ Warning: Library '{name}' exposes raw binary tasks {raw_funcs} but " 66 f"does not export 'pyroxide_plugin_free'. Memory leaks will occur " 67 f"if these functions return heap-allocated pointers.", 68 file=sys.stderr, 69 ) 70 71 if not out_path: 72 out_path = f"{name}_proxy.pyi" 73 74 lines = [ 75 "# Auto-generated by Pyroxide. Do not edit directly.", 76 "from typing import Any", 77 "from pyroxide import TaskHandle", 78 "", 79 f"class {class_name}:", 80 ' """', 81 f" Type-annotated proxy for registered {library_type} module '{name}'.", 82 ' """', 83 ] 84 85 if not exports: 86 lines.append(" pass") 87 else: 88 for symbol in exports: 89 if symbol in sigs: 90 91 def _map_ffi(ffi_type: str) -> str: 92 if ffi_type in FFI_PYTHON_TYPES: 93 return FFI_PYTHON_TYPES[ffi_type] 94 raise ValueError( 95 f"Unsupported FFI type '{ffi_type}' in metadata for symbol '{symbol}'." 96 ) 97 98 _map_ffi(sigs[symbol]["ret"]) 99 arg_list = [ 100 f"arg{i}: {_map_ffi(t)}" for i, t in enumerate(sigs[symbol]["args"]) 101 ] 102 args_str = ", ".join(arg_list) 103 if args_str: 104 lines.append( 105 f" def {symbol}(self, {args_str}) -> TaskHandle: ..." 106 ) 107 else: 108 lines.append(f" def {symbol}(self) -> TaskHandle: ...") 109 else: 110 lines.append(f" def {symbol}(self, payload: Any) -> TaskHandle: ...") 111 112 lines.append("") 113 # Add a typed factory helper or hint 114 lines.append(f"def {factory_name}(isolated: bool = ...) -> {class_name}: ...") 115 116 content = "\n".join(lines) + "\n" 117 with open(out_path, "w") as f: 118 f.write(content) 119 120 # Write runtime .py helper file alongside the stub to prevent ModuleNotFoundError 121 py_path = os.path.splitext(out_path)[0] + ".py" 122 if library_type.lower() == "wasm": 123 py_lines = [ 124 "# Auto-generated by Pyroxide. Do not edit directly.", 125 "from pyroxide.wasm import WasmProxy", 126 "", 127 f"class {class_name}(WasmProxy):", 128 " def __init__(self, isolated: bool = False):", 129 f' super().__init__("{name}", isolated=isolated)', 130 "", 131 f"def {factory_name}(isolated: bool = False) -> {class_name}:", 132 f" return {class_name}(isolated=isolated)", 133 ] 134 else: 135 py_lines = [ 136 "# Auto-generated by Pyroxide. Do not edit directly.", 137 "from pyroxide.plugins import DylibProxy", 138 "", 139 f"class {class_name}(DylibProxy):", 140 " def __init__(self, isolated: bool = False):", 141 f" signatures = {repr(sigs)}", 142 f' super().__init__("{name}", signatures=signatures, isolated=isolated)', 143 "", 144 f"def {factory_name}(isolated: bool = False) -> {class_name}:", 145 f" return {class_name}(isolated=isolated)", 146 ] 147 148 py_content = "\n".join(py_lines) + "\n" 149 with open(py_path, "w") as f: 150 f.write(py_content) 151 152 return os.path.abspath(out_path)
Generates a Python PEP 484 type stub (.pyi) file for a registered dynamic library or WASM module, providing full IDE autocompletion and hover documentation for all dynamic exports.
Args: name: The registered name of the dynamic library or WASM module. library_type: Either 'dylib' or 'wasm'. out_path: Optional output file path. Defaults to '{name}_proxy.pyi' in the current directory.
Returns: The absolute path to the generated type stub file.
def
set_wasm_limits( memory_limit_bytes: Optional[int] = None, timeout_ms: Optional[int] = None):
154def set_wasm_limits( 155 memory_limit_bytes: Optional[int] = None, 156 timeout_ms: Optional[int] = None, 157): 158 """Sets global WebAssembly sandbox execution limits.""" 159 if memory_limit_bytes is not None: 160 set_global_wasm_memory_limit_bytes( 161 _wasm_memory_bytes(memory_limit_bytes, "memory_limit_bytes") 162 ) 163 if timeout_ms is not None: 164 set_global_wasm_timeout_ms(_positive_int(timeout_ms, "timeout_ms"))
Sets global WebAssembly sandbox execution limits.
def
set_queue_timeout(timeout_ms: int):
167def set_queue_timeout(timeout_ms: int): 168 """Sets the global task submission queue timeout in milliseconds.""" 169 set_global_queue_timeout_ms(_nonnegative_int(timeout_ms, "timeout_ms"))
Sets the global task submission queue timeout in milliseconds.
@contextmanager
def
scoped( wasm_timeout_ms: Optional[int] = None, wasm_memory_limit_bytes: Optional[int] = None, queue_timeout_ms: Optional[int] = None):
172@contextmanager 173def scoped( 174 wasm_timeout_ms: Optional[int] = None, 175 wasm_memory_limit_bytes: Optional[int] = None, 176 queue_timeout_ms: Optional[int] = None, 177): 178 """ 179 Context manager to temporarily override execution limits or queue timeouts 180 for the current thread or asyncio task. 181 """ 182 tokens = [] 183 if wasm_timeout_ms is not None: 184 val = _positive_int(wasm_timeout_ms, "wasm_timeout_ms") 185 tokens.append((_wasm_timeout_var, _wasm_timeout_var.set(val))) 186 if wasm_memory_limit_bytes is not None: 187 val = _wasm_memory_bytes(wasm_memory_limit_bytes, "wasm_memory_limit_bytes") 188 tokens.append((_wasm_memory_limit_var, _wasm_memory_limit_var.set(val))) 189 if queue_timeout_ms is not None: 190 val = _nonnegative_int(queue_timeout_ms, "queue_timeout_ms") 191 tokens.append((_queue_timeout_var, _queue_timeout_var.set(val))) 192 193 try: 194 yield 195 finally: 196 for var, token in reversed(tokens): 197 var.reset(token)
Context manager to temporarily override execution limits or queue timeouts for the current thread or asyncio task.
def
is_free_threaded() -> bool:
124def is_free_threaded() -> bool: 125 """ 126 Returns True if running under a free-threaded CPython build (PEP 703, Python 3.13+) 127 with the Global Interpreter Lock (GIL) disabled. 128 """ 129 if hasattr(sys, "_is_gil_enabled"): 130 try: 131 return not sys._is_gil_enabled() 132 except Exception: 133 return False 134 return False
Returns True if running under a free-threaded CPython build (PEP 703, Python 3.13+) with the Global Interpreter Lock (GIL) disabled.
def
stats() -> dict:
137def stats() -> dict: 138 """ 139 Return process-local engine gauges and lifetime task counters. 140 141 Gauges include worker count, queue capacity, queued, running, and retained 142 active tasks. Counters include submitted, rejected, completed, failed, and 143 cancelled tasks. 144 145 Fields are read independently. During concurrent activity the returned 146 mapping is an approximate cross-field snapshot and may combine values from 147 nearby moments; use quiescent readings for drain or leak checks. 148 """ 149 from ._pyroxide import get_engine_stats 150 151 return get_engine_stats()
Return process-local engine gauges and lifetime task counters.
Gauges include worker count, queue capacity, queued, running, and retained active tasks. Counters include submitted, rejected, completed, failed, and cancelled tasks.
Fields are read independently. During concurrent activity the returned mapping is an approximate cross-field snapshot and may combine values from nearby moments; use quiescent readings for drain or leak checks.
class
CompilerNotFoundError(builtins.RuntimeError):
Raised when a required compiler binary (cargo, gcc, clang, zig) is missing from PATH.