feat(workers): Python skeleton + config + structlog
Plan 4 Phase A scaffolding. void-workers package at /workers/, sibling of /lib/. pyproject.toml pins Python 3.12 with separate extras for pdf / image / video / test. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
1
workers/.gitignore
vendored
Normal file
1
workers/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
|||||||
|
.venv/
|
||||||
23
workers/README.md
Normal file
23
workers/README.md
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
# void-workers
|
||||||
|
|
||||||
|
Python ML ingest service alongside `void-server` (Node). Sibling of `lib/` in the void-v2 repo.
|
||||||
|
|
||||||
|
## Local dev
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd workers
|
||||||
|
python3.12 -m venv .venv
|
||||||
|
. .venv/bin/activate
|
||||||
|
pip install -e ".[all]"
|
||||||
|
export DATABASE_URL="postgres://..."
|
||||||
|
python -m void_workers.runner
|
||||||
|
```
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install -e ".[test,all]"
|
||||||
|
DATABASE_URL="postgres://..." pytest -v
|
||||||
|
```
|
||||||
|
|
||||||
|
See `../docs/superpowers/plans/2026-06-01-void-v2-plan4-workers.md` for the full plan and `../docs/superpowers/specs/2026-06-01-void-v2-plan4-workers.md` for the design.
|
||||||
23
workers/pyproject.toml
Normal file
23
workers/pyproject.toml
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
[project]
|
||||||
|
name = "void-workers"
|
||||||
|
version = "0.1.0"
|
||||||
|
requires-python = ">=3.12"
|
||||||
|
dependencies = [
|
||||||
|
"psycopg[binary,pool]>=3.2",
|
||||||
|
"structlog>=24.1",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
pdf = ["pdfplumber>=0.11", "pytesseract>=0.3.13", "pillow>=10.3"]
|
||||||
|
image = ["pytesseract>=0.3.13", "pillow>=10.3"]
|
||||||
|
video = ["yt-dlp>=2024.10.0", "faster-whisper>=1.0.3"]
|
||||||
|
test = ["pytest>=8.0", "pytest-asyncio>=0.23"]
|
||||||
|
all = ["void-workers[pdf,image,video,test]"]
|
||||||
|
|
||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=68"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[tool.setuptools.packages.find]
|
||||||
|
where = ["."]
|
||||||
|
include = ["void_workers*"]
|
||||||
1
workers/void_workers/__init__.py
Normal file
1
workers/void_workers/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
__version__ = "0.1.0"
|
||||||
26
workers/void_workers/config.py
Normal file
26
workers/void_workers/config.py
Normal file
@@ -0,0 +1,26 @@
|
|||||||
|
import os
|
||||||
|
|
||||||
|
def env(name, default=None, required=False):
|
||||||
|
v = os.environ.get(name, default)
|
||||||
|
if required and v is None:
|
||||||
|
raise RuntimeError(f"env {name} is required")
|
||||||
|
return v
|
||||||
|
|
||||||
|
def env_int(name, default):
|
||||||
|
return int(os.environ.get(name, default))
|
||||||
|
|
||||||
|
DATABASE_URL = env("DATABASE_URL", required=True)
|
||||||
|
BLOB_ROOT = env("BLOB_ROOT", "/var/lib/void/blobs")
|
||||||
|
WHISPER_MODEL = env("WHISPER_MODEL", "small.en")
|
||||||
|
WHISPER_CACHE = env("WHISPER_CACHE", "/var/lib/void/whisper-models")
|
||||||
|
ALLOW_PRIVATE = env("VOID_INGEST_ALLOW_PRIVATE", "false") == "true"
|
||||||
|
|
||||||
|
CONCURRENCY = {
|
||||||
|
"extract.pdf": env_int("VOID_CONCURRENCY_EXTRACT_PDF", 2),
|
||||||
|
"extract.image": env_int("VOID_CONCURRENCY_EXTRACT_IMAGE", 2),
|
||||||
|
"ingest.video": env_int("VOID_CONCURRENCY_INGEST_VIDEO", 1),
|
||||||
|
"sync.source_doc": env_int("VOID_CONCURRENCY_SYNC_SOURCE_DOC", 1),
|
||||||
|
"echo": env_int("VOID_CONCURRENCY_ECHO", 1),
|
||||||
|
}
|
||||||
|
|
||||||
|
POLL_INTERVAL_MS = env_int("VOID_POLL_INTERVAL_MS", 1000)
|
||||||
16
workers/void_workers/log.py
Normal file
16
workers/void_workers/log.py
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
import logging
|
||||||
|
import structlog
|
||||||
|
|
||||||
|
def init():
|
||||||
|
structlog.configure(
|
||||||
|
processors=[
|
||||||
|
structlog.contextvars.merge_contextvars,
|
||||||
|
structlog.processors.add_log_level,
|
||||||
|
structlog.processors.TimeStamper(fmt="iso"),
|
||||||
|
structlog.processors.JSONRenderer(),
|
||||||
|
],
|
||||||
|
wrapper_class=structlog.make_filtering_bound_logger(logging.INFO),
|
||||||
|
)
|
||||||
|
return structlog.get_logger()
|
||||||
|
|
||||||
|
log = init()
|
||||||
Reference in New Issue
Block a user