QTZPOC-20

QTZPOC-20: Implement model import workflow for .sientia bundles

Introduces the import_model workflow and its associated activities to enable importing .sientia bundles into MLflow and MongoDB. This includes:

*   New core workflow and activity logic for handling bundle download, decryption, validation, MLflow provisioning (experiments, runs, registered versions), and MongoDB model document creation.
*   Comprehensive E2E and unit test suites, including a dedicated test harness and a manual run script for development.
*   Updated .env.example and README.md with new configurations and usage instructions.
*   Ensures correct MLflow model source resolution for legacy loader compatibility and robust MongoDB document management.
*   Refactors existing utility scripts into a scripts/ directory.
This commit is contained in:
2026-08-25 17:07:42 -03:00
parent 0e445de386
commit dac4f1718f
41 changed files with 8168 additions and 1045 deletions

View File

@@ -1,10 +1,27 @@
"""Queue names and how many workers a runtime starts.
Every queue is runtime-scoped, so two runtimes never poll each other's work. The import worker is
the only conditional one: unset, the four default workers are exactly what they were, and no import
queue is polled at all.
"""
import ast
from pathlib import Path
from pytest import mark
from sientia_do.temporal.worker.prepare_worker import build_queue_name
from laborious.worker.worker import import_worker_enabled
from laborious.workflows.drift import Drift
from laborious.workflows.import_model import ImportModel
from laborious.workflows.minimal_retrain import MinimalRetrain
from laborious.workflows.predictions_batch import PredictionsBatch
from laborious.workflows.simple_metrics import SimpleMetrics
WORKER_MODULE = Path(__file__).resolve().parents[3] / 'laborious/worker/worker.py'
DEFAULT_WORKFLOWS = (PredictionsBatch, MinimalRetrain, Drift, SimpleMetrics)
def test_runtime_scoped_queue_names():
runtime = 'prod-a'
@@ -12,3 +29,86 @@ def test_runtime_scoped_queue_names():
assert build_queue_name(MinimalRetrain.__name__, runtime) == 'minimal_retrain-prod-a-queue'
assert build_queue_name(Drift.__name__, runtime) == 'drift-prod-a-queue'
assert build_queue_name(SimpleMetrics.__name__, runtime) == 'simple_metrics-prod-a-queue'
def test_the_import_queue_shares_the_runtime_suffix():
assert build_queue_name(ImportModel.__name__, 'prod-a') == 'import_model-prod-a-queue'
def test_every_queue_carries_the_same_runtime():
runtime = 'prod-a'
queues = [
build_queue_name(workflow.__name__, runtime)
for workflow in (*DEFAULT_WORKFLOWS, ImportModel)
]
assert all(queue.endswith(f'-{runtime}-queue') for queue in queues)
assert len(set(queues)) == 5
@mark.parametrize('value', ['1', 'true', 'TRUE', 'yes', 'on', ' true '])
def test_the_import_worker_is_registered_when_the_flag_says_so(monkeypatch, value: str):
monkeypatch.setenv('IMPORT_MODEL_ENABLED', value)
assert import_worker_enabled() is True
@mark.parametrize('value', ['', '0', 'false', 'no', 'off', 'maybe'])
def test_the_import_worker_stays_out_by_default(monkeypatch, value: str):
monkeypatch.setenv('IMPORT_MODEL_ENABLED', value)
assert import_worker_enabled() is False
def test_an_unset_flag_leaves_the_import_worker_out(monkeypatch):
monkeypatch.delenv('IMPORT_MODEL_ENABLED', raising=False)
assert import_worker_enabled() is False
def _prepare_worker_calls(tree: ast.Module) -> tuple[int, int]:
"""How many workers are built unconditionally, and how many behind the flag."""
conditional = 0
for node in ast.walk(tree):
if not isinstance(node, ast.If):
continue
if 'import_worker_enabled' not in ast.unparse(node.test):
continue
conditional += sum(
1
for inner in ast.walk(node)
if isinstance(inner, ast.Call) and ast.unparse(inner.func) == 'prepare_worker'
)
total = sum(
1
for node in ast.walk(tree)
if isinstance(node, ast.Call) and ast.unparse(node.func) == 'prepare_worker'
)
return total - conditional, conditional
def test_four_workers_by_default_and_five_with_the_flag():
unconditional, conditional = _prepare_worker_calls(ast.parse(WORKER_MODULE.read_text()))
assert unconditional == 4
assert conditional == 1
def test_the_import_worker_registers_only_the_import_activities():
"""The containment boundary: the import runtime polls one queue and holds one workflow."""
tree = ast.parse(WORKER_MODULE.read_text())
activities: list[str] = []
for node in ast.walk(tree):
if not isinstance(node, ast.If) or 'import_worker_enabled' not in ast.unparse(node.test):
continue
for inner in ast.walk(node):
if isinstance(inner, ast.keyword) and inner.arg == 'activities':
activities = [ast.unparse(element) for element in inner.value.elts]
if isinstance(inner, ast.keyword) and inner.arg == 'main_workflow':
assert ast.unparse(inner.value) == 'ImportModel'
if isinstance(inner, ast.keyword) and inner.arg == 'other_workflows':
assert ast.unparse(inner.value) == '[]'
assert len(activities) == 13
assert all(name.startswith('activities.') for name in activities)
assert all('import' in name for name in activities)