QTZPOC-20

QTZPOC-20: Implement .sientia model bundle import workflow

Introduces a new Temporal workflow and activities to import .sientia model bundles. This enables:
- Secure decryption and validation of bundles through a seven-gate process.
- Provisioning imported models in MLflow and updating the MongoDB model listing.
- Comprehensive status tracking in the public.experiment_run table with new IMPORT_SUCCESS/IMPORT_ERROR states.
- Dedicated metrics, robust error handling, and configurable settings.
- Includes updated e2e database fixtures and extensive unit/integration tests for bundle format, reader, and overall workflow.

tests/fixtures/bundle/regenerate.py

@@ -0,0 +1,185 @@
+"""Regenerate the golden `.sientia` fixtures using the producer's own source.
+
+This script is the auditable half of the anti-drift contract described in the change
+`import-model-workflow` (design D1): the committed fixtures are bytes produced by
+`sientia-projects-templates/app/src/operations/model_export/{bundle,crypto}.py`, loaded here by
+path so no constant is re-implemented. Regenerating is a deliberate act — see the README next to
+this file for the recorded producer commit and the fixture password.
+
+Usage (from the repository root, inside the venv):
+
+    PRODUCER_REPO=../sientia-projects-templates ./venv/bin/python tests/fixtures/bundle/regenerate.py
+
+The producer's own venv is not used: it does not have `pynacl` installed, while this repository
+pins the producer's exact version (`pynacl==1.5.0`). The bytes are therefore produced by the
+producer's code on the producer's dependency pin, which is what the fixture is for.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import random
+import shutil
+import subprocess
+import sys
+import tempfile
+import types
+from pathlib import Path
+
+FIXTURE_DIR = Path(__file__).resolve().parent
+DEFAULT_PRODUCER_REPO = FIXTURE_DIR.parents[2].parent / 'sientia-projects-templates'
+
+# Recorded in the README alongside the fixtures; the password satisfies the producer's own
+# export-side strength rule (>= 12 chars, upper, lower, digit, symbol).
+FIXTURE_PASSWORD = 'Import1Model!Fixture'
+
+# Deterministic filler so a regeneration differs only in the salt and the export timestamp.
+FILLER_SEED = 20260825
+
+
+def _load_producer_module(repo: Path, name: str) -> types.ModuleType:
+    """Import one producer module by path, without importing the Streamlit app."""
+    path = repo / 'app' / 'src' / 'operations' / 'model_export' / f'{name}.py'
+    if not path.is_file():
+        raise SystemExit(f'producer module not found: {path}')
+    spec = importlib.util.spec_from_file_location(f'producer_{name}', path)
+    if spec is None or spec.loader is None:
+        raise SystemExit(f'cannot load producer module: {path}')
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _producer_commit(repo: Path) -> str:
+    result = subprocess.run(  # noqa: S603 - fixed argv, no shell
+        ['git', '-C', str(repo), 'rev-parse', 'HEAD'],  # noqa: S607 - git from PATH is intended
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    return result.stdout.strip() or 'unknown'
+
+
+def _write_model_dir(root: Path, flavor: str, pickle_bytes: bytes) -> None:
+    """Write the files an MLflow 2.x model directory carries for this platform."""
+    root.mkdir(parents=True, exist_ok=True)
+    (root / 'MLmodel').write_text(
+        f'artifact_path: {flavor}\n'
+        'flavors:\n'
+        '  python_function:\n'
+        '    loader_module: mlflow.sklearn\n'
+        '    model_path: model.pkl\n'
+        '  sklearn:\n'
+        '    pickled_model: model.pkl\n'
+        '    serialization_format: cloudpickle\n'
+        '    sklearn_version: 1.4.2\n'
+        'mlflow_version: 2.10.1\n'
+        'model_uuid: 0f9c1c1f9c1f4c0f9c1c1f9c1f4c0f9c\n'
+        'utc_time_created: "2026-08-25 12:00:00.000000"\n',
+        encoding='utf-8',
+    )
+    (root / 'model.pkl').write_bytes(pickle_bytes)
+    (root / 'conda.yaml').write_text(
+        'channels:\n- conda-forge\ndependencies:\n- python=3.11\n- pip\n- pip:\n'
+        '  - mlflow==2.10.1\n  - scikit-learn==1.4.2\nname: mlflow-env\n',
+        encoding='utf-8',
+    )
+    (root / 'python_env.yaml').write_text(
+        'python: 3.11.15\nbuild_dependencies:\n- pip==24.0\ndependencies:\n- -r requirements.txt\n',
+        encoding='utf-8',
+    )
+    (root / 'requirements.txt').write_text(
+        'mlflow==2.10.1\nscikit-learn==1.4.2\ncloudpickle==3.0.0\n', encoding='utf-8'
+    )
+
+
+def _write_csv(path: Path, rows: int, rng: random.Random) -> None:
+    lines = ['datetime,feature_a,feature_b,SE']
+    for i in range(rows):
+        lines.append(
+            f'2026-08-{(i % 27) + 1:02d}T00:00:00,'
+            f'{rng.uniform(0, 100):.6f},{rng.uniform(0, 100):.6f},{rng.uniform(0, 100):.6f}'
+        )
+    path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
+
+
+def _build_artifact_tree(root: Path) -> None:
+    rng = random.Random(FILLER_SEED)  # noqa: S311 - test filler, not key material
+    # Incompressible filler keeps the zip above the 64 KiB chunk size, so the golden fixture
+    # exercises multi-chunk `MESSAGE`/`FINAL` framing with real producer bytes.
+    big_pickle = bytes(rng.getrandbits(8) for _ in range(90 * 1024))
+    small_pickle = bytes(rng.getrandbits(8) for _ in range(8 * 1024))
+    _write_model_dir(root / 'prediction_model', 'prediction_model', big_pickle)
+    _write_model_dir(root / 'data_model', 'data_model', small_pickle)
+    (root / 'model_card').mkdir(parents=True, exist_ok=True)
+    (root / 'model_card' / 'model_card.json').write_text(
+        json.dumps(
+            {
+                'model_name': 'fixture_model',
+                'target': 'SE',
+                'created_by': 'fixture',
+                'creation_method': 'time_series_template',
+            }
+        ),
+        encoding='utf-8',
+    )
+    _write_csv(root / 'train_data.csv', 120, rng)
+    _write_csv(root / 'test_data.csv', 40, rng)
+    _write_csv(root / 'raw_data.csv', 200, rng)
+
+
+RUN_PARAMETERS = {
+    'model_name': 'fixture_model',
+    'target_variable': 'SE',
+    'input_variables': "['feature_a', 'feature_b']",
+    'models_params': "{'alpha': 0.1}",
+    'nan_treatment': 'drop',
+    'scaler_name': 'StandardScaler',
+    'train_size': '0.8',
+    'test_size': '0.2',
+    'include_ar': 'False',
+    'retrain': 'False',
+    'start_date': '2026-01-01 00:00:00',
+    'end_date': '2026-08-01 00:00:00',
+}
+
+METADATA_FIELDS = {
+    'model_name': 'fixture_model',
+    'experiment_name': 'fixture_experiment',
+    'run_id': '8f14e45fceea167a5a36dedd4bea2543',
+    'model_version': '3',
+    'model_project': 'fixture_project',
+}
+
+
+def main() -> int:
+    repo = Path(os.getenv('PRODUCER_REPO', str(DEFAULT_PRODUCER_REPO))).resolve()
+    bundle = _load_producer_module(repo, 'bundle')
+    crypto = _load_producer_module(repo, 'crypto')
+
+    crypto.validate_password_strength(FIXTURE_PASSWORD)
+
+    staging = Path(tempfile.mkdtemp(prefix='sientia_fixture_'))
+    try:
+        _build_artifact_tree(staging)
+        metadata = bundle.build_metadata_json(run_parameters=RUN_PARAMETERS, **METADATA_FIELDS)
+        zip_bytes = bundle.build_zip(staging, metadata)
+        sientia_bytes = crypto.encrypt_archive(zip_bytes, FIXTURE_PASSWORD)
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)
+
+    target = FIXTURE_DIR / 'golden_model_v3.sientia'
+    target.write_bytes(sientia_bytes)
+    (FIXTURE_DIR / 'golden_model_v3.metadata.json').write_text(
+        json.dumps(metadata, indent=2) + '\n', encoding='utf-8'
+    )
+    print(f'wrote {target} ({len(sientia_bytes)} bytes, zip {len(zip_bytes)} bytes)')
+    print(f'producer commit: {_producer_commit(repo)}')
+    print(f'password: {FIXTURE_PASSWORD}')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())