chore: bump version to 0.9.2

Point Deploy Workflow to laborious-legacy-worker Release

.env

@@ -1,4 +0,0 @@
-# === Simulator Git Repo ===
-# Use SSH format because the Dockerfile uses SSH to clone
-SIMULATOR_GIT_REPO=git@github.com:Aignosi/sientia-dataops-opc_simulator.git
-SIMULATOR_GIT_BRANCH=main

.env.example

@@ -0,0 +1,73 @@
+POSTGRES_HOST="paradedb-rw.paradedb.svc.cluster.local"
+POSTGRES_PORT="5432"
+POSTGRES_USER="sientia"
+POSTGRES_PASSWORD="password"
+POSTGRES_DBNAME="sientia"
+POSTGRES_MIN_CONNECTIONS="10"
+POSTGRES_MAX_CONNECTIONS="30"
+
+MLFLOW_HOST="http://sientia-tracker-mlflow-tracking.sientia-tracker.svc.cluster.local"
+MLFLOW_PORT="80"
+MLFLOW_USERNAME="aignosi"
+MLFLOW_PASSWORD="mlflow_password"
+
+OPC_ID="1"
+OPC_URL="opc.tcp://sientia-opc-simulator-opc.sientia.svc.cluster.local:4840"
+
+LOG_LEVEL="DEBUG"
+HTTP_METRICS_PORT="9090"
+HTTP_SDK_METRICS_PORT="9091"
+PROJECT_NAME="sientia-laborious"
+
+TEMPORAL_HOST="temporal-frontend.temporal.svc.cluster.local:7233"
+TEMPORAL_NAMESPACE="laborious"
+
+MONGODB_USERNAME="mongo_user"
+MONGODB_PASSWORD="mongo_db_password"
+MONGODB_URL="my-release-mongodb.mongodb.svc.cluster.local:27017"
+MONGODB_DATABASE="sientia"
+MONGODB_TTL_INDEX_HOURS="1"
+
+MINIO_ENDPOINT_URL="http://localhost:9000"
+MINIO_ACCESS_KEY="sientia"
+MINIO_SECRET_KEY="sientia"
+MINIO_REGION_NAME="sa-east-1"
+MINIO_DEFAULT_BUCKET="sientia"
+
+# Suffix for every Temporal task queue name; required, non-empty.
+# The import worker listens on "import_model-${RUNTIME}-queue".
+RUNTIME="local"
+
+# --- Model import (.sientia bundles) -----------------------------------------
+# The import worker is registered only when IMPORT_MODEL_ENABLED is truthy
+# (1/true/yes/on). Do not turn it on before the V12 migration is applied to
+# public.experiment_run in the BFF database.
+IMPORT_MODEL_ENABLED="false"
+
+# base64 of the 32-byte AES-256-GCM key that wraps bundle passwords.
+# Generate a throwaway one with: openssl rand -base64 32
+IMPORT_PASSWORD_KEY="replace-with-base64-of-32-random-bytes"
+
+# Where the frontend uploads the bundle. The object key must sit under the prefix.
+IMPORT_BUNDLE_BUCKET="sientia"
+IMPORT_BUNDLE_PREFIX="imported_models/"
+
+# MongoDB collection holding the model listing the import writes its document into.
+IMPORT_MODELS_COLLECTION="models"
+
+# The import log lives in the Spring Boot BFF database on the POSTGRES_* server
+# above: same host, port, user and password, another database name. Only that
+# name is configuration here; the schema (public) and the table (experiment_run)
+# are constants of the code. Set IMPORT_STATUS_DB_{HOST,PORT,USER,PASSWORD} only
+# if the log really is on a different server than POSTGRES_* — they override it.
+IMPORT_STATUS_DB_NAME="sientia-core-mlops-bff"
+
+# --- scripts/import_model_manual_run.py only ---------------------------------
+# Inputs for the manual run that stands in for the frontend. The script writes
+# one object and one row to whatever the variables above name.
+IMPORT_SCRIPT_BUNDLE_PATH="tests/fixtures/bundle/golden_model_v3.sientia"
+IMPORT_SCRIPT_BUNDLE_PASSWORD="the-password-the-bundle-was-written-with"
+IMPORT_SCRIPT_USERNAME="manual.run@example.com"
+# Cell 2's guard: restate POSTGRES_HOST here (or in the cell) after reading what cell 1
+# printed. Left empty on purpose — the script refuses to write until it matches.
+IMPORT_SCRIPT_CONFIRM_TARGET=""

.github/workflows/deploy.yml

@@ -0,0 +1,38 @@
+name: Deploy DataOps Worker
+
+on:
+  pull_request:
+    types: [closed]
+    branches: [main, 'release/**', 'feature/**']
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  id-token: write
+  pull-requests: read
+
+jobs:
+  deploy:
+    name: Deploy
+    if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true
+    uses: Aignosi/github_workflow_templates/.github/workflows/reusable-deploy-dataops.yml@main
+    with:
+      branch_name: ${{ github.event.pull_request.head.ref }}
+      project_name: 'laborious-legacy'
+      # This repo deploys its own opt-in release, laborious-legacy-worker: a single Deployment
+      # (no runtimes map, like scouter-worker/orchestrator-worker) with RUNTIME=legacy as the
+      # queue suffix. The new code lives in sientia-dataops-laborious (release laborious-worker).
+      # The Gitea mirror target is derived from this repo's name:
+      # gitea_admin/sientia-dataops-laborious-legacy. GITHUB_BRANCH is bumped in the chart base
+      # .env (release is opt-in; air-gapped profiles pin their own version) and in the gcp
+      # profile's laborious-legacy-worker.env.
+      chart_name: 'laborious-legacy-worker'
+      env_list_path: '[".env"]'
+      profile: 'gcp'
+      profile_env_list_path: '[".env"]'
+      update_version_in: '["pyproject"]'
+      create_tarball: ${{ github.event_name == 'pull_request' }}
+      mirror: true
+      bump_chart: true
+      deploy: true
+    secrets: inherit

.github/workflows/quality-gate.yml

@@ -4,71 +4,28 @@ on:
   push:
     branches:
       - main
+      - 'release/**'
+      - 'feature/**'
   pull_request:
     branches:
       - main
+      - 'release/**'
+      - 'feature/**'
     types: [ opened, synchronize, reopened ]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
-  sonar:
-    name: SonarQube Analysis
-    runs-on: ubuntu-latest
-    permissions: write-all
-    steps:
-      - name: ⬇️ Checkout Code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          persist-credentials: false
-
-      - name: Generate App Token
-        id: generate-app-token
-        uses: actions/create-github-app-token@v1
-        with:
-          app-id: ${{ secrets.APP_ID }}
-          private-key: ${{ secrets.APP_PRIVATE_KEY }}
-          owner: 'Aignosi'
-          repositories: 'sientia-dataops-library,sientia-mlops-library'
-
-      - name: Prepare requirements.txt
-        id: prepare-requirements
-        run: |
-          sed -e "s|git+ssh://git@github.com/|git+https://github.com/|g" \
-              -e "s|git@github.com:|git+https://github.com/|g" \
-              requirements.txt > requirements_prepared.txt
-          echo "PROCESSED_REQUIREMENTS_FILE=requirements_prepared.txt" >> $GITHUB_OUTPUT
-
-      - name: Configure Git to use App Token
-        env:
-          GH_APP_TOKEN: ${{ steps.generate-app-token.outputs.token }}
-        run: |
-          git config --global url."https://oauth2:${GH_APP_TOKEN}@github.com/".insteadOf "https://github.com/"
-
-      - name: 🔧 Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-
-      - name: 🗄️ Cache Python dependencies
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles(steps.prepare-requirements.outputs.PROCESSED_REQUIREMENTS_FILE) }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-
-      - name: 📦 Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r ${{ steps.prepare-requirements.outputs.PROCESSED_REQUIREMENTS_FILE }}
-          pip install pytest pytest-cov pytest-asyncio
-
-      - name: 🧪 Run Tests with Pytest
-        run: |
-          pytest tests --junitxml=pytest.xml --cov=laborious --cov-report=xml --cov-report=term
-
-      - name: Run SonarQube Analysis
-        uses: SonarSource/sonarqube-scan-action@v5
-        env:
-          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-          SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
+  quality-gate:
+    uses: Aignosi/github_workflow_templates/.github/workflows/python-quality-gate.yml@main
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
+    with:
+      project_name: 'laborious'
+      repositories: 'sientia-dataops-library, sientia-mlops-library'
+      requirements_file: 'requirements-light.txt'
+    secrets: inherit

.github/workflows/require-chart-ref-label.yml

@@ -0,0 +1,16 @@
+name: Require chart-ref label
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
+    branches:
+      - main
+      - 'release/**'
+      - 'feature/**'
+
+permissions:
+  pull-requests: write
+
+jobs:
+  require-chart-ref-label:
+    uses: Aignosi/github_workflow_templates/.github/workflows/require-chart-ref-label.yml@main

.gitignore

@@ -37,6 +37,23 @@ __pycache__/
 # Ignorar coverage
 htmlcov/
 .coverage
+coverage.xml
 
 # git keys
-git_key*
+git_key*
+
+git_log
+
+.env
+
+tmp/
+catboost_info/
+
+.ruff_cache/
+.mypy_cache/
+mlruns/
+
+relatorio*
+openspec/
+.cursor/
+.claude/

Dockerfile

@@ -1,83 +0,0 @@
-FROM python:3.11-bookworm
-LABEL description="Deploy Mage on ECS"
-ARG FEATURE_BRANCH
-USER root
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-
-# Definir Python 3.11 como padrão
-ENV PATH="/usr/local/bin/python3.11:$PATH"
-RUN update-alternatives --install /usr/bin/python python /usr/local/bin/python3.11 1 && \
-    update-alternatives --install /usr/bin/python3 python3 /usr/local/bin/python3.11 1 && \
-    update-alternatives --config python3 <<< '1' && \
-    update-alternatives --config python <<< '1'
-
-## System Packages
-RUN \
-  curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - && \
-  curl https://packages.microsoft.com/config/debian/11/prod.list > /etc/apt/sources.list.d/mssql-release.list && \
-  apt-get -y update && \
-  ACCEPT_EULA=Y apt-get -y install --no-install-recommends \
-  # NFS dependencies
-  nfs-common \
-  # odbc dependencies
-  msodbcsql18 \
-  unixodbc-dev \
-  graphviz \
-  # postgres dependencies
-  postgresql-client \
-  # R
-  r-base && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
-
-## R Packages
-RUN \
-  R -e "install.packages('pacman', repos='http://cran.us.r-project.org')" && \
-  R -e "install.packages('renv', repos='http://cran.us.r-project.org')"
-
-## Python Packages
-RUN \
-  pip3 install --no-cache-dir sparkmagic && \
-  mkdir ~/.sparkmagic && \
-  curl https://raw.githubusercontent.com/jupyter-incubator/sparkmagic/master/sparkmagic/example_config.json > ~/.sparkmagic/config.json && \
-  sed -i 's/localhost:8998/host.docker.internal:9999/g' ~/.sparkmagic/config.json && \
-  jupyter-kernelspec install --user "$(pip3 show sparkmagic | grep Location | cut -d' ' -f2)/sparkmagic/kernels/pysparkkernel"
-
-# Mage integrations and other related packages
-RUN \
-  pip3 install --no-cache-dir "git+https://github.com/wbond/oscrypto.git@d5f3437ed24257895ae1edd9e503cfb352e635a8" && \
-  pip3 install --no-cache-dir "git+https://github.com/dremio-hub/arrow-flight-client-examples.git#egg=dremio-flight&subdirectory=python/dremio-flight" && \
-  pip3 install --no-cache-dir "git+https://github.com/mage-ai/singer-python.git#egg=singer-python" && \
-  pip3 install --no-cache-dir "git+https://github.com/mage-ai/dbt-mysql.git#egg=dbt-mysql" && \
-  pip3 install --no-cache-dir "git+https://github.com/mage-ai/sqlglot#egg=sqlglot" && \
-  pip3 install --no-cache-dir faster-fifo && \
-  if [ -z "$FEATURE_BRANCH" ] || [ "$FEATURE_BRANCH" = "null" ]; then \
-  pip3 install --no-cache-dir "git+https://github.com/mage-ai/mage-ai.git#egg=mage-integrations&subdirectory=mage_integrations"; \
-  else \
-  pip3 install --no-cache-dir "git+https://github.com/mage-ai/mage-ai.git@$FEATURE_BRANCH#egg=mage-integrations&subdirectory=mage_integrations"; \
-  fi
-
-# Mage
-COPY ./mage_ai/server/constants.py /tmp/constants.py
-RUN if [ -z "$FEATURE_BRANCH" ] || [ "$FEATURE_BRANCH" = "null" ] ; then \
-  tag=$(tail -n 1 /tmp/constants.py) && \
-  VERSION=$(echo "$tag" | tr -d "'") && \
-  pip3 install --no-cache-dir "mage-ai[all]==$VERSION"; \
-  else \
-  pip3 install --no-cache-dir "git+https://github.com/mage-ai/mage-ai.git@$FEATURE_BRANCH#egg=mage-ai[all]"; \
-  fi
-
-## Startup Script
-COPY --chmod=0755 ./scripts/install_other_dependencies.py ./scripts/run_app.sh /app/
-ENV MAGE_DATA_DIR="/home/src/mage_data"
-ENV PYTHONPATH="${PYTHONPATH}:/home/src"
-WORKDIR /home/src
-EXPOSE 6789
-EXPOSE 7789
-
-# Copia o arquivo requirements.txt para o contêiner
-COPY requirements.txt /app/requirements.txt
-RUN pip3 install --no-cache-dir -r /app/requirements.txt
-
-
-CMD ["/bin/sh", "-c", "/app/run_app.sh"]

Makefile

@@ -1,7 +0,0 @@
-VERSION = 1.0.8
-name = sientia-laborious
-# ENVIRONMENT = production
-
-docker-hub:
-	@docker build --no-cache -t aignosi.azurecr.io/$(name):$(VERSION) .
-	@docker push aignosi.azurecr.io/$(name):$(VERSION)

docker-compose.yml

@@ -1,82 +0,0 @@
-version: '3.8'
-
-services:
-  postgres:
-    image: postgres:15
-    container_name: postgres
-    environment:
-      POSTGRES_USER: sientia
-      POSTGRES_PASSWORD: sientia
-      POSTGRES_DB: sientia
-    ports:
-      - "5432:5432"
-    volumes:
-      - ./postgres_data:/var/lib/postgresql/data
-    networks:
-      - sientia-network
-
-  zookeeper:
-    image: confluentinc/cp-zookeeper:7.5.1
-    container_name: zookeeper
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
-    ports:
-      - "2181:2181"
-    networks:
-      - sientia-network
-
-  kafka:
-    image: confluentinc/cp-kafka:7.5.1
-    container_name: kafka
-    depends_on:
-      - zookeeper
-    ports:
-      - "9092:9092"
-      - "29092:29092"
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
-    networks:
-      - sientia-network
-
-  kafka-ui:
-    image: provectuslabs/kafka-ui:latest
-    container_name: kafka-ui
-    ports:
-      - "8080:8080"
-    environment:
-      KAFKA_CLUSTERS_0_NAME: local
-      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092
-    networks:
-      - sientia-network
-
-  simulator:
-    build:
-      context: .
-      dockerfile: simulator/Dockerfile
-      args:
-        GIT_REPO: ${SIMULATOR_GIT_REPO}
-        GIT_BRANCH: ${SIMULATOR_GIT_BRANCH}
-    container_name: simulator
-    ports:
-      - "4840:4840"
-    depends_on:
-      - kafka
-    networks:
-      - sientia-network
-    env_file:
-      - .env
-
-
-networks:
-  sientia-network:
-    driver: bridge
-
-volumes:
-  postgres_data:
-    driver: local

docs/minio-object-expiry.md

@@ -0,0 +1,40 @@
+# Two independent MinIO expiry mechanisms
+
+This platform expires objects in MinIO through two mechanisms that were built independently, for
+different features, and remain independent today. This note exists so a reader who finds one does
+not assume it is the whole story.
+
+## `ModelImport.ensure_import_bundle_retention`
+
+`laborious/activities/model_import.py`. Applies a **bucket lifecycle rule** (a MinIO/S3-native
+`Expiration` rule, `ENABLED`, keyed by a fixed `rule_id`) to the import bundle prefix
+(`import_config['prefix']`), expiring objects after `import_config['retention_days']` days. The
+rule is declarative: once set, MinIO itself removes the object when it ages out — the workflow does
+not scan for or delete anything. The activity's job is only to ensure the rule exists and matches
+the configured `days`/`prefix`, which is what makes "lifecycle active" a criterion the code can
+verify rather than an assumption about a Helm value.
+
+## `Storage.cleanup_minio_objects_expired`
+
+`laborious/activities/storage.py`. An older, unrelated activity that **scans and deletes**: given a
+list of key prefixes and a retention window (`retention_hours`), it lists objects under each prefix,
+compares their age against `now() - retention_hours`, and deletes the ones that have aged out. There
+is no bucket-side rule — the deletion happens in this code, on whatever schedule calls the activity.
+
+## Why they are not unified today
+
+They expire different prefix families, on different units (days vs. hours), through different
+mechanisms (declarative rule vs. active scan), and were written for different features at different
+times. Neither mechanism depends on or interferes with the other; a bucket can carry an
+`ensure_import_bundle_retention` lifecycle rule for one prefix and still have
+`cleanup_minio_objects_expired` scanning a different prefix on its own cadence.
+
+## What unifying them would require
+
+Either retiring the scan-and-delete path in favor of a lifecycle rule for every prefix it currently
+handles (only possible if every one of its callers can tolerate a fixed number of retention *days*,
+since S3-style lifecycle rules do not support hour-granularity windows), or teaching the
+lifecycle-rule path to manage a per-prefix table of rules with arbitrary retention windows instead of
+the one fixed rule it manages today. Either direction touches callers and configuration this change
+does not, and neither is a refactor — this is tracked as a separate, larger change, not a task of
+this one.

docs/model-flavors.md

@@ -0,0 +1,166 @@
+# Model flavors
+
+The *flavor* is how this runtime is told to deserialize a model. It is **configuration, never
+discovery**: the value comes from the model document in the `models` collection and the code never
+reads the artifact's `MLmodel` manifest to guess it.
+
+```json
+{
+  "model_config": {
+    "transform_flavor": "sklearn",
+    "predict_flavor": "joblib",
+    "retention_minutes": 60,
+    "target": "SE"
+  }
+}
+```
+
+`predict_flavor` drives `load_predict_model` (artifact `prediction_model`); `transform_flavor` drives
+`load_transform_model` (artifact `data_model`). Both default to `sklearn` when the key is absent.
+
+## Other `model_config` flags
+
+Ad-hoc behavioral overrides, read directly with `.get(key, default)` — not part of the model
+document written by import (`model-import.md` § 12), and not validated against a schema:
+
+| Key | Default | Effect |
+|---|---|---|
+| `skip_transform` | `False` | `fit_models` (retrain) skips calling `transform` before fitting. |
+| `skip_predict_timestamp` | `False` | `request_predict` skips reinserting the `timestamp` column (from the index) into the transformed data before calling predict. |
+
+## Accepted values
+
+| Flavor | Read | Write (retrain) | How it loads |
+|---|---|---|---|
+| `sklearn` | yes | yes | `mlflow.sklearn.load_model(<uri>)` — the loader fetches what it needs |
+| `pyfunc` | yes | yes | `mlflow.pyfunc.load_model(<uri>)`; the wrapper path downloads the artifact instead and unwraps `._model_impl.python_model` |
+| `joblib` | yes | yes | downloads the same artifact and `joblib.load`s its `<artifact>/model.pkl` — **no MLflow flavor loader involved** |
+
+Anything else raises `ValueError` with `INVALID_FLAVOR_MESSAGE`, **before** any artifact is resolved
+or downloaded. The same message and the same three values govern reading and writing.
+
+`pytorch` was accepted until this change and is now rejected like any other unknown value: `torch` is
+not installed in the runtime, so `mlflow.pytorch.load_model` could never have loaded anything there.
+
+**There is no fallback between flavors.** A failing loader propagates: one attempt, one cause. A
+model configured with the wrong flavor fails visibly instead of being rescued silently.
+
+## Same artifact, three readers
+
+The flavor changes the reader, never the path. All three resolve the same two artifact directories of
+the Production run:
+
+```
+<run>/artifacts/
+├── data_model/          ← transform_flavor loads this one
+│   ├── MLmodel          ← parsed by sklearn/pyfunc; ignored by joblib
+│   ├── model.pkl        ← what joblib.load reads
+│   └── code/            ← put on sys.path by every flavor, when present
+├── prediction_model/    ← predict_flavor loads this one
+└── model_card/, *.csv
+```
+
+The run's `artifacts/` root holds **no** `.pkl` — every model pickle lives one level down, inside its
+artifact directory. Predict resolves that directory through the registry URI
+(`models:/<name>/production`, which MLflow points at the same `prediction_model`); transform builds
+the run artifact URI directly with `get_model_uri`, since the run is already resolved. Observability
+is the same for the three: `MODEL_READ_*` on load, `MODEL_WRITE_*` on `log_model`, under the base
+`operation_type` — no per-flavor metric, operation type or label.
+
+## `code/` on `sys.path`, for every flavor
+
+An artifact may ship the modules its own classes live in, under `<artifact>/code/`. Without that
+directory on `sys.path`, unpickling raises `ModuleNotFoundError` — the artifact is intact, the class
+is simply not importable.
+
+MLflow's loaders do add it, but **from the manifest**: `_add_code_from_conf_to_system_path` acts only
+when the `MLmodel` flavor config declares a `code` entry. An artifact carrying `code/` on disk
+without that entry — code logged with `log_artifacts`, a re-log that dropped the config, anything a
+`joblib` retrain wrote — loads with no module path at all. So this runtime reads the **directory**,
+which no manifest can misdescribe:
+
+- `prepend_artifact_code_dir` inserts `<artifact>/code` at the front of `sys.path`, **only when that
+  directory exists**, and only once (an entry already present is not duplicated).
+- `download_artifact_code_dir` is what feeds it on the download paths, and it is deliberately cheap:
+  the remote listing (`list_artifacts(run_id, '<artifact>')`) decides first, so a run without `code`
+  downloads **nothing**, and a run with it downloads only `<artifact>/code` — a handful of `.py`
+  files, never the model's weights.
+- `download_model` runs that step first, for every flavor and both of its branches, *before* any
+  model is loaded — the insert has to be in place before the loader unpickles.
+- The models themselves are not pre-downloaded: `mlflow.sklearn.load_model` and
+  `mlflow.pyfunc.load_model` are handed the model URI and fetch what they need. `joblib` downloads
+  its artifact because it must read the pickle, and adds the entry again inside `_load_model_pickle`
+  through the same helper, so the flavor works when called directly too.
+- When the manifest *does* declare `code`, MLflow inserts the same directory a second time. Harmless.
+
+The entry is never removed: a loaded model may import lazily, long after the load returned.
+
+The directory `download_artifact_code_dir` downloaded into is also what `download_model` returns
+next to the model — `tuple[Any, str | None]`, the second element being the artifact directory the
+retrain path re-logs `code/` from, or `None` when the artifact has no `code/`. It falls back to
+whatever the loader downloaded on its own, which is a real artifact directory for `joblib` and for
+the wrapper branch. `load_predict_model` and `load_transform_model` stay the **single** place where
+the flavor is decided: `download_model` has no `joblib` branch, it delegates by `model_type`. Its
+only remaining branch is `load_wrapper`, which is derived from `flavor == 'pyfunc'` but is otherwise
+orthogonal to the flavor.
+
+> `sys.path` is process-global: two models embedding a package with the same name (`utils/`) on the
+> same worker resolve to whichever entered first. Mitigation, if it bites: one worker per model.
+
+## `sklearn`
+
+Default. A standard MLflow sklearn artifact: `mlflow.sklearn.log_model` writes `MLmodel` +
+`model.pkl`, `mlflow.sklearn.load_model` reads it, and every outside consumer (MLflow UI, MLflow
+serving) recognizes it as a model.
+
+## `pyfunc`
+
+For models logged as `mlflow.pyfunc`, including the wrapper models this pipeline produces. Retrain
+writes through the wrapper's own `store_model(artifact_path=<artifact>, code_path=…)`. Two read
+paths, and the second is **orthogonal to the flavor**:
+
+- Plain: `mlflow.pyfunc.load_model(<uri>)`, called as `predict(context, model_input)`.
+- Wrapper (`download_model(load_wrapper=True)`): downloads to `./tmp/artifacts/<model>/<artifact>`
+  via `dowload_artifacts`, loads as pyfunc, then unwraps `._model_impl.python_model`. Inside such an
+  artifact the real pickle is **nested** — `<artifact>/artifacts/stacking_model.pkl` for predict,
+  `<artifact>/artifacts/training_transformer.pkl` for transform (`PREDICTION_COMPRESSED_PATH` /
+  `TRANSFORMED_COMPRESSED_PATH`). That nesting is why the `joblib` listing does not recurse.
+
+## `joblib`
+
+For models whose `.pkl` is in the compressed form `joblib.dump` writes, which MLflow's `pickle.load`
+does not read — the artifact is intact, but no flavor loader opens it.
+
+- **Read**: `client.download_artifacts(run_id, <artifact>)`, then
+  `joblib.load(<artifact>/model.pkl)`. `model.pkl` is `MODEL_PICKLE_NAME`, the same constant
+  `log_model` writes; another `.pkl` directly inside the artifact is a fallback, in name order.
+- **Write**: `joblib.dump(model, '<artifact>/model.pkl', compress=3)`, plus a re-log of the source
+  artifact's `code/` to `<artifact>/code`.
+- **`code/` goes on `sys.path`** before the load, when present — see the section above; the joblib
+  flavor is where that started, and it is now shared by all three.
+- **No `MLmodel` is written**, so only this runtime reads the artifact back —
+  `mlflow.sklearn.load_model` and `models:/<name>/production` both fail on it. A model retrained in
+  joblib stays joblib.
+- **The pickled object must expose `predict(data)`**, plus `fit(data)` if retrained. A pyfunc
+  `PythonModel` wrapper expecting `predict(context, model_input)` raises `TypeError`.
+
+## Switching a model to `joblib`
+
+The runtime does not migrate documents; editing `models` is an operator step.
+
+1. **Inventory before the deploy** — run predict/transform and note which models raise. There is no
+   discovery afterwards: the failure names the loader, not the fix.
+2. **Edit the document** — `predict_flavor` and/or `transform_flavor` to `'joblib'`. Imported models
+   arrive ready: `import_model` writes it itself, see [`model-import.md`](model-import.md) § 12.
+3. **Check the first retrain** — the new run must carry `prediction_model/model.pkl` and, when the
+   source had one, `prediction_model/code/`.
+4. **Rollback is not just a revert** — artifacts a joblib retrain already wrote stop being loadable,
+   so it also means promoting the previous version back and reverting the documents' flavors.
+
+## Known gap: how the run id is resolved
+
+`get_model_run_id` takes `source.split('/')[2]` of the registered version's `source`. **Not confirmed
+against the real tracking server**: no scheme reproduced locally puts the run id there (a file store
+yields `''`, S3 the bucket, `mlflow-artifacts:/` the experiment id), yet predict works in production.
+Every flavor that resolves a run goes through this line — confirm it with a **read-only** query
+before relying on it.

docs/model-import.md

@@ -0,0 +1,417 @@
+# Importing a model (`import_model`)
+
+The `import_model` workflow turns an encrypted `.sientia` object in MinIO into a registered MLflow
+model plus one document in the MongoDB `models` collection — or into a row that says, in a sentence a
+person can read, why it did not.
+
+This document is the contract for the two sides that talk to it: the **frontend**, which uploads the
+object and builds the request, and the **BFF**, which owns the import log row. Everything here is
+enforced by code in this repository; where a statement is about a column or a constraint, the
+authority is the migration named in §7.
+
+Format of the file itself: [`sientia-bundle-format.md`](sientia-bundle-format.md).
+
+---
+
+## 1. What the caller does, in order
+
+1. **Upload** the `.sientia` object to the import bucket, under the configured prefix (default
+   `imported_models/`). The key must end in `.sientia`.
+2. **Compute `expected_digest`** — the SHA-256, lowercase hex, of the **exact bytes that were
+   uploaded**. Not of a re-read, not of a re-serialisation: of what went over the wire.
+3. **Encrypt the bundle password** into a `password_envelope` (§3).
+4. **Insert the import log row** (§5) and take its `id`.
+5. **Start the workflow** on `import_model-{RUNTIME}-queue` with the input of §2.
+
+The workflow claims the row, opens the bundle, provisions MLflow, writes the model document, cleans
+up and records one terminal verdict. It never inserts a row, never promotes a version and never
+activates a model.
+
+## 2. The workflow input
+
+```json
+{
+  "import_run_id": 41,
+  "object_key": "imported_models/sales_forecast_v3.sientia",
+  "expected_digest": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08",
+  "password_envelope": "Base64OfNonceCiphertextTag==",
+  "bucket": "sientia"
+}
+```
+
+| Field | Required | Meaning |
+|---|---|---|
+| `import_run_id` | yes | primary key of the `public.experiment_run` row the BFF inserted. Must be a positive integer |
+| `object_key` | yes | the uploaded object's key. Must match the row's `file_name` when the row carries one |
+| `expected_digest` | **yes** | SHA-256 of the uploaded bytes, 64 lowercase hex characters. There is no path that skips the comparison: a missing value fails at `received`, a mismatching one at `download`, before the header is parsed |
+| `password_envelope` | yes | the bundle password, encrypted (§3). A plaintext `password` field is **refused** |
+| `bucket` | no | defaults to `IMPORT_BUNDLE_BUCKET` |
+
+**Fields that are refused rather than ignored**: `project_id`, `project_name`,
+`provisioning_target`, `schema`, `table_name`, `status_table`, `password`. An input carrying any of
+them fails at `received`, so no caller can believe it recorded something the import does not record
+(see §10).
+
+## 3. The password envelope
+
+AES-256-GCM, and the ciphertext layout is what the WebCrypto API returns natively:
+
+```
+envelope = base64( nonce(12 bytes) || ciphertext || tag(16 bytes) )
+```
+
+`crypto.subtle.encrypt({name: 'AES-GCM', iv: nonce}, key, plaintext)` returns `ciphertext || tag`
+already concatenated, so the browser side is: generate 12 random bytes, encrypt, concatenate the
+nonce in front, base64 the result.
+
+The key is a 32-byte AES-256 key shared with the worker through `IMPORT_PASSWORD_KEY` (base64). The
+worker decrypts the envelope **inside the activity that uses the password**, so no plaintext password
+is ever an activity input, an activity result or a workflow variable — and therefore never appears in
+the Temporal event history. A test asserts that for every failure path.
+
+A failure to open the envelope is recorded as a `decryption` failure, distinct in the logs from a
+wrong *bundle* password but sharing its step, because the step says where the import stopped, not who
+is at fault.
+
+## 4. Environment
+
+| Variable | Default | What it is |
+|---|---|---|
+| `IMPORT_MODEL_ENABLED` | unset | registers the fifth (import) worker. Truthy values: `1`, `true`, `yes`, `on` |
+| `IMPORT_PASSWORD_KEY` | — | base64 of the 32-byte AES-256-GCM key that wraps bundle passwords |
+| `IMPORT_BUNDLE_BUCKET` | `sientia` | bucket holding the uploaded bundles |
+| `IMPORT_BUNDLE_PREFIX` | `imported_models/` | key prefix the uploads must sit under |
+| `IMPORT_BUNDLE_RETENTION_DAYS` | `7` | lifecycle expiry the importer applies to that prefix |
+| `IMPORT_WORK_DIR` | `<tempdir>/sientia-import` | worker scratch directory |
+| `IMPORT_MAX_OBJECT_BYTES` | 1 GiB | gate 1 |
+| `IMPORT_MAX_ARCHIVE_ENTRIES` | `5000` | gate 4 |
+| `IMPORT_MAX_UNCOMPRESSED_BYTES` | 4 GiB | gate 4 |
+| `IMPORT_MAX_COMPRESSION_RATIO` | `200` | gate 4 |
+| `IMPORT_MODELS_COLLECTION` | `models` | the MongoDB collection that *is* the model listing |
+| `IMPORT_STATUS_DB_NAME` | `sientia-core-mlops-bff` | the BFF database holding the import log |
+| `IMPORT_STATUS_DB_HOST` | `POSTGRES_HOST` | override: the log is on another server |
+| `IMPORT_STATUS_DB_PORT` | `POSTGRES_PORT` | override |
+| `IMPORT_STATUS_DB_USER` | `POSTGRES_USER` | override |
+| `IMPORT_STATUS_DB_PASSWORD` | `POSTGRES_PASSWORD` | override |
+
+The import log is a **second connection but the same server**: host, port, user and password come
+from `POSTGRES_*`, and only the database name is import configuration, because the log lives in the
+BFF's database rather than in the one Laborious holds for predictions. The import writes to that
+database and no other, and to one table in it. The four `IMPORT_STATUS_DB_*` connection variables
+are overrides for a deployment whose log genuinely sits elsewhere — the e2e suite, which runs the
+two databases as two containers, is exactly that case.
+
+A missing or malformed `IMPORT_PASSWORD_KEY` is an import failure at the decryption step, not a boot
+failure: the worker starts either way.
+
+## 5. The import log row — the BFF's half
+
+The row is `public.experiment_run` in `sientia-core-mlops-bff`. **The workflow never inserts it.**
+The inserting side writes:
+
+| Column | Value |
+|---|---|
+| `run_type` | `'IMPORT'` |
+| `status` | `'PENDING'` |
+| `username` | whoever asked for the import |
+| `bucket_name` | the bucket the object was uploaded to |
+| `file_name` | the object key |
+| `request_data` | the request payload. If it carries a digest under `expected_digest`, `digest`, `sha256` or `file_digest`, the workflow checks it against the input's |
+
+Then it passes the returned `id` as `import_run_id`.
+
+The workflow's first act is to **claim** that row:
+
+```sql
+UPDATE public.experiment_run
+   SET status = 'ORCHESTRATOR_WAITING_PROC', updated_at = :now,
+       orchestrator_response_data = :detail
+ WHERE id = :import_run_id
+   AND run_type = 'IMPORT'
+   AND (status = 'PENDING'
+        OR (status = 'ORCHESTRATOR_WAITING_PROC'
+            AND orchestrator_response_data -> 'import' ->> 'workflow_id' = :workflow_id))
+```
+
+The second disjunct makes a re-execution of the same workflow idempotent instead of a conflict. Every
+later write keys on `id` alone, because ownership was settled here.
+
+**Columns the workflow never writes**: `username`, `request_data`, `bucket_name`, `file_name`,
+`run_type`, `created_at`. **Columns it does write**: `status`, `error_message`, `updated_at`,
+`orchestrator_response_data`, and — as soon as the bundle is open, before anything exists in MLflow —
+`experiment_name` (the bundle's own, verbatim) and `run_name` (`import-{model_name}-v{model_version}`,
+truncated inside the name segment to fit 50 characters). Those two are **overwritten** by the
+workflow; whatever the inserting side put there is replaced.
+
+### When the row cannot be claimed
+
+If the claim matches nothing — no such id, a `run_type='TRAIN'` row, or a row already running under
+another workflow — the import **writes nothing anywhere**, because the row belongs to someone else.
+Instead it:
+
+- fails the Temporal run with error type `ImportLogRowNotClaimableError` and code
+  `IMPORT_LOG_ROW_NOT_CLAIMABLE`, non-retryable;
+- increments the import error metric and fires a notification;
+- logs the id and the observed state (which of the three causes it was).
+
+So a row left `PENDING` with no `workflow_id` in its detail JSON means the workflow never owned it.
+**Closing that row out belongs to whichever side inserted it** — by timeout, by a sweeper, by a person
+— however that side chooses. The importer deliberately does not guess.
+
+## 6. Status vocabulary
+
+| Status | Written when | New in `V12`? |
+|---|---|---|
+| `ORCHESTRATOR_WAITING_PROC` | claimed, and while running | no — **reused** from the training flow |
+| `ORCHESTRATOR_VALIDATION_ERROR` | failure at `received` … `content_policy` — the user can act on it | no — **reused** |
+| `IMPORT_SUCCESS` | every step succeeded | yes |
+| `IMPORT_ERROR` | failure at `experiment_creation` … `model_document`, or unclassified — only support can act on it | yes |
+
+The two reused values describe a *stage*, not a flow: what says the row is an import is `run_type`,
+not the status. There is deliberately no `IMPORT_RUNNING`; it would carry no information `run_type`
+does not already carry, and it would widen both the pairing `CHECK` and the frontend's switch for
+nothing.
+
+`error_message` is the user-facing sentence, taken from the catalog in §8. On success it is written as
+`NULL`, never `''` — the column carries a `LENGTH(error_message) >= 3` check.
+
+## 7. The migration this depends on
+
+`sientia-dataops-database-migrations`,
+`postgres/sientia-core-mlops-bff/migrations/V12__add_import_statuses_to_experiment_run.sql`:
+
+- adds `IMPORT_SUCCESS` and `IMPORT_ERROR` to `chk_experiment_run_status`;
+- adds `chk_experiment_run_terminal_status_run_type`, pairing only the **terminal** values with their
+  `run_type` (`IMPORT_*` on `'IMPORT'`, `TRAINING_*` on `'TRAIN'`) and leaving the shared
+  `ORCHESTRATOR_*` values out of the rule;
+- **adds no columns at all.**
+
+`V12` must be merged and applied before `IMPORT_MODEL_ENABLED` is set anywhere. Without it, an import
+provisions the model and then cannot record its verdict: the claim still succeeds (no new columns are
+needed for it), the old `CHECK` refuses `IMPORT_SUCCESS`, and the failure surfaces as
+`IMPORT_STATUS_NOT_RECORDED` on the first attempt — no retry wait — leaving the row in
+`ORCHESTRATOR_WAITING_PROC` with a fully provisioned model behind it.
+
+## 8. `orchestrator_response_data`, and the error catalog
+
+There is no column for the error code and none for the workflow identifier (decision D21). Both live
+at fixed keys in the JSON column the workflow already owns, and the whole `import` object is rewritten
+on every write, so the identifier the claim wrote is never dropped:
+
+```json
+{
+  "import": {
+    "workflow_id": "import_model-41-01HW…",
+    "code": "IMPORT_BUNDLE_INCOMPLETE",
+    "step": "structure_validation",
+    "gate": 6,
+    "source": { "model_name": "sales_forecast", "experiment_name": "…", "…": "…" },
+    "cleanup_failed": false
+  }
+}
+```
+
+- `workflow_id` — what a user quotes to support, and what the claim's idempotency disjunct reads.
+- `code` — the frontend's i18n key. **Absent on success.**
+- `step` — where the import stopped (the values of §9).
+- `gate` — which of the seven gates rejected it, when one did.
+- `source` — the bundle's own metadata block, once it is known.
+- `cleanup_failed` — temporary files were left on the worker. Present on a **successful** import too:
+  it never changes the status, because the user's model was imported.
+
+### Code → reason
+
+`error_message` is exactly the sentence in this table; nothing is interpolated into it, and the
+exception's text — paths, table names, exception classes, tracebacks — reaches only the logs.
+
+| Code | Step | Sentence written to `error_message` |
+|---|---|---|
+| `IMPORT_REQUEST_INVALID` | `received` | The import could not be started because the request was incomplete or contradictory. Nothing was created. Submit the import again. |
+| `IMPORT_FILE_UNREADABLE` | `download` | The uploaded file could not be read, or it is not the file that was sent. Upload it again. |
+| `IMPORT_BUNDLE_CANNOT_BE_OPENED` | `decryption` | This file could not be opened: the password is wrong, or the file is damaged or was altered. Confirm the password with whoever exported the model, then try again. |
+| `IMPORT_BUNDLE_REJECTED` | `archive_inspection` | The file's contents did not pass the platform's safety checks, so it was not opened. Export the model again from the origin platform. |
+| `IMPORT_BUNDLE_UNPACK_FAILED` | `extraction` | The file could not be unpacked safely and nothing from it was kept. Export the model again from the origin platform. |
+| `IMPORT_BUNDLE_INCOMPLETE` | `structure_validation` | This file is not a complete model export — part of what the platform needs is missing from it. Export the model again from the origin platform. |
+| `IMPORT_BUNDLE_UNEXPECTED_CONTENT` | `content_policy` | The file contains something a model export should not contain, so it was rejected. Export the model again from the origin platform. |
+| `IMPORT_STORAGE_PREPARATION_FAILED` | `experiment_creation` | The platform could not prepare a place to keep this model. The import stopped and no model was created. Contact support with this import's identifier. |
+| `IMPORT_MODEL_FILES_NOT_STORED` | `artifact_upload` | The model's files could not be stored on this platform. The import stopped and the model is not available. Contact support with this import's identifier. |
+| `IMPORT_MODEL_NOT_REGISTERED` | `registration` | The model's files were stored, but the model itself could not be registered, so it cannot be used. Contact support with this import's identifier. |
+| `IMPORT_MODEL_SETTINGS_NOT_SAVED` | `model_document` | The model was registered, but the settings that tell the platform how to run it could not be saved. Contact support with this import's identifier. |
+| `IMPORT_UNEXPECTED_ERROR` | — | The import stopped for an unexpected reason and no model was created. Contact support with this import's identifier. |
+
+Three codes never reach a row's `error_message`, because there is no row to write or no failure to
+report to a user. They exist in the logs, the metric and the notification only:
+
+| Code | When |
+|---|---|
+| `IMPORT_LOG_ROW_NOT_CLAIMABLE` | the row is not this workflow's (§5) |
+| `IMPORT_STATUS_NOT_RECORDED` | the verdict itself could not be written; the log line carries the verdict it could not record |
+| `IMPORT_CLEANUP_INCOMPLETE` | temporary files were left behind; also flagged as `cleanup_failed` in the detail |
+
+## 9. Steps
+
+`received`, `download`, `decryption`, `archive_inspection`, `extraction`, `structure_validation`,
+`content_policy`, `experiment_creation`, `artifact_upload`, `registration`, `model_document`,
+`cleanup`. The first seven are the gates (see the format document); a failure in any of them is a
+`ORCHESTRATOR_VALIDATION_ERROR`. The next four are provisioning; a failure there is an
+`IMPORT_ERROR`. `cleanup` is never a reported failure.
+
+## 10. The import knows nothing about projects
+
+The model listing **is** the MongoDB `models` document. No migrated relational database in this
+platform holds a project table or a model table, nothing in this repository reads a project-to-model
+link, and the workflow input carries no project target — which is why naming one is refused rather
+than accepted and ignored. Whatever the import screen does with projects is the frontend's business.
+
+The bundle's own `metadata.model_project` is logged as an MLflow parameter (`source.model_project`),
+as a record of where the model came from. It is not a link.
+
+## 11. What MLflow ends up holding
+
+| Thing | Value |
+|---|---|
+| Experiment | `metadata.experiment_name` from the bundle, created if absent, reused if present |
+| Run | always **new**, created with the `run_name` already written on the record |
+| Artifacts | the bundle's whole `artifacts/` tree, with `prediction_model` and `data_model` at the run's artifact root |
+| Parameters | the bundle's `parameters`, verbatim, plus the origin metadata as `source.*` (including `source.run_id`, which is never dereferenced) |
+| Registered version | created on demand; source is the run's **resolved artifact URI** with `/prediction_model` appended, e.g. `mlflow-artifacts:/17/<run_id>/artifacts/prediction_model` |
+| Stage | **untouched** |
+
+The source shape matters and is easy to get wrong: the legacy loader extracts the run id as
+`source.split('/')[2]`, which is the run id in the resolved artifact URI but the *artifact path* in a
+literal `runs:/<run_id>/prediction_model`. Registering with the unresolved URI breaks loading
+silently — nothing raises, the loader just returns the wrong string. This is the same value
+`mlflow.register_model` stores for the training flow, because it resolves the `runs:/` URI before
+writing it; `MlflowClient.create_model_version` does not, so the import resolves it itself.
+
+## 12. The model document
+
+Written **last**, into the collection named by `IMPORT_MODELS_COLLECTION` (default `models`):
+
+```json
+{
+  "id": "31",
+  "name": "sales_forecast",
+  "active": false,
+  "model_config": {
+    "transform_flavor": "sklearn",
+    "predict_flavor": "joblib",
+    "retention_minutes": 60,
+    "target": "SE"
+  }
+}
+```
+
+- `id` is `str(max(int(id) for id in collection) + 1)` — computed in Python, from the ids that are
+  numbers. Never a `count`, never a lexicographic sort (which would answer `"9" > "30"`). Ids that
+  are not decimal numbers are ignored for the maximum and counted in the log.
+- `name` is `metadata.model_name` from the bundle, verbatim.
+- `active` is `false`, always. No code path in the import writes or updates a document with
+  `active: true`.
+- `target` comes from `parameters.target_variable`; nothing else supplies it.
+- The `models` collection carries a **unique sparse index on `id`**, created by the MongoDB baseline
+  migration. A duplicate-key insert is reported as a `model_document` failure — no retry, no id
+  recomputation.
+
+**A name already in the collection is left exactly as it is.** The import writes nothing, modifies
+nothing, logs the skip and succeeds: `active`, `target` and the flavors of a model already on the
+platform are operator-owned.
+
+## 13. After a successful import — the manual steps
+
+A `COMPLETED` import is **not** a servable model. Stated plainly:
+
+- the registered version is **unstaged**, and `load_predict_model` only sees a version once it is in
+  stage `Production`;
+- the model document is **inactive** (`active: false`).
+
+So an operator must promote the version to `Production` and set `active: true`. This workflow never
+promotes and never activates — that runbook is QTZPOC-21's.
+
+One more thing to know before serving an imported model: `predict_flavor: 'joblib'` is what the
+artifact actually is, and it loads, predicts **and** retrains — `log_model` writes that flavor too,
+as `model.pkl` + `code/`. The artifact it writes carries no `MLmodel` manifest, so an imported model
+stays readable only by this runtime, before and after a retrain. Full contract:
+[`model-flavors.md`](model-flavors.md).
+
+## 14. Retry, and what does not retry
+
+Only the status writes retry, and only because `UPDATE ... SET <fixed values> WHERE id` leaves the row
+the same however many times it runs:
+
+| Call | Attempts | Why |
+|---|---|---|
+| claim, terminal write | 10, 1 s → 30 s backoff (≈151 s envelope) | the verdict must land; the database being briefly unavailable is exactly what a retry fixes |
+| progress writes, names | 3, capped at 2 s | a hint nobody reads — the terminal write rewrites it — must not stall a working import |
+| everything else | 1 | a repeated registry call or Mongo insert can write twice |
+
+`ImportLogRowNotClaimableError` and `ImportStatusRejectedError` are non-retryable: the first would
+read the same rows and reach the same conclusion, and the second means the statement itself was
+refused (a `CHECK` violation, an undefined column, a value too long), which a second attempt cannot
+change.
+
+There is no rollback, no compensation, no resume and no reconciliation. A failed import stays failed;
+trying again is a new upload and a new row.
+
+## 15. What a failure looks like in the logs
+
+Every failure — a row that cannot be claimed, a gate that rejected the bundle, an MLflow that refused
+the connection, a verdict that could not be written — goes through the one channel in
+`laborious/utils/models/import_errors.py`, which produces exactly two `logfmt` lines: the technical
+one, whose message
+carries the traceback (or the failure chain, root cause first, for anything that crossed an activity
+boundary), and the user-facing sentence the notification logs. **Both carry the same fields**, so one
+query returns the pair:
+
+| Field | What it holds |
+|---|---|
+| `import_run_id` | the record's primary key — the id a user quotes to support |
+| `import_workflow_id` | the Temporal workflow id |
+| `import_step` | the step that failed, or `not_claimable` / `status_write` |
+| `import_code` | the code of § 8, the same one written to the row |
+| `error_type` | the failing exception's own type — `MlflowException`, not the `ApplicationError` wrapping it |
+| `error_summary` | one bounded line of what it said |
+| `import_gate` | the gate that rejected the bundle, when one did |
+| `import_verdict_not_recorded` | the verdict a failed terminal write could not store |
+
+The workflow metadata every other line already carries (`pod_id`, `runtime`, `model_id`, `model_name`,
+`workflow_name`, `schedule_name`) is kept alongside them.
+
+A failure at any pipeline step is reported from the workflow's `except`, before cleanup and before the
+terminal write, so the log reads in the order things happened and the line's step can never disagree
+with the row's. That report never changes the outcome: its own failure is swallowed, and the failure it
+was reporting is re-raised untouched. The Temporal SDK still prints its own traceback for each failed
+attempt — that is the per-attempt record, and it is not structured; the channel's line is the one to
+query.
+
+## 16. How this contract is verified
+
+Two suites, and they prove different things.
+
+**Unit** — `pytest tests/`, part of the standard gate (`validate --project-name=laborious`). Covers
+the bundle reader against committed golden bytes, every gate, the failure catalog, the status SQL and
+the workflow's ordering, all against fakes.
+
+**End to end** — `pytest e2e -m import_e2e`, opt-in and outside `testpaths`. Runs the real workflow on
+a real worker against real MinIO, a real Postgres holding this document's `public.experiment_run`
+(with and without `V12`), a real MongoDB with the unique index on `models.id`, and a real
+`mlflow server`. Nothing on the import path is mocked. This is what proves the claims a fake cannot:
+that the `CHECK` constraints of § 6 accept every status written, that the registry source of § 11
+resolves through the existing loader, that the `V12` failure shape of § 7 is what it says, and that no
+plaintext password reaches the Temporal event history.
+
+Requires Docker plus one one-time setup per machine:
+
+```bash
+echo "ryuk.disabled=true" >> ~/.testcontainers.properties
+```
+
+Without it every e2e test fails in fixture setup with a Docker mount error — see
+[`README.md` § End-to-End Tests](../README.md#end-to-end-tests-e2e) for why and what it costs.
+
+Scenario catalogue: [`e2e/scenarios.md`](../e2e/scenarios.md) § 4.
+Requirement-to-test mapping: `openspec/changes/import-model-e2e-suite/traceability.md`.
+
+Doing one import by hand, standing in for the frontend that does not exist yet:
+[`scripts/import_model_manual_run.py`](../scripts/import_model_manual_run.py) — see
+[`README.md` § Running One Model Import by Hand](../README.md#running-one-model-import-by-hand).

docs/opc-communication.md

@@ -0,0 +1,170 @@
+# OPC UA communication (Laborious)
+
+Laborious exports predictions to OPC UA servers through `OpcRepository` ([`laborious/utils/repository/opc_repository.py`](../laborious/utils/repository/opc_repository.py)) and the Temporal activity layer in [`laborious/activities/opc.py`](../laborious/activities/opc.py).
+
+Implementation plan for session/channel recovery on Tier-1 `Bad*` errors: [`.cursor/plans/opc_bad_reconnect_ac4c6045.plan.md`](../.cursor/plans/opc_bad_reconnect_ac4c6045.plan.md).
+
+## Architecture
+
+```text
+Worker (long-lived)
+  └── OpcRepository per OPC server id (from OPC_CONFIG / env)
+        ├── connect / disconnect / validate_connection (read-only)
+        ├── _connect_locked / _reconnect_locked (under _connection_lock)
+        ├── write_data (single attempt per call)
+        └── background reconnect on Tier-1 Bad*, protocol closed, or stale session
+
+Temporal activity write_opc_data
+  └── OPC.manage_output_tags → write_data per tag (sequential per activity)
+```
+
+One worker process holds one `OpcRepository` instance per configured server. Multiple Temporal activities can call `write_data` concurrently on the same repository.
+
+## Connection lifecycle
+
+| Phase | Behavior |
+|-------|----------|
+| Startup | `init_opc()` creates repositories and calls `connect()` → `_connect_locked()` |
+| Steady state | `validate_connection()` is read-only (`protocol.state` only); `_session_ready` is checked in `write_data` |
+| Tier-1 Bad* / protocol closed | `_start_reconnect` → `_run_reconnect` → `_reconnect_locked()` (respects `reconnection_interval`) |
+| Write | `write_data()` checks reconnect task, `_session_ready`, validates protocol, then one `get_node` + `write_value` |
+| Shutdown | `close()` disconnects all repositories |
+
+### Session and channel timeouts
+
+Requested session and secure-channel lifetime: **10 minutes** (`OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS` in `opc_repository.py`). The server may revise these values; negotiated values are logged after connect and exposed as `opc_session_revised_timeout_milliseconds`.
+
+### Reconnection interval
+
+`OPC_RECONNECTION_INTERVAL` is in **seconds** (default `120`). It gates **background** reconnect after Tier-1 `Bad*`, closed protocol, or stale session (`last_reconnection_time` is updated only in `_reconnect_locked()`). It limits load on the OPC server when many workflows fail at once.
+
+## Concurrency: connection lock and session readiness
+
+To allow **multiple concurrent writes** when the session is healthy, but **block all writes** while the connection is being torn down or re-established:
+
+| Primitive | Role |
+|-----------|------|
+| `_connection_lock` (`asyncio.Lock`) | Held for the entire `disconnect` → `connect` path. Only one connection-maintenance task at a time. |
+| `_session_ready` (`asyncio.Event`) | Set when a session is ready for writes; cleared before reconnect starts and set again after a successful connect. |
+
+**Connection methods (caller holds `_connection_lock` for `_*_locked` helpers):**
+
+| Method | Role |
+|--------|------|
+| `_create_client()` | Create asyncua `Client` + optional `set_security`; raises if `client` already exists |
+| `_open_session()` | `client.connect()` + metrics; raises if session already open or client missing |
+| `_connect_locked()` | `_create_client()` (when needed) + `_open_session()`; raises if already connected |
+| `_disconnect_locked()` | Teardown session and clear `client` |
+| `_reconnect_locked()` | `_disconnect_locked()` + `_connect_locked()`; sets `last_reconnection_time` |
+
+Public `connect()` / `disconnect()` acquire the lock and call `_connect_locked()` / `_disconnect_locked()`.
+
+**Write path (`write_data`):**
+
+1. If a reconnect task is **in flight** → **fail immediately** (`opc_error_kind=reconnect_in_progress`).
+2. If `_session_ready` is cleared and no task is running → schedule reconnect (`SessionNotReady`); fail with `connection_lost` or `reconnect_in_progress` if a task started.
+3. `validate_connection()` checks `protocol.state` only (read-only). If closed → schedule reconnect (`ProtocolClosed`) and fail with `opc_error_kind=connection_lost`.
+4. Single `get_node` + `write_value` (no retry). Tier-1 `Bad*` on write also schedules reconnect.
+
+**Reconnect path (`_run_reconnect`):**
+
+1. `_start_reconnect` clears `_session_ready` and schedules the task when the interval allows and `_allow_reconnect` is true.
+2. `async with _connection_lock:` → `_reconnect_locked()`.
+3. `_session_ready` is set on successful `_open_session()`.
+4. `disconnect()` sets `_allow_reconnect=False` so shutdown does not respawn sessions.
+
+A second `_connect_locked()` while a session is already open raises `OpcSessionAlreadyConnectedError` (disconnect first).
+
+**asyncua note:** Concurrent `write_value` on the same session is only safe if the stack tolerates it. If production shows issues, serialize writes with an optional `asyncio.Semaphore(1)` while keeping the connection lock semantics above.
+
+**Future threads:** replace `asyncio.Lock` / `Event` with `threading` primitives or route all OPC I/O through one dedicated loop.
+
+## Reconnect triggers
+
+Background reconnect is scheduled when:
+
+- `validate_connection()` sees a closed or missing protocol (`ProtocolClosed`).
+- `_session_ready` is clear after a failed reconnect (`SessionNotReady`).
+- A write raises a Tier-1 `UaStatusCodeError` in `RECONNECTABLE_OPC_BAD_NAMES`.
+
+For Tier-1 `Bad*` when the server invalidates the session (e.g. `BadSessionIdInvalid`) but the client still sees transport as open, `write_data` fails once, records the OPC status in metrics, and **schedules** reconnect if:
+
+- The exception is a `UaStatusCodeError` whose name is in `RECONNECTABLE_OPC_BAD_NAMES` (see plan), and
+- `reconnection_interval` has elapsed since `last_reconnection_time`, and
+- No reconnect task is already running.
+
+There is **no write retry**: the failed export is not sent again in the same activity.
+
+## Prediction confidence and PostgreSQL comments
+
+| `prediction_confidence` | Meaning |
+|-------------------------|---------|
+| (unchanged) | Successful OPC export |
+| **12** | Generic OPC write failure (`OPC_WRITTING_ERROR_CONFIDENCE`) |
+| **14** | Tier-1 session/channel `Bad*` on export (`OPC_SESSION_BAD_CONFIDENCE`) |
+| **14** | Write while reconnect in progress (`OPC_SESSION_BAD_CONFIDENCE`, comment `OPC UA reconnect in progress`) |
+| **13** | PI Web API write failure (separate path) |
+
+Session/channel errors use a stable comment for counting:
+
+```text
+OPC UA session/channel error: BadSessionIdInvalid
+```
+
+Reconnect-in-progress exports use:
+
+```text
+OPC UA reconnect in progress
+```
+
+Example SQL:
+
+```sql
+SELECT count(*) FROM predictions WHERE prediction_confidence = 14;
+SELECT count(*) FROM predictions WHERE comments LIKE 'OPC UA session/channel error:%';
+```
+
+## Prometheus metrics (`opc_*`)
+
+Defined in [`laborious/metrics.py`](../laborious/metrics.py). Do not rename in production without a dashboard migration.
+
+| Metric | Purpose |
+|--------|---------|
+| `opc_connections_initiated_total` | Connection attempts |
+| `opc_connections_failed_total` | Failed connects |
+| `opc_connection_status` | Gauge 1=connected, 0=disconnected |
+| `opc_session_created_total` | Session established after connect |
+| `opc_session_closed_total` | Disconnect initiated |
+| `opc_session_revised_timeout_milliseconds` | Negotiated session timeout (ms) |
+| `opc_write_attempts_total` | Per write; label `result` = `OK` or exception name |
+| `opc_write_inter_arrival_over_session_timeout_total` | Successful writes spaced longer than revised session timeout |
+
+Legacy activity metrics: `laborious_prediction_opc_writing_count`, `laborious_prediction_opc_writing_response_time_monitor`.
+
+## Environment variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `OPC_CONFIG` | — | JSON map of server configs (overrides single-server env) |
+| `OPC_ID` | `1` | Server id |
+| `OPC_URL` | `opc.tcp://localhost:4840` | Endpoint |
+| `OPC_SERVER_NAME` | `default_server` | Label for metrics/logs |
+| `OPC_SERVER_URI` | same as URL | Application URI / cert SAN |
+| `OPC_CERT_PATH` | — | Client certificate (secure mode) |
+| `OPC_PRIVATE_KEY_PATH` | — | Client private key |
+| `OPC_SERVER_CERT_PATH` | — | Server certificate |
+| `OPC_RECONNECTION_INTERVAL` | `120` | Minimum seconds between reconnects |
+
+## Operations checklist
+
+- Correlate `BadSessionIdInvalid` in `opc_write_attempts_total` with `opc_session_closed_total` / `opc_session_created_total` (reconnect may finish after the row is stored with confidence 14).
+- Use confidence **14** and comment prefix for session invalidation rates; use **12** for other OPC failures.
+- Respect `OPC_RECONNECTION_INTERVAL` under parallel load; bursts of confidence 14 are expected until the next successful cycle.
+
+## Related tests
+
+- Unit: [`tests/laborious/utils/repository/test_opc_repository.py`](../tests/laborious/utils/repository/test_opc_repository.py)
+- Unit: [`tests/laborious/activities/test_opc.py`](../tests/laborious/activities/test_opc.py)
+- E2E (mock OPC): [`e2e/test_predictions_batch_format_export.py`](../e2e/test_predictions_batch_format_export.py)
+- E2E (in-process asyncua server + real `OpcRepository`): [`e2e/test_opc_real_server.py`](../e2e/test_opc_real_server.py) — scenarios 3.1.2, 3.2.2, 3.2.4, 3.2.5
+- Scenarios: [`e2e/scenarios.md`](../e2e/scenarios.md)

docs/sientia-bundle-format.md

@@ -0,0 +1,100 @@
+# The `.sientia` bundle format
+
+A `.sientia` file is one exported MLflow model: its whole artifact tree, its run parameters and its
+origin metadata, zipped and then encrypted under a password the exporter chose.
+
+**It is written by the Streamlit platform, `sientia-projects-templates`** (`app/src/operations/
+model_export/`). That application owns the export — its screen, its flow, its code — and documents
+it. This page is the short version: the parameters both sides have to agree on, and what this
+repository's reader accepts.
+
+## How it is generated
+
+The producer downloads the run's artifact tree whole, copies `run.data.params` **verbatim** (with
+`target_variable` among them, or the import fails at gate 6), writes `metadata.json` first into a
+`ZIP_DEFLATED` archive with the artifacts following as `artifacts/<relpath>` and no directory
+entries, encrypts that zip, and offers it as `{model_name}_v{model_version}.sientia`.
+
+`metadata.json` has exactly two keys: `parameters` — the run's parameters verbatim — and `metadata`,
+six fields: `model_name`, `experiment_name`, `run_id`, `model_version`, `model_project`,
+`export_timestamp` (UTC ISO 8601).
+
+## Encryption parameters
+
+The reader derives the key with the values the header carries, but refuses anything above the
+producer's own cost — a header declaring more is a resource-exhaustion attempt, refused before any
+memory is allocated.
+
+| | Value |
+|---|---|
+| KDF | Argon2id (`crypto_pwhash_ALG_ARGON2ID13`) |
+| memlimit | `268435456` (256 MiB) — a **ceiling** on the reader's side |
+| opslimit | `3` — likewise a ceiling |
+| parallelism | `1` (reserved, not configurable) |
+| salt | 16 bytes, fresh per bundle |
+| key | 32 bytes |
+| AEAD | XChaCha20-Poly1305 `secretstream`, 64 KiB plaintext chunks, last tagged `FINAL` |
+| digest | SHA-256 of the **pushed chunks only** — on disk, `sha256(file[164:])`, past the 140-byte header and the 24-byte stream header. Hashing from 140 rejects every valid bundle |
+| password | normalised `NFC`, encoded UTF-8 — no trimming, no case folding |
+
+The export-side password rule (12+ characters, mixed case, digit, symbol) is enforced there only: by
+the time a bundle exists, the password is whatever it was, and the reader cannot re-check it.
+
+## Shape of the file
+
+`HEADER_STRUCT` is `>8sBBB16sQQB32s64s`. Big-endian, no alignment padding, so the offsets are exact:
+
+```
+offset  width  field                 constraint the reader enforces
+------  -----  --------------------  ------------------------------------------------------
+     0      8  magic                 == b'SIENTIA1'
+     8      1  format_version        == 1
+     9      1  aead_id               == 1 (XChaCha20-Poly1305 secretstream)
+    10      1  kdf_id                == 1 (Argon2id)
+    11     16  salt                  passed to the KDF as-is
+    27      8  kdf_memlimit_bytes    <= 268435456
+    35      8  kdf_opslimit          <= 3
+    43      1  kdf_parallelism       == 1
+    44     32  ciphertext_digest     == sha256(file[164:])
+    76     64  reserved_signature    all zero
+------  -----
+   140     24  secretstream header   libsodium HEADERBYTES
+   164    ...  chunk stream          64 KiB plaintext chunks, each +17 bytes of AEAD tag
+```
+
+`format_version` is checked for equality, not for a range: an unknown version is refused at gate 2
+rather than parsed optimistically. `reserved_signature` must be all zero today, so a signed bundle
+will be a new format version and not a silent change of meaning.
+
+`laborious/utils/bundle/format.py` is the normative layout on this side, and the golden fixture is
+what keeps it and the producer from drifting apart.
+
+## How it is read
+
+`BundleReader.open()` runs seven ordered gates, each mapped to an import step and to one message a
+person can read:
+
+| Gate | Step | Refuses |
+|---|---|---|
+| 1 | `download` | object too large, wrong prefix or suffix, digest ≠ the one the uploader sent |
+| 2 | `download` | header that is not this format or not this version |
+| 3 | `decryption` | wrong password, or altered bytes — the AEAD cannot tell them apart, and the message says so |
+| 4 | `archive_inspection` | zip bomb, absolute or `..` entries, symlinks — read from the central directory, before extracting |
+| 5 | `extraction` | anything that escapes the fresh work directory, or overwrites a file |
+| 6 | `structure_validation` | missing `metadata.json` fields, missing `target_variable`, missing `prediction_model`/`data_model` |
+| 7 | `content_policy` | a file that is not a known model file by name or suffix |
+
+Gates 1 and 2 share the `download` step on purpose: neither may borrow the `decryption` sentence,
+which implicates the password. A gate-6 rejection naming `target_variable` means the origin run never
+logged it — the fix is a re-export, on the producer's side.
+
+Error codes, sentences and the ceilings the gates read from configuration:
+[`model-import.md`](model-import.md).
+
+## Fixtures
+
+- `tests/fixtures/bundle/` — a real bundle emitted by the producer, with its password and producer
+  commit recorded in that directory's `README.md`. It pays the producer's real Argon2id cost, so it
+  is used where the real bytes matter and not in every test.
+- `tests/helpers/bundle_factory.py` — the test-only writer: same header at the same offsets, cheap
+  KDF parameters, and a knob for every hostile variant the gates above refuse.

e2e/__init__.py

@@ -0,0 +1,3 @@
+"""
+End-to-end tests for laborious temporal workflows.
+"""

e2e/conftest.py

@@ -0,0 +1,860 @@
+"""
+Pytest configuration and fixtures for E2E tests.
+"""
+
+import importlib
+import sys
+from unittest.mock import AsyncMock, MagicMock, patch
+
+# Load every BSON codec now, for its side effect only. `pymongo` imports `bson.objectid`,
+# `bson.timestamp`, `bson.decimal128` and a dozen more lazily, and the first time is while an
+# activity is running — after Temporal's workflow sandbox has snapshotted `sys.modules`. The sandbox
+# then warns once per submodule ("Module bson.timestamp was imported after initial workflow load"),
+# which was ~20 lines of noise per test that touches MongoDB, enough to bury a warning worth reading.
+# This has to happen **before** the workflow modules are imported below, which is why it is here and
+# not in `import_harness.py` — pytest loads `pytest_plugins` only after this module's body has run.
+import bson.json_util  # noqa: F401
+
+# E2E workflows under test do not run ModelAnalysis; stub before Activities import.
+_model_analysis_module = MagicMock()
+_model_analysis_module.ModelAnalysis = MagicMock
+sys.modules.setdefault('sientia', MagicMock())
+sys.modules.setdefault('sientia.ModelAnalysis', _model_analysis_module)
+from io import BytesIO
+
+import joblib
+import pandas as pd
+import pytest
+import pytest_asyncio
+from sientia_do.notifications.handlers import CoreNotificationHandler
+from sientia_do.observability.metrics_controller import MetricsController
+from sqlalchemy import create_engine, text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+from testcontainers.community.minio import MinioContainer
+from testcontainers.community.postgres import PostgresContainer
+
+from e2e.opc_test_server import OpcE2ETestServer
+from laborious.activities.activities import Activities
+from laborious.workflows.predictions_batch import PredictionsBatch
+from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction
+from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
+
+# The `.sientia` import fixtures live in their own module rather than here: four containers' worth
+# of setup, an MLflow server subprocess and an ImportModel worker would double this file. Every
+# fixture it adds is `import_`-prefixed, so nothing below changes meaning.
+pytest_plugins = ['e2e.import_harness']
+
+# Test constants
+TEST_MONGODB_CONNECTION_STRING = 'mongodb://localhost:27017'
+TEST_DATABASE_NAME = 'test_db'
+
+# Dedicated model served through the explicit `joblib` flavor, so no MLflow flavor loader runs.
+JOBLIB_FLAVOR_MODEL_NAME = 'joblib_flavor_model'
+
+# Module carrying the classes of the joblib artifacts' models. It ships inside the artifact's
+# `code/` directory and is importable only while that directory is on `sys.path` — which is what
+# the joblib flavor arranges before deserializing.
+JOBLIB_FLAVOR_CODE_MODULE = 'joblib_flavor_probe'
+
+JOBLIB_FLAVOR_WEIGHTS = {'feature_1': 2.0, 'feature_2': 3.0}
+
+JOBLIB_FLAVOR_CODE_SOURCE = '''"""Model classes that exist only inside the artifact's `code/` directory."""
+
+import pandas as pd
+
+FEATURES = ('feature_1', 'feature_2')
+
+
+class JoblibFlavorPredictor:
+    """Prediction model: a weighted sum of the transform output columns."""
+
+    def __init__(self, weights):
+        self.weights = weights
+
+    def fit(self, data):
+        return self
+
+    def predict(self, data):
+        frame = pd.DataFrame(data)
+        return [
+            sum(row[name] * self.weights[name] for name in FEATURES)
+            for _, row in frame.iterrows()
+        ]
+
+
+class JoblibFlavorTransformer:
+    """Transform model: emits the fixed feature frame the prediction model expects."""
+
+    target_variable = 'target'
+
+    def fit(self, data):
+        return self.predict(data)
+
+    def predict(self, data):
+        frame = pd.DataFrame(data)
+        result = pd.DataFrame(
+            {'feature_1': [0.234] * len(frame), 'feature_2': [0.783] * len(frame)}
+        )
+        result.index = frame.index
+        return result
+'''
+
+
+@pytest_asyncio.fixture(scope='session')
+def minio_container():
+    """
+    MinIO S3-compatible storage for E2E tests that exercise real offload uploads.
+    """
+    minio = MinioContainer()
+    minio.start()
+    yield minio
+    minio.stop()
+
+
+@pytest_asyncio.fixture(scope='session')
+def postgres_container():
+    """
+    Create a PostgreSQL container using testcontainers.
+
+    This fixture creates a real PostgreSQL database in a Docker container
+    that will be used for all tests in the session.
+
+    `log_statement=all` is the one thing the import suite needs from this fixture: proving an import
+    issues no statement at all against the predictions database requires a log that would have
+    shown one. It changes what the server writes to its own log and nothing a test observes.
+    """
+    postgres = PostgresContainer('postgres:15').with_command('postgres -c log_statement=all')
+    postgres.start()
+    yield postgres
+    postgres.stop()
+
+
+@pytest_asyncio.fixture
+def postgres_engine(postgres_container):
+    """
+    Create SQLAlchemy engine for PostgreSQL test database.
+
+    This fixture creates a connection to the PostgreSQL container
+    created by the postgres_container fixture.
+    """
+    engine = create_engine(postgres_container.get_connection_url())
+
+    yield engine
+
+    engine.dispose()
+
+
+def _create_schema_and_tables(engine):
+    """
+    Helper function to create schema and tables in the given engine.
+
+    Creates predictions_schema with:
+    - laborious_data: Input data table for queries
+    - predictions: Output predictions table
+    - transformed_data: Output transformed data table
+    """
+    # Use begin() to ensure transaction is properly committed
+    with engine.begin() as conn:
+        # Create predictions_schema
+        conn.execute(text('CREATE SCHEMA IF NOT EXISTS predictions_schema'))
+
+        # Create laborious_data table (input data from sensors)
+        create_laborious_data_sql = """
+            CREATE TABLE IF NOT EXISTS predictions_schema.laborious_data (
+                id SERIAL NOT NULL,
+                model_id int4 NOT NULL,
+                variable text NOT NULL,
+                value numeric NULL,
+                "timestamp" timestamptz NOT NULL,
+                created_at timestamptz NOT NULL,
+                PRIMARY KEY (id)
+            );
+        """
+        conn.execute(text(create_laborious_data_sql))
+
+        # Create predictions table
+        create_predictions_sql = """
+            CREATE TABLE if not exists predictions_schema.predictions (
+                id SERIAL NOT NULL ,
+                model_id int4 NOT NULL,
+                prediction numeric NULL,
+                prediction_confidence numeric NOT NULL,
+                response_time numeric NOT NULL,
+                prediction_status text NOT NULL,
+                "timestamp" timestamptz NOT NULL,
+                created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+                "comments" text NULL,
+                PRIMARY KEY (id, created_at)
+            );
+        """
+        conn.execute(text(create_predictions_sql))
+
+        # Create transformed_data table
+        create_transformed_sql = """
+            CREATE TABLE IF NOT EXISTS predictions_schema.transformed_data (
+                id SERIAL NOT NULL,
+                model_id int4 NOT NULL,
+                variable text NOT NULL,
+                value numeric NULL,
+                "timestamp" timestamptz NOT NULL,
+                created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+                PRIMARY KEY (id)
+            );
+        """
+        conn.execute(text(create_transformed_sql))
+
+
+@pytest_asyncio.fixture(autouse=True)
+def setup_postgres_schema_and_tables(postgres_engine):
+    """
+    Automatically create necessary schema and tables before each test.
+
+    This fixture runs automatically (autouse=True) and ensures
+    that the predictions_schema and tables exist with the correct structure.
+    """
+    _create_schema_and_tables(postgres_engine)
+    yield
+
+
+@pytest_asyncio.fixture
+def mock_logger():
+    """Mock logger for testing."""
+
+    def message(message):
+        print(f'[LOG] {message}')
+
+    def custom_message(message, _metadata=None):
+        print(f'[LOG] {message}')
+
+    logger = MagicMock()
+    logger.info = MagicMock(side_effect=message)
+    logger.debug = MagicMock(side_effect=message)
+    logger.error = MagicMock(side_effect=message)
+    logger.warning = MagicMock(side_effect=message)
+    logger.custom_info = MagicMock(side_effect=custom_message)
+    logger.custom_debug = MagicMock(side_effect=custom_message)
+    logger.custom_error = MagicMock(side_effect=custom_message)
+    logger.custom_warning = MagicMock(side_effect=custom_message)
+    return logger
+
+
+@pytest_asyncio.fixture
+def mock_mongo_client():
+    """
+    Mock MongoDB client to avoid real connections.
+
+    This fixture mocks the pymongo.MongoClient used by CoreNotificationHandler,
+    allowing us to use a real NotificationHandler instance without connecting to MongoDB.
+    """
+    mock_client = MagicMock()
+    mock_db = MagicMock()
+    mock_collection = MagicMock()
+
+    # Configure the mock chain: client[database] -> db[collection] -> collection
+    mock_client.__getitem__.return_value = mock_db
+    mock_db.__getitem__.return_value = mock_collection
+
+    # Mock server_info() to avoid connection attempts
+    mock_client.server_info = MagicMock()
+
+    # Mock insert_one for notifications
+    mock_collection.insert_one = MagicMock()
+
+    return mock_client
+
+
+@pytest.fixture
+def notification_inserts(mock_mongo_client):
+    """
+    Mongo insert_one mock used by CoreNotificationHandler for notification persistence.
+
+    Yields:
+        MagicMock for insert_one, reset before each test.
+    """
+    mock_db = mock_mongo_client.__getitem__.return_value
+    mock_collection = mock_db.__getitem__.return_value
+    mock_collection.insert_one.reset_mock()
+    yield mock_collection.insert_one
+
+
+@pytest_asyncio.fixture
+def notification_handler(mock_logger, mock_mongo_client):
+    """
+    Create a real NotificationHandler instance with mocked MongoDB client.
+
+    This fixture creates a real CoreNotificationHandler instance but mocks
+    the underlying MongoDB connection to avoid real database connections.
+    """
+    # Patch MongoClient where it's imported in the handlers module
+    with patch('sientia_do.notifications.handlers.MongoClient', return_value=mock_mongo_client):
+        handler = CoreNotificationHandler(
+            connection_string=TEST_MONGODB_CONNECTION_STRING,
+            database=TEST_DATABASE_NAME,
+            logger=mock_logger,
+            project_name='laborious',
+        )
+        yield handler
+        handler.shutdown()
+
+
+@pytest_asyncio.fixture
+def metrics_controller(mock_logger):
+    """Create a real MetricsController instance."""
+    return MetricsController(logger=mock_logger)
+
+
+@pytest_asyncio.fixture
+def mock_minio_repository():
+    """Mock MinIO repository for object storage operations."""
+    mock_repo = MagicMock()
+
+    # Provide at least valid parquet bytes so that MinioDataFramePayload.retrieve()
+    # can decode the payload if offloading is exercised in an integration scenario.
+    parquet_df = pd.DataFrame({'a': [1]})
+    parquet_buffer = BytesIO()
+    parquet_df.to_parquet(parquet_buffer, engine='pyarrow', index=True)
+    parquet_bytes = parquet_buffer.getvalue()
+
+    # sientia_do MinioRepository API
+    mock_repo.bucket = 'test-bucket'
+    mock_repo.upload_file = AsyncMock(
+        side_effect=lambda file_bytes, relative_key, content_type='application/octet-stream', bucket=None, metadata=None: {
+            'minio_object_name': f'sientia/streamlit-connectors/{relative_key}',
+            'original_filename': relative_key.rsplit('/', 1)[-1],
+            'uploaded_at': '2024-01-01T00:00:00Z',
+            'sha256_hash': 'deadbeef',
+        }
+    )
+    mock_repo.download_file = AsyncMock(return_value=parquet_bytes)
+    mock_repo.list_objects = AsyncMock(return_value=[])
+    mock_repo.delete_file = AsyncMock()
+    mock_repo.close = MagicMock()
+
+    return mock_repo
+
+
+@pytest_asyncio.fixture
+def mock_pi_web_api_repository():
+    """Mock PI Web API repository for PI Web API operations."""
+    mock_repo = MagicMock()
+
+    async def _write_value(web_ids, value, metadata=None, **kwargs):
+        """
+        Mirror successful PI writes: one response item per requested web_id.
+
+        write_pi_web_api_data passes the list into process_pi_web_api_response (not a
+        wrapped {'Items': ...} envelope).
+        """
+        return [{'WebId': wid, 'Errors': []} for wid in web_ids]
+
+    mock_repo.write_value = AsyncMock(side_effect=_write_value)
+    mock_repo.close = MagicMock()
+    return mock_repo
+
+
+@pytest_asyncio.fixture
+def mock_opc_repository():
+    """Mock OPC repository for OPC operations."""
+    mock_repo = MagicMock()
+    mock_repo.write_data = AsyncMock(return_value=(True, {'response_time': 0.1}))
+    mock_repo.disconnect = AsyncMock()
+    return mock_repo
+
+
+@pytest_asyncio.fixture
+async def opc_e2e_server():
+    """
+    In-process asyncua OPC UA server for E2E tests against OpcRepository.
+    """
+    server = OpcE2ETestServer()
+    await server.start()
+    try:
+        yield server
+    finally:
+        await server.stop()
+
+
+@pytest_asyncio.fixture
+def patch_create_engine(postgres_engine):
+    """Patch create_engine to return test postgres_engine."""
+    with patch(
+        'sientia_do.temporal.activities.postgres.create_engine', return_value=postgres_engine
+    ):
+        yield
+
+
+@pytest_asyncio.fixture
+def patch_minio_repository(mock_minio_repository):
+    """Patch MinioRepository to return mock."""
+    # Patch where Activities resolves the symbol (import binds the original class).
+    with patch(
+        'laborious.activities.activities.MinioRepository', return_value=mock_minio_repository
+    ):
+        yield
+
+
+@pytest_asyncio.fixture
+def patch_pi_web_api_repository(mock_pi_web_api_repository):
+    """Patch MLflowRepository to return mock."""
+    with patch('laborious.activities.api.PIWebAPIClient', return_value=mock_pi_web_api_repository):
+        yield
+
+
+@pytest_asyncio.fixture
+def mock_mlflow_models():
+    """Create mock models for MLflow load_model methods."""
+    # Mock transform model - returns DataFrame with same index as input
+    mock_transform_model = MagicMock()
+
+    def mock_transform_predict(data):
+        num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1
+        print(data.to_csv())
+        print(data.index)
+        result = pd.DataFrame(
+            {
+                'feature_1': [0.234] * num_rows,
+                'feature_2': [0.783] * num_rows,
+            }
+        )
+        result.index = data.index
+        return result
+
+    mock_transform_model.predict = MagicMock(side_effect=mock_transform_predict)
+
+    # Mock predict model - returns array/list of predictions
+    mock_predict_model = MagicMock()
+
+    def mock_predict_predict(data):
+        num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1
+        return [0.5] * num_rows
+
+    mock_predict_model.predict = MagicMock(side_effect=mock_predict_predict)
+
+    # Mock PyFuncModel for compressed models
+    mock_pyfunc_model = MagicMock()
+    mock_pyfunc_model._model_impl = MagicMock()
+    mock_pyfunc_model._model_impl.python_model = mock_transform_model
+
+    return {
+        'transform_model': mock_transform_model,
+        'predict_model': mock_predict_model,
+        'pyfunc_model': mock_pyfunc_model,
+    }
+
+
+def _write_artifact_code_dir(artifact_dir):
+    """Write the artifact's `code/` directory, carrying the module with the model classes."""
+    code_dir = artifact_dir / 'code'
+    code_dir.mkdir()
+    (code_dir / f'{JOBLIB_FLAVOR_CODE_MODULE}.py').write_text(JOBLIB_FLAVOR_CODE_SOURCE)
+    return code_dir
+
+
+def _build_joblib_flavor_artifact(artifact_dir, class_name, *args):
+    """
+    Build one joblib artifact: `code/` with the class module, plus the compressed `model.pkl`.
+
+    Instantiating **and** dumping happen while `code/` is on `sys.path` — `pickle` refuses to
+    serialize a class it cannot import. Both are undone on the way out, so from here the pickled
+    class is unreachable until something puts the artifact's `code/` back on the path. That
+    something is the flavor under test.
+    """
+    code_dir = _write_artifact_code_dir(artifact_dir)
+
+    sys.path.insert(0, str(code_dir))
+    try:
+        module = importlib.import_module(JOBLIB_FLAVOR_CODE_MODULE)
+        model = getattr(module, class_name)(*args)
+        joblib.dump(model, artifact_dir / 'model.pkl', compress=3)
+    finally:
+        sys.path.remove(str(code_dir))
+        sys.modules.pop(JOBLIB_FLAVOR_CODE_MODULE, None)
+        importlib.invalidate_caches()
+
+    return {'path': str(artifact_dir), 'code_dir': str(code_dir), 'model': model}
+
+
+@pytest.fixture
+def joblib_flavor_artifacts(tmp_path):
+    """
+    Real artifact directories for the `joblib` flavor, one per model type.
+
+    Each holds a top-level `model.pkl` written by `joblib.dump(..., compress=3)` and a real
+    `code/` directory with the module where the model's class lives. No `MLmodel` is written:
+    the artifact this flavor reads — and the one a joblib retrain writes — has none.
+
+    Return:
+        dict keyed by artifact name ('prediction_model', 'data_model'), each value carrying
+        'path', 'code_dir' and 'model' (the object serialized, so tests can compute the
+        expected prediction).
+    """
+    original_sys_path = list(sys.path)
+
+    builds = {
+        'prediction_model': ('JoblibFlavorPredictor', (JOBLIB_FLAVOR_WEIGHTS,)),
+        'data_model': ('JoblibFlavorTransformer', ()),
+    }
+
+    artifacts = {}
+
+    for artifact_name, (class_name, args) in builds.items():
+        artifact_dir = tmp_path / artifact_name
+        artifact_dir.mkdir()
+
+        artifacts[artifact_name] = _build_joblib_flavor_artifact(artifact_dir, class_name, *args)
+
+    yield artifacts
+
+    # The flavor prepends `code/` to `sys.path` and deliberately never removes it.
+    sys.path[:] = original_sys_path
+    sys.modules.pop(JOBLIB_FLAVOR_CODE_MODULE, None)
+    importlib.invalidate_caches()
+
+
+@pytest.fixture
+def joblib_flavor_artifact_without_pickle(tmp_path):
+    """Artifact directory with no top-level `.pkl`, so the joblib flavor fails on it."""
+    artifact_dir = tmp_path / 'prediction_model_without_pickle'
+    artifact_dir.mkdir()
+    (artifact_dir / 'MLmodel').write_text('flavors:\n  sklearn: {}\n')
+    return str(artifact_dir)
+
+
+@pytest_asyncio.fixture
+def patch_mlflow(mock_mlflow_models, joblib_flavor_artifacts):
+    """Patch mlflow module in repository with load_model mocks."""
+    mock_mlflow = MagicMock()
+
+    # Mock sklearn.load_model
+    def mock_sklearn_load_model(model_uri):
+        if 'data_model' in model_uri or 'transform' in model_uri.lower():
+            return mock_mlflow_models['transform_model']
+        return mock_mlflow_models['predict_model']
+
+    mock_mlflow.sklearn = MagicMock()
+    mock_mlflow.sklearn.load_model = MagicMock(side_effect=mock_sklearn_load_model)
+
+    # Mock pyfunc.load_model
+    def mock_pyfunc_load_model(model_uri):
+        if 'artifacts' in model_uri or 'tmp' in model_uri:
+            return mock_mlflow_models['pyfunc_model']
+        if 'data_model' in model_uri or 'transform' in model_uri.lower():
+            return mock_mlflow_models['transform_model']
+        return mock_mlflow_models['predict_model']
+
+    mock_mlflow.pyfunc = MagicMock()
+    mock_mlflow.pyfunc.load_model = MagicMock(side_effect=mock_pyfunc_load_model)
+
+    mock_mlflow.artifacts = MagicMock()
+
+    # Mock other mlflow methods that might be called
+    mock_mlflow.set_tracking_uri = MagicMock()
+    mock_mlflow.get_run = MagicMock(
+        return_value=MagicMock(info=MagicMock(artifact_uri='mlflow-artifacts:/test_run_id'))
+    )
+    mock_mlflow.tracking = MagicMock()
+    mock_mlflow_client = MagicMock(
+        search_registered_models=MagicMock(return_value=[MagicMock(name='test_model')]),
+        search_model_versions=MagicMock(
+            return_value=[
+                MagicMock(
+                    current_stage='Production', version='1', source='runs:/artifacts/test_run_id'
+                )
+            ]
+        ),
+    )
+
+    # `download_model` asks whether the run's artifact carries a `code` directory before it loads
+    # anything, and downloads only that directory when it does. Answering "no" here keeps the
+    # download counts of these tests about the model itself; a test that exercises the `code` step
+    # overrides `list_artifacts` to answer "yes".
+    def mock_list_artifacts(_run_id, artifact_path=None):
+        if artifact_path:
+            return []
+        return [MagicMock(path=artifact_name) for artifact_name in joblib_flavor_artifacts]
+
+    mock_mlflow_client.list_artifacts = MagicMock(side_effect=mock_list_artifacts)
+
+    # The joblib flavor downloads a whole model artifact, one per model type — a single return
+    # value would serve `data_model` from `prediction_model`. The `code` step asks for the nested
+    # `<artifact>/code` instead, and gets that directory back, as MLflow would return it.
+    def mock_download_artifacts(_run_id, artifact_name):
+        if artifact_name.endswith('/code'):
+            return joblib_flavor_artifacts[artifact_name.removesuffix('/code')]['code_dir']
+        return joblib_flavor_artifacts[artifact_name]['path']
+
+    mock_mlflow_client.download_artifacts = MagicMock(side_effect=mock_download_artifacts)
+    mock_mlflow.tracking.MlflowClient = MagicMock(return_value=mock_mlflow_client)
+
+    with patch('laborious.utils.repository.model_repository.mlflow', new=mock_mlflow):
+        yield mock_mlflow
+
+
+@pytest.fixture
+def mlflow_client(patch_mlflow):
+    """The MlflowClient mock the repository holds as `self.client`."""
+    return patch_mlflow.tracking.MlflowClient.return_value
+
+
+@pytest_asyncio.fixture(scope='function')
+async def test_activities(
+    postgres_engine,
+    postgres_container,
+    mock_logger,
+    notification_handler,
+    metrics_controller,
+    mock_minio_repository,
+    patch_create_engine,
+    patch_minio_repository,
+    patch_mlflow,
+    patch_pi_web_api_repository,
+    mock_opc_repository,
+):
+    """
+    Create Activities instance with test dependencies.
+
+    This fixture creates a real Activities instance with:
+    - PostgreSQL database (via testcontainers)
+    - Mocked MinIO client
+    - Real NotificationHandler and MetricsController (with mocked underlying services)
+    """
+    activities = Activities(
+        postgres_config={
+            'host': 'localhost',
+            'port': postgres_container.get_exposed_port(5432),
+            'user': 'test',
+            'password': 'test',
+            'dbname': 'test',
+            'min_connections': 1,
+            'max_connections': 5,
+        },
+        mlflow_config={
+            'host': 'http://localhost',
+            'port': '5000',
+            'username': 'test',
+            'password': 'test',
+        },
+        minio_config={
+            # Host:port only; Minio() prepends http(s):// from the secure flag.
+            'endpoint_url': 'localhost:9000',
+            'access_key': 'test',
+            'secret_key': 'test',
+            'default_bucket': 'test-bucket',
+            'retention_hours': 24,
+            'secure': False,
+        },
+        opc_config={},
+        pi_web_api_config={
+            'base_url': 'http://localhost:8080',
+            'auth_type': 'bearer',
+            'auth_token': 'test_token',
+        },
+        logger=mock_logger,
+        notification_handler=notification_handler,
+    )
+
+    activities.opc_repository = {
+        '1': mock_opc_repository,
+    }
+
+    try:
+        yield activities
+    finally:
+        # Cleanup - ALWAYS runs, even if test fails
+        await activities.shutdown()
+
+
+@pytest_asyncio.fixture(scope='function')
+async def test_activities_real_minio(
+    postgres_engine,
+    postgres_container,
+    minio_container,
+    mock_logger,
+    notification_handler,
+    metrics_controller,
+    patch_create_engine,
+    patch_mlflow,
+    patch_pi_web_api_repository,
+    mock_opc_repository,
+):
+    """
+    Activities with a real MinIO testcontainer (no MinioRepository patch) for offload tests.
+    """
+    minio_client = minio_container.get_client()
+    if not minio_client.bucket_exists('test-bucket'):
+        minio_client.make_bucket('test-bucket')
+    minio_port = minio_container.get_exposed_port(9000)
+    activities = Activities(
+        postgres_config={
+            'host': 'localhost',
+            'port': postgres_container.get_exposed_port(5432),
+            'user': 'test',
+            'password': 'test',
+            'dbname': 'test',
+            'min_connections': 1,
+            'max_connections': 5,
+        },
+        mlflow_config={
+            'host': 'http://localhost',
+            'port': '5000',
+            'username': 'test',
+            'password': 'test',
+        },
+        minio_config={
+            'endpoint_url': f'localhost:{minio_port}',
+            'access_key': 'minioadmin',
+            'secret_key': 'minioadmin',
+            'default_bucket': 'test-bucket',
+            'retention_hours': 24,
+            'secure': False,
+        },
+        opc_config={},
+        pi_web_api_config={
+            'base_url': 'http://localhost:8080',
+            'auth_type': 'bearer',
+            'auth_token': 'test_token',
+        },
+        logger=mock_logger,
+        notification_handler=notification_handler,
+    )
+    activities.opc_repository = {'1': mock_opc_repository}
+    try:
+        yield activities
+    finally:
+        await activities.shutdown()
+
+
+def _worker_activity_list(test_activities: Activities):
+    return [
+        test_activities.load_custom_query,
+        test_activities.load_query_with_minio_offload,
+        test_activities.cleanup_minio_objects_expired,
+        test_activities.input_gate,
+        test_activities.request_transform,
+        test_activities.mlflow_response_gate,
+        test_activities.mlflow_content_gate,
+        test_activities.request_predict,
+        test_activities.repeat_last_prediction,
+        test_activities.format_prediction,
+        test_activities.format_transformed_data,
+        test_activities.format_default_prediction,
+        test_activities.write_pi_web_api_data,
+        test_activities.write_opc_data,
+        test_activities.export_data_to_postgres,
+        test_activities.export_payload_to_postgres,
+        test_activities.write_metrics,
+    ]
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_test_env():
+    """Create Temporal test environment."""
+    env = await WorkflowEnvironment.start_time_skipping()
+    async with env:
+        yield env
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker(temporal_test_env, test_activities):
+    """Create Temporal worker with test activities."""
+    async with Worker(
+        temporal_test_env.client,
+        task_queue='test-queue',
+        workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
+        activities=_worker_activity_list(test_activities),
+    ) as worker:
+        yield worker
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker_real_minio(temporal_test_env, test_activities_real_minio):
+    """Temporal worker backed by Activities using real MinIO testcontainer."""
+    async with Worker(
+        temporal_test_env.client,
+        task_queue='test-queue',
+        workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
+        activities=_worker_activity_list(test_activities_real_minio),
+    ) as worker:
+        yield worker
+
+
+@pytest_asyncio.fixture(scope='function')
+async def test_activities_real_opc(
+    postgres_engine,
+    postgres_container,
+    opc_e2e_server: OpcE2ETestServer,
+    mock_logger,
+    notification_handler,
+    metrics_controller,
+    patch_create_engine,
+    patch_minio_repository,
+    patch_mlflow,
+    patch_pi_web_api_repository,
+):
+    """
+    Activities with a real OpcRepository connected to the in-process OPC UA server.
+    """
+    activities = Activities(
+        postgres_config={
+            'host': 'localhost',
+            'port': postgres_container.get_exposed_port(5432),
+            'user': 'test',
+            'password': 'test',
+            'dbname': 'test',
+            'min_connections': 1,
+            'max_connections': 5,
+        },
+        mlflow_config={
+            'host': 'http://localhost',
+            'port': '5000',
+            'username': 'test',
+            'password': 'test',
+        },
+        minio_config={
+            'endpoint_url': 'localhost:9000',
+            'access_key': 'test',
+            'secret_key': 'test',
+            'default_bucket': 'test-bucket',
+            'retention_hours': 24,
+            'secure': False,
+        },
+        opc_config={
+            '1': {
+                'id': '1',
+                'server_name': 'e2e-opc',
+                'url': opc_e2e_server.url,
+                'server_uri': opc_e2e_server.url,
+                'cert_path': None,
+                'private_key_path': None,
+                'server_cert_path': None,
+                'reconnection_interval': 0,
+            }
+        },
+        pi_web_api_config={
+            'base_url': 'http://localhost:8080',
+            'auth_type': 'bearer',
+            'auth_token': 'test_token',
+        },
+        logger=mock_logger,
+        notification_handler=notification_handler,
+    )
+    await activities.init_opc()
+    repo = activities.opc_repository['1']
+    assert repo._session_ready.is_set(), 'OPC E2E server connection failed during init_opc'
+    try:
+        yield activities
+    finally:
+        await activities.shutdown()
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker_real_opc(temporal_test_env, test_activities_real_opc):
+    """Temporal worker backed by Activities using the in-process OPC UA server."""
+    async with Worker(
+        temporal_test_env.client,
+        task_queue='test-queue',
+        workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
+        activities=_worker_activity_list(test_activities_real_opc),
+    ) as worker:
+        yield worker

e2e/fixtures/experiment_run.sql

@@ -0,0 +1,65 @@
+-- public.experiment_run as migrations V1 … V11 leave it, for the e2e Postgres container.
+--
+-- Source of truth: sientia-dataops-database-migrations,
+-- postgres/sientia-core-mlops-bff/migrations/ — this file mirrors the *final state* of:
+--
+--   V1  create_table_experiment_run                     (13 columns, one index)
+--   V2  update_check_constraint_on_status               (named the status CHECK)
+--   V3  update_check_constraints                        (named every other CHECK)
+--   V4  update_status_column_size_and_add_orchestrator_validation_error  (status -> VARCHAR(50))
+--   V5  migrate_mage_statuses_to_orchestrator           (data only)
+--   V6  rename_mage_response_data_column                (-> orchestrator_response_data)
+--   V7  migrate_mlflow_statuses_to_tracking             (data only)
+--   V8  remove_orchestrator_request_sent_status         (status vocabulary)
+--   V9  remove_file_upload_statuses                     (status vocabulary)
+--   V10 remove_unused_statuses                          (status vocabulary)
+--   V11 add_run_type_to_experiment_run                  (run_type + reindexed)
+--
+-- V12 (add_import_statuses_to_experiment_run) lives in `experiment_run_v12.sql`, applied on top of
+-- this file. It is separate because one e2e test — the evidence for why the migration gates
+-- enabling the import worker — has to run against a table that does *not* have it, and a second
+-- whole-schema copy with V12 removed would rot the moment either copy were edited.
+--
+-- The same drift trade already accepted for the bundle fixtures: this is a copy, and the tests
+-- that read it are what make the copy auditable. If a migration lands there, update this file and
+-- the version list above. Nothing in this repository owns the DDL — no import ever executes it
+-- outside the test suite.
+
+CREATE TABLE IF NOT EXISTS public.experiment_run (
+    id BIGSERIAL CONSTRAINT pk_experiment_run PRIMARY KEY,
+    experiment_name VARCHAR(50) NOT NULL,
+    run_name VARCHAR(50),
+    username VARCHAR(320) NOT NULL,
+    status VARCHAR(50) NOT NULL,
+    error_message VARCHAR(1024),
+    created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+    updated_at TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+    bucket_name VARCHAR(63),
+    file_name VARCHAR(1024),
+    request_data JSONB NOT NULL,
+    orchestrator_response_data JSONB,
+    run_type VARCHAR(20) NOT NULL DEFAULT 'TRAIN',
+
+    CONSTRAINT chk_experiment_run_experiment_name_min_length CHECK (LENGTH(experiment_name) >= 3),
+    CONSTRAINT chk_experiment_run_run_name_min_length CHECK (LENGTH(run_name) >= 3),
+    CONSTRAINT chk_experiment_run_username_min_length CHECK (LENGTH(username) >= 3),
+    CONSTRAINT chk_experiment_run_username_pattern
+        CHECK (username ~* '^[a-zA-Z0-9_!#$%&''*+/=?`{|}~^.-]+@[a-zA-Z0-9.-]+$'),
+    CONSTRAINT chk_experiment_run_error_message_min_length CHECK (LENGTH(error_message) >= 3),
+    CONSTRAINT chk_experiment_run_bucket_name_min_length CHECK (LENGTH(bucket_name) >= 3),
+    CONSTRAINT chk_experiment_run_bucket_name_pattern CHECK (bucket_name ~* '^[a-z0-9-]+$'),
+    CONSTRAINT chk_experiment_run_file_name_min_length CHECK (LENGTH(file_name) >= 3),
+    CONSTRAINT chk_experiment_run_file_name_pattern
+        CHECK (file_name ~* '^[a-zA-Z0-9 /._-]+$'),
+
+    -- The V1…V11 status vocabulary: the two shared stage values plus the training terminals.
+    -- V12 adds IMPORT_SUCCESS and IMPORT_ERROR to this list.
+    CONSTRAINT chk_experiment_run_status CHECK (status IN ('PENDING', 'ORCHESTRATOR_REQUEST_ERROR',
+                                                           'ORCHESTRATOR_WAITING_PROC',
+                                                           'ORCHESTRATOR_VALIDATION_ERROR',
+                                                           'TRAINING_SUCCESS', 'TRAINING_ERROR')),
+    CONSTRAINT chk_experiment_run_run_type CHECK (run_type IN ('TRAIN', 'IMPORT'))
+);
+
+CREATE INDEX IF NOT EXISTS idx_experiment_run_user_exp_status_date_type
+    ON public.experiment_run (username, experiment_name, status, created_at, run_type);

e2e/fixtures/experiment_run_v12.sql

@@ -0,0 +1,36 @@
+-- V12 add_import_statuses_to_experiment_run, as a delta applied on top of `experiment_run.sql`.
+--
+-- Source of truth: sientia-dataops-database-migrations,
+-- postgres/sientia-core-mlops-bff/migrations/V12__add_import_statuses_to_experiment_run.sql
+--
+-- Two things, and nothing else:
+--
+--   1. the status vocabulary gains IMPORT_SUCCESS and IMPORT_ERROR;
+--   2. chk_experiment_run_terminal_status_run_type pairs each terminal value with its own
+--      run_type, so a TRAIN row can never end IMPORT_SUCCESS and an IMPORT row can never end
+--      TRAINING_ERROR. The two shared stage values (PENDING, ORCHESTRATOR_WAITING_PROC) stay out
+--      of the pairing — both run types legitimately pass through them.
+--
+-- This file is applied separately because one e2e test runs an import against a table *without*
+-- it: that failure shape — provisioned, then unrecordable, refused on the first attempt — is the
+-- evidence for why this migration has to land before IMPORT_MODEL_ENABLED is turned on anywhere.
+
+ALTER TABLE public.experiment_run
+    DROP CONSTRAINT IF EXISTS chk_experiment_run_status;
+
+ALTER TABLE public.experiment_run
+    ADD CONSTRAINT chk_experiment_run_status CHECK (status IN ('PENDING',
+                                                               'ORCHESTRATOR_REQUEST_ERROR',
+                                                               'ORCHESTRATOR_WAITING_PROC',
+                                                               'ORCHESTRATOR_VALIDATION_ERROR',
+                                                               'TRAINING_SUCCESS', 'TRAINING_ERROR',
+                                                               'IMPORT_SUCCESS', 'IMPORT_ERROR'));
+
+ALTER TABLE public.experiment_run
+    DROP CONSTRAINT IF EXISTS chk_experiment_run_terminal_status_run_type;
+
+ALTER TABLE public.experiment_run
+    ADD CONSTRAINT chk_experiment_run_terminal_status_run_type CHECK (
+        (run_type = 'IMPORT' OR status NOT IN ('IMPORT_SUCCESS', 'IMPORT_ERROR'))
+        AND (run_type = 'TRAIN' OR status NOT IN ('TRAINING_SUCCESS', 'TRAINING_ERROR'))
+    );

e2e/helpers.py

@@ -0,0 +1,186 @@
+"""
+Shared helpers for E2E tests (Temporal workflows + PostgreSQL).
+"""
+
+import asyncio
+from datetime import datetime
+from decimal import Decimal
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+
+async def start_and_await_workflow(
+    client, workflow_run, input_data: dict, workflow_id: str, timeout: float = 60.0
+):
+    """
+    Start a workflow and wait for its result.
+
+    Args:
+        client: Temporal client from WorkflowEnvironment.
+        workflow_run: Workflow run method (e.g. PredictionsBatch.run).
+        input_data: Workflow input payload.
+        workflow_id: Unique workflow id.
+        timeout: Max seconds to wait for completion.
+
+    Return:
+        Workflow result value.
+    """
+    handle = await client.start_workflow(
+        workflow_run,
+        input_data,
+        id=workflow_id,
+        task_queue='test-queue',
+    )
+    return await asyncio.wait_for(handle.result(), timeout=timeout)
+
+
+def insert_sample_data(postgres_engine: Engine, model_id: int, values: list[Any]) -> None:
+    """
+    Replace laborious_data rows for a model_id with one row per value (sensor_1..n).
+
+    Args:
+        postgres_engine: SQLAlchemy engine.
+        model_id: Model id column value.
+        values: Per-sensor values; use string 'NULL' for SQL NULL.
+    """
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}')
+        )
+        values_sql = []
+        for i, value in enumerate(values):
+            values_sql.append(f"""
+                ({model_id}, 'sensor_{i + 1}', {value}, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
+            """)
+        insert_sql = f"""
+            INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
+            VALUES
+                {', '.join(values_sql)}
+        """
+        conn.execute(text(insert_sql))
+
+
+def assert_prediction(
+    postgres_engine: Engine,
+    model_id: int,
+    prediction: float = 0.5,
+    prediction_confidence: int | Decimal = 0,
+    prediction_status: str = 'Good',
+    comments: str | None = None,
+    comments_contains: str | None = None,
+) -> None:
+    """
+    Assert exactly one prediction row exists for model_id with expected columns.
+
+    Args:
+        postgres_engine: SQLAlchemy engine.
+        model_id: Expected model_id.
+        prediction: Expected prediction value.
+        prediction_confidence: Expected confidence (int or Decimal for numeric column).
+        prediction_status: Expected status string.
+        comments: Expected exact comments string (optional).
+        comments_contains: Substring expected in comments when queued (optional).
+    """
+
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, prediction, prediction_confidence, prediction_status, comments '
+                f'FROM predictions_schema.predictions WHERE model_id = {model_id} '
+                f'ORDER BY created_at ASC'
+            )
+        )
+        prediction_rows = result_query.fetchall()
+        assert len(prediction_rows) == 1, (
+            f'Expected one prediction record, got {len(prediction_rows)}'
+        )
+        row = prediction_rows[0]
+        assert row[0] == model_id, f'Expected model_id={model_id}, got {row[0]}'
+        assert row[1] == prediction or Decimal(str(row[1])) == Decimal(str(prediction)), (
+            f'Expected prediction={prediction}, got {row[1]}'
+        )
+        assert row[2] == prediction_confidence or Decimal(str(row[2])) == Decimal(
+            str(prediction_confidence)
+        ), f'Expected prediction_confidence={prediction_confidence}, got {row[2]}'
+        assert row[3] == prediction_status, (
+            f"Expected prediction_status='{prediction_status}', got {row[3]}"
+        )
+        if comments is not None:
+            assert row[4] == comments, f"Expected comments='{comments}', got {row[4]}"
+        if comments_contains is not None:
+            assert comments_contains in row[4], (
+                f"Expected comments to contain '{comments_contains}', got {row[4]}"
+            )
+
+
+def assert_continue(
+    postgres_engine: Engine,
+    model_id: int,
+    prediction_confidence: Decimal = Decimal(2),
+    comments: str = 'Input data with bad quality',
+) -> None:
+    """Assert one default-style prediction row after CONTINUE gate path."""
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, prediction, prediction_confidence, prediction_status, comments '
+                f'FROM predictions_schema.predictions WHERE model_id = {model_id}'
+            )
+        )
+        prediction_rows = result_query.fetchall()
+        assert len(prediction_rows) == 1, 'Expected one prediction record despite warnings'
+        row = prediction_rows[0]
+        assert row[1] == 0, f'Expected prediction=0, got {row[1]}'
+        assert row[2] == prediction_confidence, (
+            f'Expected prediction_confidence={prediction_confidence}, got {row[2]}'
+        )
+        assert row[3] == 'Bad', f"Expected prediction_status='Bad', got {row[3]}"
+        assert row[4] == comments, f"Expected comments='{comments}', got {row[4]}"
+
+
+def assert_stop(postgres_engine: Engine, model_id: int) -> None:
+    """Assert no prediction rows for model_id."""
+
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(f'SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = {model_id}')
+        )
+        count = result_query.scalar()
+        assert count == 0, f'Expected no predictions, but found {count} records'
+
+
+def assert_repeat(postgres_engine: Engine, model_id: int, last_prediction: tuple) -> None:
+    """
+    Assert two prediction rows for model_id both match last_prediction.
+
+    Rows are compared in created_at order for stability.
+
+    Args:
+        postgres_engine: SQLAlchemy engine.
+        model_id: Model id.
+        last_prediction: Tuple (model_id, prediction, confidence, status) to match both rows.
+    """
+
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, prediction, prediction_confidence, prediction_status '
+                f'FROM predictions_schema.predictions WHERE model_id = {model_id} '
+                f'ORDER BY created_at ASC'
+            )
+        )
+        prediction_rows = result_query.fetchall()
+        assert len(prediction_rows) == 2, 'Expected two prediction records'
+        assert prediction_rows[0] == last_prediction, (
+            f'Expected first row {last_prediction}, got {prediction_rows[0]}'
+        )
+        assert prediction_rows[1] == last_prediction, (
+            f'Expected second row {last_prediction}, got {prediction_rows[1]}'
+        )
+
+
+def make_workflow_id(prefix: str) -> str:
+    """Build a unique workflow id using a prefix and current timestamp."""
+    return f'{prefix}-{datetime.now().timestamp()}'

e2e/opc_test_server.py

@@ -0,0 +1,192 @@
+"""
+In-process OPC UA server for E2E tests (asyncua).
+
+Provides writable prediction/confidence nodes and optional write faults
+(Tier-1 BadSessionIdInvalid via PreWrite callback).
+"""
+
+from __future__ import annotations
+
+import socket
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from asyncua import Server, ua
+from asyncua.common.callback import CallbackType
+from asyncua.common.utils import ServiceError
+
+if TYPE_CHECKING:
+    from asyncua.common.node import Node
+
+
+UNKNOWN_NODE_ID = 'ns=99;i=9999'
+
+
+@dataclass(frozen=True)
+class OpcE2ENodeIds:
+    """NodeId strings used in opc_output_config for E2E workflows."""
+
+    prediction: str
+    confidence: str
+    unknown: str = UNKNOWN_NODE_ID
+
+
+class OpcE2ETestServer:
+    """
+    Ephemeral asyncua server with Laborious E2E variables and controllable faults.
+
+    Args:
+        host: Bind address (default 127.0.0.1).
+    """
+
+    def __init__(self, host: str = '127.0.0.1') -> None:
+        self._host = host
+        self._server: Server | None = None
+        self._prediction_node: Node | None = None
+        self._confidence_node: Node | None = None
+        self._session_bad_on_write = False
+        self._url: str | None = None
+        self._node_ids: OpcE2ENodeIds | None = None
+
+    @property
+    def url(self) -> str:
+        if self._url is None:
+            raise RuntimeError('OPC E2E server is not started')
+        return self._url
+
+    @property
+    def node_ids(self) -> OpcE2ENodeIds:
+        if self._node_ids is None:
+            raise RuntimeError('OPC E2E server is not started')
+        return self._node_ids
+
+    def set_session_bad_on_write(self, enabled: bool) -> None:
+        """
+        When enabled, every client Write is rejected with BadSessionIdInvalid.
+
+        Args:
+            enabled (bool): Turn Tier-1 session fault injection on or off.
+        """
+        self._session_bad_on_write = enabled
+
+    async def start(self) -> OpcE2ENodeIds:
+        """
+        Start the OPC UA server on a free TCP port.
+
+        Return:
+            OpcE2ENodeIds: NodeId strings for prediction and confidence tags.
+        """
+        port = _free_port(self._host)
+        self._url = f'opc.tcp://{self._host}:{port}/freeopcua/server/'
+
+        server = Server()
+        server.set_endpoint(self._url)
+        await server.init()
+        server.iserver.callback_service.addListener(
+            CallbackType.PreWrite,
+            self._pre_write_callback,
+        )
+
+        idx = await server.register_namespace('http://sientia.test/laborious-e2e')
+        e2e_object = await server.nodes.objects.add_object(idx, 'LaboriousE2E')
+        prediction = await e2e_object.add_variable(
+            idx,
+            'Prediction',
+            ua.Variant(0.0, ua.VariantType.Float),
+        )
+        confidence = await e2e_object.add_variable(
+            idx,
+            'Confidence',
+            ua.Variant(0.0, ua.VariantType.Float),
+        )
+        await prediction.set_writable()
+        await confidence.set_writable()
+
+        await server.start()
+        self._server = server
+        self._prediction_node = prediction
+        self._confidence_node = confidence
+        self._node_ids = OpcE2ENodeIds(
+            prediction=prediction.nodeid.to_string(),
+            confidence=confidence.nodeid.to_string(),
+        )
+        return self._node_ids
+
+    async def stop(self) -> None:
+        """Stop the OPC UA server and release the listening port."""
+        if self._server is not None:
+            await self._server.stop()
+            self._server = None
+        self._prediction_node = None
+        self._confidence_node = None
+        self._url = None
+        self._node_ids = None
+        self._session_bad_on_write = False
+
+    async def read_prediction(self) -> float:
+        """
+        Read the current prediction variable value from the address space.
+
+        Return:
+            float: Stored prediction value.
+        """
+        if self._prediction_node is None:
+            raise RuntimeError('OPC E2E server is not started')
+        value = await self._prediction_node.read_value()
+        return float(value)
+
+    async def read_confidence(self) -> float:
+        """
+        Read the current confidence variable value from the address space.
+
+        Return:
+            float: Stored confidence value.
+        """
+        if self._confidence_node is None:
+            raise RuntimeError('OPC E2E server is not started')
+        value = await self._confidence_node.read_value()
+        return float(value)
+
+    async def _pre_write_callback(self, event, _service) -> None:
+        # Only client writes are faulted. asyncua's own clock task writes CurrentTime
+        # through this same PreWrite dispatch, and raising there kills its time_task;
+        # the stored exception then resurfaces as a teardown failure in server.stop().
+        if self._session_bad_on_write and getattr(event, 'is_external', False):
+            raise ServiceError(ua.StatusCodes.BadSessionIdInvalid)
+
+
+def _free_port(host: str) -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind((host, 0))
+        return int(sock.getsockname()[1])
+
+
+def build_opc_output_config(
+    node_ids: OpcE2ENodeIds,
+    *,
+    prediction_tag: str | None = None,
+    confidence_tag: str | None = None,
+    prediction_only: bool = False,
+    server_key: str = '1',
+) -> dict[str, dict]:
+    """
+    Build opc_output_config for PredictionsBatch using real server NodeIds.
+
+    Args:
+        node_ids (OpcE2ENodeIds): Node ids from OpcE2ETestServer.
+        prediction_tag (str | None): Override prediction NodeId (default: node_ids.prediction).
+        confidence_tag (str | None): Override confidence NodeId (default: node_ids.confidence).
+        prediction_only (bool): When True, omit confidence_tags (single write per activity).
+        server_key (str): OPC server id key in opc_output_config.
+
+    Return:
+        dict: opc_output_config payload for workflow input.
+    """
+    pred = prediction_tag if prediction_tag is not None else node_ids.prediction
+    conf = confidence_tag if confidence_tag is not None else node_ids.confidence
+    server_config: dict = {
+        'prediction_tags': {pred: {'data_type': 'float'}},
+    }
+    if not prediction_only:
+        server_config['confidence_tags'] = {conf: {'data_type': 'float'}}
+    return {server_key: server_config}

e2e/test_child_workflows_e2e.py

@@ -0,0 +1,76 @@
+"""
+Direct E2E execution of child workflows (smaller surface than PredictionsBatch).
+"""
+
+from decimal import Decimal
+
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import make_workflow_id, start_and_await_workflow
+from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_format_and_export_prediction_default_path_e2e(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    postgres_engine,
+):
+    """
+    Run FormatAndExportPrediction with path_flag set (format_default_prediction path).
+    """
+    client = temporal_test_env.client
+    model_id = 401
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}')
+        )
+
+    metadata = {
+        'metadata': {
+            'model_id': model_id,
+            'model_name': 'test_model',
+            'schedule_name': 'test-schedule',
+            'workflow_name': 'subworkflow.format_and_export_prediction',
+        }
+    }
+    input_data = {
+        'metadata': metadata,
+        'path_flag': 'CONTINUE',
+        'data': {'last_timestamp': '2024-01-01 12:00:00+00:00'},
+        'prediction_confidence': 2,
+        'timestamp': '2024-01-01 12:00:00+00:00',
+        'model_id': model_id,
+        'model_name': 'test_model',
+        'schema': 'predictions_schema',
+        'table_name': 'predictions',
+        'transform_table_name': 'transformed_data',
+        'comment': 'e2e child workflow default path',
+        'opc_output_config': {},
+        'pi_web_api_output_config': {},
+        'prediction_store_policy': 'lts:1',
+    }
+
+    await start_and_await_workflow(
+        client,
+        FormatAndExportPrediction.run,
+        input_data,
+        make_workflow_id('e2e-format-export-child'),
+    )
+
+    with postgres_engine.connect() as conn:
+        row = conn.execute(
+            text(
+                f'SELECT prediction, prediction_confidence, prediction_status, comments '
+                f'FROM predictions_schema.predictions WHERE model_id = {model_id}'
+            )
+        ).fetchone()
+    assert row is not None
+    assert row[0] == 0
+    assert row[1] == Decimal(2)
+    assert row[2] == 'Bad'
+    assert row[3] == 'e2e child workflow default path'

e2e/test_import_harness_smoke.py

@@ -0,0 +1,196 @@
+"""Does the harness itself come up? Diagnosable separately from the suite that runs on it.
+
+Four containers, an `mlflow server` subprocess and a Temporal worker is a lot of moving parts, and
+when one of them fails to start the first *real* test fails somewhere deep inside an activity with
+a message about something else. These tests exist so that a broken harness says so in its own
+words: each one checks one piece is alive and does nothing with the import path.
+
+The last test is the exception — it runs a real workflow, but with an input designed to be rejected
+at the very first step, so what it proves is only that the wiring reaches the first activity.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import mlflow
+import pytest
+from sqlalchemy import text
+from temporalio.client import WorkflowFailureError
+
+from e2e.import_harness import (
+    BFF_DATABASE,
+    IMPORT_ACTIVITY_NAMES,
+    IMPORT_BUCKET,
+    IMPORT_PREFIX,
+    IMPORT_TASK_QUEUE,
+    SEEDED_MODEL_IDS,
+    failure_text,
+    failure_types,
+)
+
+pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
+
+
+class TestContainers:
+    """Each dependency is real, reachable, and the one the harness says it is."""
+
+    async def test_bff_container_holds_the_status_table(self, import_status_table: Any) -> None:
+        with import_status_table.begin() as connection:
+            database = connection.execute(text('SELECT current_database()')).scalar_one()
+            columns = {
+                row[0]
+                for row in connection.execute(
+                    text(
+                        'SELECT column_name FROM information_schema.columns '
+                        "WHERE table_schema = 'public' AND table_name = 'experiment_run'"
+                    )
+                )
+            }
+        assert database == BFF_DATABASE
+        assert {'id', 'status', 'run_type', 'request_data', 'orchestrator_response_data'} <= columns
+
+    async def test_bff_container_logs_every_statement(self, import_bff_container: Any) -> None:
+        """`log_statement=all` — without it the absence assertions have no vantage point."""
+        result = import_bff_container.exec(
+            ['psql', '-U', 'bff', '-d', BFF_DATABASE, '-tAc', 'SHOW log_statement']
+        )
+        assert b'all' in result.output
+
+    async def test_the_v12_delta_is_what_separates_the_two_tables(
+        self, import_status_table_pre_v12: Any
+    ) -> None:
+        """The pre-`V12` table has `run_type` and refuses the import terminal statuses."""
+        with import_status_table_pre_v12.begin() as connection:
+            constraints = {
+                row[0]
+                for row in connection.execute(
+                    text(
+                        'SELECT conname FROM pg_constraint '
+                        "WHERE conrelid = 'public.experiment_run'::regclass"
+                    )
+                )
+            }
+            columns = {
+                row[0]
+                for row in connection.execute(
+                    text(
+                        'SELECT column_name FROM information_schema.columns '
+                        "WHERE table_schema = 'public' AND table_name = 'experiment_run'"
+                    )
+                )
+            }
+        assert 'run_type' in columns
+        assert 'chk_experiment_run_terminal_status_run_type' not in constraints
+
+    async def test_the_seeded_collection_has_gapped_ids_and_a_unique_index(
+        self, import_models_collection: Any
+    ) -> None:
+        ids = {
+            str(document['id']) for document in import_models_collection.find({}, {'_id': False})
+        }
+        assert ids == set(SEEDED_MODEL_IDS)
+        unique_keys = {
+            tuple(index['key'].keys())
+            for index in import_models_collection.list_indexes()
+            if index.get('unique')
+        }
+        assert ('id',) in unique_keys
+
+    async def test_the_import_bucket_exists_and_its_prefix_is_empty(
+        self, import_bucket: Any
+    ) -> None:
+        assert import_bucket.bucket_exists(IMPORT_BUCKET)
+        assert not list(
+            import_bucket.list_objects(IMPORT_BUCKET, prefix=IMPORT_PREFIX, recursive=True)
+        )
+
+    async def test_uploading_a_bundle_returns_a_key_and_its_digest(
+        self, import_upload_bundle: Any, import_bucket: Any
+    ) -> None:
+        object_key, digest = import_upload_bundle(b'not a real bundle, just bytes')
+        assert object_key.startswith(IMPORT_PREFIX)
+        assert len(digest) == 64
+        assert import_bucket.stat_object(IMPORT_BUCKET, object_key).size > 0
+
+    async def test_the_mlflow_server_answers_and_serves_a_registry(
+        self, import_mlflow_server: dict[str, Any]
+    ) -> None:
+        client = mlflow.tracking.MlflowClient(tracking_uri=import_mlflow_server['tracking_uri'])
+        experiment_id = client.create_experiment('harness_smoke_experiment')
+        run = client.create_run(experiment_id, run_name='harness-smoke-run')
+        client.create_registered_model('harness_smoke_model')
+        version = client.create_model_version(
+            'harness_smoke_model', f'runs:/{run.info.run_id}/prediction_model', run.info.run_id
+        )
+        assert version.version == '1'
+        assert version.current_stage in (None, 'None')
+
+
+class TestActivitiesAndWorker:
+    """The object under test is built from the real dependencies, and the worker registers it."""
+
+    async def test_activities_point_at_the_real_dependencies(
+        self, import_activities: Any, import_mlflow_server: dict[str, Any]
+    ) -> None:
+        assert mlflow.get_tracking_uri() == import_mlflow_server['tracking_uri']
+        assert import_activities.import_config['bucket'] == IMPORT_BUCKET
+        assert import_activities.import_config['prefix'] == IMPORT_PREFIX
+        assert import_activities.status_config['dbname'] == BFF_DATABASE
+        assert import_activities.mongo_repository is not None
+        # Not a mock: a MagicMock would satisfy any attribute access, so the type is the assertion.
+        assert type(import_activities.import_minio_repository).__name__ == 'MinioRepository'
+
+    async def test_every_import_activity_is_registered(self, import_activities: Any) -> None:
+        for name in IMPORT_ACTIVITY_NAMES:
+            assert callable(getattr(import_activities, name)), f'{name} is missing'
+
+    async def test_the_worker_polls_the_import_queue(self, import_worker: Any) -> None:
+        assert import_worker.task_queue == IMPORT_TASK_QUEUE
+
+    async def test_a_workflow_input_reaches_the_first_activity(
+        self,
+        import_status_table: Any,
+        import_execute: Any,
+        import_build_input: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        """The claim runs, and the row it could not claim proves the wiring is live.
+
+        An id no row carries is the cheapest input that gets past workflow validation and into
+        `claim_import_status`, so a run that ends failed *there* means the client, the worker, the
+        workflow, the activity and the BFF connection are all in place. Nothing else is exercised.
+        """
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=987654321,
+                object_key=f'{IMPORT_PREFIX}nothing.sientia',
+                expected_digest='0' * 64,
+                password_envelope='not-a-real-envelope',
+            )
+        )
+        with pytest.raises(WorkflowFailureError) as failure:
+            await handle.result()
+        text_of_failure = failure_text(failure.value)
+        assert 'claim' in text_of_failure.lower()
+        assert '987654321' in text_of_failure
+        assert 'ImportLogRowNotClaimableError' in failure_types(failure.value)
+        assert import_read_status_row(987654321) is None
+
+    async def test_the_insert_helper_writes_a_claimable_row(
+        self,
+        import_status_table: Any,
+        import_insert_status_row: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        object_key = f'{IMPORT_PREFIX}claimable.sientia'
+        row_id = import_insert_status_row(object_key=object_key, expected_digest='a' * 64)
+        row = import_read_status_row(row_id)
+        assert row is not None
+        assert row['status'] == 'PENDING'
+        assert row['run_type'] == 'IMPORT'
+        assert row['file_name'] == object_key
+        request_data = row['request_data']
+        payload = json.loads(request_data) if isinstance(request_data, str) else request_data
+        assert payload['expected_digest'] == 'a' * 64

e2e/test_import_model_absence.py

@@ -0,0 +1,184 @@
+"""One table, one database — proven from the servers' own logs, not from the application's side.
+
+Decision D22 says an import writes to `public.experiment_run` and to nothing else relational: no
+project table, no model table, no row in the predictions database. That claim is only worth making
+if it is checked from somewhere that sees *every* client, and there is exactly one such place.
+
+Why not the obvious alternatives:
+
+- a SQLAlchemy `before_cursor_execute` listener sees the status engine and is blind to the psycopg
+  pool `Storage` holds — which is precisely the client that would betray the claim;
+- `pg_stat_statements` is aggregated, so it cannot be scoped to the window of one import.
+
+So both containers run with `log_statement=all` and the window is bracketed by a marker statement on
+each server. The assertions are about *which relation names appear*, not about exact statement text,
+because the text is the driver's business and changes with it.
+
+The predictions database is deliberately real and reachable here: `import_activities` is built with
+working credentials for it, so "no statement reached it" is a claim about the import's behaviour and
+not about a missing connection.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from temporalio.client import WorkflowFailureError
+
+from e2e.import_harness import names_no_relation, strip_literals
+
+pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
+
+FACTORY_PASSWORD = 'Factory1Model!Test'
+
+# Relations that exist in the predictions database and in nobody's import path.
+FOREIGN_RELATIONS = (
+    'predictions_schema',
+    'laborious_data',
+    'transformed_data',
+    'predictions',
+)
+
+# Tables a reader might expect an import to want, and which decision D22 says do not exist.
+RELATIONS_D22_DENIES = ('project', 'model_registry', 'models', 'import_run', 'model_project')
+
+
+class TestOneTableOneDatabase:
+    """A full import, from claim to terminal write, watched from both servers."""
+
+    async def test_every_statement_the_import_issues_names_experiment_run(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_log_window: Any,
+    ) -> None:
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+
+        with import_log_window() as captured:
+            handle = await import_execute(
+                import_build_input(
+                    import_run_id=import_run_id,
+                    object_key=object_key,
+                    expected_digest=digest,
+                    password_envelope=import_envelope(FACTORY_PASSWORD),
+                )
+            )
+            await handle.result()
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        assert row['status'] == 'IMPORT_SUCCESS', (
+            'the window has to bracket a *working* import, or it proves nothing'
+        )
+
+        bff_statements = captured['bff']
+        assert bff_statements, 'the status database logged nothing at all — the window is wrong'
+
+        touching_a_relation = [
+            statement for statement in bff_statements if not names_no_relation(statement)
+        ]
+        assert touching_a_relation, 'no relational statement was logged, so nothing was checked'
+        for statement in touching_a_relation:
+            assert 'experiment_run' in statement, (
+                f'a statement naming another relation reached the status database: {statement!r}'
+            )
+
+        # And no relation a reader might expect the feature to want appears anywhere in the window.
+        # Literals are stripped first: the import writes the bundle's own `model_project` value into
+        # the JSON detail, and `factory_project` is data, not a reference to a `project` table.
+        window_text = '\n'.join(strip_literals(s) for s in bff_statements).lower()
+        for relation in RELATIONS_D22_DENIES:
+            assert relation not in window_text, (
+                f'{relation!r} appears in the import window: D22 says no such table is written'
+            )
+        for relation in FOREIGN_RELATIONS:
+            assert relation not in window_text, f'{relation!r} appears in the import window'
+
+    async def test_the_predictions_database_sees_no_statement_from_an_import(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_log_window: Any,
+    ) -> None:
+        """Not "no statement against our tables" — no statement at all.
+
+        The import holds a working connection to this server (`Activities` is built with one), so
+        this is about what the import chooses to do, not about what it is able to do.
+        """
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+
+        with import_log_window() as captured:
+            handle = await import_execute(
+                import_build_input(
+                    import_run_id=import_run_id,
+                    object_key=object_key,
+                    expected_digest=digest,
+                    password_envelope=import_envelope(FACTORY_PASSWORD),
+                )
+            )
+            await handle.result()
+
+        assert import_read_status_row(import_run_id)['status'] == 'IMPORT_SUCCESS'
+
+        predictions_statements = [
+            statement for statement in captured['predictions'] if not names_no_relation(statement)
+        ]
+        assert not predictions_statements, (
+            f'the import issued statements against the predictions database: '
+            f'{predictions_statements}'
+        )
+
+    async def test_a_rejected_import_also_touches_nothing_else(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_log_window: Any,
+    ) -> None:
+        """The failure path writes more status rows than the success path, so it is worth its own case.
+
+        A rejected import runs the claim, the progress hints, cleanup and the terminal write — more
+        statements than a success, and every one of them still has to name the same table.
+        """
+        object_key, digest = import_upload_bundle(import_bundle_factory(include_metadata=False))
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+
+        with import_log_window() as captured:
+            handle = await import_execute(
+                import_build_input(
+                    import_run_id=import_run_id,
+                    object_key=object_key,
+                    expected_digest=digest,
+                    password_envelope=import_envelope(FACTORY_PASSWORD),
+                )
+            )
+            with pytest.raises(WorkflowFailureError):
+                await handle.result()
+
+        for statement in captured['bff']:
+            if names_no_relation(statement):
+                continue
+            assert 'experiment_run' in statement, statement
+        assert not [s for s in captured['predictions'] if not names_no_relation(s)]

e2e/test_import_model_cleanup.py

@@ -0,0 +1,247 @@
+"""What the worker is left holding, what the bucket is left promising, and what the history reveals.
+
+Three things an import must be true about after it finishes, whichever way it finished:
+
+- **The worker's disk is empty of it.** No extracted tree, no downloaded object, no decrypted zip.
+  The bundle is decrypted onto local disk, so a leak here is a plaintext model sitting in a scratch
+  directory for the next import to trip over — and after a *failure* is exactly when that would go
+  unnoticed.
+- **The bucket carries the retention rule.** The import never deletes the uploaded object: it is the
+  record of an attempt, and the lifecycle rule is what eventually removes it. That rule is applied
+  by the importer rather than assumed from a Helm value, so it is checkable from here.
+- **The status record outlives the object**, because it is the audit trail and the object is not.
+
+And the one claim that can only be made against a real Temporal history: no plaintext password
+anywhere in it. The workflow carries an envelope it cannot open, and the activity that opens it
+never returns what it found. That is the design; this reads the history and checks it.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+from temporalio.client import WorkflowFailureError
+
+from e2e.import_harness import IMPORT_BUCKET, IMPORT_PREFIX
+from tests.helpers.bundle_factory import DEFAULT_ENVELOPE_KEY, envelope_key_b64
+
+pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
+
+FACTORY_PASSWORD = 'Factory1Model!Test'
+RETENTION_RULE_ID = 'sientia-import-bundle-expiry'
+
+
+async def _run(
+    *,
+    payload: bytes,
+    import_upload_bundle: Any,
+    import_insert_status_row: Any,
+    import_build_input: Any,
+    import_execute: Any,
+    import_envelope: Any,
+    password: str = FACTORY_PASSWORD,
+) -> tuple[Any, int, str]:
+    object_key, digest = import_upload_bundle(payload)
+    import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+    handle = await import_execute(
+        import_build_input(
+            import_run_id=import_run_id,
+            object_key=object_key,
+            expected_digest=digest,
+            password_envelope=import_envelope(password),
+        )
+    )
+    return handle, import_run_id, object_key
+
+
+class TestWorkerFilesystem:
+    """Nothing of the import is left on the worker — on the success path or on either failure path."""
+
+    @pytest.mark.parametrize(
+        ('case', 'factory_kwargs', 'expect_success'),
+        [
+            ('success', {}, True),
+            ('rejected at decryption', {'drop_final_tag': True}, False),
+            ('rejected at structure validation', {'include_metadata': False}, False),
+        ],
+    )
+    async def test_the_work_directory_is_empty_afterwards(
+        self,
+        case: str,
+        factory_kwargs: dict[str, Any],
+        expect_success: bool,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        tmp_path: Any,
+    ) -> None:
+        """Two different failure steps, because they clean up at different points.
+
+        A decryption failure leaves a partially written zip in the staging directory; a structure
+        failure has already extracted a whole tree. The reader unwinds both, and the cleanup activity
+        removes the scratch directory after either — but they are separate code paths, so asserting
+        one would not cover the other.
+        """
+        handle, _, _ = await _run(
+            payload=import_bundle_factory(**factory_kwargs),
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        if expect_success:
+            await handle.result()
+        else:
+            with pytest.raises(WorkflowFailureError):
+                await handle.result()
+
+        work_dir = tmp_path / 'import-work'
+        leftovers = sorted(str(path) for path in work_dir.rglob('*')) if work_dir.exists() else []
+        assert not leftovers, f'{case}: the worker still holds {leftovers}'
+
+
+class TestRetention:
+    """The rule the uploaded object's eventual removal depends on."""
+
+    async def test_the_import_bucket_carries_the_retention_rule(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_bucket: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+    ) -> None:
+        handle, _, object_key = await _run(
+            payload=import_bundle_factory(),
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await handle.result()
+
+        lifecycle = import_bucket.get_bucket_lifecycle(IMPORT_BUCKET)
+        assert lifecycle is not None, 'the bucket carries no lifecycle configuration at all'
+        rules = {rule.rule_id: rule for rule in lifecycle.rules}
+        assert RETENTION_RULE_ID in rules, (
+            f'no {RETENTION_RULE_ID!r} rule; the bucket has {sorted(rules)}'
+        )
+        rule = rules[RETENTION_RULE_ID]
+        assert rule.status == 'Enabled'
+        assert rule.rule_filter.prefix == IMPORT_PREFIX
+        assert rule.expiration.days == 7
+
+        # The import did not delete the uploaded object: it is the record of an attempt, and the
+        # rule above is what removes it later.
+        assert import_bucket.stat_object(IMPORT_BUCKET, object_key).size > 0
+
+    async def test_the_status_record_outlives_the_uploaded_object(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_bucket: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        """Deleting the object is what the retention rule will eventually do; the verdict must survive.
+
+        The record is the audit trail. If reading a finished import's outcome depended on the bundle
+        still being in the bucket, every import would become unreadable after seven days.
+        """
+        handle, import_run_id, object_key = await _run(
+            payload=import_bundle_factory(),
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await handle.result()
+
+        import_bucket.remove_object(IMPORT_BUCKET, object_key)
+        with pytest.raises(Exception):  # noqa: B017 - MinIO's own not-found type is not the subject
+            import_bucket.stat_object(IMPORT_BUCKET, object_key)
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        assert row['status'] == 'IMPORT_SUCCESS'
+        assert row['file_name'] == object_key
+        assert row['run_name'] is not None
+
+
+class TestEventHistorySecrecy:
+    """The claim that only a real event history can settle."""
+
+    async def test_no_plaintext_password_or_key_appears_in_the_history(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_history_events: Any,
+    ) -> None:
+        """Dump every event of a completed import and search the whole thing.
+
+        Three values must be absent, and they are absent for three different reasons:
+
+        - the **bundle password** never crosses an activity boundary — `open_import_bundle` decrypts
+          the envelope inside itself and returns metadata, never the password;
+        - the **envelope key** is configuration, read from the environment inside the activity, so it
+          is never an argument to anything;
+        - the **decrypted password** is the first of these two by another name, and is searched for
+          separately because a future refactor that returned it "just for logging" would pass a test
+          that only looked for the literal input.
+
+        The envelope itself *is* in the history, and must be: it is how the password reaches the
+        activity. That it is there is asserted too, so this test cannot pass by finding nothing at
+        all — which is the way an assertion like this usually rots.
+        """
+        password = 'Secret1Password!InHistory'
+        handle, _, _ = await _run(
+            payload=import_bundle_factory(password=password),
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+            password=password,
+        )
+        await handle.result()
+
+        events, blob = await import_history_events(handle)
+        assert events, 'the history is empty'
+
+        assert password not in blob, 'the bundle password appears in the event history'
+        assert envelope_key_b64(DEFAULT_ENVELOPE_KEY) not in blob, (
+            'the envelope key appears in the event history'
+        )
+        assert DEFAULT_ENVELOPE_KEY.hex() not in blob.lower(), (
+            'the envelope key appears in the event history, hex-encoded'
+        )
+
+        # The negative assertions above are only meaningful if the history is actually readable and
+        # actually carries the envelope. Without this, an empty or opaque history would "pass".
+        assert 'password_envelope' in blob, (
+            'the history does not even carry the envelope, so the search above proved nothing'
+        )

e2e/test_import_model_digest.py

@@ -0,0 +1,214 @@
+"""The digest is a promise about the bytes in the bucket, and it is kept against the bucket.
+
+Three ways the promise can be broken, and where each is caught:
+
+- the uploader computed the digest over *other* bytes — caught at `download`, after the object is
+  fetched and before its header is parsed, so a wrong file never costs an Argon2id pass;
+- the object was replaced between the upload and the run — the same failure, which is the point:
+  the check is against what is stored now, not against what was stored when the row was written;
+- no digest at all — caught at `received`, before anything is downloaded, because a request with no
+  digest is not a request this workflow can honour.
+
+The first two matter because MinIO is real here. A mocked object store would return whatever the
+test told it to, and "the digest is enforced against the bytes that were uploaded" would be a
+statement about the mock.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+from typing import Any
+
+import pytest
+from temporalio.client import WorkflowFailureError
+
+from e2e.import_harness import IMPORT_BUCKET, failure_text
+from laborious.utils.bundle.steps import ImportStep
+from laborious.utils.models.import_errors import IMPORT_FAILURE_CATALOG
+
+pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
+
+VALIDATION_ERROR_STATUS = 'ORCHESTRATOR_VALIDATION_ERROR'
+FACTORY_PASSWORD = 'Factory1Model!Test'
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    return json.loads(value) if isinstance(value, str) else (value or {})
+
+
+def _assert_step(row: dict[str, Any], step: ImportStep, workflow_id: str) -> None:
+    code, reason, _responsibility = IMPORT_FAILURE_CATALOG[step]
+    detail = _as_dict(row['orchestrator_response_data'])['import']
+    assert row['status'] == VALIDATION_ERROR_STATUS, row['status']
+    assert detail['step'] == step.value, detail
+    assert detail['code'] == code.value, detail
+    assert detail['workflow_id'] == workflow_id
+    assert row['error_message'] == reason
+
+
+class TestWrongDigest:
+    """A digest computed over bytes that are not the ones in the bucket."""
+
+    async def test_a_digest_from_another_file_fails_at_download(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        """The bundle is perfectly valid; only the digest is another file's.
+
+        The row carries the same wrong digest as the input, so the request and the record agree and
+        the `received` guard passes — otherwise this would be a `received` failure about a
+        contradictory request instead of the download-step digest check it is meant to exercise.
+        """
+        payload = import_bundle_factory()
+        other_digest = hashlib.sha256(
+            import_bundle_factory(metadata={'run_id': 'other'})
+        ).hexdigest()
+        assert other_digest != hashlib.sha256(payload).hexdigest()
+
+        object_key, _ = import_upload_bundle(payload)
+        import_run_id = import_insert_status_row(
+            object_key=object_key, expected_digest=other_digest
+        )
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=other_digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        with pytest.raises(WorkflowFailureError) as failure:
+            await handle.result()
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        _assert_step(row, ImportStep.DOWNLOAD, handle.id)
+
+        # No header was parsed: the failure names the digest, not the format. If the digest check
+        # ran *after* the header the message here would be about magic bytes or a version.
+        text_of_failure = failure_text(failure.value).lower()
+        assert 'digest' in text_of_failure, text_of_failure
+        assert 'marker' not in text_of_failure and 'version' not in text_of_failure
+
+        # The names are still unwritten, which is the other half of "no header was parsed": the
+        # bundle was never opened, so nothing knew what to call it.
+        assert row['run_name'] is None
+        assert row['experiment_name'] == 'import-e2e-placeholder'
+
+        import_assert_nothing_provisioned(
+            experiment_name=import_names['experiment_name'],
+            model_name=import_names['model_name'],
+            object_key=object_key,
+        )
+
+
+class TestOverwrittenObject:
+    """The bytes changed after the row was written. Only a real object store can show this."""
+
+    async def test_replacing_the_object_after_upload_fails_at_download(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_bucket: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        original = import_bundle_factory()
+        object_key, digest = import_upload_bundle(original)
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+
+        # Same key, different bytes — also a valid bundle, so nothing but the digest can tell.
+        replacement = import_bundle_factory(metadata={'run_id': 'ff00ff00ff00ff00ff00ff00ff00ff00'})
+        assert hashlib.sha256(replacement).hexdigest() != digest
+        import_bucket.put_object(
+            IMPORT_BUCKET,
+            object_key,
+            io.BytesIO(replacement),
+            length=len(replacement),
+            content_type='application/octet-stream',
+        )
+
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        with pytest.raises(WorkflowFailureError):
+            await handle.result()
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        _assert_step(row, ImportStep.DOWNLOAD, handle.id)
+        import_assert_nothing_provisioned(
+            experiment_name=import_names['experiment_name'],
+            model_name=import_names['model_name'],
+            object_key=object_key,
+        )
+
+
+class TestMissingDigest:
+    """No digest is refused at the door, before a single byte is fetched."""
+
+    async def test_an_absent_digest_fails_at_received_with_nothing_downloaded(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+        tmp_path: Any,
+    ) -> None:
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=None,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        with pytest.raises(WorkflowFailureError) as failure:
+            await handle.result()
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        _assert_step(row, ImportStep.RECEIVED, handle.id)
+        assert 'expected_digest' in failure_text(failure.value)
+
+        # Nothing downloaded: the work directory was never even created.
+        work_dir = tmp_path / 'import-work'
+        assert not work_dir.exists() or not list(work_dir.rglob('*'))
+        import_assert_nothing_provisioned(
+            experiment_name=import_names['experiment_name'],
+            model_name=import_names['model_name'],
+            object_key=object_key,
+        )

e2e/test_import_model_gates.py

@@ -0,0 +1,679 @@
+"""One rejection per gate, against a real object, a real archive and a real filesystem.
+
+The gates are unit-tested already. What only a real run can show is that a rejection is *recorded*
+correctly and that it provisions nothing: the row ends `ORCHESTRATOR_VALIDATION_ERROR` naming that
+gate's step, its catalog code and its catalog sentence, and no experiment, run, artifact, registry
+version, model document or extracted file survives.
+
+Every case here is hostile **input** — design E7's first preference. Nothing is patched, no
+dependency is broken; the bundles are built by the shared factory and the gates do the rest. The
+one exception to "input only" is the object-size ceiling, where the ceiling itself is narrowed
+rather than a gigabyte being uploaded; that is a configuration change, not a patched code path, and
+it is scoped to the one class that needs it.
+
+`_assert_rejected` is where the shared claim of task 5.8 lives — terminal status, recorded step,
+recorded code, catalog sentence, and `assert_nothing_provisioned` — so no case can quietly assert
+four fifths of it.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+from typing import Any
+
+import pytest
+from temporalio.client import WorkflowFailureError
+
+from e2e.import_harness import (
+    IMPORT_BUCKET,
+    IMPORT_PREFIX,
+    failure_text,
+)
+from laborious.utils.bundle.steps import ImportStep
+from laborious.utils.models.import_errors import IMPORT_FAILURE_CATALOG
+
+pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
+
+VALIDATION_ERROR_STATUS = 'ORCHESTRATOR_VALIDATION_ERROR'
+FACTORY_PASSWORD = 'Factory1Model!Test'
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    return json.loads(value) if isinstance(value, str) else (value or {})
+
+
+async def _run_rejected_import(
+    *,
+    payload: bytes,
+    password: str,
+    import_upload_bundle: Any,
+    import_insert_status_row: Any,
+    import_build_input: Any,
+    import_execute: Any,
+    import_envelope: Any,
+    envelope_password: str | None = None,
+) -> tuple[Any, int, str]:
+    """Upload a hostile bundle, insert the row, run the import and expect it to fail."""
+    object_key, digest = import_upload_bundle(payload)
+    import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+    handle = await import_execute(
+        import_build_input(
+            import_run_id=import_run_id,
+            object_key=object_key,
+            expected_digest=digest,
+            password_envelope=import_envelope(
+                envelope_password if envelope_password is not None else password
+            ),
+        )
+    )
+    return handle, import_run_id, object_key
+
+
+async def _assert_rejected(
+    *,
+    handle: Any,
+    import_run_id: int,
+    object_key: str,
+    step: ImportStep,
+    import_read_status_row: Any,
+    import_assert_nothing_provisioned: Any,
+    import_names: dict[str, str],
+    experiment_name: str | None = None,
+    model_name: str | None = None,
+) -> dict[str, Any]:
+    """The whole of task 5.8, in one call. Returns the row so a case can add to it."""
+    with pytest.raises(WorkflowFailureError):
+        await handle.result()
+
+    row = import_read_status_row(import_run_id)
+    assert row is not None, 'the rejection lost the row it was recorded on'
+    assert row['status'] == VALIDATION_ERROR_STATUS, (
+        f'terminal status is {row["status"]!r}, expected {VALIDATION_ERROR_STATUS!r}'
+    )
+
+    expected_code, expected_reason, _expected_responsibility = IMPORT_FAILURE_CATALOG[step]
+    detail = _as_dict(row['orchestrator_response_data'])['import']
+    assert detail['step'] == step.value, (
+        f'recorded step is {detail["step"]!r}, expected {step.value!r}'
+    )
+    assert detail['code'] == expected_code.value, (
+        f'recorded code is {detail["code"]!r}, expected {expected_code.value!r}'
+    )
+    assert detail['workflow_id'] == handle.id
+    assert row['error_message'] == expected_reason, (
+        'error_message is not the catalog sentence for this step'
+    )
+
+    import_assert_nothing_provisioned(
+        experiment_name=experiment_name or import_names['experiment_name'],
+        model_name=model_name or import_names['model_name'],
+        object_key=object_key,
+    )
+    return row
+
+
+class TestGate1ObjectLimits:
+    """Gate 1 rejects before the object's body is fetched at all."""
+
+    @pytest.fixture
+    def import_limits_override(self) -> dict[str, str]:
+        """A 1 KiB ceiling, so a factory bundle is over it.
+
+        The production ceiling is 1 GiB. Narrowing it is the only way to cross gate 1 without
+        uploading a gigabyte, and it changes a configured limit rather than a code path — the same
+        comparison runs, against a different number.
+        """
+        return {'IMPORT_MAX_OBJECT_BYTES': '1024'}
+
+    async def test_an_oversized_object_is_rejected_at_download(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        payload = import_bundle_factory()
+        assert len(payload) > 1024, 'the fixture bundle must exceed the narrowed ceiling'
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=payload,
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.DOWNLOAD,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+
+
+class TestGate2Header:
+    """Four header rejections, all landing on `download` — the step that says "not these bytes"."""
+
+    @pytest.mark.parametrize(
+        ('case', 'header_kwargs'),
+        [
+            ('wrong magic', {'magic': b'NOTSIENT'}),
+            ('unsupported version', {'format_version': 9}),
+            ('kdf memory above the ceiling', {'declared_memlimit': 512 * 1024 * 1024}),
+            ('kdf passes above the ceiling', {'declared_opslimit': 9}),
+            ('digest not of the chunk stream', {'digest_override': bytes(32)}),
+        ],
+    )
+    async def test_a_malformed_header_is_rejected_at_download(
+        self,
+        case: str,
+        header_kwargs: dict[str, Any],
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(**header_kwargs),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        row = await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.DOWNLOAD,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+        # None of these may borrow the decryption sentence: the password is not implicated.
+        assert 'password' not in (row['error_message'] or '').lower(), case
+
+
+class TestGate3Decryption:
+    """The password, the tag and the framing — the three ways a stream refuses to open."""
+
+    async def test_a_wrong_password_is_rejected_at_decryption(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        """The envelope opens — it is the *bundle* password inside it that is wrong.
+
+        Worth separating: an envelope the worker cannot open is also a decryption-step failure, but
+        for a different reason, and only this one proves the KDF actually ran and produced a key
+        the stream rejected.
+        """
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(password=FACTORY_PASSWORD),
+            password=FACTORY_PASSWORD,
+            envelope_password='Wrong1Password!Entirely',
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.DECRYPTION,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+
+    async def test_a_tampered_chunk_is_rejected_at_decryption(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        """One flipped byte inside the stream, with the header digest repaired to match.
+
+        Repairing the digest is what makes this gate 3 rather than gate 2: the file is
+        self-consistent, so only the AEAD can tell it was altered.
+        """
+        from tests.helpers.bundle_factory import tamper_ciphertext
+
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=tamper_ciphertext(import_bundle_factory()),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.DECRYPTION,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+
+    async def test_a_stream_without_its_final_tag_is_rejected_at_decryption(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        """Every chunk framed as MESSAGE, so the stream claims more is coming and never ends.
+
+        The digest cannot see this — the bytes on disk are exactly the ones it covers. Only the
+        `secretstream` framing knows the file is incomplete.
+        """
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(drop_final_tag=True),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.DECRYPTION,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+
+
+class TestGate4ArchiveInspection:
+    """The central directory alone decides — nothing has been written to disk yet."""
+
+    @pytest.fixture
+    def import_limits_override(self) -> dict[str, str]:
+        """A 1 MiB declared-expansion ceiling, for the oversized-expansion case in this class."""
+        return {'IMPORT_MAX_UNCOMPRESSED_BYTES': str(1024 * 1024)}
+
+    @pytest.mark.parametrize(
+        ('case', 'kwargs'),
+        [
+            ('entry escaping the tree', {'traversal_entry': 'artifacts/../../escaped.csv'}),
+            ('absolute entry path', {'absolute_entry': '/opt/absolute.csv'}),
+            ('symbolic link', {'symlinks': {'artifacts/link.csv': 'artifacts/train_data.csv'}}),
+            ('declared expansion over the ceiling', {'bomb_bytes': 4 * 1024 * 1024}),
+        ],
+    )
+    async def test_a_hostile_archive_is_rejected_before_anything_is_extracted(
+        self,
+        case: str,
+        kwargs: dict[str, Any],
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+        tmp_path: Any,
+    ) -> None:
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(**kwargs),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.ARCHIVE_INSPECTION,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+        # "Nothing extracted" is stronger than "nothing provisioned": not one member reached disk.
+        work_dir = tmp_path / 'import-work'
+        assert not list(work_dir.rglob('*')) if work_dir.exists() else True, case
+
+
+class TestGate5Extraction:
+    """A member that gate 4 passed and extraction still cannot place.
+
+    Gate 5 is about *placement*, not size: containment on the resolved path, and never overwriting
+    a file already written. A zip carrying the same name twice passes gate 4 — which checks names
+    and modes, not duplicates — and is refused here on the second member.
+    """
+
+    # `zipfile` warns while *writing* the duplicate entry. That is the fixture doing exactly what
+    # this test needs, so it is silenced here rather than left to look like a problem — and scoped to
+    # this one test, so a duplicate name appearing anywhere else still gets noticed.
+    @pytest.mark.filterwarnings('ignore:Duplicate name:UserWarning')
+    async def test_a_duplicate_member_is_rejected_at_extraction(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        from tests.helpers.bundle_factory import default_tree
+
+        # `bomb_bytes` writes `artifacts/bomb.csv`, and `files` already carries that name, so the
+        # archive declares it twice. Small enough that no size ceiling is involved.
+        files = dict(default_tree())
+        files['artifacts/bomb.csv'] = b'first\n'
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(files=files, bomb_bytes=64),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.EXTRACTION,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+
+
+class TestGate6Structure:
+    """The layout and the metadata schema, checked on the extracted tree."""
+
+    @pytest.mark.parametrize(
+        ('case', 'kwargs'),
+        [
+            ('no metadata.json', {'include_metadata': False}),
+            ('metadata missing run_id', {'drop_metadata_field': 'run_id'}),
+            ('parameters missing target_variable', {'drop_parameter': 'target_variable'}),
+        ],
+    )
+    async def test_an_incomplete_bundle_is_rejected_at_structure_validation(
+        self,
+        case: str,
+        kwargs: dict[str, Any],
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(**kwargs),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.STRUCTURE_VALIDATION,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+
+    async def test_a_bundle_without_prediction_model_is_rejected(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        """Without `prediction_model` there is nothing to register a version against."""
+        from tests.helpers.bundle_factory import default_tree
+
+        files = {
+            name: content
+            for name, content in default_tree().items()
+            if not name.startswith('artifacts/prediction_model/')
+        }
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(files=files),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.STRUCTURE_VALIDATION,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+
+    async def test_an_over_long_experiment_name_never_reaches_the_varchar_column(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+        import_bff_engine: Any,
+    ) -> None:
+        """A 60-character experiment name is refused at gate 6, not by Postgres three steps later.
+
+        `experiment_name` is `VARCHAR(50)` and the import writes the bundle's own name onto the row
+        *before* provisioning. If the gate did not enforce the column's limit, the names write would
+        be a `DataError` — non-retryable, recorded as a status failure, on a row whose import was
+        otherwise fine. Asserting the row's `experiment_name` is untouched is what proves the write
+        never happened.
+        """
+        from sqlalchemy import text
+
+        long_name = 'x' * 60
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(metadata={'experiment_name': long_name}),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        row = await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.STRUCTURE_VALIDATION,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+            experiment_name=long_name,
+        )
+        assert row['experiment_name'] == 'import-e2e-placeholder', (
+            'the over-long name reached the row, so the gate did not hold the column'
+        )
+        with import_bff_engine.begin() as connection:
+            width = connection.execute(
+                text(
+                    'SELECT character_maximum_length FROM information_schema.columns '
+                    "WHERE table_name = 'experiment_run' AND column_name = 'experiment_name'"
+                )
+            ).scalar_one()
+        assert width == 50, 'the column this gate exists to protect is no longer VARCHAR(50)'
+
+
+class TestGate7ContentPolicy:
+    """A file kind an MLflow artifact tree has no reason to carry."""
+
+    async def test_a_forbidden_file_type_is_rejected_at_content_policy(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(extra_files={'artifacts/payload.exe': b'MZ\x00\x00'}),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        await _assert_rejected(
+            handle=handle,
+            import_run_id=import_run_id,
+            object_key=object_key,
+            step=ImportStep.CONTENT_POLICY,
+            import_read_status_row=import_read_status_row,
+            import_assert_nothing_provisioned=import_assert_nothing_provisioned,
+            import_names=import_names,
+        )
+
+
+class TestTheRejectionSaysNothingTechnical:
+    """`error_message` is a sentence for a person: no path, no object key, no traceback.
+
+    One case is enough to make the point structurally — the catalog is keyed by step and
+    `classify_import_failure` discards the exception's text entirely — and the per-gate cases above
+    each assert the sentence is *exactly* the catalog's, which is the same guarantee stated
+    positively.
+    """
+
+    async def test_no_technical_detail_reaches_the_recorded_reason(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        handle, import_run_id, object_key = await _run_rejected_import(
+            payload=import_bundle_factory(include_metadata=False),
+            password=FACTORY_PASSWORD,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        with pytest.raises(WorkflowFailureError) as failure:
+            await handle.result()
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        message = row['error_message'] or ''
+        for forbidden in (
+            object_key,
+            IMPORT_BUCKET,
+            IMPORT_PREFIX,
+            'experiment_run',
+            'Traceback',
+            'BundleGateError',
+            tempfile.gettempdir(),
+            'gate',
+        ):
+            assert forbidden not in message, f'{forbidden!r} leaked into error_message: {message!r}'
+
+        # The technical truth is not lost — it is on the Temporal error, where support can read it.
+        assert 'metadata' in failure_text(failure.value).lower()

e2e/test_import_model_happy_path.py

@@ -0,0 +1,315 @@
+"""An import that works, twice over: once from the producer's own bytes and once from the factory.
+
+Two origins, because they prove different things. The **golden** bundle is bytes the producer's code
+emitted — it decrypts at the producer's real Argon2id cost (256 MiB, 3 passes) and carries the tree
+the producer really writes, so it is the only case that would notice the producer's layout moving.
+The **factory** bundle is synthesised here, cheaply, with names unique to the test; it is what the
+rest of the suite runs on, so a happy path over it is what makes the negative cases' "and nothing
+was provisioned" meaningful.
+
+What "it worked" means, in full: the experiment exists, a run exists inside it carrying the derived
+run name, the artifact tree is under the run with `prediction_model` and `data_model` at its root,
+every parameter the bundle declared is logged plus the origin's identity under `source.*`, a
+registry version exists, the MongoDB document exists, and the status row ends `IMPORT_SUCCESS`.
+
+Two things a successful import must **not** have done are asserted here too, because they are the
+difference between "imported" and "live": no version is in `Production`, and the document is
+`active: false`. Promotion and activation are manual operator steps (QTZPOC-21).
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import mlflow
+import pytest
+
+from e2e.import_harness import (
+    IMPORT_BUCKET,
+    IMPORT_PREFIX,
+    SEEDED_MODEL_IDS,
+)
+
+pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
+
+GOLDEN_MODEL_NAME = 'fixture_model'
+GOLDEN_EXPERIMENT_NAME = 'fixture_experiment'
+GOLDEN_MODEL_VERSION = '3'
+GOLDEN_TARGET = 'SE'
+
+REQUIRED_ARTIFACT_ROOTS = {'prediction_model', 'data_model'}
+
+
+def _mlflow_client(server: dict[str, Any]) -> Any:
+    return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri'])
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    """`request_data` / `orchestrator_response_data` come back as a dict or as JSON text."""
+    return json.loads(value) if isinstance(value, str) else (value or {})
+
+
+async def _run_import(
+    *,
+    payload: bytes,
+    import_upload_bundle: Any,
+    import_insert_status_row: Any,
+    import_build_input: Any,
+    import_execute: Any,
+    password: str,
+    import_envelope: Any,
+) -> tuple[Any, int, str]:
+    """Upload, insert the row the BFF would have inserted, run the workflow. Return the outcome."""
+    object_key, digest = import_upload_bundle(payload)
+    import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+    handle = await import_execute(
+        import_build_input(
+            import_run_id=import_run_id,
+            object_key=object_key,
+            expected_digest=digest,
+            password_envelope=import_envelope(password),
+        )
+    )
+    return handle, import_run_id, object_key
+
+
+def _assert_provisioned(
+    *,
+    client: Any,
+    experiment_name: str,
+    model_name: str,
+    run_name: str,
+    expected_parameters: dict[str, str],
+    expected_source_fields: dict[str, str],
+) -> tuple[str, Any]:
+    """The whole positive claim about MLflow, in one place. Returns `(run_id, version)`."""
+    experiment = client.get_experiment_by_name(experiment_name)
+    assert experiment is not None, f'no experiment named {experiment_name!r}'
+
+    runs = [
+        run
+        for run in client.search_runs([experiment.experiment_id])
+        if run.info.run_name == run_name
+    ]
+    assert len(runs) == 1, f'expected exactly one run named {run_name!r}, found {len(runs)}'
+    run = runs[0]
+    assert run.info.status == 'FINISHED'
+
+    artifact_roots = {artifact.path for artifact in client.list_artifacts(run.info.run_id)}
+    assert REQUIRED_ARTIFACT_ROOTS <= artifact_roots, (
+        f'the run holds {sorted(artifact_roots)}, missing {sorted(REQUIRED_ARTIFACT_ROOTS - artifact_roots)}'
+    )
+    # `prediction_model` is a directory the legacy loader walks, not an opaque blob.
+    prediction_files = {
+        artifact.path.rsplit('/', 1)[-1]
+        for artifact in client.list_artifacts(run.info.run_id, 'prediction_model')
+    }
+    assert {'MLmodel', 'model.pkl'} <= prediction_files, prediction_files
+
+    for name, value in expected_parameters.items():
+        assert run.data.params.get(name) == value, (
+            f'parameter {name!r} is {run.data.params.get(name)!r}, expected {value!r}'
+        )
+    for name, value in expected_source_fields.items():
+        assert run.data.params.get(f'source.{name}') == value, (
+            f'source.{name} is {run.data.params.get(f"source.{name}")!r}, expected {value!r}'
+        )
+
+    versions = list(client.search_model_versions(f"name='{model_name}'"))
+    assert len(versions) == 1, f'expected one registered version, found {len(versions)}'
+    version = versions[0]
+    assert version.run_id == run.info.run_id
+    return run.info.run_id, version
+
+
+class TestGoldenBundle:
+    """The producer's own bytes, at the producer's own KDF cost."""
+
+    async def test_the_producer_artefact_imports_end_to_end(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_golden_bundle: dict[str, Any],
+        import_mlflow_server: dict[str, Any],
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        handle, import_run_id, object_key = await _run_import(
+            payload=import_golden_bundle['bytes'],
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            password=import_golden_bundle['password'],
+            import_envelope=import_envelope,
+        )
+        result = await handle.result()
+
+        assert result['model_name'] == GOLDEN_MODEL_NAME
+        assert result['import_run_id'] == import_run_id
+        assert result['version'] == '1', 'the destination numbering starts at 1, not at the bundle'
+
+        run_id, version = _assert_provisioned(
+            client=_mlflow_client(import_mlflow_server),
+            experiment_name=GOLDEN_EXPERIMENT_NAME,
+            model_name=GOLDEN_MODEL_NAME,
+            run_name=f'import-{GOLDEN_MODEL_NAME}-v{GOLDEN_MODEL_VERSION}',
+            expected_parameters={'target_variable': GOLDEN_TARGET, 'nan_treatment': 'drop'},
+            expected_source_fields={
+                'model_name': GOLDEN_MODEL_NAME,
+                'experiment_name': GOLDEN_EXPERIMENT_NAME,
+                'model_version': GOLDEN_MODEL_VERSION,
+                'model_project': 'fixture_project',
+            },
+        )
+        assert result['run_id'] == run_id
+
+        document = import_models_collection.find_one({'name': GOLDEN_MODEL_NAME}, {'_id': False})
+        assert document is not None, 'no model document was written'
+        assert document['id'] == str(max(int(i) for i in SEEDED_MODEL_IDS) + 1)
+        assert document['model_config']['target'] == GOLDEN_TARGET
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        assert row['status'] == 'IMPORT_SUCCESS'
+        assert row['error_message'] is None
+        assert row['experiment_name'] == GOLDEN_EXPERIMENT_NAME
+        assert row['run_name'] == f'import-{GOLDEN_MODEL_NAME}-v{GOLDEN_MODEL_VERSION}'
+        assert row['file_name'] == object_key
+        assert row['bucket_name'] == IMPORT_BUCKET
+        assert object_key.startswith(IMPORT_PREFIX)
+
+        # Imported, not live.
+        assert version.current_stage in (None, 'None'), version.current_stage
+        assert document['active'] is False
+
+
+class TestFactoryBundle:
+    """A synthesised tree with names unique to this test — the shape the rest of the suite runs on."""
+
+    async def test_a_factory_bundle_imports_end_to_end(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_mlflow_server: dict[str, Any],
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        payload = import_bundle_factory(parameters={'target_variable': 'SE'})
+        handle, import_run_id, _ = await _run_import(
+            payload=payload,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            password='Factory1Model!Test',
+            import_envelope=import_envelope,
+        )
+        result = await handle.result()
+
+        assert result['model_name'] == import_names['model_name']
+        run_id, version = _assert_provisioned(
+            client=_mlflow_client(import_mlflow_server),
+            experiment_name=import_names['experiment_name'],
+            model_name=import_names['model_name'],
+            # DEFAULT_METADATA declares model_version '2'.
+            run_name=f'import-{import_names["model_name"]}-v2',
+            expected_parameters={'target_variable': 'SE', 'retrain': 'False'},
+            expected_source_fields={
+                'model_name': import_names['model_name'],
+                'experiment_name': import_names['experiment_name'],
+                'model_version': '2',
+            },
+        )
+        assert result['run_id'] == run_id
+        assert result['version'] == '1'
+
+        document = import_models_collection.find_one(
+            {'name': import_names['model_name']}, {'_id': False}
+        )
+        assert document is not None
+        assert document['model_config'] == {
+            'transform_flavor': 'sklearn',
+            'predict_flavor': 'joblib',
+            'retention_minutes': 60,
+            'target': 'SE',
+            'skip_predict_timestamp': True,
+        }
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        assert row['status'] == 'IMPORT_SUCCESS'
+        detail = _as_dict(row['orchestrator_response_data'])
+        assert detail['import']['workflow_id'] == handle.id
+
+        assert version.current_stage in (None, 'None')
+        assert document['active'] is False
+
+
+class TestLegacyLoaderResolution:
+    """The registry source an import writes has to be one the *existing* loader can parse.
+
+    This is the assertion that could only ever be made against a real registry: the importer writes
+    the run's resolved artifact URI with `prediction_model` appended, because `get_model_run_id`
+    extracts the run id as `source.split('/')[2]` — an index that is the run id only in the resolved
+    form. A literal `runs:/<run_id>/prediction_model` would put `prediction_model` at index 2, and a
+    `models:/m-*` source would have none of the parts at all.
+
+    It is called with the stage the version is actually in rather than `Production`, because an
+    import deliberately promotes nothing — passing `Production` here would test the promotion step
+    this feature does not have.
+    """
+
+    async def test_the_imported_source_resolves_to_the_import_run(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_activities: Any,
+        import_mlflow_server: dict[str, Any],
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+    ) -> None:
+        payload = import_bundle_factory()
+        handle, _, _ = await _run_import(
+            payload=payload,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            password='Factory1Model!Test',
+            import_envelope=import_envelope,
+        )
+        result = await handle.result()
+        model_name = import_names['model_name']
+
+        client = _mlflow_client(import_mlflow_server)
+        version = next(iter(client.search_model_versions(f"name='{model_name}'")))
+        assert version.source.endswith('/prediction_model'), version.source
+
+        # The production method, on the production repository object, against the real registry.
+        resolved = import_activities.import_mlflow_repository.get_model_run_id(
+            model_name, stage=version.current_stage
+        )
+        assert resolved == result['run_id'], (
+            f'get_model_run_id resolved {resolved!r} from source {version.source!r}, '
+            f'but the import created run {result["run_id"]!r}'
+        )
+
+        artifact_roots = {artifact.path for artifact in client.list_artifacts(resolved)}
+        assert REQUIRED_ARTIFACT_ROOTS <= artifact_roots, sorted(artifact_roots)

e2e/test_import_model_provisioning_failures.py

@@ -0,0 +1,384 @@
+"""Each provisioning step fails as itself: its own code, its own sentence, nothing half-built.
+
+The four steps after the gates — experiment creation, artifact upload, registration, the model
+document — each get one failure, and every one lands `IMPORT_ERROR` rather than
+`ORCHESTRATOR_VALIDATION_ERROR`. That split is not cosmetic: it is the table's existing distinction
+between "the person who submitted this can fix it" and "only support can", and a provisioning
+failure is always the second kind.
+
+**How each failure is caused**, in design E7's order of preference:
+
+| Step | Cause | Preference |
+|---|---|---|
+| `experiment_creation` | the MLflow server is genuinely stopped | broken dependency |
+| `artifact_upload` | targeted patch on `log_artifacts` | patch, justified in place |
+| `registration` | targeted patch on `create_model_version` | patch, justified in place |
+| `model_document` | a real unique index refuses the activity's own insert | broken dependency |
+
+Two departures from what the tasks anticipated, both recorded in the test that found them:
+
+- **Registration needs a patch too**, not just artifact upload. One process serves both tracking and
+  the registry, so stopping it fails experiment creation three steps earlier; and MLflow validates
+  nothing about a registered model name except that it is non-empty, which gate 6 already requires.
+  Both patches are on the single client call the step *is*, never on the activity or the workflow, so
+  what is under test — the workflow attributing this step's failure to this step — is untouched.
+- **The model document's unique-index violation is on `model_config.target`**, not on `id`. The
+  duplicate-`id` state the task describes cannot be constructed at all; `TestModelDocument`'s
+  docstring shows why, and the mechanism the task wanted — a real index refusing a real insert — is
+  kept intact.
+"""
+
+from __future__ import annotations
+
+import json
+import tempfile
+from typing import Any
+
+import pytest
+from temporalio.client import WorkflowFailureError
+
+from e2e.import_harness import IMPORT_BUCKET, IMPORT_PREFIX, SEEDED_MODEL_IDS
+from laborious.utils.bundle.steps import ImportStep
+from laborious.utils.models.import_errors import IMPORT_FAILURE_CATALOG
+
+pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
+
+PLATFORM_ERROR_STATUS = 'IMPORT_ERROR'
+FACTORY_PASSWORD = 'Factory1Model!Test'
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    return json.loads(value) if isinstance(value, str) else (value or {})
+
+
+def _assert_platform_failure(
+    *,
+    row: dict[str, Any] | None,
+    step: ImportStep,
+    workflow_id: str,
+    object_key: str,
+) -> None:
+    """The shared claim of task 7.5, asserted by every case in this module.
+
+    `error_message` is the catalog sentence *exactly* — not "contains" — which is the strongest
+    possible form of "carries no path, table name, object key or traceback": there is nothing in it
+    that did not come from the catalog.
+    """
+    assert row is not None, 'the failure lost the row it was recorded on'
+    assert row['status'] == PLATFORM_ERROR_STATUS, (
+        f"status is {row['status']!r}; a provisioning failure is not the user's to fix"
+    )
+
+    code, reason, _responsibility = IMPORT_FAILURE_CATALOG[step]
+    detail = _as_dict(row['orchestrator_response_data'])['import']
+    assert detail['step'] == step.value, detail
+    assert detail['code'] == code.value, detail
+    assert detail['workflow_id'] == workflow_id, detail
+    assert row['error_message'] == reason
+
+    message = row['error_message'] or ''
+    for forbidden in (
+        object_key,
+        IMPORT_BUCKET,
+        IMPORT_PREFIX,
+        'experiment_run',
+        'Traceback',
+        'Error',
+        'mlflow',
+        'Mongo',
+        tempfile.gettempdir(),
+    ):
+        assert forbidden not in message, f'{forbidden!r} leaked into error_message: {message!r}'
+
+
+async def _upload_and_start(
+    *,
+    import_bundle_factory: Any,
+    import_upload_bundle: Any,
+    import_insert_status_row: Any,
+    import_build_input: Any,
+    import_execute: Any,
+    import_envelope: Any,
+    **factory_kwargs: Any,
+) -> tuple[Any, int, str]:
+    object_key, digest = import_upload_bundle(import_bundle_factory(**factory_kwargs))
+    import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+    handle = await import_execute(
+        import_build_input(
+            import_run_id=import_run_id,
+            object_key=object_key,
+            expected_digest=digest,
+            password_envelope=import_envelope(FACTORY_PASSWORD),
+        )
+    )
+    return handle, import_run_id, object_key
+
+
+class TestExperimentCreation:
+    """The tracking server is not there. No patch, no mock — the process is stopped."""
+
+    async def test_a_stopped_tracking_server_fails_at_experiment_creation(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_mlflow_stopped: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        with import_mlflow_stopped():
+            handle, import_run_id, object_key = await _upload_and_start(
+                import_bundle_factory=import_bundle_factory,
+                import_upload_bundle=import_upload_bundle,
+                import_insert_status_row=import_insert_status_row,
+                import_build_input=import_build_input,
+                import_execute=import_execute,
+                import_envelope=import_envelope,
+            )
+            with pytest.raises(WorkflowFailureError):
+                await handle.result()
+            row = import_read_status_row(import_run_id)
+
+        _assert_platform_failure(
+            row=row,
+            step=ImportStep.EXPERIMENT_CREATION,
+            workflow_id=handle.id,
+            object_key=object_key,
+        )
+
+        # Download, decryption, experiment creation and artifact upload are one fused activity
+        # attempt (`model-import-fix-bugs`, bug 8): the workflow only learns the bundle's names once
+        # that whole attempt returns, and an experiment-creation failure means it never does. The
+        # placeholder names inserted with the row are therefore still there — narrowed from the old
+        # "names before MLflow is touched" invariant, per `model-import-status-record`'s updated spec.
+        assert row is not None
+        assert row['experiment_name'] == 'import-e2e-placeholder'
+
+        # Nothing in Mongo: the document is the last step and never ran.
+        documents = list(import_models_collection.find({}, {'_id': False}))
+        assert {str(d['id']) for d in documents} == set(SEEDED_MODEL_IDS)
+        import_assert_nothing_provisioned(
+            experiment_name=import_names['experiment_name'],
+            model_name=import_names['model_name'],
+            object_key=object_key,
+        )
+
+
+class TestArtifactUpload:
+    """The step design E7 names as the one a patch is needed for."""
+
+    async def test_a_failing_artifact_upload_is_recorded_as_its_own_step(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_activities: Any,
+        import_mlflow_server: dict[str, Any],
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        """Why a patch: with the server up and the bundle valid, nothing else fails only this step.
+
+        Hostile input cannot reach here — a bundle that gets past gate 7 has a readable tree, and
+        one that does not never reaches artifact upload. A broken dependency cannot isolate it
+        either: the only dependency this step has is the tracking server, and stopping that fails
+        experiment creation one step earlier. So the patch is on the single client call the step
+        makes, `log_artifacts`; everything around it — the run creation, the parameters, the
+        workflow's attribution of the failure — is the real code.
+        """
+        client = import_activities.import_mlflow_repository.client
+
+        def refuse(*_args: Any, **_kwargs: Any) -> None:
+            raise OSError('artifact store is unreachable')
+
+        monkeypatch.setattr(client, 'log_artifacts', refuse)
+
+        handle, import_run_id, object_key = await _upload_and_start(
+            import_bundle_factory=import_bundle_factory,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        with pytest.raises(WorkflowFailureError):
+            await handle.result()
+
+        _assert_platform_failure(
+            row=import_read_status_row(import_run_id),
+            step=ImportStep.ARTIFACT_UPLOAD,
+            workflow_id=handle.id,
+            object_key=object_key,
+        )
+
+        # The experiment exists — it was the previous step and it succeeded. Nothing after does.
+        mlflow_client = _fresh_client(import_mlflow_server)
+        assert mlflow_client.get_experiment_by_name(import_names['experiment_name']) is not None
+        assert not list(mlflow_client.search_model_versions(f"name='{import_names['model_name']}'"))
+        assert import_models_collection.find_one({'name': import_names['model_name']}) is None
+
+
+class TestRegistration:
+    """The artifacts are stored and the model still cannot be registered."""
+
+    async def test_a_failing_registration_is_recorded_as_its_own_step(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_activities: Any,
+        import_mlflow_server: dict[str, Any],
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        """Why a patch here too, contrary to design E7's expectation that this step needs none.
+
+        Hostile input was the first thing tried: a registered model name MLflow refuses. It does not
+        exist — `_validate_model_name` rejects only the empty string, which gate 6 already refuses,
+        and the sqlite backend store does not enforce the `VARCHAR(256)` the schema declares, so
+        even a 300-character name is accepted. A broken dependency was the second: one process
+        serves both tracking and the registry, so stopping it fails experiment creation three steps
+        earlier. The patch is on `create_model_version`, the one call this step is.
+        """
+        client = import_activities.import_mlflow_repository.client
+
+        def refuse(*_args: Any, **_kwargs: Any) -> None:
+            raise RuntimeError('registry write rejected')
+
+        monkeypatch.setattr(client, 'create_model_version', refuse)
+
+        handle, import_run_id, object_key = await _upload_and_start(
+            import_bundle_factory=import_bundle_factory,
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        with pytest.raises(WorkflowFailureError):
+            await handle.result()
+
+        _assert_platform_failure(
+            row=import_read_status_row(import_run_id),
+            step=ImportStep.REGISTRATION,
+            workflow_id=handle.id,
+            object_key=object_key,
+        )
+
+        # The run and its artifacts survive — "the model's files were stored, but the model itself
+        # could not be registered" is exactly what the catalog sentence for this step says.
+        mlflow_client = _fresh_client(import_mlflow_server)
+        experiment = mlflow_client.get_experiment_by_name(import_names['experiment_name'])
+        assert experiment is not None
+        runs = mlflow_client.search_runs([experiment.experiment_id])
+        assert len(runs) == 1
+        assert {a.path for a in mlflow_client.list_artifacts(runs[0].info.run_id)} >= {
+            'prediction_model',
+            'data_model',
+        }
+        assert import_models_collection.find_one({'name': import_names['model_name']}) is None
+
+
+class TestModelDocument:
+    """The last step, failed by a real unique index refusing the real insert.
+
+    **Why the index is on `model_config.target` and not on `id`.** Task 7.4 asks for "a document
+    already holding the id the activity is about to compute". That state cannot be constructed, and
+    why is worth more than the test would have been: `_next_model_document_id` computes
+    `max(int(id)) + 1` over `distinct('id')` of the *same* collection, so every value that
+    stringifies to the colliding id is counted by that maximum. Put a document at `'8'` and the next
+    id becomes `'9'`. Make the id unparseable and it is ignored by the numbering *and* does not
+    collide. There is no third case — the numbering cannot collide with itself, and the only real
+    duplicate-key path on `id` is two imports racing, which this change lists as a non-goal. That is
+    a good property of the implementation, not a gap.
+
+    So the mechanism the task asked for is kept — a real unique index refusing the activity's own
+    insert — on a key that can actually collide. `model_config.target` is `'SE'` on the seeded
+    documents and `'SE'` on the one this import writes, so a unique index there makes the insert a
+    genuine `DuplicateKeyError` from the server. No patch, no mock: the same class of failure, from
+    the same enforcement mechanism, at the same step.
+    """
+
+    async def test_a_unique_index_violation_fails_at_model_document(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_mlflow_server: dict[str, Any],
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        # All three seeded documents carry target 'SE', so the index has to be buildable first:
+        # give two of them their own target and leave exactly one holding 'SE'. That one is what the
+        # import's own document — also target 'SE' — collides with.
+        for ordinal, seeded_id in enumerate(sorted(SEEDED_MODEL_IDS)[1:], start=1):
+            import_models_collection.update_one(
+                {'id': seeded_id}, {'$set': {'model_config.target': f'OTHER_{ordinal}'}}
+            )
+        import_models_collection.create_index('model_config.target', unique=True, name='e2e_target')
+        try:
+            handle, import_run_id, object_key = await _upload_and_start(
+                import_bundle_factory=import_bundle_factory,
+                import_upload_bundle=import_upload_bundle,
+                import_insert_status_row=import_insert_status_row,
+                import_build_input=import_build_input,
+                import_execute=import_execute,
+                import_envelope=import_envelope,
+                parameters={'target_variable': 'SE'},
+            )
+            with pytest.raises(WorkflowFailureError):
+                await handle.result()
+            row = import_read_status_row(import_run_id)
+        finally:
+            # The collection is dropped and reseeded per test, but dropping the index here keeps the
+            # failure local even if that ever changes.
+            import_models_collection.drop_index('e2e_target')
+
+        _assert_platform_failure(
+            row=row,
+            step=ImportStep.MODEL_DOCUMENT,
+            workflow_id=handle.id,
+            object_key=object_key,
+        )
+
+        # No document for this model, and the seeded ones are exactly as they were.
+        assert import_models_collection.find_one({'name': import_names['model_name']}) is None
+        assert {
+            str(document['id']) for document in import_models_collection.find({}, {'_id': False})
+        } == set(SEEDED_MODEL_IDS)
+
+        # Everything before it stands — registered, and unusable, which is the point of the split
+        # between this step's sentence and the registration step's.
+        mlflow_client = _fresh_client(import_mlflow_server)
+        versions = list(mlflow_client.search_model_versions(f"name='{import_names['model_name']}'"))
+        assert len(versions) == 1
+
+
+def _fresh_client(server: dict[str, Any]) -> Any:
+    """An MlflowClient bound to the test server, independent of the one the activities hold."""
+    import mlflow
+
+    return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri'])

e2e/test_import_model_retry_recovery.py

@@ -0,0 +1,514 @@
+"""What happens when the *record* cannot be written — the cases that gate this feature's release.
+
+Everything else in this suite asks "did the import do the right thing". This file asks the harder
+question: the import did the right thing, and then could not say so. Design D20 answers it in two
+halves, and both are release-critical.
+
+**A database that comes back.** The terminal write retries, so a status database restarted during a
+deploy still records the correct verdict. Without this, every rolling restart would leave imports
+looking unfinished forever.
+
+**A database that does not come back.** The model is fully provisioned, and the workflow must not
+claim otherwise. It **completes**, and the verdict it could not write goes out through the failure
+channel — log, metric, notification — under `IMPORT_STATUS_NOT_RECORDED`. An import that worked must
+never be reported as an import that failed just because the audit row was unreachable: somebody
+would re-import a model that is already there.
+
+That last point was worth discovering the hard way. The first version of this file asserted the run
+*failed*; it does not, and it should not. The spec says so in as many words — "the run is not
+reported as failed" — and the implementation agrees with the spec.
+
+And the case that is the evidence for a migration in another repository: **a table without `V12`**.
+The claim succeeds (`V11` already gave the table `run_type`, and `PENDING` and the running status are
+pre-`V12` values), the model is fully provisioned, and only the *terminal* value is new — so the last
+write of the import is the first thing that fails. `ImportStatusRejectedError` is non-retryable, so
+it lands on the first attempt with no wait, and the row is left running forever. That is what
+"apply `V12` before turning `IMPORT_MODEL_ENABLED` on" buys, as a test rather than a warning in a
+document.
+
+Everything here runs on a **real clock** (design E9): under time skipping the backoff sleeps are
+virtual, so a restarted container would never get wall-clock time to come back and the recovery test
+would race by construction. The retry intervals are shortened by `import_status_retry_shortening`;
+the production values stay asserted by unit test 12.1 of `import-model-workflow`, because what these
+tests are about is the behaviour, not the arithmetic.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from typing import Any
+
+import mlflow
+import pytest
+from temporalio.client import WorkflowFailureError
+
+from e2e.import_harness import failure_text, failure_types
+
+pytestmark = [pytest.mark.import_e2e, pytest.mark.slow, pytest.mark.asyncio]
+
+FACTORY_PASSWORD = 'Factory1Model!Test'
+RUNNING_STATUS = 'ORCHESTRATOR_WAITING_PROC'
+NOT_RECORDED_CODE = 'IMPORT_STATUS_NOT_RECORDED'
+STATUS_WRITE_MARKER = 'step=status_write'
+PROVISIONING_STEPS = ('experiment_creation', 'artifact_upload', 'registration', 'model_document')
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    return json.loads(value) if isinstance(value, str) else (value or {})
+
+
+def _client(server: dict[str, Any]) -> Any:
+    return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri'])
+
+
+def _channel_text(mock_logger: Any) -> str:
+    """Everything the failure channel said, as one string.
+
+    The channel is a log line, a metric and a notification, and the log line is the only sink that
+    carries the whole classification — code, marker and the verdict it could not record. Reading the
+    logger is therefore reading the channel, and it is synchronous, which the notification path is
+    not.
+    """
+    return '\n'.join(str(call) for call in mock_logger.mock_calls)
+
+
+def _assert_fully_provisioned(
+    *, server: dict[str, Any], names: dict[str, str], collection: Any
+) -> str:
+    """Everything the import was supposed to create exists. Returns the run id."""
+    client = _client(server)
+    experiment = client.get_experiment_by_name(names['experiment_name'])
+    assert experiment is not None, 'no experiment: the import never got as far as provisioning'
+    runs = client.search_runs([experiment.experiment_id])
+    assert len(runs) == 1, f'expected one run, found {len(runs)}'
+    assert {a.path for a in client.list_artifacts(runs[0].info.run_id)} >= {
+        'prediction_model',
+        'data_model',
+    }
+    versions = list(client.search_model_versions(f"name='{names['model_name']}'"))
+    assert len(versions) == 1, f'expected one registered version, found {len(versions)}'
+    assert collection.find_one({'name': names['model_name']}) is not None, 'no model document'
+    return runs[0].info.run_id
+
+
+def _assert_no_provisioning_step_is_blamed(text_of_failure: str) -> None:
+    """The import's own steps all succeeded, so none of them may appear as the cause."""
+    for step in PROVISIONING_STEPS:
+        assert step not in text_of_failure, f'{step!r} is blamed, but that step succeeded'
+
+
+async def _wait_for_claim(read_row: Any, import_run_id: int, timeout: float = 30.0) -> None:
+    """Block until the workflow has taken ownership of the row.
+
+    The earliest deterministic point after which every status write belongs to this import. Used to
+    place an outage that must cover the whole rest of the run.
+    """
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        row = read_row(import_run_id)
+        if row is not None and row['status'] == RUNNING_STATUS:
+            return
+        await asyncio.sleep(0.1)
+    raise AssertionError('the row was never claimed, so the outage could not be placed')
+
+
+async def _wait_for_provisioning(
+    *, server: dict[str, Any], names: dict[str, str], collection: Any, timeout: float = 90.0
+) -> None:
+    """Block until the model document exists — the observable end of provisioning.
+
+    Polling the two destinations rather than patching an activity: "the document exists" *is* the
+    end of provisioning, so there is nothing to hook and nothing to fake. This is what lets the
+    outage below be placed exactly on the terminal write.
+    """
+    client = _client(server)
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        document = collection.find_one({'name': names['model_name']})
+        if document is not None and list(
+            client.search_model_versions(f"name='{names['model_name']}'")
+        ):
+            return
+        await asyncio.sleep(0.2)
+    raise AssertionError('provisioning never finished, so the outage could not be placed')
+
+
+class TestTheDatabaseComesBack:
+    """Stopped and restarted inside the retry envelope: the verdict still lands."""
+
+    async def test_a_restart_inside_the_envelope_still_records_the_verdict(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_mlflow_server: dict[str, Any],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute_real_clock: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_bff_stopped: Any,
+    ) -> None:
+        """The outage lands on the terminal write, and the server is back before the retries run out.
+
+        The stop is placed after provisioning rather than at the start, so what is being tested is
+        the *terminal* write's retry envelope specifically. A stop at the start would also exercise
+        the claim's envelope, which is a different (and already covered) code path.
+        """
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+        handle = await import_execute_real_clock(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+
+        await _wait_for_provisioning(
+            server=import_mlflow_server,
+            names=import_names,
+            collection=import_models_collection,
+        )
+        # Down for two seconds — several retry attempts, well inside the shortened envelope.
+        async with import_bff_stopped():
+            await asyncio.sleep(2)
+
+        result = await handle.result()
+        assert result['model_name'] == import_names['model_name']
+
+        run_id = _assert_fully_provisioned(
+            server=import_mlflow_server,
+            names=import_names,
+            collection=import_models_collection,
+        )
+        assert result['run_id'] == run_id
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        assert row['status'] == 'IMPORT_SUCCESS', (
+            f'the verdict did not land after the restart: the row says {row["status"]!r}'
+        )
+        assert row['error_message'] is None
+        assert _as_dict(row['orchestrator_response_data'])['import']['workflow_id'] == handle.id
+
+    async def test_a_broken_progress_update_does_not_delay_the_verdict(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_mlflow_server: dict[str, Any],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute_real_clock: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_bff_stopped: Any,
+    ) -> None:
+        """A mid-flight outage that only the progress hints see. The import does not care.
+
+        Progress writes are hints nobody reads — the terminal write rewrites status, code, reason and
+        step from one classification — so they get their own short policy. If they shared the terminal
+        write's envelope, a database the import does not need would stall a working import for
+        minutes at every stage update.
+
+        The outage is placed early, so it lands on hints rather than on the terminal write, and the
+        assertion is that the import still finished with the correct verdict.
+        """
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+        handle = await import_execute_real_clock(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+
+        # Early and brief: the claim has landed, the gates are running, the terminal write is far off.
+        await asyncio.sleep(1.0)
+        async with import_bff_stopped():
+            await asyncio.sleep(1.5)
+
+        started = time.monotonic()
+        result = await handle.result()
+        elapsed = time.monotonic() - started
+
+        assert result['model_name'] == import_names['model_name']
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        assert row['status'] == 'IMPORT_SUCCESS'
+        assert elapsed < 60, (
+            f'the import took {elapsed:.1f}s after the outage ended; a lost hint should cost '
+            f'its own short policy and nothing more'
+        )
+
+
+class TestTheDatabaseStaysDown:
+    """Past the envelope. The model is real, the record is not, and the run says which is which."""
+
+    @pytest.fixture
+    def import_status_retry_shortening(self) -> dict[str, Any]:
+        """Shorter still, because these tests wait for the envelope to *expire*.
+
+        Six half-second attempts inside an eight-second envelope: long enough that retries really
+        happen, short enough that giving up is something a test suite can wait for.
+        """
+        return {
+            'initial_interval': 0.5,
+            'backoff_coefficient': 1.0,
+            'maximum_attempts': 6,
+            'hint_maximum_attempts': 2,
+            'envelope_seconds': 8,
+            'write_timeout_seconds': 3,
+        }
+
+    async def test_a_successful_import_is_not_reported_as_failed(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_mlflow_server: dict[str, Any],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute_real_clock: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_bff_stopped: Any,
+        mock_logger: Any,
+    ) -> None:
+        """The most important assertion in this file.
+
+        The model is registered, the document is written, the artifacts are stored — and the row
+        cannot be updated. The workflow **completes**: reporting failure here would tell somebody to
+        re-import a model that is already on the platform. The verdict goes out through the failure
+        channel instead, under the not-recorded code, carrying what it could not write.
+
+        The outage is placed as soon as the row has been **claimed**, not once provisioning has
+        finished. Waiting for provisioning looks more precise and is in fact a race: the terminal
+        write follows the model document by milliseconds, while `docker stop` takes seconds, so the
+        write lands before the database is actually gone and the test proves nothing. Stopping right
+        after the claim covers everything that follows — the progress hints (which are allowed to
+        fail) and the terminal write (which is the subject) — with no timing to get right.
+        """
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+        handle = await import_execute_real_clock(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+
+        await _wait_for_claim(import_read_status_row, import_run_id)
+        async with import_bff_stopped():
+            result = await handle.result()
+
+        # The run completed. This is the assertion, not an incidental detail.
+        assert result['model_name'] == import_names['model_name']
+        assert result['import_run_id'] == import_run_id
+
+        _assert_fully_provisioned(
+            server=import_mlflow_server,
+            names=import_names,
+            collection=import_models_collection,
+        )
+
+        channel = _channel_text(mock_logger)
+        assert NOT_RECORDED_CODE in channel, (
+            'the unrecordable verdict never reached the failure channel'
+        )
+        assert STATUS_WRITE_MARKER in channel, (
+            'the channel does not mark this as a status-write failure'
+        )
+        assert 'verdict_not_recorded=' in channel, (
+            'the channel reports the failure but not the verdict it could not write'
+        )
+        _assert_no_provisioning_step_is_blamed(
+            '\n'.join(
+                line for line in channel.splitlines() if 'model import failed' in line
+            ).replace('step=model_document', '')
+        )
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        assert row['status'] == RUNNING_STATUS, (
+            f'the row should still be running, not {row["status"]!r}'
+        )
+
+    async def test_a_failed_import_keeps_its_original_cause(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        import_status_table_pre_v12: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_activities: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute_real_clock: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        mock_logger: Any,
+    ) -> None:
+        """An import that already failed, whose failure then cannot be written down either.
+
+        Two failures, and only one of them answers "why did this import not work". The status write
+        is the second, and it must not displace the first: somebody told "the status could not be
+        recorded" would go and look at the database, when what actually happened is that their model's
+        files could not be stored.
+
+        **Why a pre-`V12` table instead of stopping the database.** The outage has to cover the
+        terminal write and nothing earlier, and an import that fails at a gate reaches its terminal
+        write within a second of starting — there is no window to aim at, and the first version of
+        this test proved only that a stopped database *after* a completed write changes nothing.
+        A pre-`V12` table has no timing at all: `IMPORT_ERROR` is a value it has never heard of, so
+        the terminal write of a *failed* import is refused deterministically while every write before
+        it succeeds. The failure being tested is the same one either way — the record cannot take the
+        verdict — and this version cannot race.
+        """
+        client = import_activities.import_mlflow_repository.client
+
+        def refuse(*_args: Any, **_kwargs: Any) -> None:
+            raise OSError('artifact store is unreachable')
+
+        monkeypatch.setattr(client, 'log_artifacts', refuse)
+
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+        handle = await import_execute_real_clock(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        with pytest.raises(WorkflowFailureError) as failure:
+            await handle.result()
+
+        # The reported cause is the step that actually failed.
+        text_of_failure = failure_text(failure.value)
+        assert 'artifact store is unreachable' in text_of_failure, (
+            f'the reported cause lost the original failure: {text_of_failure}'
+        )
+        assert 'ImportStatusRejectedError' not in failure_types(failure.value), (
+            f'the status write displaced the original cause: {failure_types(failure.value)}'
+        )
+        assert 'the import log database refused' not in text_of_failure, (
+            'the run is reported as a status-write failure rather than an artifact-upload failure'
+        )
+
+        # The status write still reported *itself*, through the channel, as the secondary failure.
+        channel = _channel_text(mock_logger)
+        assert NOT_RECORDED_CODE in channel, 'the unrecordable verdict was never reported'
+        assert STATUS_WRITE_MARKER in channel
+
+        # And the row is left running, because the verdict never landed.
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        assert row['status'] == RUNNING_STATUS
+
+
+class TestMissingV12:
+    """A table without `V12`. The evidence for why the migration gates enabling the worker."""
+
+    async def test_the_terminal_write_is_refused_on_its_first_attempt(
+        self,
+        import_status_table_pre_v12: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_mlflow_server: dict[str, Any],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute_real_clock: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_bff_engine: Any,
+        mock_logger: Any,
+    ) -> None:
+        """Provisioned, then unrecordable — and unrecordable *immediately*.
+
+        The shape matters as much as the outcome. `ImportStatusRejectedError` is non-retryable: a
+        `CHECK` violation is not going to start passing because it was tried again, and waiting the
+        full 151-second envelope would turn a misconfiguration into a two-and-a-half-minute stall on
+        every single import. The elapsed time is what proves it — the shortened policy still allows
+        24 attempts, so a retried failure could not finish this fast.
+        """
+        from sqlalchemy import text as sql_text
+
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+
+        started = time.monotonic()
+        handle = await import_execute_real_clock(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        result = await handle.result()
+        elapsed = time.monotonic() - started
+
+        # The run completed: an import that provisioned a model is not a failed import, even when
+        # the last thing it wanted to do was impossible.
+        assert result['model_name'] == import_names['model_name']
+
+        # The claim and the names write both landed — the pre-V12 table was usable right up to the
+        # last statement, which is exactly what makes this failure mode easy to miss.
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        assert row['status'] == RUNNING_STATUS, (
+            f'the row is {row["status"]!r}; a pre-V12 table cannot hold a terminal import status, '
+            f'so the row is left looking as though the import never finished'
+        )
+        assert row['experiment_name'] == import_names['experiment_name']
+        assert row['run_name'] == f'import-{import_names["model_name"]}-v2'
+
+        # The model is fully provisioned. This is the whole problem: the work is done and invisible.
+        _assert_fully_provisioned(
+            server=import_mlflow_server,
+            names=import_names,
+            collection=import_models_collection,
+        )
+
+        # The verdict reached the failure channel under the not-recorded code.
+        channel = _channel_text(mock_logger)
+        assert NOT_RECORDED_CODE in channel
+        assert STATUS_WRITE_MARKER in channel
+        assert 'verdict_not_recorded=' in channel
+
+        # Non-retryable: refused on the first attempt, with no retry wait.
+        assert elapsed < 60, (
+            f'the import took {elapsed:.1f}s; a non-retryable rejection must not be retried'
+        )
+
+        # And the constraint that refused it is the one V12 adds — asserted directly, so the test
+        # says *why* the write was impossible rather than only that it was.
+        with import_bff_engine.begin() as connection:
+            statuses = connection.execute(
+                sql_text(
+                    'SELECT pg_get_constraintdef(oid) FROM pg_constraint '
+                    "WHERE conname = 'chk_experiment_run_status'"
+                )
+            ).scalar_one()
+        assert 'IMPORT_SUCCESS' not in statuses, (
+            'the pre-V12 fixture already allows the import statuses, so this proved nothing'
+        )

e2e/test_import_model_second_import.py

@@ -0,0 +1,272 @@
+"""Importing the same model twice, and what the model document is allowed to say.
+
+Two claims, and they pull in opposite directions, which is why they share a file.
+
+**A second import is additive.** It creates a new run in the *same* experiment and a new registry
+version of the *same* registered model, and it promotes nothing. That is the whole of it: no
+overwrite, no stage change, no reconciliation.
+
+**A second import touches the document not at all.** `active`, `target` and the flavors of a model
+already on the platform are operator-owned — somebody turned that model on, and re-importing it must
+not turn it off, retarget it or reset how it is loaded. So the document is written on the first
+import and left byte-identical on every one after it.
+
+The document's content is asserted exactly rather than loosely, because it *is* the model listing:
+`model_config.transform_flavor`, `model_config.predict_flavor` and `model_config.target` are what
+the runtime reads to know how to load the model, and a missing or wrong one makes an imported model
+either invisible or unloadable.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import mlflow
+import pytest
+
+from e2e.import_harness import SEEDED_MODEL_IDS
+
+pytestmark = [pytest.mark.import_e2e, pytest.mark.asyncio]
+
+FACTORY_PASSWORD = 'Factory1Model!Test'
+CONTRACTED_MODEL_CONFIG_KEYS = {
+    'transform_flavor',
+    'predict_flavor',
+    'retention_minutes',
+    'target',
+    'skip_predict_timestamp',
+}
+
+
+def _client(server: dict[str, Any]) -> Any:
+    return mlflow.tracking.MlflowClient(tracking_uri=server['tracking_uri'])
+
+
+async def _import_once(
+    *,
+    payload: bytes,
+    import_upload_bundle: Any,
+    import_insert_status_row: Any,
+    import_build_input: Any,
+    import_execute: Any,
+    import_envelope: Any,
+) -> dict[str, Any]:
+    """One successful import. Raises if it did not succeed, so a caller never asserts on a failure."""
+    object_key, digest = import_upload_bundle(payload)
+    import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+    handle = await import_execute(
+        import_build_input(
+            import_run_id=import_run_id,
+            object_key=object_key,
+            expected_digest=digest,
+            password_envelope=import_envelope(FACTORY_PASSWORD),
+        )
+    )
+    result = await handle.result()
+    return {'result': result, 'import_run_id': import_run_id, 'object_key': object_key}
+
+
+class TestSecondImport:
+    """The same model, imported twice in one test — the only way to observe the second as second."""
+
+    async def test_a_second_import_adds_a_version_and_a_run_and_nothing_else(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_mlflow_server: dict[str, Any],
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        """Both bundles declare the same model and experiment, and different `model_version`s.
+
+        The second bundle declaring `model_version` '7' while the first declared '2' is the point of
+        task 8.2: the destination version number is the registry's own count, and the bundle's
+        number is a record of where the model came from, logged as `source.model_version` and used
+        for nothing else. If the two were confused, the second version here would be '7'.
+        """
+        first = await _import_once(
+            payload=import_bundle_factory(metadata={'model_version': '2'}),
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+        document_after_first = import_models_collection.find_one(
+            {'name': import_names['model_name']}, {'_id': False}
+        )
+        assert document_after_first is not None, 'the first import wrote no document'
+
+        second = await _import_once(
+            payload=import_bundle_factory(metadata={'model_version': '7'}),
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+
+        model_name = import_names['model_name']
+        client = _client(import_mlflow_server)
+
+        # 8.1 — a new version of the same registered model.
+        assert first['result']['version'] == '1'
+        assert second['result']['version'] == '2'
+        versions = list(client.search_model_versions(f"name='{model_name}'"))
+        assert len(versions) == 2, f'expected two versions, found {len(versions)}'
+        assert {v.version for v in versions} == {'1', '2'}
+
+        # 8.2 — destination numbering is independent of what the bundles declared.
+        assert second['result']['version'] != '7'
+        runs_by_id = {
+            run.info.run_id: run
+            for run in client.search_runs(
+                [client.get_experiment_by_name(import_names['experiment_name']).experiment_id]
+            )
+        }
+        assert runs_by_id[first['result']['run_id']].data.params['source.model_version'] == '2'
+        assert runs_by_id[second['result']['run_id']].data.params['source.model_version'] == '7'
+
+        # 8.1 — a new run, in the *same* experiment, and both runs are still there.
+        assert first['result']['run_id'] != second['result']['run_id']
+        assert len(runs_by_id) == 2
+        assert {run.info.run_name for run in runs_by_id.values()} == {
+            f'import-{model_name}-v2',
+            f'import-{model_name}-v7',
+        }
+
+        # 8.1 — nothing promoted, by either import.
+        assert all(version.current_stage in (None, 'None') for version in versions), [
+            (v.version, v.current_stage) for v in versions
+        ]
+
+        # 8.3 — the document is byte-identical to what the first import wrote.
+        document_after_second = import_models_collection.find_one(
+            {'name': model_name}, {'_id': False}
+        )
+        assert document_after_second == document_after_first
+
+        # And both rows record their own success.
+        for run in (first, second):
+            row = import_read_status_row(run['import_run_id'])
+            assert row is not None
+            assert row['status'] == 'IMPORT_SUCCESS'
+
+
+class TestModelDocumentContent:
+    """What the written document says, asserted exactly rather than by presence."""
+
+    async def test_the_document_continues_the_collection_and_carries_the_contract(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+    ) -> None:
+        """`id` is `max(int(id)) + 1` across gapped ids — 1, 2, 7 give 8, not 4 and not "3".
+
+        The gaps are what make this worth asserting: a count would give 4, and a lexicographic
+        maximum would give "8" here but "10" for a collection holding "9" and "30". The seeded ids
+        are strings, as the baseline migration stores them, so `int()` is doing real work.
+        """
+        await _import_once(
+            payload=import_bundle_factory(parameters={'target_variable': 'PRODUCT_QUALITY'}),
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+
+        document = import_models_collection.find_one(
+            {'name': import_names['model_name']}, {'_id': False}
+        )
+        assert document is not None
+
+        expected_id = str(max(int(seeded) for seeded in SEEDED_MODEL_IDS) + 1)
+        assert document['id'] == expected_id, (
+            f'{document["id"]!r} does not continue {SEEDED_MODEL_IDS}'
+        )
+        assert document['id'] == '8'
+
+        assert document['name'] == import_names['model_name']
+        assert document['active'] is False
+
+        assert set(document['model_config']) == CONTRACTED_MODEL_CONFIG_KEYS, (
+            f'model_config holds {sorted(document["model_config"])}, not exactly the '
+            f'contracted keys'
+        )
+        assert document['model_config'] == {
+            'transform_flavor': 'sklearn',
+            'predict_flavor': 'joblib',
+            'retention_minutes': 60,
+            'target': 'PRODUCT_QUALITY',
+            'skip_predict_timestamp': True,
+        }
+
+        # The document has exactly these fields and no others — nothing was smuggled in alongside.
+        assert set(document) == {'id', 'name', 'active', 'model_config'}
+
+    async def test_a_name_already_in_the_collection_is_left_untouched(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        """An operator-owned document with this model's name survives the import unchanged.
+
+        `active: True` and a hand-set target are exactly what a re-import must not undo: somebody
+        turned this model on and pointed it at something. The import notices the name and stops.
+        """
+        existing = {
+            'id': '42',
+            'name': import_names['model_name'],
+            'active': True,
+            'model_config': {
+                'transform_flavor': 'sklearn',
+                'predict_flavor': 'pyfunc',
+                'retention_minutes': 5,
+                'target': 'OPERATOR_CHOSE_THIS',
+            },
+        }
+        import_models_collection.insert_one(dict(existing))
+        before = list(import_models_collection.find({}, {'_id': False}))
+
+        outcome = await _import_once(
+            payload=import_bundle_factory(parameters={'target_variable': 'SOMETHING_ELSE'}),
+            import_upload_bundle=import_upload_bundle,
+            import_insert_status_row=import_insert_status_row,
+            import_build_input=import_build_input,
+            import_execute=import_execute,
+            import_envelope=import_envelope,
+        )
+
+        # The import still succeeded — a document that is already there is not a failure.
+        row = import_read_status_row(outcome['import_run_id'])
+        assert row is not None
+        assert row['status'] == 'IMPORT_SUCCESS'
+
+        after = list(import_models_collection.find({}, {'_id': False}))
+        assert after == before, 'the collection changed'
+
+        matching = [d for d in after if d['name'] == import_names['model_name']]
+        assert len(matching) == 1, 'a duplicate was inserted'
+        assert matching[0] == existing

e2e/test_import_model_status_record.py

@@ -0,0 +1,602 @@
+"""The import log row: who owns it, when, and which columns the import is allowed to touch.
+
+This is the file where the *real* `CHECK` constraints matter. Everything here would pass against a
+table without `V12` or without the length and pattern checks, and none of it would mean anything.
+
+Four claims:
+
+- **Ordering.** The claim is the workflow's first act, before any gate runs, and the names are on
+  the row while the import is still in flight — because a person watching the import list has to see
+  *which* model is being imported while it is being imported.
+- **Ownership.** The columns the inserting side owns — `username`, `request_data`, `bucket_name`,
+  `file_name`, `run_type`, `created_at` — are never rewritten. The import updates status, the
+  reason, the two names and the JSON detail, and nothing else.
+- **Refusal.** A row this workflow may not own is not written to at all. Not with an error, not with
+  a status: the loudness goes to the Temporal error, the log and the metric, because a row that
+  belongs to somebody else is not the place to be loud.
+- **Conformance.** Every status value an import writes is one the real constraint accepts, a
+  training row's vocabulary still works beside it, and the feature added no column.
+
+Two observations here are made from *inside* a running import, by wrapping the first call the step
+under observation makes. That is a patch, and it is an observer rather than a failure injector: it
+snapshots the row and delegates. Ordering cannot be asserted from a finished run — the terminal
+write is precisely what erases the intermediate state — so the assertion has to stand somewhere in
+the middle.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import pytest
+from sqlalchemy import text
+from sqlalchemy.exc import IntegrityError
+from temporalio.client import WorkflowFailureError
+
+from e2e.import_harness import IMPORT_PREFIX, failure_types
+from laborious.utils.bundle.steps import ImportStep
+from laborious.utils.models.import_errors import IMPORT_FAILURE_CATALOG
+
+# `asyncio` is applied per class rather than to the whole module: `TestConstraintConformance`
+# talks to Postgres directly and its tests are synchronous, and a module-wide `asyncio` mark on
+# a sync test is a pytest warning, not a no-op.
+pytestmark = [pytest.mark.import_e2e]
+
+FACTORY_PASSWORD = 'Factory1Model!Test'
+RUNNING_STATUS = 'ORCHESTRATOR_WAITING_PROC'
+VALIDATION_ERROR_STATUS = 'ORCHESTRATOR_VALIDATION_ERROR'
+
+# The columns the inserting side owns. The import may never rewrite one of these.
+INSERTER_OWNED_COLUMNS = (
+    'username',
+    'request_data',
+    'bucket_name',
+    'file_name',
+    'run_type',
+    'created_at',
+)
+
+# The 13 columns V1…V12 leave. D21 says the feature adds none; this is what checks it.
+EXPECTED_COLUMNS = {
+    'id',
+    'experiment_name',
+    'run_name',
+    'username',
+    'status',
+    'error_message',
+    'created_at',
+    'updated_at',
+    'bucket_name',
+    'file_name',
+    'request_data',
+    'orchestrator_response_data',
+    'run_type',
+}
+
+
+def _as_dict(value: Any) -> dict[str, Any]:
+    return json.loads(value) if isinstance(value, str) else (value or {})
+
+
+@pytest.mark.asyncio
+class TestOrdering:
+    """When each write happens, observed from inside the run rather than after it."""
+
+    async def test_the_row_is_claimed_before_the_first_gate_runs(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_activities: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        """Snapshot the row from inside `stat_object` — the first call gate 1 makes.
+
+        Standing here rather than after the run is not a convenience: the terminal write rewrites
+        status, reason and detail, so a finished row cannot say what it looked like before the gates.
+        `stat_object` is patched with a wrapper that records and then delegates — an observer, not a
+        failure. Everything the import does still really happens.
+        """
+        client = import_activities.import_minio_repository.minio_client
+        real_stat = client.stat_object
+        snapshots: list[dict[str, Any] | None] = []
+
+        def observing_stat(bucket: str, key: str, *args: Any, **kwargs: Any) -> Any:
+            snapshots.append(import_read_status_row(observed_run_id[0]))
+            return real_stat(bucket, key, *args, **kwargs)
+
+        observed_run_id: list[int] = [0]
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        observed_run_id[0] = import_insert_status_row(object_key=object_key, expected_digest=digest)
+        client.stat_object = observing_stat
+        try:
+            handle = await import_execute(
+                import_build_input(
+                    import_run_id=observed_run_id[0],
+                    object_key=object_key,
+                    expected_digest=digest,
+                    password_envelope=import_envelope(FACTORY_PASSWORD),
+                )
+            )
+            await handle.result()
+        finally:
+            client.stat_object = real_stat
+
+        assert snapshots, 'gate 1 never ran, so nothing was observed'
+        claimed = snapshots[0]
+        assert claimed is not None
+        assert claimed['status'] == RUNNING_STATUS, (
+            f'at the first gate the row was {claimed["status"]!r}, not the running status'
+        )
+        detail = _as_dict(claimed['orchestrator_response_data'])
+        assert detail['import']['workflow_id'] == handle.id
+        assert detail['import']['step'] in (
+            ImportStep.RECEIVED.value,
+            ImportStep.DOWNLOAD.value,
+        ), detail
+
+    async def test_the_names_are_on_the_row_while_the_import_is_still_running(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_activities: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        """Observed at the first call registration makes, so the experiment and run already exist.
+
+        Download, decryption, experiment creation and artifact upload are fused into one activity
+        attempt (`model-import-status-record`'s design, decision on the fused step): the names are
+        not known to the workflow, and so cannot be recorded, until that whole activity returns —
+        the guarantee is anchored to the two steps that follow it, registration and the model
+        document, not to the experiment or run, which by construction already exist by then.
+
+        This is the stronger form of the same claim task 7.1 makes from a failure: here the import
+        *succeeds*, and the names were still already written before registration ran — so their
+        presence is not an artefact of the run having stopped.
+        """
+        client = import_activities.import_mlflow_repository.client
+        real_get_registered_model = client.get_registered_model
+        snapshots: list[dict[str, Any] | None] = []
+        observed_run_id: list[int] = [0]
+
+        def observing_get_registered_model(*args: Any, **kwargs: Any) -> Any:
+            snapshots.append(import_read_status_row(observed_run_id[0]))
+            return real_get_registered_model(*args, **kwargs)
+
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        observed_run_id[0] = import_insert_status_row(object_key=object_key, expected_digest=digest)
+        client.get_registered_model = observing_get_registered_model
+        try:
+            handle = await import_execute(
+                import_build_input(
+                    import_run_id=observed_run_id[0],
+                    object_key=object_key,
+                    expected_digest=digest,
+                    password_envelope=import_envelope(FACTORY_PASSWORD),
+                )
+            )
+            await handle.result()
+        finally:
+            client.get_registered_model = real_get_registered_model
+
+        assert snapshots, 'registration never ran'
+        mid_flight = snapshots[0]
+        assert mid_flight is not None
+        assert mid_flight['status'] == RUNNING_STATUS
+        assert mid_flight['experiment_name'] == import_names['experiment_name']
+        assert mid_flight['run_name'] == f'import-{import_names["model_name"]}-v2'
+
+
+@pytest.mark.asyncio
+class TestOwnership:
+    """The inserting side's columns survive an import untouched, whatever the outcome."""
+
+    @pytest.mark.parametrize('outcome', ['success', 'failure'])
+    async def test_the_inserted_columns_are_never_rewritten(
+        self,
+        outcome: str,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        payload = (
+            import_bundle_factory()
+            if outcome == 'success'
+            else import_bundle_factory(include_metadata=False)
+        )
+        object_key, digest = import_upload_bundle(payload)
+        import_run_id = import_insert_status_row(
+            object_key=object_key,
+            expected_digest=digest,
+            username='owner.of.the.row@example.com',
+        )
+
+        # Snapshotted before the run, not after: `import_execute` waits for the workflow, so a
+        # snapshot taken from its return value would already be the post-import row and every
+        # comparison below would be a row against itself.
+        before = import_read_status_row(import_run_id)
+        assert before is not None
+        assert before['status'] == 'PENDING'
+
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+
+        if outcome == 'success':
+            await handle.result()
+        else:
+            with pytest.raises(WorkflowFailureError):
+                await handle.result()
+
+        after = import_read_status_row(import_run_id)
+        assert after is not None
+        for column in INSERTER_OWNED_COLUMNS:
+            assert after[column] == before[column], (
+                f'{column} was rewritten: {before[column]!r} -> {after[column]!r}'
+            )
+
+        # And the import *did* write what it owns, so this is not passing by doing nothing at all.
+        assert after['status'] != before['status']
+        assert after['updated_at'] >= before['updated_at']
+
+
+@pytest.mark.asyncio
+class TestClaimGuards:
+    """A row this workflow may not own is not written to. At all."""
+
+    async def test_an_unknown_id_fails_the_run_and_writes_nothing(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_bff_engine: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        # A real, claimable row exists — so "nothing was written" is a claim about *this* run
+        # choosing the right row, not about the table being empty.
+        bystander_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+        before = _snapshot_table(import_bff_engine)
+
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=bystander_id + 10_000,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        with pytest.raises(WorkflowFailureError) as failure:
+            await handle.result()
+
+        assert 'ImportLogRowNotClaimableError' in failure_types(failure.value)
+        assert _snapshot_table(import_bff_engine) == before, 'the table changed'
+        import_assert_nothing_provisioned(
+            experiment_name=import_names['experiment_name'],
+            model_name=import_names['model_name'],
+            object_key=object_key,
+        )
+
+    async def test_a_training_row_cannot_be_claimed_by_an_import(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_bff_engine: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        """`run_type='TRAIN'` is a different feature's row, and the claim will not take it."""
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        training_id = import_insert_status_row(
+            object_key=object_key,
+            expected_digest=digest,
+            run_type='TRAIN',
+            experiment_name='a-training-experiment',
+        )
+        before = _snapshot_table(import_bff_engine)
+
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=training_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        with pytest.raises(WorkflowFailureError) as failure:
+            await handle.result()
+
+        assert 'ImportLogRowNotClaimableError' in failure_types(failure.value)
+        assert _snapshot_table(import_bff_engine) == before
+        import_assert_nothing_provisioned(
+            experiment_name=import_names['experiment_name'],
+            model_name=import_names['model_name'],
+            object_key=object_key,
+        )
+
+    async def test_a_row_held_by_another_workflow_cannot_be_taken(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_bff_engine: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        """Already running under somebody else's workflow id.
+
+        The claim is idempotent for *its own* workflow id and refuses every other, which is what
+        makes a duplicated workflow start safe and a stolen row impossible.
+        """
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        held_id = import_insert_status_row(
+            object_key=object_key,
+            expected_digest=digest,
+            status=RUNNING_STATUS,
+            orchestrator_response_data={
+                'import': {'workflow_id': 'import-held-by-somebody-else', 'step': 'received'}
+            },
+        )
+        before = _snapshot_table(import_bff_engine)
+
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=held_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        with pytest.raises(WorkflowFailureError) as failure:
+            await handle.result()
+
+        assert 'ImportLogRowNotClaimableError' in failure_types(failure.value)
+        assert _snapshot_table(import_bff_engine) == before
+        import_assert_nothing_provisioned(
+            experiment_name=import_names['experiment_name'],
+            model_name=import_names['model_name'],
+            object_key=object_key,
+        )
+
+
+@pytest.mark.asyncio
+class TestInputAgainstTheRow:
+    """A request that contradicts the row it claimed — recorded *on* that row, because it owns it."""
+
+    async def test_a_file_name_disagreeing_with_the_object_key_is_recorded_on_the_row(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_names: dict[str, str],
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+        import_assert_nothing_provisioned: Any,
+    ) -> None:
+        """The difference from the claim guards above is where the error goes.
+
+        A row that could not be claimed is somebody else's, so nothing is written to it. This row
+        *was* claimed — the contradiction is only found afterwards — so the failure belongs on it,
+        where the person who submitted the import will see it.
+        """
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        import_run_id = import_insert_status_row(
+            object_key=f'{IMPORT_PREFIX}a-different-object.sientia',
+            expected_digest=digest,
+        )
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        with pytest.raises(WorkflowFailureError):
+            await handle.result()
+
+        row = import_read_status_row(import_run_id)
+        assert row is not None
+        code, reason, _responsibility = IMPORT_FAILURE_CATALOG[ImportStep.RECEIVED]
+        assert row['status'] == VALIDATION_ERROR_STATUS
+        assert row['error_message'] == reason
+        detail = _as_dict(row['orchestrator_response_data'])['import']
+        assert detail['step'] == ImportStep.RECEIVED.value
+        assert detail['code'] == code.value
+        # The record still names the object *it* was given, not the one the request named.
+        assert row['file_name'] == f'{IMPORT_PREFIX}a-different-object.sientia'
+        import_assert_nothing_provisioned(
+            experiment_name=import_names['experiment_name'],
+            model_name=import_names['model_name'],
+            object_key=object_key,
+        )
+
+
+@pytest.mark.asyncio
+class TestTrainingRowsAreUntouched:
+    """A training row sitting in the same table is never modified by an import."""
+
+    async def test_a_training_row_beside_an_import_is_unchanged(
+        self,
+        import_status_table: Any,
+        import_models_collection: Any,
+        import_bundle_factory: Any,
+        import_upload_bundle: Any,
+        import_insert_status_row: Any,
+        import_build_input: Any,
+        import_execute: Any,
+        import_envelope: Any,
+        import_read_status_row: Any,
+    ) -> None:
+        object_key, digest = import_upload_bundle(import_bundle_factory())
+        training_id = import_insert_status_row(
+            object_key=object_key,
+            expected_digest=digest,
+            run_type='TRAIN',
+            status='TRAINING_SUCCESS',
+            experiment_name='someone-elses-training',
+            username='trainer@example.com',
+        )
+        training_before = import_read_status_row(training_id)
+
+        import_run_id = import_insert_status_row(object_key=object_key, expected_digest=digest)
+        handle = await import_execute(
+            import_build_input(
+                import_run_id=import_run_id,
+                object_key=object_key,
+                expected_digest=digest,
+                password_envelope=import_envelope(FACTORY_PASSWORD),
+            )
+        )
+        await handle.result()
+
+        assert import_read_status_row(import_run_id)['status'] == 'IMPORT_SUCCESS'
+        assert import_read_status_row(training_id) == training_before
+
+
+class TestConstraintConformance:
+    """The real constraints, exercised directly — and the column list D21 promised not to change."""
+
+    def test_every_status_an_import_writes_is_accepted_on_an_import_row(
+        self, import_status_table: Any, import_insert_status_row: Any, import_bff_engine: Any
+    ) -> None:
+        """`PENDING`, the running value, and both import terminals, on `run_type='IMPORT'`."""
+        row_id = import_insert_status_row(
+            object_key=f'{IMPORT_PREFIX}conformance.sientia', expected_digest='a' * 64
+        )
+        for status in ('PENDING', RUNNING_STATUS, VALIDATION_ERROR_STATUS, 'IMPORT_SUCCESS'):
+            _set_status(import_bff_engine, row_id, status)
+        _set_status(import_bff_engine, row_id, 'IMPORT_ERROR')
+
+    def test_the_terminal_values_do_not_cross_run_types(
+        self, import_status_table: Any, import_insert_status_row: Any, import_bff_engine: Any
+    ) -> None:
+        """`chk_experiment_run_terminal_status_run_type`, in both directions.
+
+        This is the constraint `V12` adds, and it is the reason the migration gates enabling the
+        worker: without it an import could write `TRAINING_SUCCESS` and a training run could write
+        `IMPORT_ERROR`, and the import list would be unreadable.
+        """
+        import_row = import_insert_status_row(
+            object_key=f'{IMPORT_PREFIX}crossing.sientia', expected_digest='b' * 64
+        )
+        training_row = import_insert_status_row(
+            object_key=f'{IMPORT_PREFIX}crossing.sientia',
+            expected_digest='b' * 64,
+            run_type='TRAIN',
+        )
+
+        for row_id, forbidden in (
+            (import_row, 'TRAINING_SUCCESS'),
+            (import_row, 'TRAINING_ERROR'),
+            (training_row, 'IMPORT_SUCCESS'),
+            (training_row, 'IMPORT_ERROR'),
+        ):
+            with pytest.raises(IntegrityError) as refusal:
+                _set_status(import_bff_engine, row_id, forbidden)
+            assert 'chk_experiment_run_terminal_status_run_type' in str(refusal.value)
+
+        # The shared stage values are deliberately outside the pairing: both run types pass through.
+        for row_id in (import_row, training_row):
+            for shared in ('PENDING', RUNNING_STATUS, VALIDATION_ERROR_STATUS):
+                _set_status(import_bff_engine, row_id, shared)
+
+    def test_a_training_row_still_walks_its_own_vocabulary(
+        self, import_status_table: Any, import_insert_status_row: Any, import_bff_engine: Any
+    ) -> None:
+        training_row = import_insert_status_row(
+            object_key=f'{IMPORT_PREFIX}training.sientia',
+            expected_digest='c' * 64,
+            run_type='TRAIN',
+        )
+        for status in ('PENDING', RUNNING_STATUS, 'TRAINING_SUCCESS'):
+            _set_status(import_bff_engine, training_row, status)
+        _set_status(import_bff_engine, training_row, 'TRAINING_ERROR')
+
+    def test_the_feature_added_no_column(
+        self, import_status_table: Any, import_bff_engine: Any
+    ) -> None:
+        """Design D21: the error code and the workflow id live in the existing JSONB column.
+
+        A quiet `ALTER TABLE` would be the easiest way for this feature to break the BFF that owns
+        the table, and the hardest to notice, so it is asserted rather than trusted.
+        """
+        with import_bff_engine.begin() as connection:
+            columns = {
+                row[0]
+                for row in connection.execute(
+                    text(
+                        'SELECT column_name FROM information_schema.columns '
+                        "WHERE table_schema = 'public' AND table_name = 'experiment_run'"
+                    )
+                )
+            }
+        assert columns == EXPECTED_COLUMNS, (
+            f'unexpected: {sorted(columns - EXPECTED_COLUMNS)}, '
+            f'missing: {sorted(EXPECTED_COLUMNS - columns)}'
+        )
+
+
+def _set_status(engine: Any, row_id: int, status: str) -> None:
+    """Write one status onto one row, so the real constraint gets to answer."""
+    with engine.begin() as connection:
+        connection.execute(
+            text('UPDATE public.experiment_run SET status = :status WHERE id = :id'),
+            {'status': status, 'id': row_id},
+        )
+
+
+def _snapshot_table(engine: Any) -> list[dict[str, Any]]:
+    """Every row, ordered, as plain values — for "the table is byte-identical" assertions."""
+    with engine.begin() as connection:
+        return [
+            dict(row)
+            for row in connection.execute(
+                text('SELECT * FROM public.experiment_run ORDER BY id')
+            ).mappings()
+        ]

e2e/test_minio_offload.py

@@ -0,0 +1,136 @@
+"""
+E2E-style tests for MinIO offload using a real MinIO testcontainer.
+"""
+
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import insert_sample_data, make_workflow_id, start_and_await_workflow
+from laborious.activities.activities import Activities
+from laborious.utils.models import minio_dataframe_payload as mdp
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_load_query_with_minio_offload_writes_object_to_bucket(
+    postgres_engine,
+    minio_container,
+    test_activities_real_minio: Activities,
+):
+    """
+    With a tiny offload threshold, query results are uploaded as Parquet to MinIO.
+
+    Uses real MinioRepository against testcontainers MinIO (no MinIO mock).
+    """
+    model_id = 501
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}')
+        )
+    insert_sample_data(postgres_engine, model_id, [1.0, 2.0])
+
+    metadata = {
+        'metadata': {
+            'schedule_name': 'test-schedule',
+            'model_name': 'test_model',
+            'model_id': model_id,
+            'workflow_name': 'predictions_batch',
+        }
+    }
+    with patch.object(mdp, 'OFFLOAD_THRESHOLD_BYTES', 1):
+        payload = await test_activities_real_minio.load_query_with_minio_offload(
+            {
+                **metadata,
+                'query': (
+                    'SELECT timestamp, variable, value, created_at '
+                    f'FROM predictions_schema.laborious_data WHERE model_id = {model_id}'
+                ),
+                'model_name': 'test_model',
+                'datetime_columns': ['timestamp', 'created_at'],
+            }
+        )
+    assert payload.object_key, 'offloaded payload must reference a MinIO object'
+    assert payload.data is None or payload.data == {}, (
+        'large payloads should not inline tabular dict'
+    )
+
+    df = await payload.retrieve(test_activities_real_minio.minio_repository, metadata['metadata'])
+    assert len(df) >= 1
+
+    client = minio_container.get_client()
+    listed = list(client.list_objects('test-bucket', recursive=True))
+    names = [getattr(o, 'object_name', None) or getattr(o, '_object_name', '') for o in listed]
+    assert any(n and 'prediction_datasets' in n for n in names), (
+        f'unexpected object listing: {names!r}'
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_predictions_batch_with_minio_offload_path(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_minio: Worker,
+    postgres_engine,
+    test_activities_real_minio: Activities,
+):
+    """
+    Full PredictionsBatch run with offload: load step stores Parquet in MinIO; pipeline completes.
+    """
+    model_id = 502
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}')
+        )
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}')
+        )
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}')
+        )
+    insert_sample_data(postgres_engine, model_id, [10.0, 20.0, 30.0])
+
+    input_data = {
+        'schedule_name': 'test-schedule',
+        'model_name': 'test_model',
+        'model_id': model_id,
+        'query': (
+            'SELECT timestamp, variable, value, created_at '
+            f'FROM predictions_schema.laborious_data WHERE model_id = {model_id}'
+        ),
+        'schema': 'predictions_schema',
+        'table_name': 'predictions',
+        'transform_table_name': 'transformed_data',
+        'input_filters': {'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}}},
+        'mlflow_transform_filters': {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}},
+        'mlflow_predict_filters': {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}},
+        'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
+        'opc_output_config': {},
+        'pi_web_api_output_config': {},
+        'save_transform': True,
+        'prediction_store_policy': 'lts:1',
+        'model_config': {
+            'retention_minutes': 0,
+            'transform_flavor': 'sklearn',
+            'predict_flavor': 'sklearn',
+        },
+        'datetime_columns': ['timestamp', 'created_at'],
+    }
+
+    with patch.object(mdp, 'OFFLOAD_THRESHOLD_BYTES', 1):
+        await start_and_await_workflow(
+            temporal_test_env.client,
+            PredictionsBatch.run,
+            input_data,
+            make_workflow_id('test-batch-minio-offload'),
+        )
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text(f'SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = {model_id}')
+        ).scalar()
+    assert count == 1

e2e/test_opc_real_server.py

@@ -0,0 +1,211 @@
+"""
+E2E tests for OPC export using an in-process asyncua server and real OpcRepository.
+
+Covers scenarios 3.1.2, 3.2.2, 3.2.4, and 3.2.5 from e2e/scenarios.md.
+Mock-based OPC tests remain in test_predictions_batch_format_export.py.
+"""
+
+import asyncio
+
+import pytest
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import (
+    assert_prediction,
+    insert_sample_data,
+    make_workflow_id,
+    start_and_await_workflow,
+)
+from e2e.opc_test_server import UNKNOWN_NODE_ID, OpcE2ETestServer, build_opc_output_config
+from e2e.test_predictions_batch_format_export import get_base_input_data
+from laborious.activities.activities import Activities
+from laborious.activities.opc import OPC_RECONNECT_IN_PROGRESS_COMMENT
+from laborious.utils.repository.opc_repository import OpcRepository
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+
+async def _reconnect_under_held_lock(
+    repo: OpcRepository, acquired: asyncio.Event, release: asyncio.Event
+) -> None:
+    """
+    Hold the connection lock until released, then reconnect.
+
+    The lock must stay held for the whole workflow, and how long the workflow takes to
+    reach the OPC write is not bounded — a fixed sleep here lets `_reconnect_locked`
+    finish first, which sets `_session_ready` again and the write then succeeds.
+
+    Args:
+        repo (OpcRepository): Connected repository.
+        acquired (asyncio.Event): Set once the lock is held, before any write can race.
+        release (asyncio.Event): Awaited while holding the lock; set by the caller.
+    """
+    async with repo._connection_lock:
+        acquired.set()
+        await release.wait()
+        await repo._reconnect_locked()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.opc
+async def test_scenario_3_1_2_export_with_opc_only_real_server(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_opc: Worker,
+    test_activities_real_opc: Activities,
+    opc_e2e_server: OpcE2ETestServer,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.2 (real OPC): connect, write prediction and confidence, verify server values.
+    """
+    client = temporal_test_env.client
+    model_id = 412
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = build_opc_output_config(opc_e2e_server.node_ids)
+    input_data['pi_web_api_output_config'] = None
+
+    await start_and_await_workflow(
+        client,
+        PredictionsBatch.run,
+        input_data,
+        make_workflow_id('test-opc-real-happy'),
+    )
+
+    test_activities_real_opc.pi_web_api_client.write_value.assert_not_called()
+    assert await opc_e2e_server.read_prediction() == pytest.approx(0.5)
+    assert await opc_e2e_server.read_confidence() == pytest.approx(0.0)
+    assert_prediction(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.opc
+async def test_scenario_3_2_2_opc_write_error_real_server(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_opc: Worker,
+    test_activities_real_opc: Activities,
+    opc_e2e_server: OpcE2ETestServer,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.2 (real OPC): unknown NodeId yields generic write failure (confidence 12).
+    """
+    client = temporal_test_env.client
+    model_id = 422
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    node_ids = opc_e2e_server.node_ids
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = build_opc_output_config(
+        node_ids,
+        prediction_tag=UNKNOWN_NODE_ID,
+        confidence_tag=UNKNOWN_NODE_ID,
+    )
+    input_data['pi_web_api_output_config'] = None
+
+    await start_and_await_workflow(
+        client,
+        PredictionsBatch.run,
+        input_data,
+        make_workflow_id('test-opc-real-bad-node'),
+    )
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=12,
+        comments='Some data could not be written to OPC servers',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.opc
+async def test_scenario_3_2_4_opc_session_bad_real_server(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_opc: Worker,
+    test_activities_real_opc: Activities,
+    opc_e2e_server: OpcE2ETestServer,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.4 (real OPC): server PreWrite fault injects BadSessionIdInvalid (confidence 14).
+    """
+    client = temporal_test_env.client
+    model_id = 424
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    opc_e2e_server.set_session_bad_on_write(True)
+    try:
+        input_data = get_base_input_data(model_id)
+        input_data['opc_output_config'] = build_opc_output_config(
+            opc_e2e_server.node_ids,
+            prediction_only=True,
+        )
+        input_data['pi_web_api_output_config'] = None
+
+        await start_and_await_workflow(
+            client,
+            PredictionsBatch.run,
+            input_data,
+            make_workflow_id('test-opc-real-session-bad'),
+        )
+    finally:
+        opc_e2e_server.set_session_bad_on_write(False)
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=14,
+        comments_contains='OPC UA session/channel error: BadSessionIdInvalid',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.opc
+async def test_scenario_3_2_5_opc_write_blocked_during_reconnect_real_server(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_opc: Worker,
+    test_activities_real_opc: Activities,
+    opc_e2e_server: OpcE2ETestServer,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.5 (real OPC): writes rejected while reconnect holds the connection lock.
+    """
+    client = temporal_test_env.client
+    model_id = 425
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    repo = test_activities_real_opc.opc_repository['1']
+    repo._session_ready.clear()
+    acquired = asyncio.Event()
+    release = asyncio.Event()
+    reconnect_task = asyncio.create_task(_reconnect_under_held_lock(repo, acquired, release))
+    await acquired.wait()
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = build_opc_output_config(opc_e2e_server.node_ids)
+    input_data['pi_web_api_output_config'] = None
+
+    try:
+        await start_and_await_workflow(
+            client,
+            PredictionsBatch.run,
+            input_data,
+            make_workflow_id('test-opc-real-reconnect-block'),
+        )
+    finally:
+        release.set()
+        await reconnect_task
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=14,
+        comments_contains=OPC_RECONNECT_IN_PROGRESS_COMMENT,
+    )

e2e/test_predictions_batch_format_export.py

@@ -0,0 +1,800 @@
+"""
+End-to-end tests for PredictionsBatch workflow - Format and Export scenarios.
+"""
+
+from decimal import Decimal
+from typing import Any, cast
+from unittest.mock import AsyncMock, call
+
+import pytest
+from sientia_do.notifications.models import NotificationLevel
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import (
+    assert_prediction,
+    insert_sample_data,
+    make_workflow_id,
+    start_and_await_workflow,
+)
+from laborious.activities.activities import Activities
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+base_input_data = {
+    'schedule_name': 'test-schedule',
+    'model_name': 'test_model',
+    'model_id': 301,
+    'query': 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = 301',
+    'schema': 'predictions_schema',
+    'table_name': 'predictions',
+    'transform_table_name': 'transformed_data',
+    'input_filters': {
+        'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}},
+    },
+    'mlflow_transform_filters': {
+        'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+    },
+    'mlflow_predict_filters': {
+        'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+    },
+    'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
+    'opc_output_config': {},
+    'pi_web_api_output_config': {},
+    'save_transform': True,
+    'prediction_store_policy': 'lts:1',
+    'model_config': {
+        'retention_minutes': 0,
+        'transform_flavor': 'sklearn',
+        'predict_flavor': 'sklearn',
+    },
+    'datetime_columns': ['timestamp', 'created_at'],
+}
+
+base_query = 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = {model_id}'
+
+
+def get_base_input_data(model_id):
+    return {
+        **base_input_data,
+        'model_id': model_id,
+        'query': base_query.format(model_id=model_id),
+    }
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_1_default_prediction_export(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.1: Default prediction export (non-None path_flag).
+
+    Triggers input_gate CONTINUE via SPECIFIC_VARIABLES_NULL_VALUES so
+    PredictionProcess calls FormatAndExportPrediction with path_flag set.
+    That workflow uses format_default_prediction (not format_prediction) and
+    skips format_transformed_data / transform Postgres export.
+
+    Optional PI Web API and OPC outputs still run when configured.
+    """
+    client = temporal_test_env.client
+
+    model_id = 311
+
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}')
+        )
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}')
+        )
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters'] = {
+        'SPECIFIC_VARIABLES_NULL_VALUES': {
+            'POLICY': 'CONTINUE',
+            'CONFIG': {'variables': ['sensor_1']},
+        },
+    }
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+
+    wid = make_workflow_id('test-default-prediction')
+
+    await start_and_await_workflow(client, PredictionsBatch.run, input_data, wid)
+
+    test_activities.pi_web_api_client.write_value.assert_has_calls(
+        [
+            call(
+                web_ids=['web_id_1'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0,
+                },
+                metadata={
+                    'model_id': 311,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                web_ids=['web_id_2'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 2,
+                },
+                metadata={
+                    'model_id': 311,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ],
+        any_order=True,
+    )
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_has_calls(
+        [
+            call(
+                'addr_1',
+                0,
+                'float',
+                {
+                    'model_id': 311,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                'addr_2',
+                2,
+                'float',
+                {
+                    'model_id': 311,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ]
+    )
+
+    with postgres_engine.connect() as conn:
+        tf_count = conn.execute(
+            text(
+                f'SELECT COUNT(*) FROM predictions_schema.transformed_data WHERE model_id = {model_id}'
+            )
+        ).scalar()
+        assert tf_count == 0, 'transform export must be skipped when path_flag is set'
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction=0,
+        prediction_confidence=Decimal(2),
+        prediction_status='Bad',
+        comments='Input data with bad quality',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_2_export_with_opc_only(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.2: Export with OPC only
+
+    Description:
+    Export to PostgreSQL and OPC server only (no PI Web API).
+
+    Expected Behavior:
+    - Normal formatting
+    - PostgreSQL export executed
+    - OPC export executed
+    - PI Web API activity skipped
+    - Metrics written with OPC metrics
+
+    Assertions:
+    - PI Web API activity NOT called
+    - OPC activity called
+    - PostgreSQL export called
+    - Metrics written with opc_metrics populated
+    """
+    client = temporal_test_env.client
+
+    model_id = 312
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+    input_data['pi_web_api_output_config'] = None  # No PI Web API config
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-opc-only')
+    )
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_has_calls(
+        [
+            call(
+                'addr_1',
+                0.5,
+                'float',
+                {
+                    'model_id': 312,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                'addr_2',
+                0,
+                'float',
+                {
+                    'model_id': 312,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ]
+    )
+
+    test_activities.pi_web_api_client.write_value.assert_not_called()
+
+    assert_prediction(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_3_export_with_pi_web_api_only(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.3: Export with PI Web API only
+
+    Description:
+    Export to PostgreSQL and PI Web API only (no OPC).
+
+    Expected Behavior:
+    - Normal formatting
+    - PostgreSQL export executed
+    - PI Web API export executed
+    - OPC activity skipped
+    - Metrics written without OPC metrics
+
+    Assertions:
+    - OPC activity NOT called
+    - PI Web API activity called
+    - PostgreSQL export called
+    - Metrics written with empty opc_metrics
+    """
+    client = temporal_test_env.client
+
+    model_id = 313
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = None  # No OPC config
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-pi-api-only')
+    )
+
+    test_activities.pi_web_api_client.write_value.assert_has_calls(
+        [
+            call(
+                web_ids=['web_id_1'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0.5,
+                },
+                metadata={
+                    'model_id': 313,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                web_ids=['web_id_2'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0,
+                },
+                metadata={
+                    'model_id': 313,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ],
+        any_order=True,
+    )
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_not_called()
+
+    assert_prediction(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_4_export_without_optional_outputs(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.4: Export Without Optional Outputs
+
+    Description:
+    Export only to PostgreSQL (no OPC or PI Web API).
+
+    Expected Behavior:
+    - Normal formatting
+    - Only PostgreSQL export executed
+    - OPC and PI Web API activities skipped
+    - Metrics written without OPC metrics
+
+    Assertions:
+    - PI Web API activity NOT called
+    - OPC activity NOT called
+    - PostgreSQL export called
+    - Metrics written with empty opc_metrics
+    """
+    client = temporal_test_env.client
+
+    model_id = 314
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = None  # No OPC config
+    input_data['pi_web_api_output_config'] = None  # No PI Web API config
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-no-optional-outputs')
+    )
+
+    test_activities.pi_web_api_client.write_value.assert_not_called()
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_not_called()
+
+    assert_prediction(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_5_export_without_transformed_data(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.5: Export Without Transformed Data
+
+    Description:
+    Only prediction exported, no transform table.
+
+    Expected Behavior:
+    - Only prediction formatted and exported
+    - Transform export skipped
+    - Single PostgreSQL write
+
+    Assertions:
+    - format_transformed_data NOT called
+    - One PostgreSQL export
+    - Transform table remains empty
+    """
+    client = temporal_test_env.client
+
+    model_id = 315
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.transformed_data WHERE model_id = {model_id}')
+        )
+
+    input_data = get_base_input_data(model_id)
+    input_data['save_transform'] = False  # Don't save transformed data
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-no-transform-export')
+    )
+
+    test_activities.pi_web_api_client.write_value.assert_has_calls(
+        [
+            call(
+                web_ids=['web_id_1'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0.5,
+                },
+                metadata={
+                    'model_id': 315,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                web_ids=['web_id_2'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0,
+                },
+                metadata={
+                    'model_id': 315,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ],
+        any_order=True,
+    )
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_has_calls(
+        [
+            call(
+                'addr_1',
+                0.5,
+                'float',
+                {
+                    'model_id': 315,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                'addr_2',
+                0,
+                'float',
+                {
+                    'model_id': 315,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ]
+    )
+
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(
+                f'SELECT COUNT(*) FROM predictions_schema.transformed_data WHERE model_id = {model_id}'
+            )
+        )
+        count = result_query.scalar()
+        assert count == 0, f'Expected transform table to be empty, but found {count} records'
+
+    assert_prediction(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_2_1_pi_web_api_write_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    notification_inserts,
+):
+    """
+    Scenario 3.2.1: PI Web API Write Error
+
+    Export failure is handled inside the activity; there is no retry loop. The
+    workflow completes and PostgreSQL stores prediction_confidence 13 and the
+    error message in comments.
+    """
+    client = temporal_test_env.client
+
+    model_id = 321
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    test_activities.pi_web_api_client.write_value.side_effect = Exception(
+        'PI Web API service unavailable'
+    )
+
+    input_data = get_base_input_data(model_id)
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-pi-api-error')
+    )
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=13,
+        comments='PI Web API service unavailable',
+    )
+    assert notification_inserts.call_count >= 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_2_2_opc_write_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.2: OPC Write Error
+
+    OPC failure is reported without failing the workflow; there is no retry
+    loop. PostgreSQL stores prediction_confidence 12 and OPC error comments.
+    """
+    client = temporal_test_env.client
+
+    model_id = 322
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.return_value = (
+        False,
+        {
+            'notification_id': 'OPC_WRITE_DATA_ERROR_1',
+            'message': 'OPC server unavailable',
+            'block': 'opc_repository',
+            'level': NotificationLevel.ERROR,
+            'attachment_content': 'OPC server unavailable',
+        },
+    )
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-opc-error')
+    )
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=12,
+        comments='Some data could not be written to OPC servers',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_2_4_opc_session_bad_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.4: OPC session/channel Tier-1 Bad* (e.g. BadSessionIdInvalid).
+
+    PostgreSQL stores prediction_confidence 14 and a stable session error comment.
+    """
+    client = temporal_test_env.client
+
+    model_id = 324
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.return_value = (
+        False,
+        {
+            'notification_id': 'OPC_WRITE_DATA_ERROR_1',
+            'message': 'BadSessionIdInvalid',
+            'block': 'opc_repository',
+            'level': NotificationLevel.ERROR,
+            'attachment_content': 'BadSessionIdInvalid',
+            'opc_error_kind': 'session_bad',
+            'opc_status': 'BadSessionIdInvalid',
+        },
+    )
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+    input_data['pi_web_api_output_config'] = None
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-opc-session-bad')
+    )
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=14,
+        comments='OPC UA session/channel error: BadSessionIdInvalid',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_2_3_pi_web_api_partial_write_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.3: PI Web API Partial Write Error
+
+    Partial PI write: confidence 13, descriptive comments, workflow completes
+    without an activity retry loop.
+    """
+    client = temporal_test_env.client
+
+    model_id = 323
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    test_activities.pi_web_api_client.write_value = AsyncMock(
+        side_effect=[
+            # Prediction batch: two web_ids requested, only one acknowledged.
+            [{'WebId': 'web_id_1', 'Errors': []}],
+            # Confidence write succeeds.
+            [{'WebId': 'web_id_2', 'Errors': []}],
+        ]
+    )
+
+    input_data = get_base_input_data(model_id)
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1', 'tag_3': 'web_id_3'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-pi-api-partial-error')
+    )
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=13,
+        comments="The number of written tags does not match the number of tag names: Expected ['tag_1', 'tag_3'] tags, but ['tag_1'] tags were written.",
+    )

e2e/test_predictions_batch_joblib_flavor.py

@@ -0,0 +1,277 @@
+"""
+End-to-end tests for the explicit `joblib` flavor on the MLflow model load paths.
+
+`joblib.load` is NOT mocked here: the artifact fixture holds a real object serialized with
+`joblib.dump(..., compress=3)` whose class lives only inside the artifact's `code/` directory.
+A passing run therefore proves the flavor decompresses the pickle **and** resolves the module,
+without any `mlflow.*.load_model` being called for that model.
+"""
+
+from unittest.mock import MagicMock
+
+import pandas as pd
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.conftest import JOBLIB_FLAVOR_MODEL_NAME
+from e2e.helpers import (
+    assert_prediction,
+    assert_stop,
+    insert_sample_data,
+    make_workflow_id,
+    start_and_await_workflow,
+)
+from laborious.activities.activities import Activities
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+base_query = 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = {model_id}'
+
+FLAVOR_LOADERS = ('sklearn', 'pyfunc')
+
+
+def build_input_data(
+    model_id: int,
+    model_name: str,
+    transform_flavor: str = 'sklearn',
+    predict_flavor: str = 'sklearn',
+) -> dict:
+    """Workflow input for a plain transform + predict run of a given model."""
+    return {
+        'schedule_name': 'test-schedule',
+        'model_name': model_name,
+        'model_id': model_id,
+        'query': base_query.format(model_id=model_id),
+        'schema': 'predictions_schema',
+        'table_name': 'predictions',
+        'transform_table_name': 'transformed_data',
+        'input_filters': {
+            'SPECIFIC_VARIABLES_NULL_VALUES': {
+                'POLICY': 'CONTINUE',
+                'CONFIG': {'variables': ['sensor_1']},
+            },
+        },
+        'mlflow_transform_filters': {
+            'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'mlflow_predict_filters': {
+            'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
+        'opc_output_config': {},
+        'pi_web_api_output_config': {},
+        'save_transform': True,
+        'prediction_store_policy': 'lts:1',
+        'model_config': {
+            'retention_minutes': 0,
+            'transform_flavor': transform_flavor,
+            'predict_flavor': predict_flavor,
+        },
+        'datetime_columns': ['timestamp', 'created_at'],
+    }
+
+
+def expected_joblib_prediction(model) -> float:
+    """Prediction the artifact's model produces for the transform output of this suite."""
+    features = pd.DataFrame(
+        {
+            'feature_1': [0.234],
+            'feature_2': [0.783],
+            'timestamp': ['2024-01-01 12:00:00'],
+        }
+    )
+    return float(model.predict(features)[0])
+
+
+def assert_single_prediction_close(postgres_engine, model_id, expected, tolerance=1e-9):
+    """Assert exactly one Good prediction row whose value matches `expected`."""
+    with postgres_engine.connect() as conn:
+        rows = conn.execute(
+            text(
+                'SELECT prediction, prediction_status '
+                'FROM predictions_schema.predictions '
+                f'WHERE model_id = {model_id}'
+            )
+        ).fetchall()
+
+    assert len(rows) == 1, f'Expected one prediction record, got {len(rows)}'
+    assert abs(float(rows[0][0]) - expected) < tolerance, (
+        f'Expected prediction close to {expected}, got {rows[0][0]}'
+    )
+    assert rows[0][1] == 'Good', f"Expected prediction_status='Good', got {rows[0][1]}"
+
+
+def logged_messages(mock_logger, level: str) -> str:
+    """Concatenate every message logged at a given custom level."""
+    calls = getattr(mock_logger, f'custom_{level}').call_args_list
+    return ' '.join(str(logged.args[0]) for logged in calls)
+
+
+def loader_uris(patch_mlflow) -> list[str]:
+    """Every model URI handed to an MLflow flavor loader during the run."""
+    return [
+        str(loaded.args[0])
+        for flavor in FLAVOR_LOADERS
+        for loaded in getattr(patch_mlflow, flavor).load_model.call_args_list
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_prediction_workflow_with_joblib_flavor(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    joblib_flavor_artifacts,
+    mlflow_client,
+    patch_mlflow,
+):
+    """`predict_flavor: 'joblib'` deserializes the real artifact, and the prediction lands."""
+    client = temporal_test_env.client
+    model_id = 271
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = build_input_data(model_id, JOBLIB_FLAVOR_MODEL_NAME, predict_flavor='joblib')
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-flavor')
+    )
+
+    assert_single_prediction_close(
+        postgres_engine,
+        model_id,
+        expected_joblib_prediction(joblib_flavor_artifacts['prediction_model']['model']),
+    )
+
+    assert mlflow_client.download_artifacts.call_count == 1
+    assert mlflow_client.download_artifacts.call_args.args[1] == 'prediction_model'
+
+    # The joblib flavor never builds a `models:/` URI, so no flavor loader saw this model.
+    assert all(JOBLIB_FLAVOR_MODEL_NAME not in uri for uri in loader_uris(patch_mlflow))
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_prediction_workflow_with_joblib_transform_flavor(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    joblib_flavor_artifacts,
+    mlflow_client,
+    patch_mlflow,
+    mock_mlflow_models,
+):
+    """`transform_flavor: 'joblib'` downloads and deserializes `data_model` on its own."""
+    client = temporal_test_env.client
+    model_id = 274
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = build_input_data(model_id, JOBLIB_FLAVOR_MODEL_NAME, transform_flavor='joblib')
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-transform')
+    )
+
+    assert_prediction(postgres_engine, model_id, prediction=0.5)
+    mock_mlflow_models['predict_model'].predict.assert_called()
+
+    assert mlflow_client.download_artifacts.call_count == 1
+    assert mlflow_client.download_artifacts.call_args.args[1] == 'data_model'
+
+    # The joblib flavor resolves no artifact URI, so `get_run` is never reached and no flavor
+    # loader was given a `data_model` URI.
+    patch_mlflow.get_run.assert_not_called()
+    assert all(not uri.endswith('/data_model') for uri in loader_uris(patch_mlflow))
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_prediction_workflow_with_sklearn_flavor_is_unchanged(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_client,
+    mock_mlflow_models,
+):
+    """A model on an MLflow flavor keeps the current result and downloads nothing."""
+    client = temporal_test_env.client
+    model_id = 272
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = build_input_data(model_id, 'test_model')
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-sklearn-flavor')
+    )
+
+    assert_prediction(postgres_engine, model_id, prediction=0.5)
+    mock_mlflow_models['predict_model'].predict.assert_called()
+    mlflow_client.download_artifacts.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_prediction_workflow_downloads_only_the_artifact_code_directory(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    joblib_flavor_artifacts,
+    mlflow_client,
+    mock_mlflow_models,
+):
+    """A flavor model whose artifact carries `code` downloads that directory — and only it.
+
+    The flavor loaders read the model URI themselves, so nothing else is fetched through the
+    client: the `code` directory is the whole of what this workflow downloads.
+    """
+    mlflow_client.list_artifacts.side_effect = lambda _run_id, artifact_path=None: (
+        [MagicMock(path=f'{artifact_path}/code')] if artifact_path else []
+    )
+
+    client = temporal_test_env.client
+    model_id = 275
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = build_input_data(model_id, 'test_model')
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-code-only')
+    )
+
+    assert_prediction(postgres_engine, model_id, prediction=0.5)
+
+    downloaded = [loaded.args[1] for loaded in mlflow_client.download_artifacts.call_args_list]
+    assert sorted(downloaded) == ['data_model/code', 'prediction_model/code']
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_prediction_workflow_joblib_flavor_failure_reports_single_cause(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    joblib_flavor_artifact_without_pickle,
+    mlflow_client,
+    mock_logger,
+):
+    """The joblib flavor failing stops the workflow with one cause, not a chain of two."""
+    mlflow_client.download_artifacts.side_effect = lambda _run_id, _artifact_name: (
+        joblib_flavor_artifact_without_pickle
+    )
+
+    client = temporal_test_env.client
+    model_id = 273
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = build_input_data(model_id, JOBLIB_FLAVOR_MODEL_NAME, predict_flavor='joblib')
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-joblib-flavor-failed')
+    )
+
+    assert_stop(postgres_engine, model_id)
+
+    logged = logged_messages(mock_logger, 'debug')
+    assert 'No .pkl file found in prediction_model artifact' in logged
+    assert 'Original error' not in logged

e2e/test_predictions_batch_main_workflow.py

@@ -0,0 +1,302 @@
+"""
+End-to-end tests for PredictionsBatch workflow - Main workflow scenarios.
+"""
+
+import asyncio
+
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import make_workflow_id, start_and_await_workflow
+from laborious.activities.activities import Activities
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_1_1_1_happy_path_complete_success(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """Scenario 1.1.1: Happy path with SQL load, MLflow mocks, Postgres predictions and transforms."""
+    client = temporal_test_env.client
+
+    with postgres_engine.begin() as conn:
+        conn.execute(text('DELETE FROM predictions_schema.laborious_data WHERE model_id = 123'))
+        insert_sql = """
+            INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
+            VALUES
+                (123, 'sensor_1', 23.5, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
+                (123, 'sensor_2', 78.2, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
+                (123, 'sensor_3', 120.8, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
+        """
+        conn.execute(text(insert_sql))
+
+    input_data = {
+        'metadata': {
+            'metadata': {
+                'model_id': 123,
+                'model_name': 'test_model',
+                'schedule_name': 'test-schedule',
+                'workflow_name': 'predictions_batch',
+            }
+        },
+        'schedule_name': 'test-schedule',
+        'model_name': 'test_model',
+        'model_id': 123,
+        'query': 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = 123',
+        'schema': 'predictions_schema',
+        'table_name': 'predictions',
+        'transform_table_name': 'transformed_data',
+        'input_filters': {
+            'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'mlflow_transform_filters': {
+            'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'mlflow_predict_filters': {
+            'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
+        'opc_output_config': {},
+        'pi_web_api_output_config': {},
+        'save_transform': True,
+        'prediction_store_policy': 'lts:1',
+        'model_config': {
+            'retention_minutes': 0,
+            'transform_flavor': 'sklearn',
+            'predict_flavor': 'sklearn',
+        },
+        'datetime_columns': ['timestamp', 'created_at'],
+    }
+
+    await start_and_await_workflow(
+        client,
+        PredictionsBatch.run,
+        input_data,
+        make_workflow_id('test-predictions-batch'),
+    )
+
+    schema_name = 'predictions_schema'
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, prediction, prediction_confidence, response_time, prediction_status, comments '
+                f'FROM {schema_name}.predictions WHERE model_id = 123'
+            )
+        )
+        prediction_rows = result_query.fetchall()
+        assert len(prediction_rows) == 1
+        row = prediction_rows[0]
+        assert row[0] == 123
+        assert row[1] == 0.5
+        assert row[2] == 0, f'Expected prediction_confidence=0, got {row[2]}'
+        assert row[3] is not None
+        assert row[4] == 'Good'
+        assert row[5] == ''
+
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, variable, value FROM {schema_name}.transformed_data WHERE model_id = 123'
+            )
+        )
+        transformed_rows = result_query.fetchall()
+        assert len(transformed_rows) == 2
+        assert transformed_rows[0][0] == 123
+        assert transformed_rows[0][1] == 'feature_1'
+        assert float(transformed_rows[0][2]) == 0.234
+        assert transformed_rows[1][0] == 123
+        assert transformed_rows[1][1] == 'feature_2'
+        assert float(transformed_rows[1][2]) == 0.783
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_1_2_1_sql_query_execution_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """Invalid SQL: workflow may complete with early exit; no prediction rows."""
+    client = temporal_test_env.client
+
+    input_data = {
+        'metadata': {
+            'metadata': {
+                'model_id': 128,
+                'model_name': 'test_model',
+                'schedule_name': 'test-schedule',
+                'workflow_name': 'predictions_batch',
+            }
+        },
+        'schedule_name': 'test-schedule',
+        'model_name': 'test_model',
+        'model_id': 128,
+        'query': 'SELECT * FROM nonexistent_table WHERE invalid_syntax =',
+        'schema': 'predictions_schema',
+        'table_name': 'predictions',
+        'transform_table_name': 'transformed_data',
+        'input_filters': {
+            'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'mlflow_transform_filters': {
+            'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'mlflow_predict_filters': {
+            'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
+        'opc_output_config': {},
+        'pi_web_api_output_config': {},
+        'save_transform': True,
+        'prediction_store_policy': 'lts:1',
+        'model_config': {
+            'retention_minutes': 0,
+            'transform_flavor': 'sklearn',
+            'predict_flavor': 'sklearn',
+        },
+    }
+
+    await start_and_await_workflow(
+        client,
+        PredictionsBatch.run,
+        input_data,
+        make_workflow_id('test-sql-error'),
+    )
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 128')
+        ).scalar()
+    assert count == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_1_2_2_missing_required_parameters(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """Missing query: workflow does not produce predictions and is terminated explicitly."""
+    client = temporal_test_env.client
+
+    input_data = {
+        'metadata': {
+            'metadata': {
+                'model_id': 129,
+                'model_name': 'test_model',
+                'schedule_name': 'test-schedule',
+                'workflow_name': 'predictions_batch',
+            }
+        },
+        'schedule_name': 'test-schedule',
+        'model_name': 'test_model',
+        'model_id': 129,
+        'schema': 'predictions_schema',
+        'table_name': 'predictions',
+        'transform_table_name': 'transformed_data',
+    }
+
+    handle = await client.start_workflow(
+        PredictionsBatch.run,
+        input_data,
+        id=make_workflow_id('test-missing-param'),
+        task_queue='test-queue',
+    )
+
+    # Let Temporal process a few workflow tasks; for this case, result() can hang.
+    await asyncio.sleep(2.0)
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 129')
+        ).scalar()
+    assert count == 0
+
+    await handle.terminate('expected failure path in e2e test (missing required parameters)')
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_1_2_3_invalid_datetime_column_specification(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """Invalid datetime column: no predictions persisted; workflow terminated after validation."""
+    client = temporal_test_env.client
+
+    with postgres_engine.begin() as conn:
+        conn.execute(text('DELETE FROM predictions_schema.laborious_data WHERE model_id = 130'))
+        conn.execute(
+            text(
+                """
+            INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
+            VALUES (130, 'sensor_1', 23.5, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
+        """
+            )
+        )
+
+    input_data = {
+        'metadata': {
+            'metadata': {
+                'model_id': 130,
+                'model_name': 'test_model',
+                'schedule_name': 'test-schedule',
+                'workflow_name': 'predictions_batch',
+            }
+        },
+        'schedule_name': 'test-schedule',
+        'model_name': 'test_model',
+        'model_id': 130,
+        'query': 'SELECT timestamp, variable, value FROM predictions_schema.laborious_data WHERE model_id = 130',
+        'schema': 'predictions_schema',
+        'table_name': 'predictions',
+        'transform_table_name': 'transformed_data',
+        'input_filters': {
+            'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'mlflow_transform_filters': {
+            'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'mlflow_predict_filters': {
+            'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        },
+        'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
+        'opc_output_config': {},
+        'pi_web_api_output_config': {},
+        'save_transform': True,
+        'prediction_store_policy': 'lts:1',
+        'model_config': {
+            'retention_minutes': 0,
+            'transform_flavor': 'sklearn',
+            'predict_flavor': 'sklearn',
+        },
+        'datetime_columns': ['nonexistent_column'],
+    }
+
+    handle = await client.start_workflow(
+        PredictionsBatch.run,
+        input_data,
+        id=make_workflow_id('test-invalid-datetime-col'),
+        task_queue='test-queue',
+    )
+
+    # Let Temporal process and surface the failure path internally.
+    await asyncio.sleep(2.0)
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 130')
+        ).scalar()
+    assert count == 0
+
+    await handle.terminate('expected failure path in e2e test (invalid datetime column)')

e2e/test_predictions_batch_prediction_process.py

@@ -0,0 +1,385 @@
+"""
+End-to-end tests for PredictionsBatch workflow - Prediction Process scenarios.
+"""
+
+from decimal import Decimal
+from unittest.mock import MagicMock
+
+import numpy as np
+import pandas as pd
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import (
+    assert_continue,
+    assert_repeat,
+    assert_stop,
+    insert_sample_data,
+    make_workflow_id,
+    start_and_await_workflow,
+)
+from laborious.activities.activities import Activities
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+base_input_data = {
+    'schedule_name': 'test-schedule',
+    'model_name': 'test_model',
+    'model_id': 201,
+    'query': 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = 201',
+    'schema': 'predictions_schema',
+    'table_name': 'predictions',
+    'transform_table_name': 'transformed_data',
+    'input_filters': {
+        'SPECIFIC_VARIABLES_NULL_VALUES': {
+            'POLICY': 'CONTINUE',
+            'CONFIG': {'variables': ['sensor_1']},
+        },
+    },
+    'mlflow_transform_filters': {
+        'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+    },
+    'mlflow_predict_filters': {
+        'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+    },
+    'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
+    'opc_output_config': {},
+    'pi_web_api_output_config': {},
+    'save_transform': True,
+    'prediction_store_policy': 'lts:1',
+    'model_config': {
+        'retention_minutes': 0,
+        'transform_flavor': 'sklearn',
+        'predict_flavor': 'sklearn',
+    },
+    'datetime_columns': ['timestamp', 'created_at'],
+}
+
+base_query = 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = {model_id}'
+
+
+def get_base_input_data(model_id):
+    return {
+        **base_input_data,
+        'model_id': model_id,
+        'query': base_query.format(model_id=model_id),
+    }
+
+
+def insert_sample_prediction(postgres_engine, model_id):
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}')
+        )
+        insert_sql = f"""
+            INSERT INTO predictions_schema.predictions (model_id, timestamp, prediction, prediction_confidence, prediction_status, comments, response_time)
+            VALUES
+               ({model_id}, '2024-01-01 12:00:00+00:00', 10, 0, 'Good', '', 0.1)
+        """
+        conn.execute(text(insert_sql))
+    return (model_id, Decimal(10), Decimal(0), 'Good')
+
+
+@pytest.fixture
+def bad_data_model(patch_mlflow):
+    model = MagicMock(predict=MagicMock(side_effect=Exception('Bad data model')))
+    patch_mlflow.sklearn.load_model = MagicMock(return_value=model)
+    return model
+
+
+@pytest.fixture
+def bad_predict_model(patch_mlflow, mock_mlflow_models):
+    model = MagicMock(predict=MagicMock(side_effect=Exception('Bad predict model')))
+
+    def mock_sklearn_load_model(model_uri):
+        if 'data_model' in model_uri or 'transform' in model_uri.lower():
+            return mock_mlflow_models['transform_model']
+        return model
+
+    patch_mlflow.sklearn = MagicMock()
+    patch_mlflow.sklearn.load_model = MagicMock(side_effect=mock_sklearn_load_model)
+    return model
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_1_1_input_gate_triggers_continue(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mock_mlflow_models,
+):
+    """Input gate CONTINUE: export default prediction; MLflow transform/predict not used."""
+    client = temporal_test_env.client
+    model_id = 211
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+    input_data = get_base_input_data(model_id)
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-continue-policy')
+    )
+    assert_continue(postgres_engine, model_id)
+    mock_mlflow_models['transform_model'].predict.assert_not_called()
+    mock_mlflow_models['predict_model'].predict.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_1_2_input_gate_triggers_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mock_mlflow_models,
+):
+    """Input gate STOP: no export, no MLflow."""
+    client = temporal_test_env.client
+    model_id = 212
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'STOP'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-input-stop')
+    )
+    assert_stop(postgres_engine, model_id)
+    mock_mlflow_models['transform_model'].predict.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_1_3_input_gate_triggers_repeat(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mock_mlflow_models,
+):
+    """Input gate REPEAT with existing history."""
+    client = temporal_test_env.client
+    model_id = 213
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+    data = insert_sample_prediction(postgres_engine, model_id)
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'REPEAT'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-input-repeat')
+    )
+    assert_repeat(postgres_engine, model_id, data)
+    mock_mlflow_models['transform_model'].predict.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_1_4_input_gate_repeat_without_prior_prediction(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """REPEAT when no prior row in predictions: repeat_last_prediction runs; still no new duplicate export path."""
+    client = temporal_test_env.client
+    model_id = 214
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}')
+        )
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'REPEAT'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-input-repeat-no-history')
+    )
+    assert_stop(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_2_1_transform_gate_triggers_continue(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_data_model,
+):
+    client = temporal_test_env.client
+    model_id = 221
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'CONTINUE'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-continue')
+    )
+    assert_continue(
+        postgres_engine=postgres_engine,
+        model_id=model_id,
+        prediction_confidence=Decimal(10),
+        comments='Unknown MLFlow API error',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_2_2_transform_gate_triggers_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_data_model,
+    mock_mlflow_models,
+):
+    client = temporal_test_env.client
+    model_id = 222
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'STOP'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-stop')
+    )
+    assert_stop(postgres_engine, model_id)
+    mock_mlflow_models['predict_model'].predict.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_2_3_transform_gate_triggers_repeat(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_data_model,
+):
+    client = temporal_test_env.client
+    model_id = 223
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    data = insert_sample_prediction(postgres_engine, model_id)
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'REPEAT'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-repeat')
+    )
+    assert_repeat(postgres_engine, model_id, data)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_2_4_transform_content_gate_nan_values_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mock_mlflow_models,
+):
+    """mlflow_content_gate triggers STOP when transform output is all NaN (NAN_VALUES filter)."""
+    client = temporal_test_env.client
+    model_id = 224
+
+    def all_nan_transform(data):
+        num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1
+        result = pd.DataFrame({'feature_1': [np.nan] * num_rows, 'feature_2': [np.nan] * num_rows})
+        result.index = data.index
+        return result
+
+    mock_mlflow_models['transform_model'].predict = MagicMock(side_effect=all_nan_transform)
+
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters'] = {
+        'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        'NAN_VALUES': {'POLICY': 'STOP', 'CONFIG': {}},
+    }
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-content-stop')
+    )
+    assert_stop(postgres_engine, model_id)
+    mock_mlflow_models['predict_model'].predict.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_3_1_predict_gate_triggers_continue(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_predict_model,
+):
+    client = temporal_test_env.client
+    model_id = 231
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'CONTINUE'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-predict-continue')
+    )
+    assert_continue(
+        postgres_engine=postgres_engine,
+        model_id=model_id,
+        prediction_confidence=Decimal(10),
+        comments='Unknown MLFlow API error',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_3_2_predict_gate_triggers_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_predict_model,
+):
+    client = temporal_test_env.client
+    model_id = 232
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'STOP'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-predict-stop')
+    )
+    assert_stop(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_3_3_predict_gate_triggers_repeat(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_predict_model,
+):
+    client = temporal_test_env.client
+    model_id = 233
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+    data = insert_sample_prediction(postgres_engine, model_id)
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'REPEAT'
+    input_data['path_priority'] = ['REPEAT', 'STOP', 'CONTINUE']
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-predict-repeat')
+    )
+    assert_repeat(postgres_engine, model_id, data)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_4_1_input_empty_data_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """EMPTY_DATA filter with STOP when query returns no rows (offload payload empty)."""
+    client = temporal_test_env.client
+    model_id = 241
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}')
+        )
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters'] = {'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}}}
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-empty-data-stop')
+    )
+    assert_stop(postgres_engine, model_id)

e2e/test_retrain_joblib_flavor.py

@@ -0,0 +1,128 @@
+"""
+End-to-end test for retraining a model served through the explicit `joblib` flavor.
+
+Retraining is not part of the `predictions_batch` workflow, so it gets its own module. The test
+is a write→read round trip in one run: `retrain_model` loads the production artifact through the
+joblib flavor, `log_model` writes the retrained model back as `model.pkl` + `code/`, and the
+written artifact is then loaded again through the same flavor. Neither `joblib.dump` nor
+`joblib.load` is mocked; only MLflow's own I/O is.
+"""
+
+import importlib
+import shutil
+import sys
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+from e2e.conftest import JOBLIB_FLAVOR_CODE_MODULE, JOBLIB_FLAVOR_MODEL_NAME
+from laborious.activities.activities import Activities
+
+RETRAIN_INDEX = [
+    '2024-01-01 12:00:00+0000',
+    '2024-01-01 12:01:00+0000',
+    '2024-01-01 12:02:00+0000',
+]
+
+METADATA = {
+    'model_id': 'test_model',
+    'model_name': JOBLIB_FLAVOR_MODEL_NAME,
+    'workflow_name': 'test_workflow',
+    'schema_name': 'test_schedule',
+}
+
+
+def build_retrain_data() -> pd.DataFrame:
+    """Training frame indexed by timestamp strings, carrying the target column."""
+    return pd.DataFrame(
+        {
+            'sensor_1': [60.0, 61.0, 62.0],
+            'sensor_2': [78.2, 78.4, 78.6],
+            'target': [1.0, 2.0, 3.0],
+        },
+        index=RETRAIN_INDEX,
+    )
+
+
+class FakeArtifactStore:
+    """Minimal stand-in for the run's artifact tree, so the written layout is inspectable."""
+
+    def __init__(self, root: Path):
+        self.root = root
+
+    def log_artifact(self, local_path, artifact_path=None):
+        if artifact_path is None:
+            # Retraining also logs the CSVs at the run root; irrelevant to this test.
+            return
+        target = self.root / artifact_path
+        target.mkdir(parents=True, exist_ok=True)
+        shutil.copy(local_path, target / Path(local_path).name)
+
+    def log_artifacts(self, local_dir, artifact_path=None):
+        target = self.root / artifact_path if artifact_path else self.root
+        shutil.copytree(local_dir, target, dirs_exist_ok=True)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_retrain_joblib_model_writes_pickle_and_code_and_reloads(
+    test_activities: Activities,
+    joblib_flavor_artifacts,
+    mlflow_client,
+    patch_mlflow,
+    tmp_path,
+):
+    """A joblib retrain writes `model.pkl` + `code/`, and the joblib flavor reads it back."""
+    repository = test_activities.model_monitoring_repository
+
+    store = FakeArtifactStore(tmp_path / 'run-retrained')
+    patch_mlflow.log_artifact.side_effect = store.log_artifact
+    patch_mlflow.log_artifacts.side_effect = store.log_artifacts
+    patch_mlflow.get_run.return_value.data.params = {}
+    patch_mlflow.start_run.return_value.__enter__.return_value.info.run_id = 'run-retrained'
+
+    output = await repository.retrain_model(
+        data=build_retrain_data(),
+        model_name=JOBLIB_FLAVOR_MODEL_NAME,
+        model_config={
+            'target': 'target',
+            'transform_flavor': 'joblib',
+            'predict_flavor': 'joblib',
+        },
+        metadata=METADATA,
+    )
+
+    assert output['success'] is True, output.get('traceback')
+
+    # Both models were loaded through the joblib flavor, not through a flavor loader.
+    downloaded = [loaded.args[1] for loaded in mlflow_client.download_artifacts.call_args_list]
+    assert sorted(downloaded) == ['data_model', 'prediction_model']
+    assert patch_mlflow.sklearn.load_model.call_count == 0
+    assert patch_mlflow.pyfunc.load_model.call_count == 0
+
+    written = store.root / 'prediction_model'
+    assert (written / 'model.pkl').is_file()
+    assert (written / 'code' / f'{JOBLIB_FLAVOR_CODE_MODULE}.py').is_file()
+    assert not (written / 'MLmodel').exists()
+    assert (store.root / 'data_model' / 'model.pkl').is_file()
+
+    # Drop the module the first load put in place: the reload must resolve it from the `code/`
+    # the retrain wrote, not from what is already imported.
+    for code_dir in (artifact['code_dir'] for artifact in joblib_flavor_artifacts.values()):
+        if code_dir in sys.path:
+            sys.path.remove(code_dir)
+    sys.modules.pop(JOBLIB_FLAVOR_CODE_MODULE, None)
+    importlib.invalidate_caches()
+
+    mlflow_client.download_artifacts.side_effect = lambda _run_id, _artifact_name: str(written)
+
+    reloaded, _local_dir = await repository.load_predict_model(
+        JOBLIB_FLAVOR_MODEL_NAME, METADATA, 'joblib'
+    )
+
+    features = pd.DataFrame({'feature_1': [0.234], 'feature_2': [0.783]})
+    expected = joblib_flavor_artifacts['prediction_model']['model'].predict(features)
+
+    assert reloaded.predict(features) == pytest.approx(expected)
+    assert str(written / 'code') in sys.path

git-requirements-mapping.txt

@@ -0,0 +1,2 @@
+git+ssh://git@github.com/Aignosi/sientia-dataops-library.git:sientia-do
+git+ssh://git@github.com/Aignosi/sientia-mlops-library.git:sientia

input_sample.json

@@ -1,34 +0,0 @@
-{
-    "schedule_name": "scouter-opcua-pipeline",
-    "model_name": "Demo Model",
-    "model_id": 1,
-    "query": "SELECT * FROM sientia_data.laborious_data order by \"timestamp\" desc limit 30;",
-    "schema": "sientia_data",
-    "table_name": "predictions",
-    "retention_time": 3600,
-    "model_retention": 120,
-    "path_priority": ["STOP", "CONTINUE", "REPEAT"],
-    "input_filters": {
-        "SPECIFIC_VARIABLES_NULL_VALUES": {
-            "POLICY": "STOP",
-            "VARIABLES": ["Counter"]
-        },
-        "EMPTY_DATA": {
-            "POLICY": "STOP"
-        }
-    },
-    "mlflow_transform_filters": {
-        "API_ERROR": {
-            "POLICY": "CONTINUE"
-        },
-        "NAN_VALUES": {
-            "POLICY": "CONTINUE"
-        }
-    },
-    "mlflow_predict_filters": {
-        "API_ERROR": {
-            "POLICY": "CONTINUE"
-        }
-    },
-    "opc_output_config": {}
-}

laborious/activities/activities.py

@@ -1,49 +1,217 @@
-from temporalio import activity, workflow
+from sientia_do.observability.metrics_controller import MetricsController
+from temporalio import workflow
 
 with workflow.unsafe.imports_passed_through():
-    from sientia_do.temporal.activities.postgres import Postgres
-    from sientia_do.notifications.handlers import NotificationHandler
-    from sientia_do.temporal.utils.logger import Logger
-    from laborious.activities.mlflow import MLFlow
-    from laborious.activities.gates import Gates
-    from laborious.activities.opc import OPC
     from typing import Any
 
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.observability.logger import Logger
+    from sientia_do.repository.minio_repository import MinioRepository
+    from sientia_do.repository.mongodb_repository import MongoDBRepository
 
-class Activities(Postgres, MLFlow, Gates, OPC):
+    from laborious.activities.api import API
+    from laborious.activities.gates import Gates
+    from laborious.activities.mlflow import MLFlow
+    from laborious.activities.model_import import ModelImport
+    from laborious.activities.model_metrics import ModelMetrics
+    from laborious.activities.opc import OPC
+    from laborious.activities.storage import Storage
+    from laborious.utils.connectors_config import (
+        build_import_config,
+        build_import_status_config,
+    )
+    from laborious.utils.repository.model_repository import MLFlowRepository
 
-    def __init__(self,
-                 postgres_config: dict[str, Any],
-                 mlflow_config: dict[str, Any],
-                 opc_config: dict[str, Any],
-                 logger: Logger, notification_handler: NotificationHandler):
+
+class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, ModelImport):
+    """
+    Main activities orchestrator for the Laborious system.
+
+    This class combines functionality from multiple activity classes to provide
+    a unified interface for all workflow operations. It manages database connections,
+    MLFlow model interactions, data quality validation, and OPC server communications.
+
+    The class implements multiple inheritance to combine specialized functionality:
+    - Storage: Database operations and data persistence
+    - MLFlow: Model inference and transformation operations
+    - Gates: Data quality validation and filtering mechanisms
+    - OPC: Real-time data export to OPC servers
+    - ModelMetrics: Model performance metrics and drift detection
+    - API: PI Web API export operations for industrial systems
+
+    Attributes:
+        postgres_config (dict): PostgreSQL connection configuration
+        mlflow_config (dict): MLFlow server configuration
+        opc_config (dict): OPC server configuration
+        pi_web_api_config (dict): PI Web API server configuration
+        logger (Logger): Logging and observability instance
+        notification_handler (NotificationHandler): Notification management instance
+    """
+
+    def __init__(
+        self,
+        postgres_config: dict[str, Any],
+        mlflow_config: dict[str, Any],
+        minio_config: dict[str, Any],
+        opc_config: dict[str, Any],
+        pi_web_api_config: dict[str, Any],
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        mongo_config: dict[str, Any] | None = None,
+        import_config: dict[str, Any] | None = None,
+        import_status_config: dict[str, Any] | None = None,
+    ):
+        """
+        Initialize the Activities orchestrator with all required configurations.
+
+        This constructor initializes all parent classes with their respective
+        configurations and sets up the foundation for all activity operations.
+
+        Args:
+            postgres_config: PostgreSQL connection configuration dictionary
+                Required keys: host, port, user, password, dbname, min_connections, max_connections
+            mlflow_config: MLFlow server configuration dictionary
+                Required keys: host, port, username, password
+            opc_config: OPC server configuration dictionary
+                Can contain multiple server configurations
+            pi_web_api_config: PI Web API server configuration dictionary
+                Required keys: base_url, auth_type, auth_token
+            logger: Logger instance for observability and debugging
+            notification_handler: Notification handler for alerts and monitoring
+            mongo_config: MongoDB configuration for the model listing the import writes. When
+                absent, the import activities that need it refuse rather than guess — the
+                notification handler keeps its own client either way.
+            import_config: `build_import_config()`; read from the environment when absent
+            import_status_config: `build_import_status_config()`; read from the environment when
+                absent. This is the BFF database holding the import log: another database on the
+                `POSTGRES_*` server, and a second connection that is never the one Storage holds.
+
+        Raises:
+            Exception: If any parent class initialization fails
+        """
+        metrics_controller = MetricsController(logger=logger)
+
+        minio_repository = MinioRepository(
+            endpoint=minio_config['endpoint_url'],
+            access_key=minio_config['access_key'],
+            secret_key=minio_config['secret_key'],
+            bucket=minio_config['default_bucket'],
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+            secure=minio_config['secure'],
+        )
+
+        mlflow_repository = MLFlowRepository(
+            f'{mlflow_config["host"]}:{mlflow_config["port"]}',
+            mlflow_config['username'],
+            mlflow_config['password'],
+            logger,
+            notification_handler,
+            metrics_controller,
+        )
 
         # Initialize parent classes
-        Postgres.__init__(self, host=postgres_config['host'],
-                          port=postgres_config['port'],
-                          user=postgres_config['user'],
-                          password=postgres_config['password'],
-                          dbname=postgres_config['dbname'],
-                          min_connections=postgres_config['min_connections'],
-                          max_connections=postgres_config['max_connections'],
-                          logger=logger,
-                          notification_handler=notification_handler)
+        Storage.__init__(
+            self,
+            host=postgres_config['host'],
+            port=postgres_config['port'],
+            user=postgres_config['user'],
+            password=postgres_config['password'],
+            dbname=postgres_config['dbname'],
+            min_connections=postgres_config['min_connections'],
+            max_connections=postgres_config['max_connections'],
+            retention_hours=minio_config['retention_hours'],
+            minio_repository=minio_repository,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
 
-        MLFlow.__init__(self, mlflow_host=mlflow_config['host'],
-                        mlflow_port=mlflow_config['port'],
-                        mlflow_username=mlflow_config['username'],
-                        mlflow_password=mlflow_config['password'],
-                        logger=logger,
-                        notification_handler=notification_handler)
+        MLFlow.__init__(
+            self,
+            mlflow_repository=mlflow_repository,
+            minio_repository=minio_repository,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
 
-        Gates.__init__(self, logger=logger,
-                       notification_handler=notification_handler)
+        Gates.__init__(
+            self,
+            minio_repository=minio_repository,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
 
-        OPC.__init__(self,
-                     opc_servers=opc_config,
-                     logger=logger,
-                     notification_handler=notification_handler)
+        OPC.__init__(
+            self,
+            opc_servers=opc_config,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
 
-    def shutdown(self):
-        Postgres.close(self)
-        OPC.shutdown(self)
+        ModelMetrics.__init__(
+            self,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        API.__init__(
+            self,
+            base_url=pi_web_api_config['base_url'],
+            auth_type=pi_web_api_config['auth_type'],
+            auth_token=pi_web_api_config['auth_token'],
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        mongo_repository = (
+            MongoDBRepository(
+                connection_string=mongo_config['connection_string'],
+                database_name=mongo_config['database_name'],
+                logger=logger,
+                notification_handler=notification_handler,
+                metrics_controller=metrics_controller,
+            )
+            if mongo_config
+            else None
+        )
+
+        ModelImport.__init__(
+            self,
+            status_config=import_status_config or build_import_status_config(),
+            import_config=import_config or build_import_config(),
+            minio_repository=minio_repository,
+            mlflow_repository=mlflow_repository,
+            mongo_repository=mongo_repository,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+    async def shutdown(self):
+        """
+        Gracefully shutdown all activities and clean up resources.
+
+        This method ensures proper cleanup of all resources including:
+        - PostgreSQL connection pools
+        - OPC server connections
+        - PI Web API client connections
+        - MLFlow model repositories
+        - Any other resources that need explicit cleanup
+
+        The method should be called before the application terminates to ensure
+        proper resource cleanup and prevent resource leaks.
+        """
+        Storage.close(self)
+        MLFlow.close(self)
+        Gates.close(self)
+        await OPC.aclose(self)
+        ModelMetrics.close(self)
+        API.close(self)
+        ModelImport.close(self)

laborious/activities/api.py

@@ -0,0 +1,305 @@
+from temporalio import activity, workflow
+
+with workflow.unsafe.imports_passed_through():
+    import json
+    import traceback
+    from typing import Any
+
+    from pandas import DataFrame
+    from sientia_do.notifications.handlers import NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+    from sientia_do.repository.pi_web_api_client import PIWebAPIClient
+
+    from laborious import metrics
+
+
+PI_WEB_API_PREDICTION_ERROR_CONFIDENCE = 13
+
+
+class API(SientiaMonitoring):
+    """
+    PI Web API operations for writing prediction data to PI Web API.
+
+    This class provides Temporal activities for interacting with the PI Web API
+    to write prediction and confidence values to industrial systems. It handles
+    error scenarios gracefully by setting error confidence values and sending
+    notifications when write operations fail.
+
+    The class implements comprehensive error handling for both prediction and
+    confidence value writes, ensuring that partial failures are properly
+    reported and handled.
+    """
+
+    def __init__(
+        self,
+        base_url: str,
+        auth_type: str,
+        auth_token: str,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+    ) -> None:
+        """
+        Initialize API activity with PI Web API client.
+
+        Args:
+            base_url (str): Base URL of the PI Web API server
+            auth_type (str): Authentication type ('basic' or 'bearer')
+            auth_token (str): Authentication token
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+            metrics_controller (MetricsController): Controller for metrics collection
+        """
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
+        self.pi_web_api_client = PIWebAPIClient(
+            base_url=base_url,
+            auth_config={
+                'type': auth_type,
+                'token': auth_token,
+            },
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+            headers_config={
+                'Content-Type': 'application/json',
+                'Accept': 'application/json',
+                'x-requested-with': 'piwebapistreams',
+                'User-Agent': 'Aig-Laborious-Agent/1.0',
+            },
+        )
+
+    def get_pi_web_api_core_labels(
+        self,
+        metadata: dict[str, Any],
+        operation_type: str = 'write_pi_web_api_data',
+    ) -> dict[str, Any]:
+        """
+        Generate core labels for PI Web API metrics.
+
+        PI Web API metrics in laborious use the shared ``CORE_LABELS`` from
+        ``sientia_do``, which includes ``operation_type``. For this reason,
+        operation_type must always be present in emitted labels.
+
+        Args:
+         - metadata (dict[str, Any]): Workflow execution metadata used to derive labels.
+         - operation_type (str): Operation type label for metric cardinality.
+
+        Return:
+         dict[str, Any]: Core labels dictionary including operation_type.
+        """
+        return super().get_core_labels(
+            metadata=metadata,
+            operation_type=operation_type,
+        )
+
+    def close(self) -> None:
+        """
+        Close the PI Web API client and shutdown monitoring services.
+
+        This method properly closes all connections and resources associated
+        with the PI Web API client and monitoring services.
+        """
+        self.pi_web_api_client.close()
+        SientiaMonitoring.shutdown(self)
+
+    async def process_pi_web_api_response(
+        self,
+        response_data: list[dict[str, Any]],
+        tags: dict[str, str],
+        core_labels: dict[str, str],
+        metadata: dict[str, Any],
+    ) -> tuple[int, str]:
+        """
+        Process the response data from PI Web API write operation.
+
+        Validates that all tags were successfully written, emits metrics for each tag
+        (success or error), and returns the appropriate prediction confidence value.
+        Sets error confidence if any tag write fails or if the number of written tags
+        doesn't match the expected count.
+
+        Args:
+            - response_data (dict[str, Any]): The response data from the PI Web API write operation.
+            - tags (dict[str, str]): The tags that were written to the PI Web API.
+            - core_labels (dict[str, str]): The core labels of the workflow execution.
+            - metadata (dict[str, Any]): The metadata of the workflow execution.
+        Returns:
+            int: Prediction confidence value (0 for success, 13 for errors)
+        """
+
+        # Convert tags from name:webid to webid:name
+        tags = {w: t for t, w in tags.items()}
+
+        tag_names = list[str](tags.values())
+
+        confidence = 0
+
+        message = ''
+
+        # Evaluate response for each tag
+        written_tags = []
+        for item in response_data:
+            web_id = item.get('WebId')
+            if not web_id:
+                self.error('The response did not contain some WebIds', metadata)
+                continue
+            errors = item.get('Errors', [])
+            tag_name = tags.get(web_id)
+            if not tag_name:
+                self.error(
+                    f'The response did not contain the tag name for WebId {web_id}', metadata
+                )
+                continue
+            if errors:
+                self.error(
+                    f'Error writing tag {tag_name}:{web_id} to PI Web API: {errors}', metadata
+                )
+                await self.emit_metric(
+                    metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT,
+                    tags={
+                        **core_labels,
+                        'tag_name': tag_name,
+                    },
+                )
+                confidence = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+            else:
+                await self.emit_metric(
+                    metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_COUNT,
+                    tags={
+                        **core_labels,
+                        'tag_name': tag_name,
+                    },
+                )
+                written_tags.append(tag_name)
+
+        if len(written_tags) != len(tag_names):
+            message = f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written.'
+
+            self.error(
+                f'{message}\nResponse:\n {json.dumps(response_data, indent=4)}\nTags:\n {json.dumps(tags, indent=4)}',
+                metadata,
+            )
+
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
+                message=f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written.\nResponse:\n {json.dumps(response_data, indent=4)}\nTags:\n {json.dumps(tags, indent=4)}',
+                block='write_pi_web_api_data',
+                level=NotificationLevel.ERROR,
+            )
+            confidence = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+
+        return confidence, message
+
+    @activity.defn(name='write_pi_web_api_data')
+    async def write_pi_web_api_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+        """
+        Write prediction and confidence data to PI Web API.
+
+        Writes prediction values and confidence scores to PI Web API using configured
+        web IDs. Processes responses to validate writes and emit metrics. Handles errors
+        gracefully by setting error confidence values when writes fail and sending
+        notifications for both prediction and confidence write errors.
+
+        Args:
+            input_data (dict[str, Any]): The input data containing:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - pi_web_api_output_config (dict[str, Any]): PI Web API configuration with:
+                    - endpoint (str): PI Web API endpoint URL
+                    - prediction_tags (dict[str, str]): Mapping of tag names to web IDs for predictions
+                    - confidence_tags (dict[str, str]): Mapping of tag names to web IDs for confidence
+                - data (dict[str, Any]): Prediction data, its a dataframe converted to dict.
+        Returns:
+            dict[Any, Any]: Data dictionary with potentially modified confidence values
+                If prediction write fails, prediction_confidence is set to error value (13)
+        """
+        metadata = input_data['metadata']
+        data = DataFrame(input_data['data'])
+        pi_web_api_output_config = input_data['pi_web_api_output_config']
+
+        self.info(f'Writing data to PI Web API... config: {pi_web_api_output_config}', metadata)
+
+        raw_prediction_tags = pi_web_api_output_config['prediction_tags']
+        raw_confidence_tags = pi_web_api_output_config['confidence_tags']
+        prediction_tags = list[str](raw_prediction_tags.values())
+        confidence_tags = list(raw_confidence_tags.values())
+
+        core_labels = self.get_pi_web_api_core_labels(metadata)
+
+        prediction_value = data.head(1)['prediction'].values[0]
+        confidence_value = data.head(1)['prediction_confidence'].values[0]
+
+        try:
+            prediction_response = await self.pi_web_api_client.write_value(
+                web_ids=prediction_tags,
+                value={
+                    'Timestamp': data.head(1)['timestamp'].values[0],
+                    'Value': prediction_value,
+                },
+                metadata=metadata,
+            )
+
+            confidence, message = await self.process_pi_web_api_response(
+                response_data=prediction_response,
+                tags=raw_prediction_tags,
+                core_labels=core_labels,
+                metadata=metadata,
+            )
+
+            # Preserve incoming confidence/comments on successful PI writes.
+            # Only downgrade confidence or override comments when PI response
+            # explicitly reports a problem (e.g. partial write mismatch).
+            if confidence != 0:
+                data['prediction_confidence'] = confidence
+            if message:
+                data['comments'] = message
+
+        except Exception as e:
+            trace = traceback.format_exc()
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
+                message=f'Error writing prediction data to PI Web API: {e}\n Tags: {raw_prediction_tags}',
+                block='write_pi_web_api_data',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+
+            self.error(trace, metadata)
+
+            data['prediction_confidence'] = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+            data['comments'] = str(e)
+
+            return data.to_dict()
+
+        try:
+            confidence_response = await self.pi_web_api_client.write_value(
+                web_ids=confidence_tags,
+                value={
+                    'Timestamp': data.head(1)['timestamp'].values[0],
+                    'Value': float(confidence_value),
+                },
+                metadata=metadata,
+            )
+
+            await self.process_pi_web_api_response(
+                response_data=confidence_response,
+                tags=raw_confidence_tags,
+                core_labels=core_labels,
+                metadata=metadata,
+            )
+
+        except Exception as e:
+            trace = traceback.format_exc()
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='WRITE_PI_WEB_API_CONFIDENCE_ERROR',
+                message=f'Error writing confidence data to PI Web API: {e}\n Tags: {raw_confidence_tags}',
+                block='write_pi_web_api_data',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+
+        return data.to_dict()

laborious/activities/gates.py

@@ -1,309 +1,804 @@
+from sientia_do.repository.minio_repository import MinioRepository
 from temporalio import activity, workflow
 
+from laborious.utils.repository.minio_manager import MinioManager
 
 with workflow.unsafe.imports_passed_through():
     import traceback
-    from sientia_do.notifications.handlers import NotificationHandler
-    from sientia_do.notifications.models import NotificationLevel
-    from sientia_do.temporal.activities.base import BaseActivity
-    from sientia_do.temporal.utils.logger import Logger
-    from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter
+    from collections.abc import Callable, Mapping
     from typing import Any
+
+    from pandas import DataFrame
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.utils.formatters import create_sample_dict
+
+    from laborious import metrics
+    from laborious.utils.dataframe_debug import build_dataframe_debug_message
     from laborious.utils.filters.conditional_filters import (
         filter_empty_data,
-        filter_specific_variables_null_values
+        filter_specific_variables_null_values,
     )
-    from pandas import DataFrame
-    from datetime import datetime
+    from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter
+    from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
 
-input_filter_functions = {
+# Strongly-typed filter function signatures
+InputFilterFunc = Callable[[DataFrame, dict[str, Any]], bool]
+ResponseFilterFunc = Callable[[dict[str, Any], dict[str, Any]], bool]
+ContentFilterFunc = Callable[[DataFrame, dict[str, Any]], bool]
+
+# Input filter function mappings
+input_filter_functions: dict[str, InputFilterFunc] = {
     'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values,
     'EMPTY_DATA': filter_empty_data,
-    'path_confidence': {
-        'STOP': -1,
-        'CONTINUE': 2,
-        'REPEAT': -1
-    }
 }
 
-mlflow_response_filter_functions = {
+# Confidence mappings kept separate from function maps to avoid Union types
+input_path_confidence: Mapping[str, int] = {
+    'STOP': -1,
+    'CONTINUE': 2,
+    'REPEAT': -1,
+}
+
+# MLFlow response filter function mappings
+mlflow_response_filter_functions: dict[str, ResponseFilterFunc] = {
     'API_ERROR': api_error_filter,
-    'path_confidence': {
-        'STOP': -1,
-        'CONTINUE': 10,
-        'REPEAT': -1
-    },
 }
 
-mlflow_content_filter_functions = {
+mlflow_response_path_confidence: Mapping[str, int] = {
+    'STOP': -1,
+    'CONTINUE': 10,
+    'REPEAT': -1,
+}
+
+# MLFlow content filter function mappings
+mlflow_content_filter_functions: dict[str, ContentFilterFunc] = {
     'NAN_VALUES': nan_values_filter,
-    'path_confidence': {
-        'STOP': -1,
-        'CONTINUE': 18,
-        'REPEAT': -1
-    }
+    'EMPTY_DATA': filter_empty_data,
+}
+
+mlflow_content_path_confidence: Mapping[str, int] = {
+    'STOP': -1,
+    'CONTINUE': 18,
+    'REPEAT': -1,
 }
 
 
-class Gates(BaseActivity):
-    def __init__(self, logger: Logger, notification_handler: NotificationHandler):
-        BaseActivity.__init__(self, logger, notification_handler)
+class Gates(MinioManager):
+    """
+    Data quality gates and filtering activities for the Laborious system.
 
-    @activity.defn(name="input_gate")
+    This class implements comprehensive data quality validation and filtering
+    mechanisms that can be applied at different stages of the prediction pipeline.
+    It provides configurable filters with policy-based decision making to ensure
+    data integrity and quality throughout the ML workflow.
+
+    The class supports multiple filter types and implements a flexible policy
+    system that can be configured for different validation requirements. Each
+    filter returns a path decision (STOP, CONTINUE, REPEAT) along with confidence
+    scores and detailed comments for monitoring and debugging.
+
+    Attributes:
+        input_filter_functions (dict): Mapping of input filter names to functions
+        mlflow_response_filter_functions (dict): Mapping of MLFlow response filter names to functions
+        mlflow_content_filter_functions (dict): Mapping of MLFlow content filter names to functions
+    """
+
+    minio_repository: MinioRepository | None = None
+    _MAX_DEBUG_DATAFRAME_ROWS = 100
+
+    def __init__(
+        self,
+        minio_repository: MinioRepository | None = None,
+        logger: Logger | None = None,
+        notification_handler: NotificationHandler | None = None,
+        metrics_controller: MetricsController | None = None,
+    ):
+        """
+        Initialize data quality gates with logging and notification capabilities.
+
+        Args:
+            logger: Logger instance for observability and debugging
+            notification_handler: Notification handler for alerts and monitoring
+
+        Raises:
+            Exception: If BaseActivity initialization fails
+        """
+        MinioManager.__init__(
+            self, minio_repository, logger, notification_handler, metrics_controller
+        )
+
+    def close(self) -> None:
+        """
+        Close the gates activity and clean up resources.
+        """
+
+        MinioManager.close(self)
+
+    def __del__(self):
+        self.close()
+
+    def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None:
+        """
+        Log dataframe content only when row count is below the configured threshold
+
+        Args:
+         - message (str): Base log message to identify the dataframe in logs
+         - data (Any): Dataframe-like payload to be logged
+         - metadata (dict[str, Any]): Workflow metadata for contextual logging
+        """
+        self.debug(
+            build_dataframe_debug_message(
+                message=message,
+                data=data,
+                max_rows=self._MAX_DEBUG_DATAFRAME_ROWS,
+            ),
+            metadata,
+        )
+
+    @staticmethod
+    def _read_filter_entry(config: dict[str, Any]) -> tuple[str, dict[str, Any]]:
+        """
+        Read filter policy/config keys in a case-insensitive way.
+
+        Args:
+            config (dict[str, Any]): Filter configuration dictionary.
+
+        Return:
+            tuple[str, dict[str, Any]]: Parsed policy and config payload.
+        """
+        normalized = {str(key).upper(): value for key, value in config.items()}
+        policy = normalized['POLICY']
+        filter_config = normalized.get('CONFIG', {})
+        return policy, filter_config
+
+    @activity.defn(name='input_gate')
     async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
         """
-        Filters the data based on the filters. The return value is a tuple with the first element
-        being the policy and the second element being the confidence status.
+        Apply input data quality filters and validation.
+
+        This activity validates input data quality using configurable filters
+        before proceeding with ML operations. It applies multiple filter types
+        and returns a path decision based on the filter results and configured
+        policies.
+
+        The method implements a comprehensive filtering system that:
+        1. Applies configured filters to input data
+        2. Evaluates filter results against policy configurations
+        3. Determines appropriate path decisions (STOP, CONTINUE, REPEAT)
+        4. Provides confidence scores and detailed comments
+        5. Handles errors gracefully with notification integration
+
         Args:
-            - input_data (dict): The input data. Contains:
-                - filters (dict): The filters to apply.
-                The key is the filter name and the value is the filter configuration.
-                - data (dict[str, Any]): The data to filter.
-                - path_priority (list[str]): The path priority.
+            input_data: Configuration and data for input validation
+                Required keys:
+                    - metadata (dict): Workflow execution metadata
+                    - filters (dict): Filter configuration and policies
+                    - data (dict): Input data to validate
+                    - path_priority (list[str]): Priority order for path decisions
+
         Returns:
-            tuple[str | None, int, str]: (policy, confidence, comments) based in priority
-                list and filter configuration and functions.
+            tuple: (path_flag, confidence, comment)
+                - path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
+                - confidence (int): Confidence score for the decision
+                - comment (str): Detailed explanation of the decision
+
+        Raises:
+            Exception: If filter execution fails or configuration is invalid
         """
-
-        self.debug(f"Input data: {input_data}")
-
         metadata = input_data['metadata']
 
-        self.debug("Performing input gate...", metadata)
-
-        self.debug(f"Input data: {input_data}", metadata)
+        self.info('Performing input gate...', metadata)
 
         filters = input_data['filters']
-        data = DataFrame(input_data['data'])
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
+        data = await payload.retrieve(self.minio_repository, metadata)
         path_priority = input_data['path_priority']
 
         filter_output = []
 
-        self.debug(f"Input data:\n {data}", metadata)
-        self.debug(f"Filters: {filters}", metadata)
+        self._debug_dataframe('Input data:', data, metadata)
+        self.debug(f'Filters: {filters}', metadata)
 
+        # Apply each configured filter
         for fil, config in filters.items():
             if fil not in input_filter_functions:
-                self.error(f"Filter {fil} not found", metadata)
+                self.error(f'Filter {fil} not found', metadata)
                 continue
+            policy, filter_config = self._read_filter_entry(config)
             try:
-                if input_filter_functions[fil](data, config['config']):
-                    self.debug(
-                        f"Data not passed the input filter {fil}:{config}", metadata)
-                    filter_output.append(config['policy'])
+                if input_filter_functions[fil](data, filter_config):
+                    self.debug(f'Data not passed the input filter {fil}:{config}', metadata)
+                    filter_output.append(policy)
             except Exception as e:
                 trace = traceback.format_exc()
-                self.notification_handler.build_and_send_notification(
-                    notification_id=f"INTPUT_GATE_ERROR__{fil}",
-                    message=f"Error in filter {fil}:{config}: \n {e}",
-                    block="input_gate",
+                await self.send_notification_async(
+                    metadata=metadata,
+                    notification_id=f'INTPUT_GATE_ERROR__{fil}',
+                    message=f'Error in filter {fil}:{config}: \n {e}',
+                    block='input_gate',
                     level=NotificationLevel.ERROR,
-                    attachment_content=trace
+                    attachment_content=trace,
                 )
 
         for path_flag in path_priority:
             if path_flag in filter_output:
-                self.debug(f"Input gate result: {path_flag}", metadata)
-                return path_flag, input_filter_functions['path_confidence'][path_flag], \
-                    "Input data with bad quality"
+                self.info(f'Input gate result: {path_flag}', metadata)
+                return path_flag, input_path_confidence[path_flag], 'Input data with bad quality'
 
-        self.debug("Nothing was filtered by the input gate", metadata)
-        return None, 0, ""
+        self.info('Nothing was filtered by the input gate', metadata)
 
-    @activity.defn(name="mlflow_response_gate")
+        del data
+
+        return None, 0, ''
+
+    @activity.defn(name='mlflow_response_gate')
     async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
         """
-        Filters the data based on the mlflow response filters.
-        The return value is a tuple with the first element
-        being the policy and the second element being the confidence status.
+        Validate MLFlow API response quality and integrity.
+
+        This activity validates MLFlow API responses to ensure they meet quality
+        standards before proceeding with further processing. It applies response-specific
+        filters and determines appropriate path decisions based on response quality.
+
+        The method implements response validation that:
+        1. Applies MLFlow response-specific filters
+        2. Evaluates API response quality and integrity
+        3. Determines path decisions based on response validation results
+        4. Provides confidence scores and detailed validation comments
+        5. Handles API errors and response validation failures
+
         Args:
-            - input_data (dict): The input data. Contains:
-                - filters (dict): The filter configuration to apply.
-                - data (dict[str, Any]): The data to filter.
-                - path_priority (list[str]): The path priority list.
-                - type (str): The type of the gate.
+            input_data: Configuration and data for response validation
+                Required keys:
+                    - metadata (dict): Workflow execution metadata
+                    - filters (dict): Response filter configuration and policies
+                    - data (dict): MLFlow API response data to validate
+                    - type (str): Type of MLFlow operation (transform, predict)
+                    - path_priority (list[str]): Priority order for path decisions
+
         Returns:
-            tuple[str | None, int, str]: (policy, confidence, comments) based in priority list
-            and filter configuration and functions.
+            tuple: (path_flag, confidence, comment)
+                - path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
+                - confidence (int): Confidence score for the decision
+                - comment (str): Detailed explanation of the decision
+
+        Raises:
+            Exception: If response validation fails or configuration is invalid
         """
-
         metadata = input_data['metadata']
-        self.debug("Performing mlflow response gate...", metadata)
-
+        self.info('Performing mlflow response gate...', metadata)
+        raw_data = input_data['data']
         filters = input_data['filters']
-        data = input_data['data']
+
+        self.debug(
+            f'Input data: \n {create_sample_dict(raw_data, max_items=5, max_depth=5)}', metadata
+        )
+        self.debug(f'Filters: {filters}', metadata)
+
+        payload = MinioDataFramePayload.from_dict(raw_data)
+        data = await payload.retrieve(self.minio_repository, metadata)
+
         gate_type = input_data['type']
         path_priority = input_data['path_priority']
 
         filter_output = []
 
-        self.debug(f"Input data:\n {data}", metadata)
-        self.debug(f"Filters: {filters}", metadata)
-
         comments = []
+
+        status = payload.status or {}
+
         for fil, config in filters.items():
             if fil not in mlflow_response_filter_functions:
                 continue
+            policy, filter_config = self._read_filter_entry(config)
             try:
-                if mlflow_response_filter_functions[fil](data, config):
-                    filter_output.append(config['policy'])
-                    comments.append(data['content']['message'])
-                    self.notification_handler.build_and_send_notification(
-                        notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}",
-                        message=data['content']['message'],
-                        block="mlflow_gate",
-                        level=NotificationLevel.WARNING,
-                        attachment_content=data['content']['traceback']
+                if mlflow_response_filter_functions[fil](status, filter_config):
+                    filter_output.append(policy)
+                    comments.append(status.get('message', 'Unknown MLFlow API error'))
+                    await self.send_notification_async(
+                        metadata=metadata,
+                        notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
+                        message=status.get('message', 'Unknown MLFlow API error'),
+                        block='mlflow_gate',
+                        level=NotificationLevel.ERROR,
+                        attachment_content=status.get('traceback'),
                     )
             except Exception as e:
                 trace = traceback.format_exc()
-                self.notification_handler.build_and_send_notification(
-                    notification_id=f"MLFLOW_GATE_RESPONSE_FILTER__{fil}",
-                    message=f"Error in filter {fil}:{config}: \n {e}",
-                    block="mlflow_gate",
+                await self.send_notification_async(
+                    metadata=metadata,
+                    notification_id=f'MLFLOW_GATE_RESPONSE_FILTER__{fil}',
+                    message=f'Error in filter {fil}:{config}: \n {e}',
+                    block='mlflow_gate',
                     level=NotificationLevel.ERROR,
-                    attachment_content=trace
+                    attachment_content=trace,
                 )
 
         for path_flag in path_priority:
             if path_flag in filter_output:
-                self.debug(
-                    f"Mlflow response gate result: {path_flag}", metadata)
-                return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag], \
-                    ", ".join(comments)
+                self.info(f'Mlflow response gate result: {path_flag}', metadata)
+                return path_flag, mlflow_response_path_confidence[path_flag], ', '.join(comments)
 
-        self.debug("Nothing was filtered by the mlflow response gate", metadata)
-        return None, 0, ""
+        self.info('Nothing was filtered by the mlflow response gate', metadata)
 
-    @activity.defn(name="mlflow_content_gate")
+        del data
+
+        return None, 0, ''
+
+    @activity.defn(name='mlflow_content_gate')
     async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
         """
-        Filters the data based on the mlflow content filters.
-        The return value is a tuple with the first element
-        being the policy and the second element being the confidence status.
-        Args:
-            - input_data (dict): The input data. Contains:
-                - filters (dict): The filter configuration to apply.
-                - data (dict[str, Any]): The data to filter.
-                - path_priority (list[str]): The path priority list.
-                - type (str): The type of the gate.
-        Returns:
-            tuple[str | None, int, str]: (policy, confidence, comments) based in priority
-                list and filter configuration and functions.
-        """
+        Validate MLFlow prediction content quality and integrity.
 
+        This activity validates the content of MLFlow predictions to ensure they
+        meet quality standards before export and persistence. It applies content-specific
+        filters and determines appropriate path decisions based on content quality.
+
+        The method implements content validation that:
+        1. Applies MLFlow content-specific filters
+        2. Evaluates prediction content quality and integrity
+        3. Determines path decisions based on content validation results
+        4. Provides confidence scores and detailed validation comments
+        5. Handles content validation failures and quality issues
+
+        Args:
+            input_data: Configuration and data for content validation
+                Required keys:
+                    - metadata (dict): Workflow execution metadata
+                    - filters (dict): Content filter configuration and policies
+                    - data (dict): MLFlow prediction content to validate
+                    - type (str): Type of MLFlow operation (transform, predict)
+                    - path_priority (list[str]): Priority order for path decisions
+
+        Returns:
+            tuple: (path_flag, confidence, comment)
+                - path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
+                - confidence (int): Confidence score for the decision
+                - comment (str): Detailed explanation of the decision
+
+        Raises:
+            Exception: If content validation fails or configuration is invalid
+        """
         metadata = input_data['metadata']
-        self.debug("Performing mlflow content gate...", metadata)
+        self.info('Performing mlflow content gate...', metadata)
 
         filters = input_data['filters']
-        data = DataFrame(input_data['data'])
+
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
+        data = await payload.retrieve(self.minio_repository, metadata)
+
         gate_type = input_data['type']
         path_priority = input_data['path_priority']
 
         filter_output = []
 
-        self.debug(f"Input data:\n {data}", metadata)
-        self.debug(f"Filters: {filters}", metadata)
+        self._debug_dataframe('Input data:', data, metadata)
+        self.debug(f'Filters: \n {filters}', metadata)
 
         for fil, config in filters.items():
             if fil not in mlflow_content_filter_functions:
                 continue
+            policy, filter_config = self._read_filter_entry(config)
             try:
-                if mlflow_content_filter_functions[fil](data, config):
-                    filter_output.append(config['policy'])
-                    self.notification_handler.build_and_send_notification(
-                        notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}",
-                        message=f"Data not passed the content filter {fil}:{config}",
-                        block="mlflow_gate",
+                if mlflow_content_filter_functions[fil](data, filter_config):
+                    filter_output.append(policy)
+                    await self.send_notification_async(
+                        metadata=metadata,
+                        notification_id=f'{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}',
+                        message=f'Data not passed the content filter {fil}:{config}',
+                        block='mlflow_gate',
                         level=NotificationLevel.WARNING,
-                        attachment_content=data.to_string()
+                        attachment_content=data.to_string(),
                     )
             except Exception as e:
                 trace = traceback.format_exc()
-                self.notification_handler.build_and_send_notification(
-                    notification_id=f"MLFLOW_GATE_CONTENT_FILTER__{fil}",
-                    message=f"Error in filter {fil}:{config}: \n {e}",
-                    block="mlflow_gate",
+                await self.send_notification_async(
+                    metadata=metadata,
+                    notification_id=f'MLFLOW_GATE_CONTENT_FILTER__{fil}',
+                    message=f'Error in filter {fil}:{config}: \n {e}',
+                    block='mlflow_gate',
                     level=NotificationLevel.ERROR,
-                    attachment_content=trace
+                    attachment_content=trace,
                 )
 
         for path_flag in path_priority:
             if path_flag in filter_output:
-                self.debug(
-                    f"Mlflow content gate result: {path_flag}", metadata)
-                return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag], \
-                    "Transformed data not passed the content filter"
+                self.info(f'Mlflow content gate result: {path_flag}', metadata)
+                return (
+                    path_flag,
+                    mlflow_content_path_confidence[path_flag],
+                    'Transformed data not passed the content filter',
+                )
 
-        self.debug("Nothing was filtered by the mlflow content gate", metadata)
-        return None, 0, ""
+        self.info('Nothing was filtered by the mlflow content gate', metadata)
 
-    @activity.defn(name="format_prediction")
-    async def format_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+        del data
+
+        return None, 0, ''
+
+    def get_prediction_store_policy(
+        self, prediction_store_policy: str, metadata: dict[str, Any]
+    ) -> tuple[str, int]:
         """
-        Formats the prediction data.
+        Parse and validate prediction store policy configuration.
+
+        This method parses prediction store policy strings in the format 'type:value'
+        and validates them against allowed policy types and values. It provides
+        sensible defaults for invalid configurations and logs policy validation
+        failures for operational monitoring.
+
+        Supported Policy Types:
+        - 'lts': Latest timestamp - sorts data by timestamp descending
+        - 'erl': Earliest timestamp - sorts data by timestamp ascending
+
         Args:
-            - input_data (dict): The input data. Contains:
-                - data (dict[str, Any]): The data to format.
-                - timestamp (str): The timestamp of the data.
-                - model_id (str): The id of the model.
-                - prediction_confidence (float): The confidence of the prediction.
+            prediction_store_policy (str): Policy string in format 'type:value'
+            metadata (dict[str, Any]): Context metadata for logging and notifications
+
         Returns:
-            dict: The formatted data.
+            tuple[str, int]: (policy_type, policy_value)
+                - policy_type (str): Validated policy type ('lts' or 'erl')
+                - policy_value (int): Number of rows to retain
+        """
+        policy_elements = prediction_store_policy.split(':')
+
+        if len(policy_elements) < 2:
+            self.error(
+                f'Invalid prediction store policy: {prediction_store_policy}, using default policy',
+                metadata,
+            )
+            return 'lts', 1
+
+        policy_type = policy_elements[0]
+        policy_value = policy_elements[1]
+
+        # If the policy_type is not lts or erl, we use the default policy
+        # If the policty_value is not a number or 0, we use the default policy
+        if (
+            policy_type not in ['lts', 'erl']
+            or not policy_value.isdigit()
+            or int(policy_value) == 0
+        ):
+            self.error(
+                f'Invalid prediction store policy: {prediction_store_policy}, using default policy',
+                metadata,
+            )
+            return 'lts', 1
+
+        return policy_type, int(policy_value)
+
+    @activity.defn(name='format_transformed_data')
+    async def format_transformed_data(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
+        """
+        Format transformed data for storage and export operations.
+
+        This method formats transformed data from MLFlow model transformations
+        into a standardized format suitable for database storage. It converts
+        wide-format data (columns as variables) into long-format (melted)
+        with proper timestamp handling and model identification.
+
+        The formatting process includes:
+        1. Converting input data dictionary to DataFrame
+        2. Extracting timestamps from DataFrame index
+        3. Resetting index to create sequential row numbers
+        4. Melting data from wide format to long format (variable-value pairs)
+        5. Adding model_id for data lineage tracking
+
+        Args:
+            input_data (dict): Input data containing:
+                - metadata (dict): Workflow execution metadata
+                - data (dict[str, Any]): Transformed data to format (DataFrame-compatible dict)
+                - model_id (str): Unique identifier for the ML model
+
+        Returns:
+            dict: Formatted data dictionary with keys:
+                - timestamp (dict): Timestamp values indexed by row number
+                - variable (dict): Variable names indexed by row number
+                - value (dict): Variable values indexed by row number
+                - model_id (dict): Model identifiers indexed by row number
         """
         metadata = input_data['metadata']
-        self.debug("Formatting prediction...", metadata)
 
-        data = DataFrame(input_data['data'])
-        data['timestamp'] = input_data['timestamp']
+        model_id = input_data['model_id']
+
+        self.info('Formatting transformed data...', metadata)
+
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
+        data = await payload.retrieve(self.minio_repository, metadata)
+
+        data['timestamp'] = data.index
+        data = data.reset_index(drop=True)
+
+        data = data.melt(id_vars='timestamp', var_name='variable', value_name='value')
+        data['model_id'] = model_id
+
+        return await MinioDataFramePayload.from_dataframe(
+            dataframe=data,
+            minio_repo=self.minio_repository,
+            model_name=input_data['model_name'],
+            operation='transform',
+            workflow_metadata=metadata,
+            last_timestamp=payload.last_timestamp,
+            logger=self.logger,
+        )
+
+    @activity.defn(name='format_prediction')
+    async def format_prediction(self, input_data: dict[str, Any]) -> dict:
+        """
+        Format prediction data according to configured storage policies.
+
+        This method formats prediction data for storage and export operations.
+        It applies timestamp-based sorting policies, adds metadata fields,
+        and ensures data consistency before persistence. The method supports
+        multiple storage policies for flexible data retention strategies.
+
+        If only one row is present, we use the last timestamp as the timestamp
+
+        Storage Policies:
+        - 'lts:N': Latest timestamp - retains N most recent predictions
+        - 'erl:N': Earliest timestamp - retains N oldest predictions
+
+        Args:
+            input_data (dict): Input data containing:
+                - data (dict[str, Any]): Raw prediction data to format
+                - timestamp (str): Timestamp of the data
+                - model_id (str): Unique identifier for the ML model
+                - prediction_confidence (float): Confidence score for the prediction
+                - prediction_store_policy (str): Storage policy in format 'type:value'
+
+        Returns:
+            dict: Formatted prediction data ready for storage and export
+        """
+        metadata = input_data['metadata']
+        last_timestamp = input_data['timestamp']
+        prediction_store_policy = input_data['prediction_store_policy']
+        self.info('Formatting prediction...', metadata)
+
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
+        data = await payload.retrieve(self.minio_repository, metadata)
+
+        # Create timestamp column from index and reset index
+        data['timestamp'] = data.index
+        data = data.reset_index(drop=True)
+
+        self.debug(f'Prediction store policy: {prediction_store_policy}', metadata)
+        self._debug_dataframe('Prediction data:', data, metadata)
+
+        policy_type, policy_value = self.get_prediction_store_policy(
+            prediction_store_policy, metadata
+        )
+
+        # If data has no timestamp, we use the default timestamp and not sort the data
+        self.info(
+            f'Sorting data by timestamp and applying policy: {policy_type}:{policy_value}', metadata
+        )
+
+        # If policy_type is lts, we need to sort the data by timestamp descending and take the first policy_value rows
+        if policy_type == 'lts':
+            self.debug('Sorting data by timestamp descending', metadata)
+            data = data.sort_values(by='timestamp', ascending=False)
+        # If policy_type is erl, we need to sort the data by timestamp ascending and take the first policy_value rows
+        elif policy_type == 'erl':
+            self.debug('Sorting data by timestamp ascending', metadata)
+            data = data.sort_values(by='timestamp', ascending=True)
+        else:
+            self.error(f'Invalid policy type: {policy_type}, using default policy', metadata)
+            raise ValueError(f'Invalid policy type: {policy_type}')
+
+        int_policy_value = int(policy_value)
+
+        data = data.head(int_policy_value)
+        if int_policy_value == 1:
+            data['timestamp'] = last_timestamp
+
         data['model_id'] = input_data['model_id']
         data['prediction_confidence'] = input_data['prediction_confidence']
         data['prediction_status'] = 'Good'
-        data['comments'] = ""
-        data.sort_values(by='timestamp', inplace=True)
+        data['comments'] = ''
+        data = data.sort_values(by='timestamp', ascending=False)
+        data = data.reset_index(drop=True)
+
+        self.info(f'Prediction formatted: {len(data)} rows', metadata)
+        self._debug_dataframe('Prediction data:', data, metadata)
 
         return data.to_dict()
 
-    @activity.defn(name="format_default_prediction")
-    async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+    @activity.defn(name='format_default_prediction')
+    async def format_default_prediction(self, input_data: dict[str, Any]) -> dict:
         """
-        Creates and formats the default prediction data, with zero value in prediction,
-        and usefull information in the other fields.
+        Create and format default prediction data for error conditions.
+
+        This method generates default prediction data when the main prediction
+        pipeline encounters errors or quality issues. It creates a standardized
+        data structure with zero values for predictions and useful metadata
+        for operational monitoring and debugging.
+
+        The default prediction serves as a fallback mechanism to:
+        1. Maintain data pipeline continuity during failures
+        2. Provide operational visibility into prediction quality issues
+        3. Enable downstream systems to handle error conditions gracefully
+        4. Support debugging and troubleshooting efforts
 
         Args:
-            - input_data (dict): The input data. Contains:
-                - timestamp (str): The timestamp of the data.
-                - model_id (str): The id of the model.
-                - prediction_confidence (float): The confidence of the prediction.
-                - comment (str): The comment of the prediction.
+            input_data (dict): Input data containing:
+                - timestamp (str): Timestamp for the default prediction
+                - model_id (str): Unique identifier for the ML model
+                - prediction_confidence (float): Confidence score (typically low for errors)
+                - comment (str): Error description or operational comment
+
         Returns:
-            dict: The formatted data.
+            dict: Formatted default prediction data with error indicators
         """
 
         metadata = input_data['metadata']
-        self.debug("Formatting default prediction...", metadata)
+        self.debug('Formatting default prediction...', metadata)
 
-        return DataFrame({
-            'prediction': [0],
-            'response_time': [0],
-            'timestamp': [input_data['timestamp']],
-            'model_id': [input_data['model_id']],
-            'prediction_confidence': [input_data['prediction_confidence']],
-            'prediction_status': ['Bad'],
-            'comments': [input_data['comment']]
-        }).to_dict()
+        data = DataFrame(
+            {
+                'prediction': [0],
+                'response_time': [0],
+                'timestamp': [input_data['timestamp']],
+                'model_id': [input_data['model_id']],
+                'prediction_confidence': [input_data['prediction_confidence']],
+                'prediction_status': ['Bad'],
+                'comments': [input_data['comment']],
+            }
+        )
 
-    @activity.defn(name="get_last_timestamp")
-    async def get_last_timestamp(self, input_data: dict[str, Any]) -> str:
+        self.info(f'Default prediction formatted: {data.size} rows', metadata)
+        return data.to_dict()
+
+    @activity.defn(name='format_retrain_report')
+    async def format_retrain_report(self, input_data: dict[str, Any]) -> dict:
         """
-        Gets the last timestamp of the data.
+        Format retrain report data for storage and audit trail maintenance.
+
+        This method formats model retraining operation results into a standardized
+        report format suitable for database storage and operational monitoring.
+        It captures retraining status, timestamps, and model version information
+        for comprehensive audit trails and operational visibility.
+
+        The formatting process includes:
+        1. Extracting retraining experiment response data
+        2. Capturing model update report information (version, MLflow IDs)
+        3. Formatting timestamps and status information
+        4. Conditionally including version information for successful retrains
+
         Args:
-            - input_data (dict): The input data. Contains:
-                - data (dict[str, Any]): The data to get the last timestamp from.
+            input_data (dict): Input data containing:
+                - metadata (dict): Workflow execution metadata
+                - experiment_response (dict): Retraining experiment response containing:
+                    - success (bool): Retraining operation success status
+                    - timestamp (str): Timestamp of the retraining operation
+                    - message (str): Status message or error description
+                - update_report (dict): Model update report containing:
+                    - version (str): New model version identifier
+                    - mlflow_run_id (str): MLflow run identifier
+                    - mlflow_experiment_id (str): MLflow experiment identifier
+                - model_id (str): Unique identifier for the ML model
+                - model_name (str): Name of the ML model
+
         Returns:
-            str: The last timestamp of the data.
+            dict: Formatted retrain report dictionary with keys:
+                - model_id (dict): Model identifiers indexed by row number
+                - model_name (dict): Model names indexed by row number
+                - timestamp (dict): Retraining timestamps indexed by row number
+                - status (dict): Retraining status messages indexed by row number
+                - version (dict, optional): Model versions indexed by row number
+                    Only included if experiment_response['success'] is True
+                - mlflow_run_id (dict, optional): MLflow run IDs indexed by row number
+                    Only included if experiment_response['success'] is True
+                - mlflow_experiment_id (dict, optional): MLflow experiment IDs indexed by row number
+                    Only included if experiment_response['success'] is True
         """
-        data = DataFrame(input_data['data'])
-        if data.empty:
-            return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-        return max(data['timestamp'].values.tolist())
+        metadata = input_data['metadata']
+        self.info('Formatting retrain report...', metadata)
+
+        experiment_response = input_data['experiment_response']
+        update_report = input_data['update_report']
+        model_id = input_data['model_id']
+        model_name = input_data['model_name']
+
+        report = DataFrame(
+            {
+                'model_id': [model_id],
+                'model_name': [model_name],
+                'timestamp': [experiment_response['timestamp']],
+                'status': [experiment_response['message']],
+            }
+        )
+
+        if experiment_response['success']:
+            # Retrain was successfull
+            report['version'] = update_report['version']
+            report['mlflow_run_id'] = update_report['mlflow_run_id']
+            report['mlflow_experiment_id'] = update_report['mlflow_experiment_id']
+
+        self._debug_dataframe('Retrain report:', report, metadata)
+
+        return report.to_dict()
+
+    @activity.defn(name='write_metrics')
+    async def write_metrics(self, input_data: dict[str, Any]):
+        """
+        Write prediction performance metrics to Prometheus monitoring system.
+
+        This method records comprehensive metrics for prediction operations,
+        enabling operational monitoring, performance analysis, and alerting.
+        It tracks prediction counts, confidence levels, and response times
+        for each model and pipeline combination.
+
+        Metrics Recorded:
+        1. Prediction Count: Incremental counter for successful predictions
+        2. Confidence Monitor: Current confidence level for predictions
+        3. Response Time Monitor: Histogram of prediction response times
+
+        Args:
+            input_data (dict): Input data containing:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - prediction (dict[str, Any]): Prediction data with metrics
+
+        Raises:
+            Exception: If metrics writing fails or configuration is invalid
+        """
+        metadata = input_data['metadata']
+        prediction = DataFrame(input_data['prediction'])
+        prediction_confidence = prediction['prediction_confidence'].values[0]
+        response_time = prediction['response_time'].values[0]
+        opc_metrics = input_data['opc_metrics']
+
+        self.info(f'Writing metrics for model {metadata["model_name"]}', metadata)
+
+        core_tags = {
+            'pod_id': self.pod_id,
+            'runtime': self.runtime,
+            'operation_type': 'predict',
+            'model_name': metadata['model_name'],
+            'workflow_name': metadata['workflow_name'],
+        }
+        await self.emit_metric(
+            metric_object=metrics.PREDICTIONS_WRITTEN_COUNT,
+            tags=core_tags,
+        )
+
+        await self.emit_metric(
+            metric_object=metrics.PREDICTION_CONFIDENCE_MONITOR,
+            method='set',
+            tags=core_tags,
+            value=prediction_confidence,
+        )
+
+        await self.emit_metric(
+            metric_object=metrics.PREDICTION_RESPONSE_TIME_MONITOR,
+            method='observe',
+            tags=core_tags,
+            value=response_time,
+        )
+
+        for server_id, tags in opc_metrics.items():
+            for tag, response_time in tags.items():
+                if response_time is not None:
+                    await self.emit_metric(
+                        metric_object=metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR,
+                        method='observe',
+                        tags={
+                            **core_tags,
+                            'opc_server_id': server_id,
+                            'tag': tag,
+                        },
+                        value=response_time,
+                    )
+
+                    await self.emit_metric(
+                        metric_object=metrics.PREDICTION_OPC_WRITING_COUNT,
+                        tags={
+                            **core_tags,
+                            'opc_server_id': server_id,
+                            'tag': tag,
+                        },
+                    )
+
+        self.info(f'Metrics written for model {metadata["model_name"]}', metadata)

laborious/activities/mlflow.py

@@ -1,98 +1,509 @@
-import numpy as np
-from pandas import DataFrame
 from temporalio import activity, workflow
 
-
 with workflow.unsafe.imports_passed_through():
-    from sientia_do.temporal.activities.base import BaseActivity
-    from sientia_do.notifications.handlers import NotificationHandler
-    from sientia_do.temporal.utils.logger import Logger
-    from laborious.utils.repository.model_repository import MLFlowRepository
+    import traceback
     from typing import Any
 
+    import numpy as np
+    from pandas import to_datetime
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.repository.minio_repository import MinioRepository
+    from sientia_do.temporal.constants import (
+        DATETIME_FORMAT,
+        DATETIME_FORMAT_MS_WITH_TZ,
+        DATETIME_FORMAT_WITH_TZ,
+        now,
+    )
+    from sientia_do.utils.formatters import create_sample_dict
 
-class MLFlow(BaseActivity):
-    def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str,
-                 mlflow_password: str, logger: Logger, notification_handler: NotificationHandler):
-        BaseActivity.__init__(self, logger, notification_handler)
-        self.mlflow_host = mlflow_host
-        self.mlflow_port = mlflow_port
-        self.mlflow_username = mlflow_username
-        self.mlflow_password = mlflow_password
+    from laborious.utils.dataframe_debug import build_dataframe_debug_message
+    from laborious.utils.messages import MISSING_MINIO_REPOSITORY
+    from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
+    from laborious.utils.repository.minio_manager import MinioManager
+    from laborious.utils.repository.model_repository import MLFlowRepository
 
-        self.model_monitoring_repository = MLFlowRepository(
-            f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password
+
+class MLFlow(MinioManager):
+    """
+    MLFlow integration activities for model inference operations.
+
+    This class provides activities for interacting with MLFlow models, including
+    data transformation and prediction operations. It handles authentication,
+    data preprocessing, and model management with configurable retention policies.
+
+    The class implements comprehensive error handling and logging for all
+    MLFlow operations, ensuring reliable model inference in production environments.
+
+    Attributes:
+        model_monitoring_repository (MLFlowRepository): Repository for MLFlow operations
+    """
+
+    _MAX_DEBUG_DATAFRAME_ROWS = 100
+
+    def __init__(
+        self,
+        mlflow_repository: MLFlowRepository,
+        minio_repository: MinioRepository | None = None,
+        logger: Logger | None = None,
+        notification_handler: NotificationHandler | None = None,
+        metrics_controller: MetricsController | None = None,
+    ):
+        """
+        Initialize MLFlow activities with an already-built repository.
+
+        Args:
+            mlflow_repository: Repository for MLFlow operations, built and owned by the caller
+                (shared with other activities rather than rebuilt here)
+            logger: Logger instance for observability and debugging
+            notification_handler: Notification handler for alerts and monitoring
+        """
+        MinioManager.__init__(
+            self, minio_repository, logger, notification_handler, metrics_controller
+        )
+        self.model_monitoring_repository = mlflow_repository
+
+    def close(self) -> None:
+        """
+        Close the MLFlow activity and clean up resources.
+        """
+        MinioManager.close(self)
+
+    def __del__(self):
+        self.close()
+
+    def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None:
+        """
+        Log dataframe content only when row count is below the configured threshold
+
+        Args:
+         - message (str): Base log message to identify the dataframe in logs
+         - data (Any): Dataframe-like object expected to expose shape and to_csv
+         - metadata (dict[str, Any]): Workflow metadata for contextual logging
+        """
+        self.debug(
+            build_dataframe_debug_message(
+                message=message,
+                data=data,
+                max_rows=self._MAX_DEBUG_DATAFRAME_ROWS,
+            ),
+            metadata,
         )
 
-    @activity.defn(name="request_transform")
-    async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]:
+    @activity.defn(name='request_transform')
+    async def request_transform(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
         """
-        Access MLFlow model to get the transformed data.
+        Transform input data using MLFlow models.
+
+        This activity processes input data through MLFlow model transformation,
+        including data preprocessing, format conversion, and validation. It handles
+        data deduplication, pivoting, and cleanup to ensure optimal model performance.
+
+        The transformation process includes:
+        1. Data deduplication based on variable and timestamp
+        2. Data pivoting for model input format
+        3. Null value handling and cleanup
+        4. MLFlow model transformation request
+        5. Response validation and logging
+
         Args:
-            - input_data (dict): The input data. Contains:
-                - data (dict[str, Any]): The data to transform.
-                - model_name (str): The name of the model.
-                - model_retention (int): The retention time of the model, in minutes.
+            input_data: Configuration and data for transformation
+                Required keys:
+                    - metadata (dict): Workflow execution metadata
+                    - data (dict): Input data for transformation
+                    - model_name (str): Name of the MLFlow model to use
+                    - model_retention (int): Model retention period in minutes
+
         Returns:
-            dict[str, Any]: The transformed data.
+            dict: Transformed data from MLFlow model
+
+        Raises:
+            Exception: If transformation fails or MLFlow model is unavailable
         """
         metadata = input_data['metadata']
-        self.debug('Transforming data...', metadata)
-        data = DataFrame(input_data['data'])
-        model_name = input_data['model_name']
-        model_retention = input_data['model_retention']
+        self.info('Transforming data...', metadata)
 
-        self.debug("Raw input data:", metadata)
-        self.debug(data, metadata)
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
+        data = await payload.retrieve(self.minio_repository, metadata)
+
+        model_name = input_data['model_name']
+        model_config = input_data.get('model_config', {})
+
+        self._debug_dataframe('Raw input data:', data, metadata)
 
         # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
         data = data.sort_values('created_at', ascending=False).drop_duplicates(
             subset=['variable', 'timestamp'], keep='first'
         )
 
-        data = data.pivot(
-            index='timestamp', columns='variable',
-            values='value')
+        # Pivot data for model input format
+        data = data.pivot(index='timestamp', columns='variable', values='value')
         data.fillna(np.nan, inplace=True)
-        data.reset_index(inplace=True)
+
         data.columns.name = None
+        data.index.name = None
 
-        self.debug("Processed input data:", metadata)
-        self.debug(data, metadata)
+        data['timestamp'] = data.index
 
-        response_data = self.model_monitoring_repository.transform(
-            model_name, data, model_retention)
+        self._debug_dataframe('Processed input data:', data, metadata)
 
-        self.debug("Response data:", metadata)
-        self.debug(response_data, metadata)
+        # Request transformation from MLFlow model
+        response_data = await self.model_monitoring_repository.transform(
+            model_name, data, model_config, metadata
+        )
 
-        return response_data
+        self.debug(
+            f'Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}',
+            metadata,
+        )
 
-    @activity.defn(name="request_predict")
-    async def request_predict(self, input_data: dict[str, Any]) -> dict[str, Any]:
+        self.debug(
+            f'Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}',
+            metadata,
+        )
+
+        self.info('Data transformed successfully', metadata)
+
+        if not response_data.get('success', False):
+            return await MinioDataFramePayload.from_dataframe(
+                dataframe=None,
+                minio_repo=self.minio_repository,
+                model_name=model_name,
+                operation='transform',
+                status=response_data,
+                workflow_metadata=metadata,
+                last_timestamp=payload.last_timestamp,
+                logger=self.logger,
+            )
+
+        return await MinioDataFramePayload.from_dataframe(
+            dataframe=response_data['content'],
+            minio_repo=self.minio_repository,
+            model_name=model_name,
+            operation='transform',
+            workflow_metadata=metadata,
+            status={
+                'success': True,
+            },
+            last_timestamp=payload.last_timestamp,
+            logger=self.logger,
+        )
+
+    @activity.defn(name='request_predict')
+    async def request_predict(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
         """
-        Access MLFlow model to get the predicted data.
+        Execute predictions using MLFlow models.
+
+        This activity performs ML model inference using MLFlow models with the
+        transformed data. It handles data format conversion, null value processing,
+        and model prediction requests with comprehensive error handling.
+
+        The prediction process includes:
+        1. Data format validation and cleanup
+        2. Null value handling for model compatibility
+        3. MLFlow model prediction request
+        4. Response validation and logging
+        5. Performance monitoring and metrics
+
         Args:
-            - input_data (dict): The input data. Contains:
-                - data (dict[str, Any]): The data to predict.
-                - model_name (str): The name of the model.
-                - model_retention (int): The retention time of the model, in minutes.
+            input_data: Configuration and data for prediction
+                Required keys:
+                    - metadata (dict): Workflow execution metadata
+                    - data (dict): Transformed data for prediction
+                    - model_name (str): Name of the MLFlow model to use
+                    - model_retention (int): Model retention period in minutes
+                Optional keys (in model_config):
+                    - skip_predict_timestamp (bool): If True, don't reinsert the
+                      timestamp column from the index before predicting. Default False.
+
         Returns:
-            dict[str, Any]: The predicted data.
+            dict: Prediction results from MLFlow model
+
+        Raises:
+            Exception: If prediction fails or MLFlow model is unavailable
         """
         metadata = input_data['metadata']
-        self.debug('Predicting data...', metadata)
-        data = DataFrame(input_data['data'])
+        self.info('Predicting data...', metadata)
+
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
+        data = await payload.retrieve(self.minio_repository, metadata)
+
         model_name = input_data['model_name']
-        model_retention = input_data['model_retention']
+        model_config = input_data.get('model_config', {})
 
-        self.debug(data, metadata)
+        self._debug_dataframe('Input data for prediction:', data, metadata)
 
+        # Convert numpy.nan to None for model compatibility
         data.replace(np.nan, None, inplace=True)
 
-        response_data = self.model_monitoring_repository.predict(
-            model_name, data, model_retention)
+        if not model_config.get('skip_predict_timestamp', False):
+            data['timestamp'] = data.index
+            data['timestamp'] = to_datetime(
+                data['timestamp'], format=DATETIME_FORMAT_WITH_TZ
+            ).dt.strftime(DATETIME_FORMAT)
 
-        self.debug(response_data, metadata)
+        # Request prediction from MLFlow model
+        response_data = await self.model_monitoring_repository.predict(
+            model_name, data, model_config, metadata
+        )
 
-        return response_data
+        self.debug(
+            f'Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}',
+            metadata,
+        )
+
+        self.info('Data predicted successfully', metadata)
+
+        if not response_data.get('success', False):
+            return await MinioDataFramePayload.from_dataframe(
+                dataframe=None,
+                minio_repo=self.minio_repository,
+                model_name=model_name,
+                operation='predict',
+                status=response_data,
+                workflow_metadata=metadata,
+                last_timestamp=payload.last_timestamp,
+                logger=self.logger,
+            )
+
+        return await MinioDataFramePayload.from_dataframe(
+            dataframe=response_data['content'],
+            minio_repo=self.minio_repository,
+            model_name=model_name,
+            operation='predict',
+            workflow_metadata=metadata,
+            status={
+                'success': True,
+            },
+            last_timestamp=payload.last_timestamp,
+            logger=self.logger,
+        )
+
+    @activity.defn(name='retrain_model')
+    async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
+        """
+        Retrain MLFlow models with updated training data.
+
+        This activity orchestrates the complete model retraining process,
+        including data preparation, model retraining execution, and result
+        validation. It handles data preprocessing, column cleanup, and
+        comprehensive error handling for production model management.
+
+        The retraining process includes:
+        1. Data timestamp extraction and validation
+        2. Column cleanup and data preparation
+        3. Data pivoting for model input format
+        4. MLFlow model retraining execution
+        5. Result validation and error handling
+
+        Args:
+            input_data (dict): Input data containing:
+                - metadata (dict): Workflow execution metadata
+                - data (dict[str, Any]): Training data for model retraining
+                - model_name (str): Name of the MLFlow model to retrain
+
+        Returns:
+            dict: Retraining results containing:
+                - status (str): Retraining operation status
+                - timestamp (str): Timestamp of the retraining operation
+                - experiment (str): MLFlow experiment identifier
+
+        Raises:
+            Exception: If retraining fails or encounters critical errors
+        """
+
+        if self.minio_repository is None:
+            raise ValueError(MISSING_MINIO_REPOSITORY)
+
+        metadata = input_data['metadata']
+
+        try:
+            # Payload-based retrain input (inline dict or MinIO offloaded).
+            payload = MinioDataFramePayload.from_dict(input_data['data'])
+            data = await payload.retrieve(self.minio_repository, metadata)
+
+        except Exception as e:
+            trace = traceback.format_exc()
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='ERROR_LOADING_RETRAIN_DATA',
+                message=f'Error loading retrain data: {e}',
+                block='retrain_model',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+            self.error(trace, metadata)
+            return {
+                'success': False,
+                'message': f'Error loading retrain data: {e}',
+                'traceback': trace,
+                'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ),
+            }
+
+        self.debug(f'Retrain data loaded successfully: shape {data.shape}', metadata)
+
+        model_name = input_data['model_name']
+        model_config = input_data.get('model_config', {})
+
+        self.info(f'Retraining model {model_name}...', metadata)
+
+        timestamp = data['timestamp'].max()
+        self.debug(f'Timestamp: {timestamp}', metadata)
+
+        # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
+        if 'created_at' in data.columns:
+            data = data.sort_values('created_at', ascending=False).drop_duplicates(
+                subset=['variable', 'timestamp'], keep='first'
+            )
+        else:
+            data = data.drop_duplicates(subset=['variable', 'timestamp'], keep='first')
+
+        data.drop(columns=['model_id'], inplace=True, errors='ignore')
+        data.drop(columns=['created_at'], inplace=True, errors='ignore')
+
+        # Pivot data for model input format
+        data = data.pivot(index='timestamp', columns='variable', values='value')
+        data.fillna(np.nan, inplace=True)
+        # data.reset_index(inplace=True)
+        data.columns.name = None
+
+        data['timestamp'] = data.index
+        data['timestamp'] = to_datetime(
+            data['timestamp'], format=DATETIME_FORMAT_WITH_TZ
+        ).dt.strftime(DATETIME_FORMAT)
+        data['timestamp'] = to_datetime(data['timestamp'], format=DATETIME_FORMAT)
+
+        data.columns.name = None
+
+        retrain_output = await self.model_monitoring_repository.retrain_model(
+            data=data, model_name=model_name, model_config=model_config, metadata=metadata
+        )
+
+        if not retrain_output['success']:
+            trace = retrain_output['traceback']
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='RETRAIN_MODEL_ERROR',
+                message=f'Error retraining model {model_name}: {retrain_output["message"]}',
+                block='retrain_model',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+            self.error(trace, metadata=metadata)
+
+        return {**retrain_output, 'timestamp': timestamp}
+
+    @activity.defn(name='update_production_model')
+    async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+        """
+        Update production model with newly trained model version.
+
+        This activity manages the critical process of updating production
+        models with newly trained versions. It handles model deployment,
+        status tracking, and comprehensive reporting for operational
+        visibility and audit trails.
+
+        The update process includes:
+        1. Production model update execution
+        2. Status and metadata tracking
+        3. Comprehensive reporting and logging
+        4. Error handling and notification
+        5. Audit trail maintenance
+
+        Args:
+            input_data (dict): Input data containing:
+                - metadata (dict): Workflow execution metadata
+                - model_name (str): Name of the MLFlow model to update
+                - experiment (str): MLFlow experiment identifier
+                - model_id (str): Unique identifier for the model version
+                - timestamp (str): Timestamp of the update operation
+                - status (str): Current status of the model update
+
+        Returns:
+            dict[Any, Any]: Comprehensive update report containing:
+                - model_id (str): Model version identifier
+                - model_name (str): Name of the updated model
+                - timestamp (str): Update operation timestamp
+                - status (str): Update operation status
+                - Additional MLFlow response metadata
+
+        Raises:
+            Exception: If production model update fails
+        """
+        metadata = input_data['metadata']
+        model_name = input_data['model_name']
+        experiment = input_data['experiment']
+        self.info(
+            f'Updating production model {model_name} from experiment {experiment}...', metadata
+        )
+
+        try:
+            response = await self.model_monitoring_repository.update_production_model(
+                experiment=experiment, model_name=model_name, metadata=metadata
+            )
+
+            self.info(f'Production model {model_name} updated successfully', metadata)
+            return response
+
+        except Exception as e:
+            trace = traceback.format_exc()
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='UPDATE_PRODUCTION_MODEL_ERROR',
+                message=f'Error updating production model {model_name}: {e}',
+                block='update_production_model',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+            self.error(trace, metadata=metadata)
+            raise e
+
+    @activity.defn(name='get_reference_data')
+    async def get_reference_data(self, input_data: dict[str, Any]) -> list[dict] | None:
+        """
+        Get reference data from the MLflow Model Registry.
+
+        This method retrieves evaluation reference data stored as artifacts in the
+        MLflow Model Registry. The reference data is typically used for model
+        drift detection, performance comparison, and quality validation. The method
+        loads the data from a CSV artifact file and formats timestamps for
+        consistent processing.
+
+        The method handles:
+        1. Loading evaluation data artifact from MLflow Model Registry
+        2. Timestamp parsing and formatting for consistency
+        3. Data conversion to dictionary format for workflow consumption
+        4. Graceful handling of missing reference data
+
+        Args:
+            input_data (dict): Input data containing:
+                - metadata (dict): Workflow execution metadata
+                - model_name (str): Name of the MLFlow model to get reference data from
+
+        Returns:
+            list[dict[Hashable, Any]] | None: Reference data from the MLflow Model Registry
+                as a list of dictionaries. Returns None if reference data is not found
+                or if the artifact does not exist.
+
+        Raises:
+            Exception: If artifact loading fails or encounters errors during processing
+        """
+
+        metadata = input_data['metadata']
+        model_name = input_data['model_name']
+        artifact = 'evaluation_data.csv'
+
+        reference_data = await self.model_monitoring_repository.load_artifact_dataframe(
+            model_name=model_name, artifact_path=artifact, metadata=metadata
+        )
+
+        if reference_data is None:
+            self.warning(f'Reference data not found for model {model_name}', metadata)
+            return None
+
+        reference_data['timestamp'] = to_datetime(reference_data['timestamp'])
+        reference_data['timestamp'] = reference_data['timestamp'].dt.strftime(DATETIME_FORMAT)
+
+        return reference_data.to_dict(orient='records')

laborious/activities/model_metrics.py

@@ -0,0 +1,364 @@
+from temporalio import activity, workflow
+
+with workflow.unsafe.imports_passed_through():
+    import time
+    import traceback
+    import warnings
+    from typing import Any
+
+    import numpy as np
+    from pandas import DataFrame, Index, to_datetime
+    from sientia.ModelAnalysis import ModelAnalysis
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+    from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
+
+    from laborious import metrics
+    from laborious.utils.dataframe_debug import build_dataframe_debug_message
+
+warnings.filterwarnings('ignore', category=RuntimeWarning, message='Degrees of freedom <= 0')
+warnings.filterwarnings(
+    'ignore', category=RuntimeWarning, message='invalid value encountered in scalar divide'
+)
+
+
+class ModelMetrics(SientiaMonitoring):
+    """
+    Metrics activities for the Laborious system.
+
+    This class provides activities for writing metrics to the Prometheus monitoring system.
+    """
+
+    _MAX_DEBUG_DATAFRAME_ROWS = 100
+
+    def __init__(
+        self,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+    ):
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
+
+    def close(self) -> None:
+        """
+        Close the model metrics activity and clean up resources.
+        """
+        SientiaMonitoring.shutdown(self)
+
+    def __del__(self):
+        self.close()
+
+    def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None:
+        """
+        Log dataframe content only when row count is below the configured threshold
+
+        Args:
+         - message (str): Base log message to identify the dataframe in logs
+         - data (Any): Dataframe-like payload to be logged
+         - metadata (dict[str, Any]): Workflow metadata for contextual logging
+        """
+        self.debug(
+            build_dataframe_debug_message(
+                message=message,
+                data=data,
+                max_rows=self._MAX_DEBUG_DATAFRAME_ROWS,
+            ),
+            metadata,
+        )
+
+    async def get_drift_metrics(
+        self,
+        reference_data: DataFrame,
+        target_data: DataFrame,
+        target_name: str,
+        reference_columns: Index,
+        drift_metrics: list[str],
+        chunk_period: str,
+        metadata: dict[str, Any],
+    ) -> DataFrame:
+        """
+        Calculate univariate drift metrics for a model.
+        Args:
+            model_analysis (ModelAnalysis): Model analysis object
+            reference_data (DataFrame): Reference data
+            target_data (DataFrame): Target data
+            reference_columns (list[str]): Reference columns
+            drift_metrics (list[str]): Drift metrics
+            metadata (dict[str, Any]): Workflow execution metadata
+        """
+
+        config = {
+            'target': target_name,
+            'prediction': 'prediction',
+            'timestamp': 'timestamp',
+            'features': reference_columns,
+        }
+
+        model_analysis = ModelAnalysis(config=config)
+
+        self._debug_dataframe(
+            f'Reference data: Size {reference_data.shape}', reference_data, metadata
+        )
+
+        self._debug_dataframe(f'Target data: Size {target_data.shape}', target_data, metadata)
+
+        core_labels = self.get_core_labels(metadata, operation_type='detect_univariate_drift')
+        start_time = time.time()
+        try:
+            univariate_drift = model_analysis.detect_univariate_drift(
+                reference_df=reference_data,
+                analysis_df=target_data,
+                features=reference_columns,
+                timestamp_col=config['timestamp'],
+                methods=drift_metrics,
+                chunk_period=chunk_period,
+            )
+        except Exception as e:
+            self.error(f'Error detecting univariate drift: {e}', metadata)
+            await self.emit_metric(
+                metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
+            )
+            raise e
+        await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
+        await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
+
+        core_labels = self.get_core_labels(metadata, operation_type='detect_multivariate_drift')
+        start_time = time.time()
+        try:
+            multivariate_drift = model_analysis.detect_multivariate_drift(
+                reference_df=reference_data,
+                analysis_df=target_data,
+                features=reference_columns,
+                timestamp_col=config['timestamp'],
+                chunk_period=chunk_period,
+            )
+        except Exception as e:
+            self.error(f'Error detecting multivariate drift: {e}', metadata)
+            await self.emit_metric(
+                metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
+            )
+            raise e
+        await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
+        await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
+
+        start_time = time.time()
+        core_labels = self.get_core_labels(metadata, operation_type='get_drift_metrics_dataframe')
+        try:
+            drift_df = model_analysis.get_drift_metrics_dataframe(
+                univariate_drift=univariate_drift,
+                multivariate_drift=multivariate_drift,
+            )
+        except Exception as e:
+            self.error(f'Error getting drift metrics: {e}', metadata)
+            await self.emit_metric(
+                metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
+            )
+            raise e
+        await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
+        await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
+
+        self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
+
+        return drift_df
+
+    @activity.defn(name='calculate_drift')
+    async def calculate_drift(self, input_data: dict[str, Any]) -> list[dict]:
+        """
+        Calculate drift metrics for a model.
+
+        Args:
+            input_data (dict[str, Any]): Input data containing:
+                - metadata (dict): Workflow execution metadata
+                - model_name (str): Name of the MLFlow model to calculate drift for
+                - reference_data (pd.DataFrame): Reference data for the model
+                - target_data (pd.DataFrame): Target data for calculating drift
+                - target_name (str): Name of the target column
+                - drift_metrics (list[str]): List of drift metrics to calculate
+        """
+        metadata = input_data['metadata']
+        model_name = input_data['model_name']
+        model_id = input_data['model_id']
+        reference_raw_data = input_data['reference_data']
+        target_data = DataFrame(input_data['target_data'])
+        target_name = input_data['target_name']
+        drift_metrics = input_data['drift_metrics']
+        chunk_period = input_data['chunk_period']
+
+        if chunk_period not in ['min', 's']:
+            self.error(f'Invalid chunk period: {chunk_period}', metadata)
+            raise ValueError(f'Invalid chunk period: {chunk_period}, must be "min" or "s"')
+
+        self.info(f'Calculating drift for model {model_name}', metadata)
+
+        target_data = target_data.pivot(index='timestamp', columns='variable', values='value')
+        target_data['timestamp'] = target_data.index
+        target_data['timestamp'] = to_datetime(target_data['timestamp'])
+        target_data['timestamp'] = target_data['timestamp'].dt.strftime(DATETIME_FORMAT)
+        target_data = target_data.reset_index(drop=True)
+        target_data.dropna(inplace=True)
+
+        if reference_raw_data is not None:
+            self.info('Using reference data', metadata)
+            reference_data = DataFrame(reference_raw_data)
+            accurate = True
+        else:
+            # Get 30% first rows of target_data
+            self.warning('Using 30% first rows of target data as reference data', metadata)
+            target_data.sort_values(by='timestamp', ascending=True, inplace=True)
+            reference_data = target_data.head(int(len(target_data) * 0.3))
+            accurate = False
+
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING',
+                message='Using 30% first rows of target data as reference data',
+                block='model_metrics',
+                level=NotificationLevel.WARNING,
+                attachment_content=reference_data.to_csv(),
+            )
+
+        reference_columns = reference_data.drop(
+            columns=[target_name, 'timestamp', 'target', 'prediction'], errors='ignore'
+        ).columns
+
+        try:
+            drift_df = await self.get_drift_metrics(
+                reference_data=reference_data,
+                target_data=target_data,
+                target_name=target_name,
+                reference_columns=reference_columns,
+                drift_metrics=drift_metrics,
+                chunk_period=chunk_period,
+                metadata=metadata,
+            )
+        except Exception as e:
+            self.error(f'Error getting drift metrics: {e}', metadata)
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='MODEL_METRICS_GET_DRIFT_METRICS_ERROR',
+                message=f'Error getting drift metrics: {e}',
+                block='model_metrics',
+                level=NotificationLevel.ERROR,
+                attachment_content=traceback.format_exc(),
+            )
+            return []
+
+        if drift_df.empty:
+            self.warning('No drift metrics found', metadata)
+            return []
+
+        # Drop unnecessary columns
+        drift_df.drop(columns=['p_value'], inplace=True)
+
+        # Extract timestamps only until minutes
+        if chunk_period == 'min':
+            target_timestamps = target_data['timestamp'].apply(lambda x: x[:16])
+        else:
+            target_timestamps = target_data['timestamp']
+
+        # Drop rows where timestamp is not in target data, to avoid save drift from reference
+        drift_df = drift_df[drift_df['timestamp'].isin(target_timestamps)]
+
+        if drift_df.empty:
+            self.warning(
+                'No drift metrics found after dropping rows where timestamp is not in target data',
+                metadata,
+            )
+            return []
+
+        # Rename columns to match database columns
+        drift_df.rename(
+            columns={
+                'metric': 'method',
+                'statistic': 'value',
+            },
+            inplace=True,
+        )
+
+        # Drop duplicates
+        drift_df.drop_duplicates(
+            subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
+        )
+
+        drift_df['model_id'] = model_id
+        drift_df['accurate'] = accurate
+
+        drift_df['timestamp'] = to_datetime(drift_df['timestamp'])
+        drift_df['timestamp'] = drift_df['timestamp'].dt.tz_localize('UTC')
+        drift_df['timestamp'] = drift_df['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
+
+        self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
+
+        return drift_df.to_dict(orient='records')
+
+    @activity.defn(name='calculate_simple_metrics')
+    async def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]:
+        """
+        Calculate simple metrics for a model. Metrics available are:
+            - rmse
+            - mse
+            - mae
+            - r2
+            - accuracy
+            - precision
+            - recall
+            - f1
+        Args:
+            input_data (dict[str, Any]): Input data containing:
+                - metadata (dict): Workflow execution metadata
+                - model_id (str): ID of the MLFlow model
+                - target_data (pd.DataFrame): Target data for calculating metrics, containing target and prediction columns
+                - metrics (list[str]): List of metrics to calculate
+        Returns:
+            dict[Hashable, Any]: Dictionary containing the calculated metrics
+        """
+
+        metadata = input_data['metadata']
+        model_id = input_data['model_id']
+        target_data = DataFrame(input_data['target_data'])
+        metrics = input_data['metrics']
+        interval_minutes = input_data['interval_minutes']
+
+        data_size = target_data.shape[0]
+
+        output_data = []
+
+        diff = target_data['target'] - target_data['prediction']
+        diff_squared = diff**2
+
+        self.info(f'Calculating simple metrics for model {model_id}: {metrics}', metadata)
+
+        for metric in metrics:
+            if metric == 'rmse':
+                output_data.append({'metric': 'rmse', 'value': np.sqrt(np.mean(diff_squared))})
+            elif metric == 'mse':
+                output_data.append({'metric': 'mse', 'value': np.mean(diff_squared)})
+            elif metric == 'mae':
+                output_data.append({'metric': 'mae', 'value': np.mean(np.abs(diff))})
+            elif metric == 'r2':
+                y_true = target_data['target']
+                y_mean = np.mean(y_true)
+
+                ss_res = np.sum(diff_squared)
+                ss_tot = np.sum((y_true - y_mean) ** 2)
+
+                # Evita divisão por zero
+                if ss_tot == 0:
+                    r2_score = 0.0
+                else:
+                    r2_score = 1 - (ss_res / ss_tot)
+
+                output_data.append({'metric': 'r2', 'value': r2_score})
+
+        data = DataFrame(output_data)
+        data['model_id'] = model_id
+        data['timestamp'] = target_data['timestamp'].max()
+        data['data_size'] = data_size
+        data['interval_minutes'] = interval_minutes
+
+        self._debug_dataframe(f'Simple metrics dataframe: Size {data.shape}', data, metadata)
+
+        return data.to_dict(orient='records')

laborious/activities/opc.py

@@ -1,31 +1,132 @@
 from temporalio import activity, workflow
 
-
 with workflow.unsafe.imports_passed_through():
-    from sientia_do.notifications.handlers import NotificationHandler
-    from sientia_do.notifications.models import NotificationLevel
-    from sientia_do.temporal.activities.base import BaseActivity
-    from sientia_do.temporal.utils.logger import Logger
-    from laborious.utils.repository.opc_repository import OpcRepository
-    from typing import Any
     import traceback
+    from collections.abc import Hashable
+    from typing import Any
+
     from pandas import DataFrame
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+
+    from laborious.utils.repository.opc_repository import OpcRepository
 
 OPC_WRITTING_ERROR_CONFIDENCE = 12
+OPC_SESSION_BAD_CONFIDENCE = 14
+OPC_SESSION_BAD_COMMENT_PREFIX = 'OPC UA session/channel error:'
+OPC_WRITTING_ERROR_MESSAGE = 'Some data could not be written to OPC servers'
+OPC_RECONNECT_IN_PROGRESS_COMMENT = 'OPC UA reconnect in progress'
+OPC_COMMENT_SEPARATOR = ' | '
 
 
-class OPC(BaseActivity):
-    def __init__(self, opc_servers: dict[str, dict[str, Any]],
-                 logger: Logger, notification_handler: NotificationHandler):
+def _opc_session_bad_comment(opc_status: str | None) -> str:
+    status = opc_status or 'Unknown'
+    return f'{OPC_SESSION_BAD_COMMENT_PREFIX} {status}'
 
+
+def _apply_opc_write_error(
+    error_info: dict[str, Any] | None,
+    session_bad_seen: bool,
+    session_bad_status: str | None,
+    reconnect_in_progress_seen: bool,
+) -> tuple[bool, str | None, bool]:
+    """
+    Update session/reconnect flags from an OPC write error payload.
+
+    Args:
+        error_info: Repository error details, or None when the write succeeded.
+        session_bad_seen: Whether a session_bad error was seen so far.
+        session_bad_status: Last known OPC status for session errors.
+        reconnect_in_progress_seen: Whether reconnect_in_progress was seen so far.
+
+    Return:
+        Updated (session_bad_seen, session_bad_status, reconnect_in_progress_seen).
+    """
+    if not error_info:
+        return session_bad_seen, session_bad_status, reconnect_in_progress_seen
+
+    kind = error_info.get('opc_error_kind')
+    if kind == 'session_bad':
+        return True, error_info.get('opc_status', session_bad_status), reconnect_in_progress_seen
+    if kind == 'reconnect_in_progress':
+        return session_bad_seen, session_bad_status, True
+    return session_bad_seen, session_bad_status, reconnect_in_progress_seen
+
+
+class OPC(SientiaMonitoring):
+    """
+    OPC server integration activities for real-time data export.
+
+    This class provides comprehensive OPC UA client functionality for connecting
+    to multiple OPC servers and writing prediction data in real-time. It implements
+    secure communication with certificate-based authentication and automatic
+    reconnection capabilities.
+
+    The class supports multiple OPC servers with individual configurations and
+    provides robust error handling and monitoring for production environments.
+
+    Attributes:
+        opc_servers (dict): Configuration for multiple OPC servers
+        opc_repository (dict): Active OPC repository connections
+        logger (Logger): Logging and observability instance
+        notification_handler (NotificationHandler): Notification management instance
+    """
+
+    def __init__(
+        self,
+        opc_servers: dict[str, dict[str, Any]],
+        logger: Logger,
+        notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
+    ):
         self.logger = logger
         self.notification_handler = notification_handler
         self.opc_servers = opc_servers
 
-        self.opc_repository = {}
-        for id, server in opc_servers.items():
-            self.opc_repository[id] = OpcRepository(
-                id=server['id'],
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
+
+        self.opc_repository: dict[str, OpcRepository] = {}
+
+    async def init_opc(self):
+        """
+        Initialize OPC server connections and establish communication channels.
+
+        This method iterates through all configured OPC servers and attempts to
+        establish secure connections using certificate-based authentication.
+        Each server connection is managed independently, and connection failures
+        are reported through the notification system.
+
+        The method performs the following operations:
+        1. Creates OpcRepository instances for each configured server
+        2. Establishes secure connections with certificate validation
+        3. Reports connection success/failure through notifications
+        4. Logs connection status for operational visibility
+
+        Raises:
+            ValueError: A server whose `cert_path` is set but whose `server_uri`
+                (or private key) is missing fails `OpcRepository.set_security()`
+                with no try/except between here and `main()`. This is a
+                configuration error, not a connectivity one, and is intentionally
+                fatal to the whole worker process — fix the deployment config
+                rather than catch it here.
+
+        Note:
+            Connection failures (server unreachable, timeout, etc.) are logged
+            and reported but do not prevent the initialization of other OPC
+            servers — each server's `connect()` call catches those on its own.
+            This isolation does NOT extend to configuration errors raised before
+            a connection is attempted (see `Raises` above): a single
+            misconfigured server intentionally stops every other server, and the
+            whole worker, from starting.
+        """
+        self.info('Initializing OPC servers...')
+        for opc_id, server in self.opc_servers.items():
+            self.opc_repository[opc_id] = OpcRepository(
+                opc_id=server['id'],
+                server_name=server['server_name'],
                 url=server['url'],
                 logger=self.logger,
                 server_uri=server['server_uri'],
@@ -34,44 +135,239 @@ class OPC(BaseActivity):
                 server_cert_path=server['server_cert_path'],
                 notification_handler=self.notification_handler,
                 reconnection_interval=server['reconnection_interval'],
+                metrics_controller=self.metrics_controller,
             )
-            self.opc_repository[id].connect()
+            is_connected, error_data = await self.opc_repository[opc_id].connect()
+            if not is_connected:
+                await self.send_notification_async(
+                    metadata={
+                        'model_id': '-',
+                        'model_name': '-',
+                        'workflow_name': '-',
+                        'schedule_name': 'INITIALIZATION',
+                    },
+                    notification_id=error_data['notification_id'],
+                    message=error_data['message'],
+                    block=error_data['block'],
+                    level=error_data.get('level', NotificationLevel.ERROR),
+                    attachment_content=error_data.get('attachment_content', None),
+                )
+            else:
+                self.info(f'OPC server {opc_id}:{server["server_name"]} connected successfully.')
 
-        BaseActivity.__init__(self, logger, notification_handler)
-
-    def write_data(self, server_id: str, tag: str, data: Any,
-                   data_type: str, tag_type: str) -> bool:
+    async def write_data(
+        self,
+        server_id: str,
+        tag: str,
+        data: Any,
+        data_type: str,
+        tag_type: str,
+        metadata: dict[str, Any],
+    ) -> tuple[float | None, dict[str, Any] | None]:
         """
-        Write data to OPC server.
+        Write data to a specific OPC server tag with comprehensive error handling.
 
-        Args:
-            - server_id (str): The id of the OPC server.
-            - tag (str): The tag to write to.
-            - data (Any): The data to write.
-            - data_type (str): The data type.
-            - tag_type (str): The tag type.
-
-        Returns:
-            - bool: True if the data was written successfully, False otherwise.
+        Return:
+            tuple[float | None, dict[str, Any] | None]: Response time on success, or
+                (None, error info_data) on repository failure.
         """
 
         try:
-            return self.opc_repository[server_id].write_data(
-                tag, data, data_type)
-            self.logger.debug(f"Wrote {tag_type} to {tag}")
+            is_success, info_data = await self.opc_repository[server_id].write_data(
+                tag, data, data_type, metadata
+            )
+            if not is_success:
+                await self.send_notification_async(
+                    metadata=metadata,
+                    notification_id=info_data['notification_id'],
+                    message=info_data['message'],
+                    block=info_data['block'],
+                    level=info_data.get('level', NotificationLevel.ERROR),
+                    attachment_content=info_data.get('attachment_content', None),
+                )
+                return None, info_data
+            return info_data['response_time'], None
         except Exception as e:
             trace = traceback.format_exc()
-            self.notification_handler.build_and_send_notification(
-                notification_id=f"WRITE_OPC_{tag_type.upper()}_ERROR",
-                message=f"Error writing data to OPC server: {e}",
-                block="write_opc_data",
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id=f'WRITE_OPC_{tag_type.upper()}_ERROR',
+                message=f'Error writing data to OPC server: {e}',
+                block='write_opc_data',
                 level=NotificationLevel.ERROR,
-                attachment_content=trace
+                attachment_content=trace,
             )
             raise e
 
+    async def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool:
+        """
+        Validate that an OPC server is available and configured for write operations.
+
+        This method checks if the specified OPC server exists in the active
+        repository and is available for data writing operations. It provides
+        immediate feedback for server availability and logs validation failures
+        for operational monitoring.
+
+        Args:
+            server_id (str): Unique identifier for the OPC server to validate
+            metadata (dict[str, Any]): Context metadata for logging and notifications
+
+        Returns:
+            bool: True if server is available, False otherwise
+
+        Note:
+            Server validation failures are automatically reported through the
+            notification system with detailed information about available servers.
+            This helps operators quickly identify configuration issues.
+        """
+        if self.opc_repository.get(server_id) is None:
+            message = f'OPC server {server_id} not found to perform write operation.'
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='OPC_SERVER_NOT_FOUND',
+                message=message,
+                block='write_opc_data',
+                level=NotificationLevel.ERROR,
+                attachment_content=f'OPC servers: {list(self.opc_repository.keys())}',
+            )
+            return False
+        return True
+
+    async def _write_tags_from_config(
+        self,
+        server_id: str,
+        tags_config: dict[str, dict[str, Any]],
+        data: DataFrame,
+        data_column: str,
+        tag_type: str,
+        log_label: str,
+        metadata: dict[str, Any],
+    ) -> tuple[dict[str, float | None], bool, str | None, bool]:
+        """
+        Write a group of OPC tags and collect response times and error flags.
+
+        Args:
+            server_id: Target OPC server identifier.
+            tags_config: Tag name to configuration mapping.
+            data: DataFrame with prediction/confidence columns.
+            data_column: Column name whose first row value is written.
+            tag_type: Tag category passed to write_data ('prediction' or 'confidence').
+            log_label: Human-readable label for success logs.
+            metadata: Context metadata for logging and notifications.
+
+        Return:
+            (response_times, session_bad_seen, session_bad_status, reconnect_in_progress_seen)
+        """
+        response_times: dict[str, float | None] = {}
+        session_bad_seen = False
+        session_bad_status: str | None = None
+        reconnect_in_progress_seen = False
+
+        for tag, tag_config in tags_config.items():
+            response_time, error_info = await self.write_data(
+                server_id=server_id,
+                tag=tag,
+                data=data.head(1)[data_column].values[0],
+                data_type=tag_config['data_type'],
+                tag_type=tag_type,
+                metadata=metadata,
+            )
+            session_bad_seen, session_bad_status, reconnect_in_progress_seen = (
+                _apply_opc_write_error(
+                    error_info,
+                    session_bad_seen,
+                    session_bad_status,
+                    reconnect_in_progress_seen,
+                )
+            )
+            if response_time is not None:
+                self.info(
+                    f'{log_label} written to OPC server {server_id} for tag {tag}.',
+                    metadata,
+                )
+            response_times[tag] = response_time
+
+        return response_times, session_bad_seen, session_bad_status, reconnect_in_progress_seen
+
+    async def manage_output_tags(
+        self,
+        server_id: str,
+        config: dict[str, Any],
+        data: DataFrame,
+        metadata: dict[str, Any],
+    ) -> tuple[bool, dict[str, float | None], bool, str | None, bool]:
+        """
+        Manage the writing of prediction and confidence data to OPC server tags.
+
+        This method orchestrates the writing of multiple data types to OPC servers
+        based on configuration. It handles both prediction data and confidence
+        values independently, allowing for flexible tag configuration and
+        comprehensive error handling.
+
+        The method supports two main tag types:
+        1. Prediction tags: Write actual prediction values to configured OPC tags
+        2. Confidence tags: Write confidence scores to separate OPC tags
+
+        Args:
+            server_id (str): Unique identifier for the target OPC server
+            config (dict[str, Any]): OPC tag configuration containing:
+                - prediction_tags (dict, optional): Prediction tag configurations
+                - confidence_tags (dict, optional): Confidence tag configurations
+            data (DataFrame): DataFrame containing prediction and confidence data
+            metadata (dict[str, Any]): Context metadata for logging and notifications
+            success (bool): Current success status to maintain across operations
+
+        Returns:
+            tuple[bool, int]: (overall_success, total_tags_written)
+                - overall_success: True if all configured tags were written successfully
+                - total_tags_written: Count of successfully written tags
+        """
+        response_times: dict[str, float | None] = {}
+        session_bad_seen = False
+        session_bad_status: str | None = None
+        reconnect_in_progress_seen = False
+
+        tag_groups = (
+            ('prediction_tags', 'prediction', 'prediction', 'Prediction data'),
+            ('confidence_tags', 'prediction_confidence', 'confidence', 'Confidence data'),
+        )
+        for config_key, data_column, tag_type, log_label in tag_groups:
+            if config_key not in config:
+                continue
+            (
+                group_times,
+                group_session_bad,
+                group_status,
+                group_reconnect,
+            ) = await self._write_tags_from_config(
+                server_id=server_id,
+                tags_config=config[config_key],
+                data=data,
+                data_column=data_column,
+                tag_type=tag_type,
+                log_label=log_label,
+                metadata=metadata,
+            )
+            response_times.update(group_times)
+            if group_session_bad:
+                session_bad_seen = True
+                session_bad_status = group_status or session_bad_status
+            if group_reconnect:
+                reconnect_in_progress_seen = True
+
+        success = None not in response_times.values()
+        return (
+            success,
+            response_times,
+            session_bad_seen,
+            session_bad_status,
+            reconnect_in_progress_seen,
+        )
+
     @activity.defn(name='write_opc_data')
-    async def write_opc_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+    async def write_opc_data(
+        self, input_data: dict[str, Any]
+    ) -> tuple[dict[Hashable, Any], dict[str, dict[str, float | None]]]:
         """
         Write prediction and confidence data to OPC servers. The two writing
         operations are optional and independent of each other.
@@ -90,67 +386,136 @@ class OPC(BaseActivity):
 
         """
         metadata = input_data['metadata']
-        self.debug("Writing data to OPC servers...", metadata)
+        self.info('Writing data to OPC servers...', metadata)
         data = DataFrame(input_data['data'])
         opc_output_config = input_data['opc_output_config']
-        self.debug(data, metadata)
+        self.info(f'Data to write: {data.size} rows', metadata)
 
         success = True
+        session_bad_seen = False
+        session_bad_status: str | None = None
+        reconnect_in_progress_seen = False
+
+        metrics: dict[str, dict[str, float | None]] = {}
 
         for server_id, config in opc_output_config.items():
-            if self.opc_repository.get(server_id) is None:
-                self.error(f"OPC server {server_id} not found", metadata)
+            if not await self.validate_server(server_id, metadata):
+                success = False
                 continue
 
-            if 'prediction_tags' in config:
-                for tag, tag_config in config['prediction_tags'].items():
-                    success = success and self.write_data(
-                        server_id=server_id,
-                        tag=tag,
-                        data=data.head(1)['prediction'].values[0],
-                        data_type=tag_config['data_type'],
-                        tag_type='prediction'
-                    )
+            (
+                local_success,
+                local_response_times,
+                local_session_bad,
+                local_status,
+                local_reconnect_in_progress,
+            ) = await self.manage_output_tags(server_id, config, data, metadata)
+            metrics[server_id] = local_response_times
+            local_count = len(local_response_times)
+            success = success and local_success
+            if local_session_bad:
+                session_bad_seen = True
+                session_bad_status = local_status or session_bad_status
+            if local_reconnect_in_progress:
+                reconnect_in_progress_seen = True
 
-            if 'confidence_tags' in config:
-                for tag, tag_config in config['confidence_tags'].items():
-                    self.write_data(
-                        server_id=server_id,
-                        tag=tag,
-                        data=data.head(1)['prediction_confidence'].values[0],
-                        data_type=tag_config['data_type'],
-                        tag_type='confidence'
-                    )
+            self.info(
+                f'Process completed for OPC server {server_id}: {local_count} of {len(config.get("prediction_tags", []))} prediction tags and {len(config.get("confidence_tags", []))} confidence tags',
+                metadata,
+            )
 
-        return self.process_confidence(data, success, metadata)
+        return (
+            self.process_confidence(
+                data,
+                success,
+                metadata,
+                session_bad=session_bad_seen,
+                opc_status=session_bad_status,
+                reconnect_in_progress=reconnect_in_progress_seen,
+            ),
+            metrics,
+        )
 
-    def process_confidence(self, data: DataFrame, success: bool, metadata: dict[str, Any]) -> dict[Any, Any]:
+    def process_confidence(
+        self,
+        data: DataFrame,
+        success: bool,
+        metadata: dict[str, Any],
+        *,
+        session_bad: bool = False,
+        opc_status: str | None = None,
+        reconnect_in_progress: bool = False,
+    ) -> dict[Hashable, Any]:
         """
-        Processes the confidence of OPC server write operations and updates the DataFrame accordingly.
+        Process prediction confidence based on OPC write operation success.
 
-        If the write operation was not successful, sets the 'prediction_confidence' column in the DataFrame
-        to a predefined error confidence value and logs a debug message. Otherwise, logs a success message.
+        This method updates the prediction confidence values in the DataFrame
+        based on the success status of OPC server write operations. If any
+        write operations failed, it sets the confidence to a predefined error
+        value to indicate data quality issues.
+
+        The method implements a confidence degradation strategy:
+        - Success: Maintains original confidence values
+        - Failure: Sets confidence to error value for operational awareness
 
         Args:
-            data (DataFrame): The DataFrame containing the data to be processed.
-            success (bool): Indicates whether the data was successfully written to the OPC servers.
+            data (DataFrame): DataFrame containing prediction and confidence data
+            success (bool): Overall success status of OPC write operations
+            metadata (dict[str, Any]): Context metadata for logging and notifications
 
         Returns:
-            dict[Any, Any]: The processed data as a dictionary.
+            dict[Any, Any]: Processed data as a dictionary with updated confidence values
+
+        Note:
+            The error confidence value (OPC_WRITTING_ERROR_CONFIDENCE = 12) is
+            used to indicate that data was not successfully exported to OPC servers.
+            This allows downstream systems to handle data quality appropriately.
         """
 
         if not success:
-            data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE
-            self.debug(
-                "Some data could not be written to OPC servers, setting confidence to "
-                f"{OPC_WRITTING_ERROR_CONFIDENCE}."
-            )
+            comment_parts: list[str] = []
+            confidence = OPC_WRITTING_ERROR_CONFIDENCE
 
+            if session_bad:
+                comment_parts.append(_opc_session_bad_comment(opc_status))
+                confidence = OPC_SESSION_BAD_CONFIDENCE
+            if reconnect_in_progress:
+                comment_parts.append(OPC_RECONNECT_IN_PROGRESS_COMMENT)
+                confidence = OPC_SESSION_BAD_CONFIDENCE
+            if not comment_parts:
+                comment_parts.append(OPC_WRITTING_ERROR_MESSAGE)
+
+            comments = OPC_COMMENT_SEPARATOR.join(comment_parts)
+            data['prediction_confidence'] = confidence
+            data['comments'] = comments
+            self.debug(
+                f'OPC write issues, confidence={confidence}, comments={comments}',
+                metadata,
+            )
         else:
-            self.debug("Data written to OPC servers successfully.", metadata)
+            self.debug('Data written to OPC servers successfully.', metadata)
 
         return data.to_dict()
 
-    def shutdown(self):
+    async def aclose(self):
+        """
+        Gracefully shutdown all OPC server connections and cleanup resources.
+
+        This method ensures proper cleanup of all active OPC server connections
+        by calling the disconnect method on each repository instance. It's
+        designed to be called during application shutdown to prevent resource
+        leaks and ensure clean termination.
+
+        The method performs the following cleanup operations:
+        1. Iterates through all active OPC repository connections
+        2. Calls disconnect() on each repository instance
+        3. Allows for graceful connection termination
+        4. Prevents resource leaks and connection hanging
+
+        Note:
+            This method should be called during application shutdown to ensure
+            proper cleanup. It handles all active connections regardless of
+            their current state and provides a clean shutdown experience.
+        """
         for opc in self.opc_repository.values():
-            opc.disconnect()
+            await opc.disconnect()

laborious/activities/storage.py

@@ -0,0 +1,215 @@
+from temporalio import activity, workflow
+
+from laborious.utils.repository.minio_manager import MinioManager
+
+with workflow.unsafe.imports_passed_through():
+    # Extend the Temporal Postgres activities for convenient query -> MinIO export
+    import traceback
+    from datetime import timedelta
+    from typing import Any
+
+    import pandas as pd
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.repository.minio_repository import MinioRepository
+    from sientia_do.temporal.activities.postgres import Postgres
+    from sientia_do.temporal.constants import now
+
+    from laborious.utils.messages import MISSING_MINIO_REPOSITORY
+    from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
+    from laborious.utils.time import to_naive
+
+_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
+
+
+class Storage(Postgres, MinioManager):
+    """
+    Extensions for Postgres activities with a helper to export query results
+    directly to MinIO as Parquet and return the object name.
+    """
+
+    minio_repository: MinioRepository | None = None
+
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        user: str,
+        password: str,
+        dbname: str,
+        min_connections: int,
+        max_connections: int,
+        retention_hours: int = 24,
+        minio_repository: MinioRepository | None = None,
+        logger: Logger | None = None,
+        notification_handler: NotificationHandler | None = None,
+        metrics_controller: MetricsController | None = None,
+    ):
+        self.retention_hours = retention_hours
+        Postgres.__init__(
+            self,
+            host=host,
+            port=port,
+            user=user,
+            password=password,
+            dbname=dbname,
+            min_connections=min_connections,
+            max_connections=max_connections,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+        MinioManager.__init__(
+            self, minio_repository, logger, notification_handler, metrics_controller
+        )
+
+    @activity.defn(name='load_query_with_minio_offload')
+    async def load_query_with_minio_offload(
+        self, input_data: dict[str, Any]
+    ) -> MinioDataFramePayload:
+        """
+        Run the custom SQL load, then return a MinIO-aware dataframe wire dict.
+
+        Args (input_data):
+            metadata (dict): Workflow metadata (same as load_custom_query).
+            query (str): SQL query.
+            datetime_columns (list[str], optional): Datetime column names.
+            model_name (str): Model name for object key basename.
+            key_prefix (str, optional): Directory prefix inside the bucket.
+            size_threshold_bytes (int, optional): Override env offload threshold.
+
+        Returns:
+            dict[str, Any]: Flat ``MinioDataFramePayload`` dict or ``success: False`` on failure.
+        """
+        if self.minio_repository is None:
+            raise ValueError(MISSING_MINIO_REPOSITORY)
+
+        metadata: dict = input_data.get('metadata', {})
+        model_name = input_data['model_name']
+
+        rows = await self.load_custom_query(
+            input_data,
+        )
+        if not rows:
+            self.error(
+                'load_query_with_minio_offload failed: No data returned from query', metadata
+            )
+            dataframe = None
+        else:
+            dataframe = pd.DataFrame(rows)
+
+        return await MinioDataFramePayload.from_dataframe(
+            dataframe,
+            minio_repo=self.minio_repository,
+            workflow_metadata=metadata,
+            model_name=model_name,
+            operation='initial',
+            logger=self.logger,
+        )
+
+    @activity.defn(name='export_payload_to_postgres')
+    async def export_payload_to_postgres(self, input_data: dict[str, Any]) -> dict:
+        """
+        Export a payload to PostgreSQL.
+        """
+        metadata = input_data.get('metadata')
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
+        data = await payload.retrieve(self.minio_repository, metadata)
+
+        return await self.export_data_to_postgres(
+            {
+                **input_data,
+                'data': data,
+            }
+        )
+
+    @activity.defn(name='cleanup_minio_objects_expired')
+    async def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
+        """
+        Delete objects under the given prefixes that are older than the retention window.
+
+        See also: `ModelImport.ensure_import_bundle_retention`
+        (laborious/activities/model_import.py) — a newer, unrelated lifecycle-policy expiry
+        mechanism for the import bundle prefix. The two are independent by design for now; see
+        docs/minio-object-expiry.md.
+
+        Args (input_data):
+            metadata (dict): Workflow metadata for logging and metrics.
+            prefixes (list[str]): Key prefixes to scan (one level or subtree per prefix).
+
+        Returns:
+            dict[str, Any]: ``success``, ``deleted_count``, and optional ``message``.
+        """
+        if self.minio_repository is None:
+            raise ValueError(MISSING_MINIO_REPOSITORY)
+
+        metadata = input_data.get('metadata', {})
+        payload = MinioDataFramePayload.from_dict(input_data['data'])
+        prefix = payload.cleanup_prefix()
+        base = now()
+        cutoff = to_naive(base) - timedelta(hours=self.retention_hours)
+
+        report: dict[str, Any] = {
+            'failed': {},
+            'deleted': {},
+            'failed_count': 0,
+            'deleted_count': 0,
+        }
+        try:
+            keys = await self.minio_repository.list_objects(
+                prefix=prefix,
+                recursive=True,
+                metadata=metadata,
+            )
+            for key in keys:
+                try:
+                    ts = MinioDataFramePayload.parse_object_timestamp(key)
+                    if ts is None:
+                        continue
+                    if ts >= cutoff:
+                        continue
+                    await self.minio_repository.delete_file(
+                        object_name=key,
+                        metadata=metadata,
+                    )
+                except Exception as e:
+                    report['failed'][key] = {
+                        'success': False,
+                        'message': str(e),
+                    }
+                    report['failed_count'] += 1
+                    continue
+                report['deleted'][key] = {
+                    'success': True,
+                    'message': 'Deleted',
+                }
+                report['deleted_count'] += 1
+        except Exception as e:
+            trace = traceback.format_exc()
+            await self.send_notification_async(
+                metadata=metadata,
+                notification_id='ERROR_CLEANUP_MINIO_OBJECTS_EXPIRED',
+                message=f'Error cleaning up MinIO objects: {e}',
+                block='cleanup_minio_objects_expired',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+            self.error(trace, metadata)
+        else:
+            # Cleanup success is expected in normal flow; avoid noisy INFO notifications
+            # that do not impact behavior and can flood observability in test runs.
+            self.info('MinIO objects cleaned up successfully', metadata)
+
+        return report
+
+    def close(self) -> None:
+        """Close Storage resources (MinIO client and Postgres engine)."""
+        if hasattr(self, 'engine'):
+            Postgres.close(self)
+        MinioManager.close(self)
+
+    def __del__(self):
+        self.close()

laborious/metrics.py

@@ -0,0 +1,239 @@
+"""
+Laborious Metrics Module
+
+This module defines all Prometheus metrics used by the Sientia DataOps Laborious system
+for monitoring and observability. The metrics provide insights into system performance,
+prediction quality, and operational health.
+
+The metrics are designed to be scraped by Prometheus and can be visualized in
+Grafana or other monitoring dashboards to provide real-time visibility into
+the system's operation.
+
+Key Metric Categories:
+- Application Health: Overall system status and availability
+- Prediction Operations: Count and performance of prediction operations
+- Data Quality: Confidence levels and validation results
+- Export Operations: Database and OPC export performance
+- Response Times: Performance monitoring for various operations
+
+Metric Labels:
+- pod_id: Kubernetes pod identifier for multi-instance deployments
+- runtime: Runtime / environment identifier (matches ``RUNTIME`` env, see ``SientiaMonitoring``)
+- model_name: Name of the ML model being used
+- workflow_name: Name of the prediction pipeline
+- opc_server_id: Identifier for OPC server operations
+"""
+
+from prometheus_client import Counter, Gauge, Histogram
+from sientia_do.observability.metrics import CORE_LABELS
+
+# Application health metric
+APP_UP = Gauge(
+    'app_up',
+    'Indicates if the application is running (1) or shutting down (0)',
+    ['pod_id'],
+)
+
+# Prediction operation metrics
+PREDICTIONS_WRITTEN_COUNT = Counter(
+    'laborious_predictions_written_count',
+    'Number of predictions written to the database table predictions',
+    CORE_LABELS,
+)
+
+# Prediction quality metrics
+PREDICTION_CONFIDENCE_MONITOR = Gauge(
+    'laborious_prediction_confidence_monitor',
+    'Current confidence of each prediction',
+    CORE_LABELS,
+)
+
+# Prediction total response time
+PREDICTION_RESPONSE_TIME_MONITOR = Histogram(
+    'laborious_prediction_response_time_monitor',
+    'Current response time of each prediction',
+    CORE_LABELS,
+    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
+)
+
+
+# ================== OPC metrics ==================
+
+
+PREDICTION_OPC_WRITING_COUNT = Counter(
+    'laborious_prediction_opc_writing_count',
+    'Number of predictions written to the OPC server',
+    [*CORE_LABELS, 'opc_server_id', 'tag'],
+)
+
+PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram(
+    'laborious_prediction_opc_writing_response_time_monitor',
+    'Current response time of each prediction written to the OPC server',
+    [*CORE_LABELS, 'opc_server_id', 'tag'],
+    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
+)
+
+OPC_CONNECTIONS_TOTAL = Counter(
+    'opc_connections_initiated_total',
+    'Total connection attempts to OPC servers',
+    ['pod_id', 'server_name'],
+)
+OPC_CONNECTIONS_FAILED = Counter(
+    'opc_connections_failed_total',
+    'Total failed connection attempts to OPC servers',
+    ['pod_id', 'server_name'],
+)
+OPC_CONNECTION_STATUS = Gauge(
+    'opc_connection_status',
+    'Connection status with the OPC server (1=connected, 0=disconnected)',
+    ['pod_id', 'server_name', 'server_url'],
+)
+
+_OPC_SESSION_DEBUG_LABELS = ['pod_id', 'server_name', 'runtime', 'opc_server_id', 'session_id']
+
+OPC_SESSION_CREATED_TOTAL = Counter(
+    'opc_session_created_total',
+    'OPC UA sessions established (after successful connect)',
+    _OPC_SESSION_DEBUG_LABELS,
+)
+
+OPC_SESSION_CLOSED_TOTAL = Counter(
+    'opc_session_closed_total',
+    'OPC UA client disconnects completed (session tear-down initiated)',
+    _OPC_SESSION_DEBUG_LABELS,
+)
+
+OPC_SESSION_REVISED_TIMEOUT_MS = Gauge(
+    'opc_session_revised_timeout_milliseconds',
+    'Server-revised OPC UA session timeout (RevisedSessionTimeout) in ms after connect',
+    _OPC_SESSION_DEBUG_LABELS,
+)
+
+OPC_WRITE_ATTEMPT_LABELS = [*_OPC_SESSION_DEBUG_LABELS, 'model_id', 'model_name', 'result']
+
+OPC_WRITE_ATTEMPTS_TOTAL = Counter(
+    'opc_write_attempts_total',
+    'OPC UA write attempts with session and outcome (result=OK or exception class name)',
+    OPC_WRITE_ATTEMPT_LABELS,
+)
+
+OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL = Counter(
+    'opc_write_inter_arrival_over_session_timeout_total',
+    'Successful writes where seconds since the previous successful write exceeded RevisedSessionTimeout (ms)',
+    _OPC_SESSION_DEBUG_LABELS,
+)
+
+# ================== Model metrics ==================
+
+MODEL_READ_LAG = Histogram(
+    'laborious_model_read_lag',
+    'Lag between the start and read of read operations',
+    CORE_LABELS,
+    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
+)
+
+MODEL_WRITE_LAG = Histogram(
+    'laborious_model_write_lag',
+    'Lag between the start and end of write operations',
+    CORE_LABELS,
+    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
+)
+
+MODEL_READ_COUNT = Counter(
+    'laborious_model_read_count',
+    'Number of reads from the model',
+    CORE_LABELS,
+)
+
+MODEL_WRITE_COUNT = Counter(
+    'laborious_model_write_count',
+    'Number of writes to the model',
+    CORE_LABELS,
+)
+
+MODEL_READ_ERROR_COUNT = Counter(
+    'laborious_model_read_error_count',
+    'Number of errors reading from the model',
+    CORE_LABELS,
+)
+
+MODEL_WRITE_ERROR_COUNT = Counter(
+    'laborious_model_write_error_count',
+    'Number of errors writing to the model',
+    CORE_LABELS,
+)
+
+MODEL_ANALYZE_LAG = Histogram(
+    'laborious_model_analyze_lag',
+    'Lag between the start and end of analyze operations',
+    CORE_LABELS,
+    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
+)
+
+MODEL_ANALYZE_COUNT = Counter(
+    'laborious_model_analyze_count',
+    'Number of analyze operations',
+    CORE_LABELS,
+)
+
+MODEL_ANALYZE_ERROR_COUNT = Counter(
+    'laborious_model_analyze_error_count',
+    'Number of errors during analyze operations',
+    CORE_LABELS,
+)
+
+
+# ================== Model import metrics ==================
+#
+# The failing step and the error code travel in `operation_type`, the one label the shared
+# `CORE_LABELS` leaves free. Nothing else about a failure is a label: an error reason is a sentence
+# and an exception is a log line, neither of which belongs in a metric's cardinality.
+
+MODEL_IMPORT_STARTED_COUNT = Counter(
+    'laborious_model_import_started_count',
+    'Number of model imports that claimed their import log row',
+    CORE_LABELS,
+)
+
+MODEL_IMPORT_COMPLETED_COUNT = Counter(
+    'laborious_model_import_completed_count',
+    'Number of model imports that reached a registered version and a model document',
+    CORE_LABELS,
+)
+
+MODEL_IMPORT_ERROR_COUNT = Counter(
+    'laborious_model_import_error_count',
+    'Number of failed model imports, with the failing step or error code in operation_type',
+    CORE_LABELS,
+)
+
+MODEL_IMPORT_KDF_DURATION = Histogram(
+    'laborious_model_import_kdf_duration_seconds',
+    'Time spent deriving the bundle key with Argon2id',
+    CORE_LABELS,
+    buckets=[0.5, 1.0, 2.0, 3.0, 5.0, 8.0, 13.0, 21.0],
+)
+
+MODEL_IMPORT_STEP_DURATION = Histogram(
+    'laborious_model_import_step_duration_seconds',
+    'Duration of each model import step, named in operation_type',
+    CORE_LABELS,
+    buckets=[0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0],
+)
+
+
+# ================== PI Web API metrics ==================
+
+PI_WEB_API_LABELS = [*CORE_LABELS, 'tag_name']
+
+PI_WEB_API_PREDICTION_WRITTEN_COUNT = Counter(
+    'laborious_pi_web_api_prediction_written_count',
+    'Number of predictions written to the PI Web API',
+    PI_WEB_API_LABELS,
+)
+
+PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT = Counter(
+    'laborious_pi_web_api_prediction_written_error_count',
+    'Number of errors writing predictions to the PI Web API',
+    PI_WEB_API_LABELS,
+)

laborious/utils/bundle/__init__.py

@@ -0,0 +1,27 @@
+"""Reading `.sientia` bundles: the wire format, the seven gates and the step vocabulary.
+
+There is no writer here. The producer is the Streamlit app in `sientia-projects-templates`
+(`app/src/operations/model_export/`), and a second production writer for a format with one producer
+would be a fork waiting to happen. What stands in for it in the tests is
+`tests/helpers/bundle_factory.py` plus the golden fixtures under `tests/fixtures/bundle/`.
+"""
+
+from laborious.utils.bundle.format import BundleFormatError, BundleHeader
+from laborious.utils.bundle.reader import (
+    BundleGateError,
+    BundleLimits,
+    BundleReader,
+    OpenedBundle,
+)
+from laborious.utils.bundle.steps import GATE_TO_STEP, ImportStep
+
+__all__ = [
+    'GATE_TO_STEP',
+    'BundleFormatError',
+    'BundleGateError',
+    'BundleHeader',
+    'BundleLimits',
+    'BundleReader',
+    'ImportStep',
+    'OpenedBundle',
+]

laborious/utils/bundle/format.py

@@ -0,0 +1,216 @@
+"""The `.sientia` wire format, as the producer writes it.
+
+Every constant here is copied from the producer that emits the bundles this reader opens — the
+Streamlit app in `sientia-projects-templates`, modules
+`app/src/operations/model_export/crypto.py` and `app/src/operations/model_export/bundle.py`. The
+producer file each value came from is named next to it. The golden fixtures under
+`tests/fixtures/bundle/` are the anti-drift device: if the producer's layout moves, the tests here
+fail against the committed bytes instead of failing in production.
+
+This module deliberately imports nothing from Laborious, Temporal, MLflow or libsodium: it is pure
+`struct` and `hashlib` over bytes, so it stays liftable into a shared package if the format ever
+gets one (the cancelled QTZPOC-15). `reader.py` is where the crypto lives.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import struct
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final
+
+# --- Producer: app/src/operations/model_export/crypto.py ---
+MAGIC: Final[bytes] = b'SIENTIA1'
+FORMAT_VERSION: Final[int] = 1
+AEAD_ID_XCHACHA20POLY1305_SECRETSTREAM: Final[int] = 1
+KDF_ID_ARGON2ID: Final[int] = 1
+SALT_SIZE: Final[int] = 16
+DIGEST_SIZE: Final[int] = 32
+RESERVED_SIGNATURE_SIZE: Final[int] = 64
+# `KDF_MEMLIMIT_BYTES` / `KDF_OPSLIMIT` on the producer: what it writes is also the ceiling the
+# reader refuses to exceed, so header bytes cannot buy an arbitrary allocation on the worker.
+MAX_KDF_MEMLIMIT: Final[int] = 256 * 1024 * 1024  # 268435456
+MAX_KDF_OPSLIMIT: Final[int] = 3
+# `KDF_PARALLELISM_RESERVED`: libsodium's `crypto_pwhash` has no lane parameter, so the field is
+# reserved and must be exactly 1.
+PARALLELISM: Final[int] = 1
+CHUNK_SIZE: Final[int] = 64 * 1024  # 65536, fixed by the format
+KEY_SIZE: Final[int] = 32  # crypto_secretstream_xchacha20poly1305_KEYBYTES
+
+# magic(8s) version(B) aead_id(B) kdf_id(B) salt(16s) memlimit(Q) opslimit(Q)
+# parallelism(B) digest(32s) reserved_signature(64s) — producer's `_HEADER_STRUCT`.
+HEADER_STRUCT: Final[struct.Struct] = struct.Struct('>8sBBB16sQQB32s64s')
+HEADER_SIZE: Final[int] = HEADER_STRUCT.size  # 140
+
+# --- libsodium constants the framing depends on ---
+# Not producer constants, but constants of the primitive the producer uses. Kept as literals so
+# this module stays import-free; `tests/laborious/utils/bundle/test_format.py` asserts they equal
+# the values `nacl.bindings` reports, which is what makes the literals safe.
+STREAM_HEADER_SIZE: Final[int] = 24  # crypto_secretstream_xchacha20poly1305_HEADERBYTES
+ABYTES: Final[int] = 17  # crypto_secretstream_xchacha20poly1305_ABYTES
+
+# The pre-upload check prefix of `08-security-and-encryption.md` § 5.3: enough bytes to hold the
+# header, the stream header and one full chunk. Derived from constants, never read from the file.
+PREFIX_SIZE: Final[int] = HEADER_SIZE + STREAM_HEADER_SIZE + CHUNK_SIZE + ABYTES  # 65717
+
+# The digest covers the chunk stream only. The producer computes `sha256(ciphertext)` over the
+# joined chunks and *then* writes `header + stream_header + ciphertext`, so on disk the covered
+# range starts after both headers. Verified against a bundle produced by the producer's own code:
+# `sha256(file[164:])` matches the header field and `sha256(file[140:])` does not.
+DIGEST_OFFSET: Final[int] = HEADER_SIZE + STREAM_HEADER_SIZE  # 164
+
+MIN_FILE_SIZE: Final[int] = HEADER_SIZE + STREAM_HEADER_SIZE
+
+SUPPORTED_FORMAT_VERSIONS: Final[frozenset[int]] = frozenset({FORMAT_VERSION})
+SUPPORTED_AEAD_IDS: Final[frozenset[int]] = frozenset({AEAD_ID_XCHACHA20POLY1305_SECRETSTREAM})
+SUPPORTED_KDF_IDS: Final[frozenset[int]] = frozenset({KDF_ID_ARGON2ID})
+
+_HASH_BLOCK_SIZE: Final[int] = 1024 * 1024
+
+
+class BundleFormatError(Exception):
+    """The bytes are not a `.sientia` bundle this reader implements.
+
+    Raised for every header-level rejection: wrong magic, unknown version, unknown primitive ids,
+    cost parameters above the ceiling, a reserved field carrying a value, a file too short to hold
+    a header, or a chunk stream whose digest does not match the header's. `reader.py` turns this
+    into a gate-2 rejection; nothing here knows about gates or import steps.
+    """
+
+
+@dataclass(frozen=True)
+class BundleHeader:
+    """The 140-byte plaintext header, parsed and validated."""
+
+    format_version: int
+    aead_id: int
+    kdf_id: int
+    salt: bytes
+    kdf_memlimit_bytes: int
+    kdf_opslimit: int
+    kdf_parallelism: int
+    ciphertext_digest: bytes
+    reserved_signature: bytes
+
+
+def parse_header(raw: bytes) -> BundleHeader:
+    """Parse and validate the plaintext header.
+
+    Everything the reader trusts about the file's shape is decided here, before any offset past
+    the header is used and before a key is derived.
+
+    Args:
+        raw: at least `HEADER_SIZE` bytes read from the start of the object.
+
+    Returns:
+        BundleHeader: the validated header.
+
+    Raises:
+        BundleFormatError: on any invalid or unsupported field.
+    """
+    if len(raw) < HEADER_SIZE:
+        raise BundleFormatError(
+            f'file is shorter than the {HEADER_SIZE}-byte header ({len(raw)} bytes available)'
+        )
+
+    (
+        magic,
+        format_version,
+        aead_id,
+        kdf_id,
+        salt,
+        kdf_memlimit_bytes,
+        kdf_opslimit,
+        kdf_parallelism,
+        ciphertext_digest,
+        reserved_signature,
+    ) = HEADER_STRUCT.unpack(raw[:HEADER_SIZE])
+
+    if magic != MAGIC:
+        raise BundleFormatError('file does not start with the expected bundle marker')
+    if format_version not in SUPPORTED_FORMAT_VERSIONS:
+        raise BundleFormatError(f'unsupported bundle format version: {format_version}')
+    if aead_id not in SUPPORTED_AEAD_IDS:
+        raise BundleFormatError(f'unsupported encryption identifier: {aead_id}')
+    if kdf_id not in SUPPORTED_KDF_IDS:
+        raise BundleFormatError(f'unsupported key derivation identifier: {kdf_id}')
+    if kdf_memlimit_bytes > MAX_KDF_MEMLIMIT:
+        raise BundleFormatError(
+            f'declared key derivation memory {kdf_memlimit_bytes} exceeds the '
+            f'{MAX_KDF_MEMLIMIT} ceiling'
+        )
+    if kdf_opslimit > MAX_KDF_OPSLIMIT:
+        raise BundleFormatError(
+            f'declared key derivation passes {kdf_opslimit} exceeds the {MAX_KDF_OPSLIMIT} ceiling'
+        )
+    if kdf_memlimit_bytes <= 0 or kdf_opslimit <= 0:
+        raise BundleFormatError('key derivation parameters must be positive')
+    if kdf_parallelism != PARALLELISM:
+        raise BundleFormatError(
+            f'reserved parallelism field must be {PARALLELISM}, found {kdf_parallelism}'
+        )
+    if reserved_signature != bytes(RESERVED_SIGNATURE_SIZE):
+        raise BundleFormatError('reserved signature field must be empty in this format version')
+
+    return BundleHeader(
+        format_version=format_version,
+        aead_id=aead_id,
+        kdf_id=kdf_id,
+        salt=salt,
+        kdf_memlimit_bytes=kdf_memlimit_bytes,
+        kdf_opslimit=kdf_opslimit,
+        kdf_parallelism=kdf_parallelism,
+        ciphertext_digest=ciphertext_digest,
+        reserved_signature=reserved_signature,
+    )
+
+
+def read_header(path: Path | str) -> BundleHeader:
+    """Read and validate the header of the file at `path`.
+
+    Raises:
+        BundleFormatError: when the file cannot hold a header plus a stream header, or when the
+            header itself is invalid.
+    """
+    file_path = Path(path)
+    size = file_path.stat().st_size
+    if size < MIN_FILE_SIZE:
+        raise BundleFormatError(
+            f'file is {size} bytes, smaller than the {MIN_FILE_SIZE} bytes a bundle needs for its '
+            'header and encrypted stream header'
+        )
+    with file_path.open('rb') as handle:
+        return parse_header(handle.read(HEADER_SIZE))
+
+
+def compute_ciphertext_digest(path: Path | str) -> bytes:
+    """SHA-256 of the chunk stream — the bytes from `DIGEST_OFFSET` to the end of the file."""
+    digest = hashlib.sha256()
+    with Path(path).open('rb') as handle:
+        handle.seek(DIGEST_OFFSET)
+        while block := handle.read(_HASH_BLOCK_SIZE):
+            digest.update(block)
+    return digest.digest()
+
+
+def verify_ciphertext_digest(path: Path | str, header: BundleHeader) -> None:
+    """Compare the file's chunk-stream digest against the header's.
+
+    Raises:
+        BundleFormatError: when they differ — a truncated, overwritten or corrupted transfer. This
+            runs before any key derivation, so a damaged file never costs an Argon2id pass.
+    """
+    if compute_ciphertext_digest(path) != header.ciphertext_digest:
+        raise BundleFormatError(
+            'the encrypted content does not match the digest recorded in the bundle header'
+        )
+
+
+def file_digest(path: Path | str) -> str:
+    """Hex SHA-256 of the whole file, for the `expected_digest` comparison of gate 1."""
+    digest = hashlib.sha256()
+    with Path(path).open('rb') as handle:
+        while block := handle.read(_HASH_BLOCK_SIZE):
+            digest.update(block)
+    return digest.hexdigest()

laborious/utils/bundle/reader.py

@@ -0,0 +1,495 @@
+"""Reading a `.sientia` bundle: the seven gates, in order, in one function.
+
+`BundleReader.open()` is the only entry point. Gate ordering is a property of that function rather
+than of the caller, so it cannot be reordered by accident and can be unit-tested without Temporal:
+
+1. object-level limits (`check_object_limits`, called by the download activity before the body is
+   fetched — the only gate that does not need the file)
+2. header validation, including the chunk-stream digest, before any key derivation
+3. AEAD decryption — the integrity check; a wrong password and a tampered file fail identically
+4. archive inspection over the central directory, writing nothing to disk
+5. extraction into a fresh, isolated directory
+6. structure and schema of the extracted tree
+7. content policy over the file set
+
+Nothing here logs, notifies or emits a metric: rejections are raised as `BundleGateError` and the
+activity layer owns the one failure channel.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import stat
+import time
+import unicodedata
+import zipfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import nacl.bindings as sodium
+import nacl.exceptions
+
+from laborious.utils.bundle import format as fmt
+from laborious.utils.bundle.steps import GATE_TO_STEP, ImportStep
+
+# Gate 5 — chunk size for the byte-counted extraction copy loop.
+_EXTRACT_CHUNK_BYTES = 1024 * 1024
+
+# Gate 6 — the layout `bundle.build_zip` writes and `model_repository.py` reads back.
+METADATA_FILE_NAME = 'metadata.json'
+ARTIFACTS_DIR_NAME = 'artifacts'
+REQUIRED_TOP_LEVEL_KEYS = frozenset({'parameters', 'metadata'})
+REQUIRED_METADATA_FIELDS = (
+    'model_name',
+    'experiment_name',
+    'run_id',
+    'model_version',
+    'model_project',
+    'export_timestamp',
+)
+REQUIRED_PARAMETERS = ('target_variable',)
+REQUIRED_MODEL_DIRS = ('prediction_model', 'data_model')
+REQUIRED_MODEL_FILES = ('MLmodel', 'model.pkl')
+
+# The import record's `experiment_name` column is `VARCHAR(50)` with a length-3 floor, and the name
+# is written onto the record before provisioning starts — so the column's limits are enforced here,
+# not three activities later as a constraint violation.
+EXPERIMENT_NAME_MIN_LENGTH = 3
+EXPERIMENT_NAME_MAX_LENGTH = 50
+
+# Gate 7 — an MLflow 2.x artifact tree for this platform: the model directories' known files, the
+# model card, the CSVs the training template logs and the bundle's own metadata.
+ALLOWED_FILE_NAMES = frozenset(
+    {
+        'MLmodel',
+        'metadata.json',
+        'model.pkl',
+        'conda.yaml',
+        'python_env.yaml',
+        'requirements.txt',
+        'model_card.json',
+        'model_card.svg',
+    }
+)
+ALLOWED_FILE_SUFFIXES = frozenset({'.csv', '.json', '.yaml', '.yml', '.txt', '.pkl', '.svg', '.md'})
+
+_DECRYPT_FAILURE_MESSAGE = (
+    'could not open this bundle: wrong password, or the file is damaged or was altered'
+)
+
+
+class BundleGateError(Exception):
+    """A gate rejected the bundle.
+
+    Carries the gate number and the import step that gate maps onto, so the workflow never has to
+    guess which step to record.
+    """
+
+    def __init__(self, message: str, *, gate: int, step: ImportStep | None = None) -> None:
+        super().__init__(message)
+        self.gate = gate
+        self.step = step if step is not None else GATE_TO_STEP[gate]
+
+
+@dataclass(frozen=True)
+class BundleLimits:
+    """Ceilings enforced by gates 1 and 4. Values come from `build_import_config()`."""
+
+    max_object_bytes: int = 1024 * 1024 * 1024
+    max_entries: int = 5000
+    max_uncompressed_bytes: int = 4 * 1024 * 1024 * 1024
+    max_compression_ratio: float = 200.0
+    object_prefix: str = 'imported_models/'
+    object_suffix: str = '.sientia'
+
+
+@dataclass(frozen=True)
+class OpenedBundle:
+    """What a bundle yields once every gate has passed. Nothing secret is carried."""
+
+    extracted_dir: Path
+    metadata: dict[str, Any] = field(default_factory=dict)
+    parameters: dict[str, Any] = field(default_factory=dict)
+    digest: str = ''
+    header: fmt.BundleHeader | None = None
+    # How long Argon2id took, so the worker's most expensive step is measurable without the
+    # activity layer having to reach inside the reader.
+    kdf_seconds: float = 0.0
+
+
+class BundleReader:
+    """Opens `.sientia` bundles produced by the exporter in `sientia-projects-templates`."""
+
+    def __init__(self, limits: BundleLimits | None = None) -> None:
+        self.limits = limits or BundleLimits()
+
+    # ------------------------------------------------------------------ gate 1
+
+    def check_object_limits(self, *, size: int, key: str) -> None:
+        """Gate 1, object level: reject before the object's body is fetched.
+
+        Args:
+            size: the object's size as reported by `stat_object`.
+            key: the object key.
+
+        Raises:
+            BundleGateError: oversized object, key outside the import prefix, or wrong suffix.
+        """
+        if size > self.limits.max_object_bytes:
+            raise BundleGateError(
+                f'uploaded object is {size} bytes, above the '
+                f'{self.limits.max_object_bytes} byte limit',
+                gate=1,
+            )
+        if size <= 0:
+            raise BundleGateError('uploaded object is empty', gate=1)
+        if not key.startswith(self.limits.object_prefix):
+            raise BundleGateError('uploaded object is not under the import prefix', gate=1)
+        if not key.endswith(self.limits.object_suffix):
+            raise BundleGateError(
+                f'uploaded object does not end in {self.limits.object_suffix}', gate=1
+            )
+
+    # --------------------------------------------------------------------- open
+
+    def open(
+        self,
+        encrypted_path: Path | str,
+        password: str,
+        extract_root: Path | str,
+        work_dir: Path | str | None = None,
+    ) -> OpenedBundle:
+        """Run gates 2 to 7 in order and return the opened bundle.
+
+        Args:
+            encrypted_path: the downloaded `.sientia` file.
+            password: the plaintext bundle password. Never stored, never logged, never returned.
+            extract_root: the directory to extract into. It MUST NOT exist yet (gate 5).
+            work_dir: where the decrypted zip is written. Defaults to `extract_root`'s parent.
+
+        Returns:
+            OpenedBundle: extracted tree, `metadata`, `parameters`, digest and header.
+
+        Raises:
+            BundleGateError: with the gate that rejected the bundle and its import step.
+        """
+        encrypted = Path(encrypted_path)
+        root = Path(extract_root)
+        staging = Path(work_dir) if work_dir is not None else root.parent
+
+        header = self._gate_2_header(encrypted)
+        zip_path, kdf_seconds = self._gate_3_decrypt(encrypted, password, header, staging)
+        try:
+            entries = self._gate_4_inspect(zip_path)
+            self._gate_5_extract(zip_path, root, entries)
+            metadata, parameters = self._gate_6_structure(root)
+            self._gate_7_content_policy(root)
+        except BaseException:
+            # A rejection after decryption leaves nothing extracted behind (gate 5's contract) and
+            # never leaves a decrypted zip on the worker for the next import to trip over.
+            shutil.rmtree(root, ignore_errors=True)
+            zip_path.unlink(missing_ok=True)
+            raise
+        zip_path.unlink(missing_ok=True)
+
+        return OpenedBundle(
+            extracted_dir=root,
+            metadata=metadata,
+            parameters=parameters,
+            digest=header.ciphertext_digest.hex(),
+            header=header,
+            kdf_seconds=kdf_seconds,
+        )
+
+    # ------------------------------------------------------------------ gate 2
+
+    def _gate_2_header(self, encrypted: Path) -> fmt.BundleHeader:
+        """Parse the header and check the chunk-stream digest, before deriving anything."""
+        try:
+            header = fmt.read_header(encrypted)
+            fmt.verify_ciphertext_digest(encrypted, header)
+        except fmt.BundleFormatError as error:
+            raise BundleGateError(str(error), gate=2) from error
+        except OSError as error:
+            raise BundleGateError('downloaded bundle could not be read', gate=2) from error
+        return header
+
+    # ------------------------------------------------------------------ gate 3
+
+    def _gate_3_decrypt(
+        self, encrypted: Path, password: str, header: fmt.BundleHeader, staging: Path
+    ) -> tuple[Path, float]:
+        """Derive the key with the header's parameters and decrypt the stream to a file.
+
+        The AEAD *is* the integrity check: a wrong password and a tampered byte fail here with the
+        same message, which never claims to distinguish them.
+        """
+        staging.mkdir(parents=True, exist_ok=True)
+        zip_path = staging / 'bundle.zip'
+        try:
+            started = time.monotonic()
+            key = self._derive_key(password, header)
+            kdf_seconds = time.monotonic() - started
+            self._pull_stream(encrypted, key, zip_path)
+        except BundleGateError:
+            zip_path.unlink(missing_ok=True)
+            raise
+        except (nacl.exceptions.CryptoError, ValueError, RuntimeError) as error:
+            zip_path.unlink(missing_ok=True)
+            raise BundleGateError(_DECRYPT_FAILURE_MESSAGE, gate=3) from error
+        except OSError as error:
+            zip_path.unlink(missing_ok=True)
+            raise BundleGateError('decrypted bundle could not be written', gate=3) from error
+        return zip_path, kdf_seconds
+
+    @staticmethod
+    def _derive_key(password: str, header: fmt.BundleHeader) -> bytes:
+        """Argon2id over the NFC-normalised password, with the header's own cost parameters.
+
+        The password is normalised and **not** trimmed, matching the producer's `_derive_key`: the
+        same password typed on another OS must derive the same key, while a trailing space is part
+        of the secret.
+        """
+        normalized = unicodedata.normalize('NFC', password)
+        return sodium.crypto_pwhash_alg(
+            fmt.KEY_SIZE,
+            normalized.encode('utf-8'),
+            header.salt,
+            header.kdf_opslimit,
+            header.kdf_memlimit_bytes,
+            sodium.crypto_pwhash_ALG_ARGON2ID13,
+        )
+
+    @staticmethod
+    def _pull_stream(encrypted: Path, key: bytes, zip_path: Path) -> None:
+        """Decrypt the chunk stream into `zip_path`, requiring the FINAL tag."""
+        chunk_size = fmt.CHUNK_SIZE + fmt.ABYTES
+        state = sodium.crypto_secretstream_xchacha20poly1305_state()
+        with encrypted.open('rb') as source, zip_path.open('wb') as target:
+            source.seek(fmt.HEADER_SIZE)
+            stream_header = source.read(fmt.STREAM_HEADER_SIZE)
+            sodium.crypto_secretstream_xchacha20poly1305_init_pull(state, stream_header, key)
+            last_tag: int | None = None
+            while chunk := source.read(chunk_size):
+                plaintext, last_tag = sodium.crypto_secretstream_xchacha20poly1305_pull(
+                    state, chunk
+                )
+                target.write(plaintext)
+        if last_tag != sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL:
+            # A stream that stops without its FINAL tag was cut short. The digest already catches
+            # truncation of a stored file; this catches a stream that was framed to look complete.
+            raise BundleGateError(_DECRYPT_FAILURE_MESSAGE, gate=3)
+
+    # ------------------------------------------------------------------ gate 4
+
+    def _gate_4_inspect(self, zip_path: Path) -> list[zipfile.ZipInfo]:
+        """Inspect the central directory only. Nothing is written to disk by this gate."""
+        try:
+            with zipfile.ZipFile(zip_path) as archive:
+                entries = archive.infolist()
+        except zipfile.BadZipFile as error:
+            raise BundleGateError('bundle content is not a readable archive', gate=4) from error
+
+        if len(entries) > self.limits.max_entries:
+            raise BundleGateError(
+                f'archive declares {len(entries)} entries, above the '
+                f'{self.limits.max_entries} limit',
+                gate=4,
+            )
+        if not entries:
+            raise BundleGateError('archive is empty', gate=4)
+
+        uncompressed = sum(entry.file_size for entry in entries)
+        compressed = sum(entry.compress_size for entry in entries)
+        if uncompressed > self.limits.max_uncompressed_bytes:
+            raise BundleGateError(
+                f'archive declares {uncompressed} uncompressed bytes, above the '
+                f'{self.limits.max_uncompressed_bytes} limit',
+                gate=4,
+            )
+        ratio = uncompressed / max(compressed, 1)
+        if ratio > self.limits.max_compression_ratio:
+            raise BundleGateError(
+                f'archive compression ratio {ratio:.1f} is above the '
+                f'{self.limits.max_compression_ratio} limit',
+                gate=4,
+            )
+
+        for entry in entries:
+            self._check_entry_name(entry.filename)
+            self._check_entry_mode(entry)
+
+        return entries
+
+    @staticmethod
+    def _check_entry_name(name: str) -> None:
+        """Reject absolute names and traversal, from the declared name alone."""
+        normalised = name.replace('\\', '/')
+        if normalised.startswith('/') or (len(normalised) > 1 and normalised[1] == ':'):
+            raise BundleGateError('archive declares an absolute entry path', gate=4)
+        if any(part == '..' for part in normalised.split('/')):
+            raise BundleGateError('archive declares an entry escaping its own tree', gate=4)
+
+    @staticmethod
+    def _check_entry_mode(entry: zipfile.ZipInfo) -> None:
+        """Reject symlinks and anything that is neither a regular file nor a directory."""
+        mode = entry.external_attr >> 16
+        if stat.S_IFMT(mode) == 0:
+            # No file-type bits stored. This is the normal case for two kinds of entry: a zip
+            # written by a tool that records no Unix mode at all (`mode == 0`), and
+            # `ZipFile.writestr`, which stores permissions only — the producer's `metadata.json`
+            # arrives as `0o600 << 16`. Nothing to check: the member is extracted as a plain file
+            # with the worker's own permissions either way.
+            return
+        if stat.S_ISLNK(mode):
+            raise BundleGateError('archive declares a symbolic link', gate=4)
+        if not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)):
+            raise BundleGateError('archive declares an entry that is not a regular file', gate=4)
+
+    # ------------------------------------------------------------------ gate 5
+
+    def _gate_5_extract(self, zip_path: Path, root: Path, entries: list[zipfile.ZipInfo]) -> None:
+        """Extract into a fresh directory, containing every member by resolved path.
+
+        A pre-existing extraction root is a failure rather than something to clear: it means
+        another import, or a previous attempt, owns that path. `entries` is the central-directory
+        listing gate 4 already read; the archive is still opened here to read member bytes, but its
+        `infolist()` is not read a second time.
+        """
+        if root.exists():
+            raise BundleGateError('extraction directory already exists', gate=5)
+        root.mkdir(parents=True)
+        resolved_root = root.resolve()
+        try:
+            with zipfile.ZipFile(zip_path) as archive:
+                for entry in entries:
+                    self._extract_member(archive, entry, root, resolved_root)
+        except BundleGateError:
+            shutil.rmtree(root, ignore_errors=True)
+            raise
+        except (OSError, zipfile.BadZipFile) as error:
+            shutil.rmtree(root, ignore_errors=True)
+            raise BundleGateError('bundle content could not be unpacked', gate=5) from error
+
+    @staticmethod
+    def _extract_member(
+        archive: zipfile.ZipFile, entry: zipfile.ZipInfo, root: Path, resolved_root: Path
+    ) -> None:
+        """Write one member, re-checking containment on the resolved path."""
+        name = entry.filename.replace('\\', '/')
+        target = root / name
+        resolved = Path(os.path.realpath(target))
+        if resolved != resolved_root and resolved_root not in resolved.parents:
+            raise BundleGateError('archive member resolves outside the extraction root', gate=5)
+        if entry.is_dir():
+            resolved.mkdir(parents=True, exist_ok=True)
+            return
+        if resolved.exists():
+            raise BundleGateError('archive member would overwrite an existing file', gate=5)
+        resolved.parent.mkdir(parents=True, exist_ok=True)
+        # Written as a plain file: the archive's stored permissions are never restored, so no
+        # member can arrive executable.
+        #
+        # Copied in chunks, counting bytes actually written, instead of `shutil.copyfileobj`:
+        # `entry.file_size` is central-directory metadata gate 4 already checked in aggregate, but a
+        # DEFLATE stream can be crafted to decompress to far more bytes than its own header
+        # declares. Aborting as soon as the running count exceeds `file_size` catches that member
+        # before the excess ever reaches disk, rather than trusting the declared size alone.
+        written = 0
+        with archive.open(entry) as source, resolved.open('wb') as sink:
+            while chunk := source.read(_EXTRACT_CHUNK_BYTES):
+                written += len(chunk)
+                if written > entry.file_size:
+                    raise BundleGateError(
+                        f'archive member {entry.filename!r} decompresses to more than its '
+                        f'declared {entry.file_size} bytes',
+                        gate=5,
+                    )
+                sink.write(chunk)
+
+    # ------------------------------------------------------------------ gate 6
+
+    def _gate_6_structure(self, root: Path) -> tuple[dict[str, Any], dict[str, Any]]:
+        """Validate the extracted layout and the metadata schema."""
+        metadata_path = root / METADATA_FILE_NAME
+        artifacts_dir = root / ARTIFACTS_DIR_NAME
+        if not metadata_path.is_file():
+            raise BundleGateError('bundle has no metadata document at its root', gate=6)
+        if not artifacts_dir.is_dir():
+            raise BundleGateError('bundle has no artifacts directory', gate=6)
+
+        document = self._load_metadata_document(metadata_path)
+        metadata = document['metadata']
+        parameters = document['parameters']
+
+        self._check_metadata_fields(metadata)
+        self._check_parameters(parameters)
+        self._check_model_directories(artifacts_dir)
+        return metadata, parameters
+
+    @staticmethod
+    def _load_metadata_document(metadata_path: Path) -> dict[str, Any]:
+        """Read `metadata.json` and check its two top-level keys."""
+        try:
+            document = json.loads(metadata_path.read_text(encoding='utf-8'))
+        except (json.JSONDecodeError, UnicodeDecodeError, OSError) as error:
+            raise BundleGateError('bundle metadata is not readable JSON', gate=6) from error
+        if not isinstance(document, dict) or set(document) != REQUIRED_TOP_LEVEL_KEYS:
+            raise BundleGateError(
+                'bundle metadata does not carry exactly the parameters and metadata blocks', gate=6
+            )
+        if not isinstance(document['metadata'], dict) or not isinstance(
+            document['parameters'], dict
+        ):
+            raise BundleGateError('bundle metadata blocks are not objects', gate=6)
+        return document
+
+    @staticmethod
+    def _check_metadata_fields(metadata: dict[str, Any]) -> None:
+        """Every origin field the import records must be present and usable."""
+        for field_name in REQUIRED_METADATA_FIELDS:
+            value = metadata.get(field_name)
+            if not isinstance(value, str) or not value.strip():
+                raise BundleGateError(f'bundle metadata is missing the {field_name} field', gate=6)
+        experiment_name = metadata['experiment_name']
+        if not (EXPERIMENT_NAME_MIN_LENGTH <= len(experiment_name) <= EXPERIMENT_NAME_MAX_LENGTH):
+            raise BundleGateError(
+                "bundle experiment name does not fit the platform's limits", gate=6
+            )
+
+    @staticmethod
+    def _check_parameters(parameters: dict[str, Any]) -> None:
+        """`target_variable` is what `model_config.target` is built from; nothing else supplies it."""
+        for name in REQUIRED_PARAMETERS:
+            value = parameters.get(name)
+            if value is None or not str(value).strip():
+                raise BundleGateError(f'bundle parameters are missing {name}', gate=6)
+
+    @staticmethod
+    def _check_model_directories(artifacts_dir: Path) -> None:
+        """`prediction_model` and `data_model` must each be a loadable MLflow model directory."""
+        for directory in REQUIRED_MODEL_DIRS:
+            model_dir = artifacts_dir / directory
+            if not model_dir.is_dir():
+                raise BundleGateError(f'bundle has no {directory} artifacts', gate=6)
+            for required in REQUIRED_MODEL_FILES:
+                if not (model_dir / required).is_file():
+                    raise BundleGateError(f'bundle {directory} artifacts are incomplete', gate=6)
+
+    # ------------------------------------------------------------------ gate 7
+
+    def _gate_7_content_policy(self, root: Path) -> None:
+        """Only the file kinds an MLflow artifact tree calls for may be present."""
+        for path in root.rglob('*'):
+            if path.is_dir():
+                continue
+            if not path.is_file():
+                raise BundleGateError('bundle contains an entry that is not a file', gate=7)
+            if path.name in ALLOWED_FILE_NAMES:
+                continue
+            if path.suffix.lower() in ALLOWED_FILE_SUFFIXES:
+                continue
+            raise BundleGateError('bundle contains a file a model export should not carry', gate=7)

laborious/utils/bundle/steps.py

@@ -0,0 +1,55 @@
+"""The import pipeline's step vocabulary, and the gate-to-step mapping.
+
+`ImportStep` is the join between three things: the machine-readable step written into the import
+record's JSON detail, the key of the error catalog (one code and one sentence per step) and the
+value the frontend switches on. It lives in the bundle package rather than in the activities module
+because the reader raises with it — `laborious/activities/model_import.py` imports it from here so
+there is exactly one definition.
+
+It holds **pipeline** steps only. The two failures that are not steps of the pipeline — failing to
+claim the record and failing to write it — are reported by the failure channel under their own
+markers (`not_claimable`, `status_write`), deliberately outside this enum so the
+catalog-completeness test keeps meaning "every pipeline step has a sentence".
+"""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from typing import Final
+
+
+class ImportStep(StrEnum):
+    """Where an import stopped. Values are what gets written to the record."""
+
+    RECEIVED = 'received'
+    DOWNLOAD = 'download'
+    DECRYPTION = 'decryption'
+    ARCHIVE_INSPECTION = 'archive_inspection'
+    EXTRACTION = 'extraction'
+    STRUCTURE_VALIDATION = 'structure_validation'
+    CONTENT_POLICY = 'content_policy'
+    EXPERIMENT_CREATION = 'experiment_creation'
+    ARTIFACT_UPLOAD = 'artifact_upload'
+    REGISTRATION = 'registration'
+    MODEL_DOCUMENT = 'model_document'
+    CLEANUP = 'cleanup'
+
+
+# The seven gates map onto the first seven steps, with gates 1 and 2 sharing one: both say "these
+# are not the bytes that were sent" — an object over the size ceiling, a key with the wrong shape, a
+# digest that disagrees with what the uploader computed, a wrong magic, an unsupported version, a
+# chunk stream that does not match the header's digest. The catalog's `download` sentence ("The
+# uploaded file could not be read, or it is not the file that was sent. Upload it again.") is true
+# for all of them, and none of them may borrow the `decryption` sentence, which implicates the
+# password.
+GATE_TO_STEP: Final[dict[int, ImportStep]] = {
+    1: ImportStep.DOWNLOAD,
+    2: ImportStep.DOWNLOAD,
+    3: ImportStep.DECRYPTION,
+    4: ImportStep.ARCHIVE_INSPECTION,
+    5: ImportStep.EXTRACTION,
+    6: ImportStep.STRUCTURE_VALIDATION,
+    7: ImportStep.CONTENT_POLICY,
+}
+
+GATE_COUNT: Final[int] = 7

laborious/utils/connectors_config.py

@@ -1,46 +1,179 @@
-"""
-Builds the configuration for the connectors.
-"""
-
-from os import getenv
 import json
+import tempfile
+from os import getenv, path
+from typing import Any
 
 
-def build_postgres_config():
-    return {
-        'host': getenv('POSTGRES_HOST', 'localhost'),
-        'port': int(getenv('POSTGRES_PORT', '5432')),
-        'user': getenv('POSTGRES_USER', 'sientia'),
-        'password': getenv('POSTGRES_PASSWORD', 'sientia'),
-        'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
-        'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
-        'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20'))
-    }
+def build_mlflow_config() -> dict[str, Any]:
+    """
+    Build MLFlow server configuration from environment variables.
 
+    This function constructs an MLFlow configuration dictionary from
+    environment variables with sensible defaults for local development.
+    It handles server connection and authentication parameters.
 
-def build_mlflow_config():
+    Environment Variables:
+        MLFLOW_HOST: MLFlow server hostname (default: http://localhost)
+        MLFLOW_PORT: MLFlow server port (default: 5080)
+        MLFLOW_USERNAME: MLFlow username (default: aignosi)
+        MLFLOW_PASSWORD: MLFlow password (default: aignosi)
+
+    Returns:
+        dict: MLFlow configuration dictionary with all required parameters
+    """
     return {
         'host': getenv('MLFLOW_HOST', 'http://localhost'),
         'port': int(getenv('MLFLOW_PORT', '5080')),
         'username': getenv('MLFLOW_USERNAME', 'aignosi'),
-        'password': getenv('MLFLOW_PASSWORD', 'aignosi')
+        'password': getenv('MLFLOW_PASSWORD', 'aignosi'),
     }
 
 
-def build_opc_config():
+def build_opc_config() -> dict[str, Any]:
+    """
+    Build OPC server configuration from environment variables.
+
+    This function constructs an OPC server configuration dictionary from
+    environment variables. It supports both single server and multi-server
+    configurations with flexible parameter handling.
+
+    Environment Variables:
+        OPC_CONFIG: JSON string containing multiple OPC server configurations
+        OPC_ID: OPC server ID (fallback, default: 1)
+        OPC_URL: Single OPC server URL (fallback, default: opc.tcp://localhost:4840)
+        OPC_SERVER_URI: Single OPC server URI (fallback, default: opc.tcp://localhost:4840)
+        OPC_CERT_PATH: Client certificate path (fallback, default: None)
+        OPC_PRIVATE_KEY_PATH: Client private key path (fallback, default: None)
+        OPC_SERVER_CERT_PATH: Server certificate path (fallback, default: None)
+        OPC_RECONNECTION_INTERVAL: Reconnection interval in seconds (fallback, default: 120)
+
+    Returns:
+        dict: OPC server configuration dictionary
+    """
     opc_raw = getenv('OPC_CONFIG', None)
 
     if opc_raw:
         return json.loads(opc_raw)
 
     return {
-        'opc': {
+        getenv('OPC_ID', '1'): {
             'id': getenv('OPC_ID', '1'),
+            'server_name': getenv('OPC_SERVER_NAME', 'default_server'),
             'url': getenv('OPC_URL', 'opc.tcp://localhost:4840'),
             'server_uri': getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'),
             'cert_path': getenv('OPC_CERT_PATH', None),
             'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None),
             'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None),
-            'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120'))
+            'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120')),
         }
     }
+
+
+def build_minio_config() -> dict[str, Any]:
+    """
+    Build MinIO (S3-compatible) configuration from environment variables.
+
+    Environment Variables:
+        MINIO_ENDPOINT: MinIO endpoint including scheme (default: http://localhost:9000)
+        MINIO_ACCESS_KEY: Access key (default: minioadmin)
+        MINIO_SECRET_KEY: Secret key (default: minioadmin)
+        MINIO_REGION: Region name for S3 client (default: us-east-1)
+        MINIO_BUCKET_DEFAULT: Default bucket for uploads (default: laborious)
+        MINIO_SECURE: Whether to use HTTPS (default: false)
+    Returns:
+        dict: MinIO configuration dictionary
+    """
+    return {
+        'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'),
+        'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'),
+        'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'),
+        'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious'),
+        'retention_hours': int(getenv('MINIO_RETENTION_HOURS', '24')),
+        'secure': getenv('MINIO_SECURE', 'false') == 'true',
+    }
+
+
+def build_import_config() -> dict[str, Any]:
+    """
+    Build the `.sientia` model-import configuration from environment variables.
+
+    The password key is read as the raw base64 string and **not** validated here: the worker must
+    start whether or not `IMPORT_MODEL_ENABLED` is set, and a missing or malformed key is an import
+    failure at the decryption step (`laborious.utils.import_password.load_envelope_key`), not a
+    boot failure.
+
+    Environment Variables:
+        IMPORT_PASSWORD_KEY: base64 of the 32-byte AES-256-GCM key that wraps bundle passwords
+        IMPORT_BUNDLE_BUCKET: bucket holding the uploaded bundles (default: sientia)
+        IMPORT_BUNDLE_PREFIX: key prefix the uploads must sit under (default: imported_models/)
+        IMPORT_WORK_DIR: worker scratch directory (default: <tempdir>/sientia-import)
+        IMPORT_MAX_OBJECT_BYTES: gate 1 object size ceiling (default: 1 GiB)
+        IMPORT_MAX_ARCHIVE_ENTRIES: gate 4 entry count ceiling (default: 5000)
+        IMPORT_MAX_UNCOMPRESSED_BYTES: gate 4 uncompressed size ceiling (default: 4 GiB)
+        IMPORT_MAX_COMPRESSION_RATIO: gate 4 compression ratio ceiling (default: 200)
+        IMPORT_MODELS_COLLECTION: MongoDB collection holding the model listing (default: models)
+        IMPORT_MODEL_ID_COUNTERS_COLLECTION: MongoDB collection holding the atomic id counters that
+            assign `models` document ids (default: model_id_counters). **External dependency**: this
+            collection and its seed value (`seq = max(int(id) for id in models)`) are not created by
+            this repository — a migration must add them to
+            `sientia-dataops-database-migrations/mongodb/migrations/`, following the `models`
+            baseline's `ensureCollection`/`ensureIndexes` convention, before any environment's counter
+            is first incremented. See `openspec/changes/model-import-fix-bugs/design.md`, Decision 3.
+        IMPORT_BUNDLE_RETENTION_DAYS: lifecycle expiry for the uploaded bundle (default: 7)
+
+    Returns:
+        dict: import configuration dictionary
+    """
+    prefix = getenv('IMPORT_BUNDLE_PREFIX', 'imported_models/')
+    return {
+        'password_key': getenv('IMPORT_PASSWORD_KEY'),
+        'bucket': getenv('IMPORT_BUNDLE_BUCKET', 'sientia'),
+        'prefix': prefix if prefix.endswith('/') else f'{prefix}/',
+        'work_dir': getenv('IMPORT_WORK_DIR', path.join(tempfile.gettempdir(), 'sientia-import')),
+        'max_object_bytes': int(getenv('IMPORT_MAX_OBJECT_BYTES', str(1024 * 1024 * 1024))),
+        'max_entries': int(getenv('IMPORT_MAX_ARCHIVE_ENTRIES', '5000')),
+        'max_uncompressed_bytes': int(
+            getenv('IMPORT_MAX_UNCOMPRESSED_BYTES', str(4 * 1024 * 1024 * 1024))
+        ),
+        'max_compression_ratio': float(getenv('IMPORT_MAX_COMPRESSION_RATIO', '200')),
+        'models_collection': getenv('IMPORT_MODELS_COLLECTION', 'models'),
+        'counters_collection': getenv('IMPORT_MODEL_ID_COUNTERS_COLLECTION', 'model_id_counters'),
+        'retention_days': int(getenv('IMPORT_BUNDLE_RETENTION_DAYS', '7')),
+    }
+
+
+def build_import_status_config() -> dict[str, Any]:
+    """
+    Build the connection to the import log database — the POSTGRES_* server, another database.
+
+    The import log is `public.experiment_run` in the database owned by the Spring Boot BFF
+    (`sientia-core-mlops-bff`), which sits on the same Postgres server as the `sientia` database
+    Laborious itself uses. Only the database name differs, so only the database name has a variable
+    of its own; host, port, user and password come from POSTGRES_*. The table and schema are
+    constants of the code, never inputs.
+
+    The four IMPORT_STATUS_DB_* connection variables remain as overrides, for a deployment where the
+    log genuinely lives on another server — the e2e suite is exactly that, two containers.
+
+    Environment Variables:
+        IMPORT_STATUS_DB_NAME: database (default: sientia-core-mlops-bff)
+        POSTGRES_HOST: server host, unless IMPORT_STATUS_DB_HOST overrides it (default: localhost)
+        POSTGRES_PORT: server port, unless IMPORT_STATUS_DB_PORT overrides it (default: 5432)
+        POSTGRES_USER: user, unless IMPORT_STATUS_DB_USER overrides it (default: sientia)
+        POSTGRES_PASSWORD: password, unless IMPORT_STATUS_DB_PASSWORD overrides it (default: empty)
+
+    Returns:
+        dict: import status database configuration dictionary
+    """
+    # Written as `override or fallback or default` rather than
+    # `getenv(override) or getenv(fallback, default)`: inside an `or`, the two-argument `getenv`
+    # widens to `str | None` for mypy, so the `int(...)` below fails to type-check. Putting the
+    # default last as its own term keeps every value a plain `str` and reads in precedence order.
+    return {
+        'host': getenv('IMPORT_STATUS_DB_HOST') or getenv('POSTGRES_HOST') or 'localhost',
+        'port': int(getenv('IMPORT_STATUS_DB_PORT') or getenv('POSTGRES_PORT') or '5432'),
+        'dbname': getenv('IMPORT_STATUS_DB_NAME') or 'sientia-core-mlops-bff',
+        'user': getenv('IMPORT_STATUS_DB_USER') or getenv('POSTGRES_USER') or 'sientia',
+        'password': getenv('IMPORT_STATUS_DB_PASSWORD') or getenv('POSTGRES_PASSWORD') or '',
+        'schema': 'public',
+    }

laborious/utils/dataframe_debug.py

@@ -0,0 +1,34 @@
+from typing import Any
+
+from pandas import DataFrame
+
+DEFAULT_MAX_DEBUG_DATAFRAME_ROWS = 100
+
+
+def build_dataframe_debug_message(
+    message: str,
+    data: Any,
+    max_rows: int = DEFAULT_MAX_DEBUG_DATAFRAME_ROWS,
+) -> str:
+    """
+    Build a safe debug message for dataframe payloads
+
+    Args:
+     - message (str): Base message to identify the logged payload
+     - data (Any): Payload to evaluate for dataframe-aware logging
+     - max_rows (int): Maximum dataframe row count allowed for full payload logging
+
+    Return:
+     Formatted debug message with full dataframe content or compact summary
+    """
+    if not isinstance(data, DataFrame):
+        return f'{message} {data}'
+
+    rows = data.shape[0]
+    if rows <= max_rows:
+        return f'{message}\n{data.to_csv()}'
+
+    return (
+        f'{message} skipped because dataframe has {rows} rows '
+        f'(max: {max_rows}). Shape: {data.shape}'
+    )

laborious/utils/filters/conditional_filters.py

@@ -3,28 +3,46 @@ from pandas import DataFrame
 
 def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool:
     """
-    Returns True if the specific columns have null values, False otherwise.
+    Filter to check if specific variables contain null values.
+
+    This function examines a DataFrame to determine if any of the specified variables
+    contain null (NaN) values. It returns True if null values are found for any of
+    the specified variables, False otherwise.
 
     Args:
-        - data (DataFrame): The data to filter.
-        - config (dict): The configuration.
+        data (DataFrame): The pandas DataFrame to be examined. Must contain columns
+            named 'variable' and 'value'.
+        config (dict): Configuration dictionary containing the following key:
+            - variables (list): List of variable names to check for null values
 
     Returns:
-        bool: True if the specific columns have null values, False otherwise.
+        bool: True if any of the specified variables contain null values,
+              False if none of the specified variables contain null values.
+
     """
-    return not data[
-        data['variable'].isin(config['variables']) & data['value'].isna()].empty
+
+    if data.empty:
+        return False
+
+    return not data[data['variable'].isin(config['variables']) & data['value'].isna()].empty
 
 
 def filter_empty_data(data: DataFrame, _config: dict) -> bool:
     """
-    Returns True if the data is empty, False otherwise.
+    Filter to check if the DataFrame is empty.
+
+    This function determines whether the provided DataFrame contains any data.
+    It's a simple utility function that can be used in conditional logic to
+    handle cases where no data is available.
 
     Args:
-        - data (DataFrame): The data to filter.
-        - _config (dict): The configuration.
+        data (DataFrame): The pandas DataFrame to be checked for emptiness.
+        _config (dict): Configuration dictionary (unused in this function).
+            The underscore prefix indicates this parameter is required for
+            interface consistency but not used in the implementation.
 
     Returns:
-        bool: True if the data is empty, False otherwise.
+        bool: True if the DataFrame is empty (has no rows), False if it contains data.
+
     """
     return data.empty

laborious/utils/filters/mlflow_filters.py

@@ -2,16 +2,26 @@ import numpy as np
 from pandas import DataFrame
 
 
-def api_error_filter(response: dict, _config: dict):
+def api_error_filter(response: dict, _config: dict) -> bool:
     """
-    Returns True if the API response is empty or the 'success' key is False, False otherwise.
+    Filter MLFlow API responses for error conditions.
+
+    This function analyzes MLFlow API responses to detect error conditions
+    and determine if the response should be filtered out due to quality
+    or reliability issues.
+
 
     Args:
-        - response (dict): The API response.
-        - _config (dict): The configuration.
+        response: MLFlow API response data (dict)
+        _config: Filter configuration dictionary
+            Required keys:
+                - error_codes (list, optional): List of error codes to detect
+                - error_keywords (list, optional): List of error keywords to detect
+                - check_structure (bool, optional): Whether to validate response structure
 
     Returns:
-        bool: True if the API response is empty or the 'success' key is False, False otherwise.
+        bool: True if data should be filtered (contains errors), False otherwise
+
     """
     if not response:
         return True
@@ -22,19 +32,31 @@ def api_error_filter(response: dict, _config: dict):
     return False
 
 
-def nan_values_filter(predictions: DataFrame, _config: dict):
+def nan_values_filter(predictions: DataFrame, _config: dict) -> bool:
     """
-    Returns True if the predictions DataFrame contains only NaN values, False otherwise.
+    Filter data for NaN (Not a Number) values.
+
+    This function detects NaN values in MLFlow prediction results and
+    determines if the data quality is sufficient for further processing
+    or export operations.
 
     Args:
-        - predictions (DataFrame): The predictions DataFrame.
-        - _config (dict): The configuration.
+        predictions: DataFrame containing prediction data to check for NaN values
+        _config: Filter configuration dictionary
+            Required keys:
+                - max_nan_ratio (float, optional): Maximum allowed NaN value ratio (0.0 to 1.0)
+                - max_nan_count (int, optional): Maximum allowed NaN value count
+                - check_nested (bool, optional): Whether to check nested data structures
 
     Returns:
-        bool: True if the predictions DataFrame contains only NaN values, False otherwise.
+        bool: True if data should be filtered (too many NaN values), False otherwise
+
     """
-    data = predictions.replace({None: np.nan}).drop(
-        columns=['timestamp'], errors='ignore').infer_objects(copy=False)
+    data = (
+        predictions.replace({None: np.nan})
+        .drop(columns=['timestamp'], errors='ignore')
+        .infer_objects()
+    )
 
     if data.isna().all().all():
         return True

laborious/utils/import_password.py

@@ -0,0 +1,97 @@
+"""The bundle password envelope: AES-256-GCM, opened inside the activity that uses the password.
+
+The workflow only ever carries `password_envelope`, so no plaintext password reaches the Temporal
+event history. This module is imported by the bundle-opening activity and by nothing else — in
+particular not by `laborious/workflows/import_model.py`, which a test asserts.
+
+Envelope layout, chosen because WebCrypto gives the Angular frontend AES-GCM natively:
+
+    base64( nonce(12 bytes) || ciphertext || GCM tag(16 bytes) )
+
+The key is a 32-byte value held in a Kubernetes Secret and exposed as `IMPORT_PASSWORD_KEY`,
+base64-encoded. Neither the key, the envelope nor the recovered password is ever logged, notified,
+used as a metric label or written to the import record.
+"""
+
+from __future__ import annotations
+
+import base64
+from typing import Final
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+NONCE_SIZE: Final[int] = 12
+TAG_SIZE: Final[int] = 16
+KEY_SIZE: Final[int] = 32
+MIN_ENVELOPE_SIZE: Final[int] = NONCE_SIZE + TAG_SIZE
+
+
+class EnvelopeError(Exception):
+    """The envelope or its key is unusable.
+
+    Deliberately one type for every cause — missing key, wrong key, malformed base64, altered
+    bytes — because the failure is reported at the decryption step either way. What it must never
+    be confused with is a wrong *bundle* password: that one comes from the reader's gate 3, and the
+    two carry different sentences.
+    """
+
+
+def load_envelope_key(raw_key: str | None) -> bytes:
+    """Decode the configured envelope key.
+
+    Args:
+        raw_key: the value of `IMPORT_PASSWORD_KEY` — base64 of exactly 32 bytes.
+
+    Returns:
+        bytes: the 32-byte key.
+
+    Raises:
+        EnvelopeError: when the value is absent, not base64, or not 32 bytes long. There is no
+            fallback interpretation: an unset key never means "the envelope is plaintext".
+    """
+    if not raw_key:
+        raise EnvelopeError('the import password key is not configured')
+    try:
+        key = base64.b64decode(raw_key, validate=True)
+    except ValueError as error:
+        raise EnvelopeError('the import password key is not valid base64') from error
+    if len(key) != KEY_SIZE:
+        raise EnvelopeError(f'the import password key must be {KEY_SIZE} bytes')
+    return key
+
+
+def decrypt_password_envelope(envelope: str, key: bytes) -> str:
+    """Recover the bundle password from its envelope.
+
+    Args:
+        envelope: base64 of `nonce || ciphertext || tag`.
+        key: the 32-byte key from `load_envelope_key`.
+
+    Returns:
+        str: the plaintext password, exactly as it was typed — not trimmed, not case-folded. NFC
+            normalisation happens at key derivation, where the producer does it.
+
+    Raises:
+        EnvelopeError: malformed envelope, wrong key or altered bytes (AES-GCM authentication).
+    """
+    if not envelope:
+        raise EnvelopeError('the import request carries no password envelope')
+    try:
+        payload = base64.b64decode(envelope, validate=True)
+    except ValueError as error:
+        raise EnvelopeError('the password envelope is not valid base64') from error
+    if len(payload) < MIN_ENVELOPE_SIZE:
+        raise EnvelopeError('the password envelope is too short to hold a nonce and a tag')
+
+    nonce, sealed = payload[:NONCE_SIZE], payload[NONCE_SIZE:]
+    try:
+        plaintext = AESGCM(key).decrypt(nonce, sealed, None)
+    except InvalidTag as error:
+        raise EnvelopeError('the password envelope could not be authenticated') from error
+    except ValueError as error:
+        raise EnvelopeError('the password envelope could not be opened') from error
+    try:
+        return plaintext.decode('utf-8')
+    except UnicodeDecodeError as error:
+        raise EnvelopeError('the recovered password is not valid UTF-8') from error

laborious/utils/messages.py

@@ -0,0 +1 @@
+MISSING_MINIO_REPOSITORY = 'Minio repository not initialized'

laborious/utils/models/import_errors.py

@@ -0,0 +1,546 @@
+"""The import's failure vocabulary: a closed catalog, typed errors and one reporting channel.
+
+Three things live here and nowhere else:
+
+- **the catalog** — one stable `error_code` and one plain-language `error_reason` per pipeline step.
+  The reason is written for a person who does not know this system exists: what happened, what state
+  the model is in, what to do next. It never carries a path, a table name, an object key, a Python
+  exception class or a traceback, and it is never built from `str(exception)`.
+- **the typed errors** — the two failures that must not be retried, because repeating them cannot
+  change the answer: a record this workflow does not own, and a statement the database refuses.
+- **the channel** — `report_import_failure`, the single funnel every failure passes through. It is
+  the only place in the import path that logs an error or sends a notification, which a test
+  asserts, so no failure is reported twice or half-reported.
+"""
+
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    import traceback
+    from collections import deque
+    from collections.abc import Iterator, Mapping
+    from dataclasses import dataclass
+    from enum import StrEnum
+    from types import MappingProxyType
+    from typing import Any, Final, Protocol
+
+    from temporalio.exceptions import ApplicationError
+
+    from laborious import metrics
+    from laborious.utils.bundle.steps import ImportStep
+
+
+class ImportErrorCode(StrEnum):
+    """The machine-readable half of a failure: the frontend's i18n key and support's triage key.
+
+    Written under `import.code` in the record's JSON detail — not in a column of its own, so the
+    import adds nothing to a table it shares with the training flow.
+    """
+
+    REQUEST_INVALID = 'IMPORT_REQUEST_INVALID'
+    FILE_UNREADABLE = 'IMPORT_FILE_UNREADABLE'
+    BUNDLE_CANNOT_BE_OPENED = 'IMPORT_BUNDLE_CANNOT_BE_OPENED'
+    BUNDLE_REJECTED = 'IMPORT_BUNDLE_REJECTED'
+    BUNDLE_UNPACK_FAILED = 'IMPORT_BUNDLE_UNPACK_FAILED'
+    BUNDLE_INCOMPLETE = 'IMPORT_BUNDLE_INCOMPLETE'
+    BUNDLE_UNEXPECTED_CONTENT = 'IMPORT_BUNDLE_UNEXPECTED_CONTENT'
+    STORAGE_PREPARATION_FAILED = 'IMPORT_STORAGE_PREPARATION_FAILED'
+    MODEL_FILES_NOT_STORED = 'IMPORT_MODEL_FILES_NOT_STORED'
+    MODEL_NOT_REGISTERED = 'IMPORT_MODEL_NOT_REGISTERED'
+    MODEL_SETTINGS_NOT_SAVED = 'IMPORT_MODEL_SETTINGS_NOT_SAVED'
+    UNEXPECTED_ERROR = 'IMPORT_UNEXPECTED_ERROR'
+    # Codes that never reach a record's `error_message`, and are therefore absent from the catalog:
+    # one names the failure to own a row, one the failure to write one, one the temporary files left
+    # on the worker after an import that otherwise succeeded. All three live on the channel — log,
+    # metric, notification, Temporal error — and the third is additionally flagged in the record's
+    # detail as `cleanup_failed`, without a code and without changing the status.
+    LOG_ROW_NOT_CLAIMABLE = 'IMPORT_LOG_ROW_NOT_CLAIMABLE'
+    STATUS_NOT_RECORDED = 'IMPORT_STATUS_NOT_RECORDED'
+    CLEANUP_INCOMPLETE = 'IMPORT_CLEANUP_INCOMPLETE'
+
+
+class ImportResponsibility(StrEnum):
+    """Who can act on a step's failure: the user (re-export, re-type, re-submit) or only support."""
+
+    USER = 'user'
+    SUPPORT = 'support'
+
+
+@dataclass(frozen=True)
+class ImportFailure:
+    """One classification of one failure: the step, its code, its sentence and whose to fix."""
+
+    step: ImportStep | None
+    code: ImportErrorCode
+    reason: str
+    responsibility: ImportResponsibility = ImportResponsibility.SUPPORT
+
+
+_CONTACT_SUPPORT = "Contact support with this import's identifier."
+_RE_EXPORT = 'Export the model again from the origin platform.'
+
+IMPORT_FAILURE_CATALOG: Final[
+    Mapping[ImportStep, tuple[ImportErrorCode, str, ImportResponsibility]]
+] = MappingProxyType(
+    {
+        ImportStep.RECEIVED: (
+            ImportErrorCode.REQUEST_INVALID,
+            'The import could not be started because the request was incomplete or '
+            'contradictory. Nothing was created. Submit the import again.',
+            ImportResponsibility.USER,
+        ),
+        ImportStep.DOWNLOAD: (
+            ImportErrorCode.FILE_UNREADABLE,
+            'The uploaded file could not be read, or it is not the file that was sent. '
+            'Upload it again.',
+            ImportResponsibility.USER,
+        ),
+        ImportStep.DECRYPTION: (
+            ImportErrorCode.BUNDLE_CANNOT_BE_OPENED,
+            'This file could not be opened: the password is wrong, or the file is damaged or '
+            'was altered. Confirm the password with whoever exported the model, then try '
+            'again.',
+            ImportResponsibility.USER,
+        ),
+        ImportStep.ARCHIVE_INSPECTION: (
+            ImportErrorCode.BUNDLE_REJECTED,
+            "The file's contents did not pass the platform's safety checks, so it was not "
+            f'opened. {_RE_EXPORT}',
+            ImportResponsibility.USER,
+        ),
+        ImportStep.EXTRACTION: (
+            ImportErrorCode.BUNDLE_UNPACK_FAILED,
+            f'The file could not be unpacked safely and nothing from it was kept. {_RE_EXPORT}',
+            ImportResponsibility.USER,
+        ),
+        ImportStep.STRUCTURE_VALIDATION: (
+            ImportErrorCode.BUNDLE_INCOMPLETE,
+            'This file is not a complete model export — part of what the platform needs is '
+            f'missing from it. {_RE_EXPORT}',
+            ImportResponsibility.USER,
+        ),
+        ImportStep.CONTENT_POLICY: (
+            ImportErrorCode.BUNDLE_UNEXPECTED_CONTENT,
+            'The file contains something a model export should not contain, so it was '
+            f'rejected. {_RE_EXPORT}',
+            ImportResponsibility.USER,
+        ),
+        ImportStep.EXPERIMENT_CREATION: (
+            ImportErrorCode.STORAGE_PREPARATION_FAILED,
+            'The platform could not prepare a place to keep this model. The import stopped '
+            f'and no model was created. {_CONTACT_SUPPORT}',
+            ImportResponsibility.SUPPORT,
+        ),
+        ImportStep.ARTIFACT_UPLOAD: (
+            ImportErrorCode.MODEL_FILES_NOT_STORED,
+            "The model's files could not be stored on this platform. The import stopped and "
+            f'the model is not available. {_CONTACT_SUPPORT}',
+            ImportResponsibility.SUPPORT,
+        ),
+        ImportStep.REGISTRATION: (
+            ImportErrorCode.MODEL_NOT_REGISTERED,
+            "The model's files were stored, but the model itself could not be registered, so "
+            f'it cannot be used. {_CONTACT_SUPPORT}',
+            ImportResponsibility.SUPPORT,
+        ),
+        ImportStep.MODEL_DOCUMENT: (
+            ImportErrorCode.MODEL_SETTINGS_NOT_SAVED,
+            'The model was registered, but the settings that tell the platform how to run it '
+            f'could not be saved. {_CONTACT_SUPPORT}',
+            ImportResponsibility.SUPPORT,
+        ),
+    }
+)
+
+UNCLASSIFIED_FAILURE: Final[tuple[ImportErrorCode, str]] = (
+    ImportErrorCode.UNEXPECTED_ERROR,
+    f'The import stopped for an unexpected reason and no model was created. {_CONTACT_SUPPORT}',
+)
+
+# The three explicit-code failures are not pipeline steps, so `classify_import_failure` falls
+# through to `UNCLASSIFIED_FAILURE`'s reason for all of them. That sentence says "no model was
+# created", which is false for a cleanup-only failure (the model was imported) and misleading for
+# the other two (neither ever attempted the pipeline). Each gets its own reason here instead.
+_OVERRIDE_REASONS: Final[Mapping[ImportErrorCode, str]] = MappingProxyType(
+    {
+        ImportErrorCode.CLEANUP_INCOMPLETE: (
+            'The model was imported successfully; only its temporary files could not be removed '
+            'from the worker. No action needed.'
+        ),
+        ImportErrorCode.LOG_ROW_NOT_CLAIMABLE: 'The import request could not be started.',
+        ImportErrorCode.STATUS_NOT_RECORDED: 'The import outcome could not be recorded.',
+    }
+)
+
+# `cleanup` deletes temporary files on the worker. It is never the reported failure: when every
+# other step succeeded and only cleanup failed, the import is a success and the leak is a log line.
+# So it deliberately has no user-facing sentence, and the catalog-completeness test knows it.
+STEPS_WITHOUT_A_USER_FACING_REASON: Final[frozenset[ImportStep]] = frozenset({ImportStep.CLEANUP})
+
+
+def classify_import_failure(step: ImportStep | None) -> ImportFailure:
+    """Turn a failure into the one code and the one sentence that will be recorded.
+
+    The step alone decides the outcome — an exception's text carries paths, table names and type
+    names, and belongs in the log, never in the record.
+
+    Args:
+        step: the step that was being attempted, or `None` when none had been entered.
+
+    Returns:
+        ImportFailure: step, code and the reason to store.
+    """
+    if step is None or step not in IMPORT_FAILURE_CATALOG:
+        code, reason = UNCLASSIFIED_FAILURE
+        return ImportFailure(step=step, code=code, reason=reason)
+    code, reason, responsibility = IMPORT_FAILURE_CATALOG[step]
+    return ImportFailure(step=step, code=code, reason=reason, responsibility=responsibility)
+
+
+class ImportLogRowNotClaimableError(ApplicationError):
+    """The record named by `import_run_id` is not this workflow's to write.
+
+    Raised when the claim affects zero rows: no such row, a `TRAIN` row, or a row already running
+    under another workflow. Non-retryable on purpose — a second attempt would read the same rows and
+    reach the same conclusion — and loud everywhere except the row itself, which belongs to someone
+    else.
+    """
+
+    def __init__(self, message: str, *, import_run_id: Any = None, observed: str = '') -> None:
+        super().__init__(
+            message,
+            type='ImportLogRowNotClaimableError',
+            non_retryable=True,
+        )
+        self.import_run_id = import_run_id
+        self.observed = observed
+
+
+class ImportStatusRejectedError(ApplicationError):
+    """The database accepted the connection and refused the statement.
+
+    A `CHECK` violation, an undefined column, a value too long for its column. `V12` not being
+    applied yet lands here, at the terminal write, and it must fail on the first attempt instead of
+    spending the whole retry envelope to say so.
+    """
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message, type='ImportStatusRejectedError', non_retryable=True)
+
+
+class ImportMisconfiguredError(ApplicationError):
+    """A required repository was not injected into this worker. A deploy defect, never transient."""
+
+    def __init__(self, message: str) -> None:
+        super().__init__(message, type='ImportMisconfiguredError', non_retryable=True)
+
+
+class ImportInputError(ApplicationError):
+    """The request is incomplete or contradictory. Fails at `received`, on the claimed row."""
+
+    def __init__(self, message: str) -> None:
+        super().__init__(
+            message,
+            {'step': ImportStep.RECEIVED.value, 'gate': None},
+            type='ImportInputError',
+            non_retryable=True,
+        )
+
+
+class ImportGateRejectedError(ApplicationError):
+    """A gate rejected the bundle, and the error says which one.
+
+    Gates 2 to 7 all run inside one activity, so the step the workflow was *attempting* is not
+    precise enough to record. An exception loses its Python type crossing an activity boundary but
+    keeps its `details`, so the step and the gate travel there and the terminal write can name them.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        step: ImportStep,
+        gate: int | None = None,
+        cause: str | None = None,
+    ) -> None:
+        super().__init__(
+            message,
+            {'step': step.value, 'gate': gate, 'cause': cause},
+            type='ImportGateRejectedError',
+            non_retryable=True,
+        )
+        self.step = step
+        self.gate = gate
+
+
+_SUMMARY_LIMIT = 400
+_CHAIN_LIMIT = 10
+_UNKNOWN_ERROR_TYPE = 'unavailable'
+
+
+def walk_cause_chain(
+    exception: BaseException, limit: int = _CHAIN_LIMIT
+) -> Iterator[BaseException]:
+    """Yield `exception`, then each `__cause__`, up to `limit` links. Bounded against a cycle."""
+    candidate: BaseException | None = exception
+    for _ in range(limit):
+        if candidate is None:
+            return
+        yield candidate
+        candidate = candidate.__cause__
+
+
+def _root_cause(exception: BaseException) -> BaseException:
+    """The deepest `__cause__` in the chain: the failure that actually happened.
+
+    A `MlflowException` reaches this channel wrapped — by an `ActivityError`, by an
+    `ApplicationError`, by `requests`' own re-raises — and the outermost type is the wrapper's, which
+    says nothing an operator can act on. The chain is walked to its end, and bounded, because a cycle
+    in `__cause__` would otherwise hang the reporting of a failure.
+    """
+    return deque(walk_cause_chain(exception), maxlen=1)[-1]
+
+
+def _type_name(exception: BaseException) -> str:
+    """The name to report a failure under: `ApplicationError.type` when there is one.
+
+    An activity's own exception type is the one thing worth grouping failures by, and Temporal keeps
+    it in `ApplicationError.type` rather than in the Python class — a `MlflowException` raised in an
+    activity reaches the workflow as an `ApplicationError` whose `type` still says `MlflowException`.
+    Reporting the class name here would say `ApplicationError` for every activity failure there is.
+    """
+    return str(getattr(exception, 'type', None) or type(exception).__name__)
+
+
+def _one_line(text: str) -> str:
+    """The first non-empty line, bounded — a log *field* has to stay one field.
+
+    The traceback still reaches the log in full, in the message. This is the value a dashboard groups
+    by and an alert quotes, so a 40-frame `urllib3` chain would make it useless.
+    """
+    for line in text.splitlines():
+        stripped = line.strip()
+        if stripped:
+            return stripped[:_SUMMARY_LIMIT]
+    return ''
+
+
+def summarize_failure(
+    exception: BaseException | None,
+    detail_text: str | None = None,
+    error_type: str | None = None,
+) -> tuple[str, str]:
+    """`(error_type, error_summary)` — the two technical fields every failure log carries.
+
+    Args:
+        exception: the failure, when the object itself is to hand.
+        detail_text: the failure as text, for a caller whose exception crossed an activity boundary.
+        error_type: the type name that travelled with such a text, when it is known.
+
+    Returns:
+        tuple: the exception type name and a single-line summary of what it said.
+    """
+    if exception is not None:
+        cause = _root_cause(exception)
+        return _type_name(cause), _one_line(str(cause))
+    return error_type or _UNKNOWN_ERROR_TYPE, _one_line(detail_text or '')
+
+
+def describe_failure_chain(exception: BaseException) -> tuple[str, str]:
+    """`(error_type, detail)` for a failure that is about to lose its exception object.
+
+    The workflow is the one place that holds the whole chain — `ActivityError` wrapping the
+    `ApplicationError` that carries the activity's own type and message — and it cannot hand that
+    object to the channel, because the channel runs in an activity. So the chain is flattened here,
+    **root cause first**, one link per line: the first line is what actually broke, which is what
+    becomes `error_summary`, and the wrappers follow for whoever reads further.
+
+    `ApplicationError.type` is preferred over the Python class name because it is the only place the
+    activity's original exception type survives the boundary: `MlflowException` arrives as an
+    `ApplicationError` whose `type` says so.
+    """
+    links: list[str] = []
+    for candidate in walk_cause_chain(exception):
+        kind = _type_name(candidate)
+        message = _one_line(str(candidate))
+        # `ApplicationError.__str__` already prefixes its own type, and `MlflowException:
+        # MlflowException: …` reads like two failures rather than one.
+        if not message:
+            links.append(kind)
+        elif message.startswith(f'{kind}:'):
+            links.append(message)
+        else:
+            links.append(f'{kind}: {message}')
+    links.reverse()
+    root = links[0].split(':', 1)[0] if links else _UNKNOWN_ERROR_TYPE
+    return root, '\n'.join(links)
+
+
+def _log_fields(
+    *,
+    failure: ImportFailure,
+    where: str,
+    import_run_id: Any,
+    workflow_id: str,
+    gate: int | None,
+    error_type: str,
+    error_summary: str,
+    verdict: str | None,
+    metadata: dict[str, Any] | None,
+) -> dict[str, Any]:
+    """The workflow metadata plus this failure's own fields, as `logfmt` keys.
+
+    Every key is prefixed `import_` or `error_` so none of them can collide with the metadata the
+    rest of the runtime already logs (`model_id`, `model_name`, `schedule_name`, `workflow_name`) or
+    with the formatter's own (`at`, `msg`, `timestamp`). Values are single-line by construction: the
+    traceback stays in the message, where a newline costs nothing.
+    """
+    fields: dict[str, Any] = {
+        **(metadata or {}),
+        'import_run_id': import_run_id,
+        'import_workflow_id': workflow_id,
+        'import_step': where,
+        'import_code': failure.code.value,
+        'error_type': error_type,
+    }
+    if error_summary:
+        fields['error_summary'] = error_summary
+    if gate is not None:
+        fields['import_gate'] = gate
+    if verdict:
+        fields['import_verdict_not_recorded'] = verdict
+    return fields
+
+
+def failure_location(exception: BaseException | None) -> tuple[str | None, int | None]:
+    """Read the step and gate an activity's failure carries, if it carried any.
+
+    Walks the `__cause__` chain because Temporal wraps an activity failure in an `ActivityError`.
+    Returns `(None, None)` when the failure did not say where it happened, and the caller then falls
+    back to the step it was attempting.
+    """
+    if exception is None:
+        return None, None
+    for candidate in walk_cause_chain(exception):
+        details = getattr(candidate, 'details', None)
+        if details and isinstance(details[0], dict) and 'step' in details[0]:
+            location = details[0]
+            gate = location.get('gate')
+            return location.get('step'), int(gate) if isinstance(gate, int) else None
+    return None, None
+
+
+class FailureSinks(Protocol):
+    """What the channel needs from its caller: the repository's existing observability surface."""
+
+    def error(self, message: str, metadata: dict | None = None) -> None: ...
+
+    def get_core_labels(self, metadata: dict | None = None, operation_type: str = '-') -> dict: ...
+
+    async def emit_metric(
+        self,
+        metric_object: Any,
+        tags: dict[str, Any],
+        method: str = 'inc',
+        value: float | None = None,
+    ) -> None: ...
+
+    async def send_notification_async(
+        self,
+        metadata: dict,
+        notification_id: str,
+        message: str,
+        block: str,
+        level: str = 'INFO',
+        attachment_content: str | None = None,
+    ) -> None: ...
+
+
+async def report_import_failure(
+    sinks: FailureSinks,
+    step: ImportStep | None,
+    exception: BaseException | None,
+    *,
+    import_run_id: Any,
+    workflow_id: str,
+    metadata: dict[str, Any] | None = None,
+    code: ImportErrorCode | None = None,
+    marker: str | None = None,
+    verdict: str | None = None,
+    detail_text: str | None = None,
+    error_type: str | None = None,
+    gate: int | None = None,
+) -> ImportFailure:
+    """Report one failure through every sink, exactly once.
+
+    Sinks, in order: a structured log carrying the ids, the step, the code **and** the exception
+    with its traceback (the only sink that gets the exception text); the import error metric, with
+    the code in `operation_type`; a notification through the existing handler. The exception itself
+    reaches Temporal by being raised, and the record is the fifth, optional sink — the terminal
+    write, when the row can be written at all.
+
+    Both log lines this produces — the technical one here and the user-facing sentence the
+    notification logs — carry the same `logfmt` fields, built once by `_log_fields`. That is what
+    makes the two greppable and joinable: `import_run_id=29` finds the pair, `import_code=` groups
+    them, and `error_type=` says what actually broke without reading a traceback. The message text is
+    kept as it was, ids and all, so a query written against it still matches.
+
+    Args:
+        sinks: the activity providing logger, metrics and notifications.
+        step: the pipeline step, or `None` for a failure that is not one (see `marker`).
+        exception: the failure.
+        import_run_id: the record's id, for correlation.
+        workflow_id: the identifier a user quotes to support.
+        metadata: workflow metadata for labels and the notification.
+        code: an explicit code, for the two failures that are not pipeline steps.
+        marker: `not_claimable` or `status_write` — what a step name would have been.
+        verdict: for a failed terminal write, the verdict it could not record, so the outcome
+            survives in text even when it cannot survive in the row.
+        detail_text: the failure as text, for the callers whose exception object is already gone —
+            anything that crossed an activity boundary. Used in place of a traceback.
+        error_type: the exception type name that travelled with such a text, when it is known.
+        gate: the gate that rejected the bundle, when one did — a field, never a sentence.
+
+    Returns:
+        ImportFailure: the classification that was reported.
+    """
+    failure = classify_import_failure(step)
+    if code is not None:
+        failure = ImportFailure(
+            step=step, code=code, reason=_OVERRIDE_REASONS.get(code, failure.reason)
+        )
+
+    where = marker or (step.value if step else 'unknown')
+    if exception is not None:
+        detail = ''.join(traceback.format_exception(exception)).strip()
+    else:
+        detail = detail_text or 'no exception object available'
+    kind, summary = summarize_failure(exception, detail_text, error_type)
+    fields = _log_fields(
+        failure=failure,
+        where=where,
+        import_run_id=import_run_id,
+        workflow_id=workflow_id,
+        gate=gate,
+        error_type=kind,
+        error_summary=summary,
+        verdict=verdict,
+        metadata=metadata,
+    )
+    verdict_line = f' verdict_not_recorded={verdict}' if verdict else ''
+    sinks.error(
+        f'model import failed: step={where} code={failure.code.value} '
+        f'import_run_id={import_run_id} workflow_id={workflow_id}{verdict_line}\n{detail}',
+        fields,
+    )
+
+    labels = sinks.get_core_labels(metadata or {}, operation_type=failure.code.value)
+    await sinks.emit_metric(metric_object=metrics.MODEL_IMPORT_ERROR_COUNT, tags=labels)
+    await sinks.send_notification_async(
+        metadata=fields,
+        notification_id=failure.code.value,
+        message=failure.reason,
+        block=where,
+        level='ERROR',
+    )
+    return failure

laborious/utils/models/minio_dataframe_payload.py

@@ -0,0 +1,348 @@
+"""
+MinIO-backed DataFrame payload for Temporal workflows.
+
+Data is never stored as a pandas ``DataFrame`` field on the dataclass.
+Instead, the DataFrame is only provided as an input to:
+`from_dataframe` / `from_dataframe_to_dict`.
+
+At build time, the DataFrame is evaluated for its serialized size; if it exceeds
+the configured threshold, it is serialized to parquet bytes and uploaded to MinIO.
+Otherwise, it is inlined as a Temporal-friendly ``dict``.
+"""
+
+import pickle
+import re
+from collections.abc import Hashable
+from dataclasses import dataclass
+from datetime import datetime
+from io import BytesIO
+from os import getenv
+from typing import Any, Literal
+
+from pandas import DataFrame, read_parquet
+from sientia_do.observability.logger import Logger
+from sientia_do.repository.minio_repository import MinioRepository
+from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
+
+# Keys that are part of the serialized wire format (not arbitrary metadata).
+_SERIALIZED_FIELD_KEYS = frozenset({'data', 'bucket', 'object_key', 'object_prefix', 'uri'})
+
+_OBJECT_TIMESTAMP_PATTERN = re.compile(
+    r'-(?:initial|transform)-(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.parquet$'
+)
+
+OFFLOAD_THRESHOLD_BYTES = int(
+    float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1.5')) * 1024 * 1024
+)
+
+# Relative prefix used for storing offloaded prediction datasets in MinIO.
+# It is also the root directory for retention cleanup listing.
+PREDICTION_DATASETS_PREFIX = 'prediction_datasets'
+
+OperationKind = Literal['initial', 'transform', 'predict']
+
+
+def _build_object_key(
+    model_name: str, operation: OperationKind, timestamp: str
+) -> tuple[str, str | None]:
+    """
+    Build the MinIO object key and the directory prefix used for retention listing.
+
+    Args:
+        model_name: Registered model name used in the pipeline.
+        operation: Either initial (pre-transform load) or transform (post-MLFlow transform).
+        timestamp: Filename timestamp segment from DATETIME_FORMAT_FILENAME.
+
+    Return:
+        tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root).
+    """
+    # Naming convention:
+    #   - Directory is always `prediction_datasets/<model_name>`
+    #   - Filename follows the retention-parsing pattern
+    basename = f'{model_name}-{operation}-{timestamp}.parquet'
+    model_dir = model_name.strip().strip('/')
+    prefix = f'{PREDICTION_DATASETS_PREFIX}/{model_dir}'
+    return f'{prefix}/{basename}', prefix
+
+
+@dataclass
+class MinioDataFramePayload:
+    """
+    Serializable payload after a DataFrame was evaluated: inline tabular dict and/or MinIO keys.
+
+    Build from a live DataFrame only via `from_dataframe` / `from_dataframe_to_dict`.
+    Rehydrate from Temporal via `from_dict`. The DataFrame is not a field on this class.
+    """
+
+    last_timestamp: str
+    status: dict[str, Any] | None = None
+    data: dict[Hashable, Any] | None = None
+    bucket: str | None = None
+    object_key: str | None = None
+    object_prefix: str | None = None
+    uri: str | None = None
+
+    @staticmethod
+    def _debug(
+        logger: Logger | None,
+        message: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """
+        Emit debug logs only when logger is provided
+
+        Args:
+         - logger (Logger | None): Logger instance used for debug messages
+         - message (str): Message to be logged
+         - metadata (dict[str, Any] | None): Optional workflow metadata context
+        """
+        if logger is None:
+            return
+        logger.custom_debug(message, metadata)
+
+    @classmethod
+    def from_dict(cls, raw: 'dict[str, Any] | MinioDataFramePayload') -> 'MinioDataFramePayload':
+        """
+        Reconstruct a MinioDataFramePayload from a plain dict produced by Temporal serialization.
+
+        Temporal converts dataclass return values into plain dicts when crossing
+        workflow/activity boundaries. This method rebuilds the typed instance so
+        that methods like ``retrieve``, ``cleanup_prefix`` and ``has_data`` are
+        available on the receiving side.
+
+        If the argument is already a MinioDataFramePayload, it is returned as-is.
+
+        Args:
+            raw: Dict with keys matching the dataclass fields
+                (last_timestamp, status, data, bucket, object_key, object_prefix, uri),
+                or an existing MinioDataFramePayload instance.
+
+        Return:
+            MinioDataFramePayload: Reconstructed (or original) instance.
+        """
+        if isinstance(raw, MinioDataFramePayload):
+            return raw
+        return cls(
+            last_timestamp=raw['last_timestamp'],
+            status=raw.get('status'),
+            data=raw.get('data'),
+            bucket=raw.get('bucket'),
+            object_key=raw.get('object_key'),
+            object_prefix=raw.get('object_prefix'),
+            uri=raw.get('uri'),
+        )
+
+    @staticmethod
+    def estimate_size_bytes(
+        df: DataFrame,
+        metadata: dict[str, Any] | None = None,
+        logger: Logger | None = None,
+    ) -> int:
+        """
+        Approximate serialized size of the DataFrame as the default-orient dict.
+
+        Args:
+            df: DataFrame whose tabular content size is estimated.
+
+        Return:
+            int: Estimated size in bytes (pickle of dict representation).
+        """
+        try:
+            size = len(pickle.dumps(df.to_dict()))
+        except Exception:
+            size = len(pickle.dumps(df))
+
+        MinioDataFramePayload._debug(
+            logger,
+            f'DataFrame size: {size} bytes',
+            metadata,
+        )
+        return size
+
+    @staticmethod
+    def parse_object_timestamp(object_key: str) -> datetime | None:
+        """
+        Parse the timestamp embedded in the object key basename (before .parquet).
+
+        Args:
+            object_key: S3/MinIO object key whose basename follows
+                ``{model}-{initial|transform}-{DATETIME_FORMAT_FILENAME}.parquet``.
+
+        Return:
+            datetime | None: Parsed UTC-naive datetime from the key, or None if not matched.
+        """
+        basename = object_key.rsplit('/', 1)[-1]
+        match = _OBJECT_TIMESTAMP_PATTERN.search(basename)
+        if not match:
+            return None
+        try:
+            return datetime.strptime(match.group(1), DATETIME_FORMAT_FILENAME)
+        except ValueError:
+            return None
+
+    def cleanup_prefix(self) -> str | None:
+        """
+        Return True if cleanup is enabled for this payload.
+        """
+        if self.object_key is not None and self.data is None:
+            return self.object_prefix
+        return None
+
+    def has_data(self) -> bool:
+        """
+        Return True if the payload has some data internally or in MinIO.
+        """
+        return (self.data is not None and self.data != {}) or self.object_key is not None
+
+    @classmethod
+    async def from_dataframe(
+        cls,
+        dataframe: DataFrame | None,
+        minio_repo: MinioRepository,
+        model_name: str,
+        operation: OperationKind,
+        status: dict[str, Any] | None = None,
+        workflow_metadata: dict | None = None,
+        last_timestamp: str | None = None,
+        logger: Logger | None = None,
+    ) -> 'MinioDataFramePayload':
+        """
+        Evaluate the DataFrame size, then either inline dict or upload parquet to MinIO.
+
+        The DataFrame is not stored on the returned instance.
+
+        Args:
+            dataframe: Tabular data to evaluate and persist (inline or MinIO).
+            metadata: Small metadata dict merged into the payload (e.g. success, message).
+            minio_repo: sientia_do MinioRepository (or compatible) with `upload_file()`.
+            workflow_metadata: Metadata passed to MinIO store for logging/metrics.
+            model_name: Registered model name used in the object basename.
+            operation: Either ``initial`` (query load) or ``transform`` (post-transform).
+            key_prefix: Backward-compatible parameter (currently ignored for object naming).
+            size_threshold_bytes: Byte limit before offload. When None, the module-level
+                environment-derived default is used.
+
+        Return:
+            MinioDataFramePayload: Instance with data and/or MinIO fields set.
+        """
+
+        if dataframe is None or dataframe.empty:
+            cls._debug(
+                logger,
+                'MinioDataFramePayload.from_dataframe received empty dataframe, returning empty payload',
+                workflow_metadata,
+            )
+            return cls(
+                data=None, last_timestamp=now().strftime(DATETIME_FORMAT_WITH_TZ), status=status
+            )
+
+        if last_timestamp is None:
+            last_timestamp = max(dataframe['timestamp'].values.tolist())
+
+        dataframe_size = cls.estimate_size_bytes(dataframe, workflow_metadata, logger)
+        cls._debug(
+            logger,
+            (
+                f'MinioDataFramePayload.from_dataframe estimated size: {dataframe_size} bytes '
+                f'(threshold: {OFFLOAD_THRESHOLD_BYTES} bytes)'
+            ),
+            workflow_metadata,
+        )
+
+        if dataframe_size <= OFFLOAD_THRESHOLD_BYTES:
+            cls._debug(
+                logger,
+                'MinioDataFramePayload.from_dataframe using inline payload',
+                workflow_metadata,
+            )
+            return cls(data=dataframe.to_dict(), last_timestamp=last_timestamp, status=status)
+
+        timestamp = now().strftime(DATETIME_FORMAT_FILENAME)
+        object_key, object_prefix = _build_object_key(model_name, operation, timestamp)
+        cls._debug(
+            logger,
+            (
+                'MinioDataFramePayload.from_dataframe offloading payload to MinIO '
+                f'with key {object_key}'
+            ),
+            workflow_metadata,
+        )
+
+        # Upload using the relative object key. The upstream repository will
+        # prefix it internally under its MinIO namespace.
+        parquet_buffer = BytesIO()
+        dataframe.to_parquet(parquet_buffer, engine='pyarrow', index=True)
+        file_bytes = parquet_buffer.getvalue()
+
+        upload_result = await minio_repo.upload_file(
+            file_bytes=file_bytes,
+            relative_key=object_key,
+            metadata=workflow_metadata,
+        )
+
+        bucket = minio_repo.bucket
+        object_key_full = upload_result.get('minio_object_name', object_key)
+        uri = f's3://{bucket}/{object_key_full}' if bucket else None
+        cls._debug(
+            logger,
+            f'MinioDataFramePayload.from_dataframe upload completed: {uri}',
+            workflow_metadata,
+        )
+
+        return cls(
+            data=None,
+            bucket=bucket,
+            object_key=object_key_full,
+            object_prefix=object_prefix,
+            uri=uri,
+            last_timestamp=last_timestamp,
+            status=status,
+        )
+
+    async def retrieve(
+        self,
+        minio_repo: MinioRepository,
+        workflow_metadata: dict[str, Any] | None = None,
+        logger: Logger | None = None,
+    ) -> DataFrame:
+        """
+        Load parquet from MinIO when object_key is set and populate inline data.
+
+        Args:
+            minio_repo: sientia_do MinioRepository (or compatible) with download_file().
+            workflow_metadata: Metadata passed to MinIO read for logging/metrics.
+
+        Return:
+            dict[str, Any]: Flat dict with data filled (same keys as to_dict after load).
+        """
+        if self.data is not None:
+            self._debug(
+                logger,
+                'MinioDataFramePayload.retrieve using inline payload data',
+                workflow_metadata,
+            )
+            return DataFrame(self.data)
+
+        if not self.has_data():
+            self._debug(
+                logger,
+                'MinioDataFramePayload.retrieve found no payload data, returning empty dataframe',
+                workflow_metadata,
+            )
+            return DataFrame()
+
+        self._debug(
+            logger,
+            f'MinioDataFramePayload.retrieve downloading object from MinIO: {self.object_key}',
+            workflow_metadata,
+        )
+        file_bytes = await minio_repo.download_file(
+            object_name=self.object_key, metadata=workflow_metadata
+        )
+        df = read_parquet(BytesIO(file_bytes))
+        self._debug(
+            logger,
+            f'MinioDataFramePayload.retrieve loaded dataframe from MinIO with shape {df.shape}',
+            workflow_metadata,
+        )
+        return df

laborious/utils/repository/minio_manager.py

@@ -0,0 +1,32 @@
+from sientia_do.notifications.handlers import NotificationHandler
+from sientia_do.observability.logger import Logger
+from sientia_do.observability.metrics_controller import MetricsController
+from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+from sientia_do.repository.minio_repository import MinioRepository
+
+
+class MinioManager(SientiaMonitoring):
+    minio_repository: MinioRepository | None = None
+
+    def __init__(
+        self,
+        minio_repository: MinioRepository | None = None,
+        logger: Logger | None = None,
+        notification_handler: NotificationHandler | None = None,
+        metrics_controller: MetricsController | None = None,
+    ):
+        if self.minio_repository is None:
+            self.minio_repository = minio_repository
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
+
+    def close(self) -> None:
+        """
+        Close the MinioManager and clean up resources.
+        """
+        if self.minio_repository is not None:
+            try:
+                self.minio_repository.close()
+            finally:
+                self.minio_repository = None
+
+        SientiaMonitoring.shutdown(self)

laborious/utils/time.py

@@ -0,0 +1,6 @@
+from datetime import datetime
+
+
+def to_naive(moment: datetime) -> datetime:
+    """Strip tzinfo, if any. Postgres `TIMESTAMP WITHOUT TIME ZONE` columns need naive values."""
+    return moment.replace(tzinfo=None) if moment.tzinfo else moment

laborious/worker/worker.py

@@ -1,64 +1,236 @@
-from temporalio import workflow, client
-from temporalio.worker import Worker
+"""
+Laborious Worker Module
 
+This module provides the main worker implementation for the Sientia DataOps Laborious system.
+It orchestrates Temporal workers, manages task queues, and handles the lifecycle of
+prediction and retraining workflows.
+
+The worker supports multiple runtime-scoped task queues (via ``sientia_do.temporal.worker.prepare_worker``):
+- predictions_batch-{runtime}-queue: Batch prediction workflows (heavy workload)
+- minimal_retrain-{runtime}-queue: Model retraining workflows
+- drift-{runtime}-queue: Drift detection workflows
+- simple_metrics-{runtime}-queue: Simple metrics workflows
+- import_model-{runtime}-queue: `.sientia` model import — registered **only** when
+  ``IMPORT_MODEL_ENABLED`` is truthy, so the containment boundary stays a deployment decision
+
+``RUNTIME`` must be set; it is passed to every ``prepare_worker`` call. Schedulers must use the
+same queue names (breaking change vs legacy ``drift-queue`` / ``simple_metrics-queue``).
+
+Key Features:
+- Resource-based scaling with WorkerTuner (CPU and memory aware)
+- Automatic polling scaling with PollerBehaviorAutoscaling
+- Prometheus metrics integration
+- Comprehensive error handling and logging
+- Graceful shutdown with cleanup
+- Multiple worker instances for different workflow types
+
+Environment Variables:
+- RUNTIME: Required non-empty string; suffix for all task queue names
+- TEMPORAL_HOST: Temporal server address (default: localhost:7233)
+- TEMPORAL_NAMESPACE: Temporal namespace (default: laborious)
+- POD_ID: Kubernetes pod identifier for metrics
+- HTTP_METRICS_PORT: Prometheus metrics server port (default: 9090)
+- HTTP_SDK_METRICS_PORT: Temporal SDK metrics port (default: 9091)
+- PROJECT_NAME: Project name for notifications (default: laborious)
+- IMPORT_MODEL_ENABLED: register the fifth (import) worker; unset by default
+- IMPORT_PASSWORD_KEY / IMPORT_BUNDLE_* / IMPORT_STATUS_DB_NAME: import configuration, see
+  ``laborious.utils.connectors_config`` (the import log's server is POSTGRES_*)
+"""
+
+from temporalio import client, workflow
+from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig
 
 with workflow.unsafe.imports_passed_through():
+    import asyncio
     import os
     import sys
-    import asyncio
-    from laborious.workflows.predictions_batch import PredictionsBatch
-    from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
-    from laborious.workflows.sub_workflows.format_and_export_prediction import \
-        FormatAndExportPrediction
+
+    from prometheus_client import start_http_server
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.observability.logger import get_logger
+    from sientia_do.temporal.worker.prepare_worker import prepare_worker
+    from sientia_do.utils.connectors_config import (
+        build_api_config,
+        build_mongodb_config,
+        build_postgres_config,
+    )
+
+    from laborious import metrics
     from laborious.activities.activities import Activities
     from laborious.utils.connectors_config import (
-        build_postgres_config,
+        build_import_config,
+        build_import_status_config,
+        build_minio_config,
         build_mlflow_config,
-        build_opc_config
+        build_opc_config,
     )
-    from sientia_do.notifications.handlers import NotificationHandler
-    from sientia_do.temporal.utils.logger import get_logger
+    from laborious.workflows.drift import Drift
+    from laborious.workflows.import_model import ImportModel
+    from laborious.workflows.minimal_retrain import MinimalRetrain
+    from laborious.workflows.predictions_batch import PredictionsBatch
+    from laborious.workflows.simple_metrics import SimpleMetrics
+    from laborious.workflows.sub_workflows.format_and_export_prediction import (
+        FormatAndExportPrediction,
+    )
+    from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
+
+POD_ID = os.getenv('HOSTNAME')
+SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091'))
+
+# The values that turn the import worker on. Anything else — unset, empty, `0`, `false`, a typo —
+# leaves the four default workers alone and polls no import queue at all.
+IMPORT_ENABLED_VALUES = ('1', 'true', 'yes', 'on')
+
+
+def import_worker_enabled() -> bool:
+    """Whether this runtime registers the fifth (import) worker.
+
+    Read at worker start, not per workflow: containment (QTZPOC-19) wants the importer in its own
+    runtime with its own credentials, and this flag is what makes that a deployment decision.
+    """
+    return os.getenv('IMPORT_MODEL_ENABLED', '').strip().lower() in IMPORT_ENABLED_VALUES
 
 
 async def main():
+    """
+    Main entry point for the Laborious worker application.
+
+    This function initializes and starts all components of the worker:
+    1. Sets up logging and metadata
+    2. Starts Prometheus metrics server
+    3. Initializes notification handler
+    4. Creates and configures activities
+    5. Initializes OPC connections
+    6. Starts Temporal client and workers
+    7. Manages worker lifecycle and graceful shutdown
+
+    The function runs indefinitely until interrupted or an error occurs.
+    On error, it performs cleanup and exits with a non-zero status code.
+
+    Raises:
+        Exception: Any unhandled exception during worker execution
+        SystemExit: On graceful shutdown or error conditions
+    """
     host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
     logger = get_logger(__name__)
 
-    logger.info('Starting Worker...')
+    metadata = {
+        'pod_id': POD_ID,
+        'model_name': '-',
+        'model_id': '-',
+        'workflow_name': '-',
+        'schedule_name': '-',
+    }
 
-    logger.info('Starting Notification Handler...')
+    logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata)
 
+    runtime = os.getenv('RUNTIME', '').strip()
+    if not runtime:
+        logger.custom_critical(
+            'RUNTIME environment variable is required and must be non-empty',
+            metadata,
+        )
+        metrics.APP_UP.labels(pod_id=POD_ID).set(0)
+        sys.exit(1)
+
+    metadata_runtime = {**metadata, 'runtime': runtime}
+
+    logger.custom_info('Starting prometheus client...', metadata_runtime)
+    start_prometheus_server(logger, metadata_runtime)
+
+    logger.custom_info('Starting Notification Handler...', metadata_runtime)
+
+    mongo_config = build_mongodb_config()
     notification_handler = NotificationHandler(
-        servers=os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'http://localhost:9092'),
+        connection_string=mongo_config['connection_string'],
+        database=mongo_config['database_name'],
         logger=logger,
         project_name=os.getenv('PROJECT_NAME', 'laborious'),
     )
 
-    logger.info('Starting Activities...')
+    logger.custom_info('Starting Activities...', metadata_runtime)
 
     activities = Activities(
         postgres_config=build_postgres_config(),
         mlflow_config=build_mlflow_config(),
+        minio_config=build_minio_config(),
         opc_config=build_opc_config(),
+        pi_web_api_config=build_api_config(),
         logger=logger,
-        notification_handler=notification_handler
+        notification_handler=notification_handler,
+        mongo_config=mongo_config,
+        import_config=build_import_config(),
+        import_status_config=build_import_status_config(),
     )
 
-    logger.info('Starting Temporal Client...')
+    logger.custom_info('Initializing OPC...', metadata_runtime)
+    await activities.init_opc()
+
+    logger.custom_info(
+        f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...',
+        metadata_runtime,
+    )
+
+    new_runtime = Runtime(
+        telemetry=TelemetryConfig(
+            metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}')
+        )
+    )
+
+    logger.custom_info(f'Starting Temporal Client at {host}...', metadata_runtime)
 
     temporal_client = await client.Client.connect(
         target_host=host,
-        namespace=os.getenv('TEMPORAL_NAMESPACE', 'default')
+        namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'),
+        runtime=new_runtime,
     )
 
-    logger.info('Starting Workers...')
+    logger.custom_info(f'Starting Workers (runtime={runtime})...', metadata_runtime)
 
     workers = [
-        Worker(
-            temporal_client,
-            task_queue='predictions_batch-queue',
-            workflows=[PredictionsBatch, PredictionProcess,
-                       FormatAndExportPrediction],
+        prepare_worker(
+            temporal_client=temporal_client,
+            main_workflow=MinimalRetrain,
+            other_workflows=[],
+            activities=[
+                activities.load_query_with_minio_offload,
+                activities.retrain_model,
+                activities.update_production_model,
+                activities.format_retrain_report,
+                activities.export_data_to_postgres,
+            ],
+            logger=logger,
+            runtime=runtime,
+        ),
+        prepare_worker(
+            temporal_client=temporal_client,
+            main_workflow=SimpleMetrics,
+            other_workflows=[],
+            activities=[
+                activities.load_custom_query,
+                activities.calculate_simple_metrics,
+                activities.export_data_to_postgres,
+            ],
+            logger=logger,
+            runtime=runtime,
+        ),
+        prepare_worker(
+            temporal_client=temporal_client,
+            main_workflow=Drift,
+            other_workflows=[],
+            activities=[
+                activities.load_custom_query,
+                activities.get_reference_data,
+                activities.calculate_drift,
+                activities.export_data_to_postgres,
+            ],
+            logger=logger,
+            runtime=runtime,
+        ),
+        prepare_worker(
+            temporal_client=temporal_client,
+            main_workflow=PredictionsBatch,
+            other_workflows=[PredictionProcess, FormatAndExportPrediction],
             activities=[
                 # MLFlow
                 activities.request_predict,
@@ -67,38 +239,117 @@ async def main():
                 activities.input_gate,
                 activities.mlflow_response_gate,
                 activities.mlflow_content_gate,
+                activities.format_transformed_data,
                 activities.format_prediction,
                 activities.format_default_prediction,
-                activities.get_last_timestamp,
                 # OPC
                 activities.write_opc_data,
-                # Postgres
-                activities.load_custom_query,
+                # Postgres / MinIO offload
+                activities.load_query_with_minio_offload,
+                activities.cleanup_minio_objects_expired,
                 activities.repeat_last_prediction,
-                activities.export_data_to_postgres
-            ]
-        )
+                activities.export_data_to_postgres,
+                activities.export_payload_to_postgres,
+                activities.write_metrics,
+                # Pi Web API
+                activities.write_pi_web_api_data,
+            ],
+            logger=logger,
+            runtime=runtime,
+        ),
     ]
 
+    # The fifth worker is opt-in: containment (QTZPOC-19) wants the importer in its own runtime with
+    # its own credentials, and a flag lets that be a deployment decision instead of forcing every
+    # Laborious runtime to poll the import queue today. Unset, the four workers above are unaffected
+    # and no import queue is polled at all.
+    if import_worker_enabled():
+        logger.custom_info('Import model worker enabled', metadata_runtime)
+        workers.append(
+            prepare_worker(
+                temporal_client=temporal_client,
+                main_workflow=ImportModel,
+                other_workflows=[],
+                activities=[
+                    # Import log row (the BFF database, update-only)
+                    activities.claim_import_status,
+                    activities.record_import_status,
+                    activities.record_import_names,
+                    activities.record_import_terminal_status,
+                    activities.report_import_status_write_failure,
+                    activities.report_import_pipeline_failure,
+                    # Bundle download/open, MLflow experiment creation and artifact upload — one
+                    # fused activity, so they never split across worker processes.
+                    activities.prepare_and_upload_import_bundle,
+                    # MLflow provisioning
+                    activities.register_import_model_version,
+                    # The model listing
+                    activities.write_import_model_document,
+                    # Cleanup and retention
+                    activities.cleanup_import_files,
+                    activities.ensure_import_bundle_retention,
+                ],
+                logger=logger,
+                runtime=runtime,
+            )
+        )
+
     handlers = []
     for w in workers:
         handlers.append(w.run())
 
-    logger.info('Workers started successfully')
+    logger.custom_info('Workers started successfully', metadata_runtime)
 
+    exit_code = 0
     try:
         # This will run the workers and wait for them to complete.
         # If an exception occurs in any of the worker handlers, it will be propagated here.
         await asyncio.gather(*handlers)
-    except BaseException as e:
-        logger.error(f"An unhandled exception occurred: {e}")
+    except Exception as e:
+        logger.custom_error(f'An unhandled exception occurred: {e}', metadata_runtime)
+        exit_code = 1
     finally:
+        # No `except (SystemExit, KeyboardInterrupt)` on purpose: `finally` runs on every path,
+        # caught or not, so cleanup still happens for a deliberate shutdown (Ctrl+C, a handler's
+        # own sys.exit) without us having to catch and re-raise it ourselves — which would only
+        # let us relabel its exit code before `sys.exit(exit_code)` below discards it anyway.
+        # Letting it propagate on its own keeps the original code (e.g. a handler's sys.exit(N))
+        # intact instead of collapsing it to 0.
         if notification_handler:
             notification_handler.shutdown()
         if activities:
-            activities.shutdown()
-        # Exit with a non-zero status code to indicate failure to Kubernetes
-        sys.exit(1)
+            await activities.shutdown()
+        metrics.APP_UP.labels(pod_id=POD_ID).set(0)  # Mark app as DOWN
+
+    sys.exit(exit_code)
+
+
+def start_prometheus_server(logger, metadata: dict):
+    """
+    Starts the Prometheus metrics server for monitoring and observability.
+
+    This function initializes the Prometheus HTTP server on the configured port
+    and sets the application health metric to indicate the service is running.
+
+    The server exposes metrics that can be scraped by Prometheus for monitoring
+    the health and performance of the Laborious worker.
+
+    Environment Variables:
+        HTTP_METRICS_PORT: Port for the metrics server (default: 9090)
+        POD_ID: Pod identifier for metrics labeling
+
+    Raises:
+        SystemExit: If the metrics server fails to start
+    """
+    try:
+        port = int(os.getenv('HTTP_METRICS_PORT', 9090))
+        start_http_server(port)
+        logger.custom_info(f'Prometheus server started on port {port}.', metadata)
+        metrics.APP_UP.labels(pod_id=POD_ID).set(1)  # Mark app as UP
+    except Exception as e:
+        logger.custom_critical(f'Failed to start Prometheus server: {e}', metadata)
+        os._exit(1)
+
 
 if __name__ == '__main__':
     asyncio.run(main())

laborious/workflows/drift.py

@@ -0,0 +1,107 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
+    from sientia_do.temporal.policies import retry_policy
+
+    from laborious.activities.activities import Activities
+
+
+@workflow.defn(name='drift')
+class Drift:
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+        """
+        Execute the drift workflow.
+
+        This method orchestrates the complete drift process by:
+        1. Loading data using the provided custom SQL query
+        2. Preparing prediction configuration and filters
+        3. Delegating to the PredictionProcess workflow for ML operations
+        """
+
+        metadata = {
+            'metadata': {
+                'schedule_name': input_data['schedule_name'],
+                'model_name': input_data['model_name'],
+                'model_id': input_data['model_id'],
+                'workflow_name': 'drift',
+            }
+        }
+
+        print(f'Input data: {input_data}', metadata)
+
+        model_config = input_data['model_config']
+        target_name = model_config['target']
+
+        gathering_query = f"""
+            SELECT *
+            FROM "{input_data['schema']}"."{input_data['source_table_name']}"
+            WHERE
+                model_id = '{input_data['model_id']}' AND
+                timestamp > NOW() - INTERVAL '{input_data['interval']} minutes'
+            ORDER BY timestamp ASC
+        """  # nosec B608 - values come from internal Temporal workflow config, not user input
+
+        target_data_handler = workflow.start_activity_method(
+            Activities.load_custom_query,
+            {
+                **metadata,
+                'query': gathering_query,
+                'datetime_columns': ['timestamp', 'created_at'],
+                'orient': 'records',
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=300),
+        )
+
+        reference_data_handler = workflow.start_activity_method(
+            Activities.get_reference_data,
+            {**metadata, 'model_name': input_data['model_name']},
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=300),
+        )
+
+        target_data = await target_data_handler
+        reference_data = await reference_data_handler
+
+        if not target_data:
+            return
+
+        drift_data = await workflow.execute_local_activity_method(
+            Activities.calculate_drift,
+            {
+                **metadata,
+                'target_data': target_data,
+                'reference_data': reference_data,
+                'model_name': input_data['model_name'],
+                'model_id': input_data['model_id'],
+                'target_name': target_name,
+                'drift_metrics': input_data.get(
+                    'drift_metrics', ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']
+                ),
+                'chunk_period': input_data.get('chunk_period', 'min'),
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=300),
+        )
+
+        if drift_data:
+            await workflow.execute_activity_method(
+                Activities.export_data_to_postgres,
+                {
+                    **metadata,
+                    'data': drift_data,
+                    'schema': input_data['schema'],
+                    'table_name': input_data['target_table_name'],
+                    'timestamp_conversion': {
+                        'column': 'timestamp',
+                        'format': DATETIME_FORMAT_WITH_TZ,
+                    },
+                },
+                retry_policy=retry_policy,
+                start_to_close_timeout=timedelta(seconds=300),
+            )

laborious/workflows/import_model.py

@@ -0,0 +1,448 @@
+"""The `import_model` workflow: a `.sientia` object becomes a registered model, or a durable ERROR.
+
+Shape of this workflow, and the reasons it has that shape:
+
+- **The claim comes first.** The record already exists — the frontend inserted it through the BFF —
+  so the workflow's first act is to take ownership of it. Everything after that failure is recorded
+  *on* the record; a failed claim is recorded nowhere, because the workflow owns no row.
+- **Fail-fast, and nothing else.** No rollback, no compensation, no resume, no reconciliation, no
+  `continue_as_new`. A failed import stays failed; trying again is a new import with a new upload.
+- **Only the status writes retry**, and only because `UPDATE ... SET <fixed values> WHERE id` is
+  idempotent. Every provisioning call keeps `maximum_attempts=1`, because a repeated registry call or
+  Mongo insert can write twice.
+- **One `finally`, two actions**: cleanup, then the single terminal write that classifies the
+  outcome once. No `except` block writes a verdict of its own.
+- **The workflow holds ids, never contents.** The decrypted bundle never crosses an activity
+  boundary, and the password reaches this module only as an envelope it cannot open.
+"""
+
+import asyncio
+from datetime import timedelta
+
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from collections.abc import Awaitable, Callable
+    from typing import Any
+
+    from sientia_do.temporal.policies import retry_policy
+    from temporalio.exceptions import is_cancelled_exception
+
+    from laborious.activities.activities import Activities
+    from laborious.activities.model_import import (
+        IMPORT_STATUS_HINT_RETRY_POLICY,
+        IMPORT_STATUS_RETRY_POLICY,
+    )
+    from laborious.utils.bundle.steps import ImportStep
+    from laborious.utils.models.import_errors import (
+        ImportInputError,
+        describe_failure_chain,
+        failure_location,
+    )
+
+# `start_to_close_timeout` is per attempt. The terminal write additionally gets a
+# `schedule_to_close_timeout` larger than its retry envelope (10 attempts, ≈151 s of backoff, ≈451 s
+# if every attempt also burns its timeout) — set it smaller and Temporal cuts the retries short,
+# which is the one configuration mistake that would silently undo design D20.
+STATUS_WRITE_TIMEOUT = timedelta(seconds=30)
+STATUS_WRITE_ENVELOPE = timedelta(seconds=900)
+# Download, decryption/extraction, experiment creation and artifact upload now run inside one
+# activity attempt (bug 8): the budget is the sum of what the four separate activities used to get,
+# since a retry of the fused activity redoes all four phases.
+PREPARE_AND_UPLOAD_TIMEOUT = (
+    timedelta(seconds=1800)
+    + timedelta(seconds=1800)
+    + timedelta(seconds=300)
+    + timedelta(seconds=1800)
+)
+REGISTRATION_TIMEOUT = timedelta(seconds=600)
+DOCUMENT_TIMEOUT = timedelta(seconds=300)
+CLEANUP_TIMEOUT = timedelta(seconds=600)
+RETENTION_TIMEOUT = timedelta(seconds=300)
+
+# Fields no import may carry. The project is a frontend concern: the model listing is the MongoDB
+# `models` document, no migrated database holds a project or model table, and nothing in this
+# repository reads a project-to-model link (design D22). An input naming one is rejected rather than
+# accepted and ignored, so no caller believes a project link was recorded. The same goes for a
+# relational target: there is none, and for a plaintext password, which does not belong in an event
+# history whatever else is true.
+REJECTED_INPUT_FIELDS = (
+    'project_id',
+    'project_name',
+    'provisioning_target',
+    'schema',
+    'table_name',
+    'status_table',
+    'password',
+)
+
+DIGEST_LENGTH = 64
+_HEX_DIGITS = frozenset('0123456789abcdef')
+
+
+@workflow.defn(name='import_model')
+class ImportModel:
+    """Turn an encrypted `.sientia` object in MinIO into a registered model, or into an ERROR."""
+
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]) -> dict[str, Any]:
+        """Execute the import.
+
+        Args:
+            input_data: the import request.
+                Required keys:
+                    - import_run_id (int): primary key of the import log row the frontend inserted
+                    - object_key (str): the uploaded object's key
+                    - expected_digest (str): the SHA-256 the uploader computed for those bytes
+                    - password_envelope (str): the encrypted bundle password, never plaintext
+                Optional keys:
+                    - bucket (str): defaults to the configured import bucket
+
+        Returns:
+            dict: `import_run_id`, `run_id`, `model_name`, `version` and the model document's id.
+
+        Raises:
+            ImportLogRowNotClaimableError: the record is not this workflow's — the run ends FAILED
+                with that error type and nothing is written to any row.
+            Exception: whatever failed. The record carries the failing step, its code and its
+                sentence; this exception carries the technical truth.
+        """
+        workflow_id = workflow.info().workflow_id
+        import_run_id = self._require_import_run_id(input_data.get('import_run_id'))
+        metadata: dict[str, Any] = {
+            'model_name': '-',
+            'model_id': str(import_run_id),
+            'workflow_name': 'import_model',
+            'schedule_name': '-',
+        }
+        base = {
+            'metadata': metadata,
+            'import_run_id': import_run_id,
+            'workflow_id': workflow_id,
+        }
+
+        # Class A. Outside the try/finally on purpose: a claim that matches nothing leaves this
+        # workflow owning no row, so there is nothing to clean up and nothing to record.
+        claim = await workflow.execute_activity_method(
+            Activities.claim_import_status,
+            base,
+            retry_policy=IMPORT_STATUS_RETRY_POLICY,
+            start_to_close_timeout=STATUS_WRITE_TIMEOUT,
+            schedule_to_close_timeout=STATUS_WRITE_ENVELOPE,
+        )
+
+        step: str = ImportStep.RECEIVED.value
+        gate: int | None = None
+        succeeded = False
+        source: dict[str, Any] = {}
+        result: dict[str, Any] = {'import_run_id': import_run_id}
+
+        try:
+            self._validate_input(input_data, claim)
+
+            # Download, decryption/extraction, experiment creation and artifact upload run inside
+            # one activity attempt (bug 8), so no later phase can depend on a file a different,
+            # separately-scheduled activity wrote to a different worker's disk. One hint covers the
+            # whole fused step; an exception it raises with no location of its own falls back to
+            # this step, since that is what the workflow was attempting.
+            step = ImportStep.DOWNLOAD.value
+            await self._record_hint(
+                base, Activities.record_import_status, {'step': step, 'source': None}
+            )
+            # Achado #9: the retention check carries no data this step consumes and already
+            # swallows its own failure (`_ensure_retention`), so it runs alongside the fused
+            # download+decrypt+provision activity instead of blocking it. Achado #8 fused
+            # `download_import_bundle` into `prepare_and_upload_import_bundle` before this change
+            # landed, so this gathers against that fused activity, not the original isolated one.
+            _, prepared = await asyncio.gather(
+                self._ensure_retention(base, input_data),
+                workflow.execute_activity_method(
+                    Activities.prepare_and_upload_import_bundle,
+                    {
+                        **base,
+                        'bucket': input_data.get('bucket'),
+                        'object_key': input_data['object_key'],
+                        'expected_digest': input_data['expected_digest'],
+                        'password_envelope': input_data['password_envelope'],
+                    },
+                    retry_policy=retry_policy,
+                    start_to_close_timeout=PREPARE_AND_UPLOAD_TIMEOUT,
+                ),
+            )
+            source = prepared['metadata']
+            parameters = prepared['parameters']
+            model_name = source['model_name']
+            metadata['model_name'] = model_name
+            run_name = prepared['run_name']
+
+            # The names are known only once the fused activity returns; they are still recorded
+            # before the two provisioning steps that follow (the registered version and the model
+            # document), which is the guarantee `model-import-status-record` now makes.
+            await self._record_hint(
+                base,
+                Activities.record_import_names,
+                {
+                    'experiment_name': source['experiment_name'],
+                    'run_name': run_name,
+                    'source': source,
+                    'step': ImportStep.EXPERIMENT_CREATION.value,
+                },
+            )
+
+            step = ImportStep.REGISTRATION.value
+            await self._record_hint(
+                base,
+                Activities.record_import_status,
+                {'step': step, 'source': source or None},
+            )
+            version = await workflow.execute_activity_method(
+                Activities.register_import_model_version,
+                {**base, 'model_name': model_name, 'run_id': prepared['run_id']},
+                retry_policy=retry_policy,
+                start_to_close_timeout=REGISTRATION_TIMEOUT,
+            )
+
+            step = ImportStep.MODEL_DOCUMENT.value
+            await self._record_hint(
+                base,
+                Activities.record_import_status,
+                {'step': step, 'source': source or None},
+            )
+            document = await workflow.execute_activity_method(
+                Activities.write_import_model_document,
+                {
+                    **base,
+                    'model_name': model_name,
+                    'target': parameters['target_variable'],
+                },
+                retry_policy=retry_policy,
+                start_to_close_timeout=DOCUMENT_TIMEOUT,
+            )
+
+            succeeded = True
+            result = {
+                'import_run_id': import_run_id,
+                'run_id': prepared['run_id'],
+                'model_name': model_name,
+                'version': version['version'],
+                'document_id': document.get('id'),
+            }
+        except BaseException as error:  # classified once below, then re-raised untouched
+            located_step, located_gate = failure_location(error)
+            step = located_step or step
+            gate = located_gate
+            await self._report_failure(base, error, step=step, gate=gate)
+            raise
+        finally:
+            cleanup = await self._cleanup(base)
+            await self._record_terminal(
+                base,
+                succeeded=succeeded,
+                step=step,
+                gate=gate,
+                source=source,
+                cleanup_failed=not cleanup.get('cleaned', True),
+            )
+
+        return result
+
+    # ------------------------------------------------------------------ validation
+
+    @staticmethod
+    def _require_import_run_id(value: Any) -> int:
+        """The one check that runs before the claim, because the claim needs its result.
+
+        A missing or unusable id means there is no record to claim and none to write: the run ends
+        FAILED with this error type, and the failure is visible in the workflow's terminal state, its
+        error type, the error metric and a notification — everywhere except a row, which is the
+        point.
+        """
+        if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+            raise ImportInputError('import_run_id must be a positive integer')
+        return value
+
+    @staticmethod
+    def _validate_input(input_data: dict[str, Any], claim: dict[str, Any]) -> None:
+        """Validate the rest of the request, and check it against the record it just claimed.
+
+        This runs **after** the claim so its failure lands on the record rather than only in the
+        logs: by now the row is this workflow's, so a bad request is reported to the person who made
+        it. Everything here fails at step `received`.
+        """
+        unknown = [name for name in REJECTED_INPUT_FIELDS if name in input_data]
+        if unknown:
+            raise ImportInputError(
+                f'the import request carries fields this workflow does not accept: '
+                f'{", ".join(sorted(unknown))}'
+            )
+
+        for required in ('object_key', 'expected_digest', 'password_envelope'):
+            if not input_data.get(required):
+                raise ImportInputError(f'the import request is missing {required}')
+
+        digest = str(input_data['expected_digest']).lower()
+        if len(digest) != DIGEST_LENGTH or not set(digest) <= _HEX_DIGITS:
+            raise ImportInputError('expected_digest must be a 64-character hexadecimal SHA-256')
+
+        object_key = input_data['object_key']
+        file_name = claim.get('file_name')
+        if file_name and file_name != object_key:
+            raise ImportInputError('the record names a different object than the request')
+
+        request_digest = claim.get('request_digest')
+        if request_digest and request_digest != digest:
+            raise ImportInputError('the record names a different digest than the request')
+
+    # --------------------------------------------------------------- status writes
+
+    async def _record_hint(
+        self,
+        base: dict[str, Any],
+        activity: Callable[..., Awaitable[Any]],
+        extra: dict[str, Any],
+    ) -> None:
+        """Record a status hint. A hint: its failure never fails the import."""
+        try:
+            await workflow.execute_activity_method(
+                activity,
+                {**base, **extra},
+                retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY,
+                start_to_close_timeout=STATUS_WRITE_TIMEOUT,
+            )
+        except BaseException as error:  # a lost hint must not fail a working import
+            if is_cancelled_exception(error):
+                raise
+            await self._report_status_write_failure(base, error)
+
+    async def _record_terminal(
+        self,
+        base: dict[str, Any],
+        *,
+        succeeded: bool,
+        step: str | None,
+        gate: int | None,
+        source: dict[str, Any],
+        cleanup_failed: bool,
+    ) -> None:
+        """The single authoritative write, and the last thing this workflow does.
+
+        Its own failure is reported through the channel with the verdict it could not write, and it
+        never turns a completed import into a failed one — nor masks the cause of a real failure.
+        """
+        try:
+            await workflow.execute_activity_method(
+                Activities.record_import_terminal_status,
+                {
+                    **base,
+                    'succeeded': succeeded,
+                    'step': step,
+                    'gate': gate,
+                    'source': source or None,
+                    'cleanup_failed': cleanup_failed,
+                },
+                retry_policy=IMPORT_STATUS_RETRY_POLICY,
+                start_to_close_timeout=STATUS_WRITE_TIMEOUT,
+                schedule_to_close_timeout=STATUS_WRITE_ENVELOPE,
+            )
+        except BaseException as error:  # the outcome stands even when it cannot be written
+            if is_cancelled_exception(error):
+                raise
+            await self._report_status_write_failure(
+                base,
+                error,
+                verdict=f'succeeded={succeeded} step={step} gate={gate}',
+            )
+
+    @staticmethod
+    async def _report_failure(
+        base: dict[str, Any],
+        error: BaseException,
+        *,
+        step: str | None,
+        gate: int | None,
+    ) -> None:
+        """Route the failure that ended this import through the one failure channel.
+
+        Runs from the `except`, before cleanup and before the terminal write, so the log reads in the
+        order things happened: what failed, then what was cleaned, then what was recorded. The step
+        and the gate are the ones already located from the failure, so the log line and the row can
+        never name different ones.
+
+        Its own failure is swallowed. The channel is the thing that logs, so a report that cannot be
+        made has nowhere to say so, and the failure it was reporting is re-raised either way — the
+        event history and the record both still carry it.
+        """
+        error_type, detail = describe_failure_chain(error)
+        try:
+            await workflow.execute_activity_method(
+                Activities.report_import_pipeline_failure,
+                {**base, 'step': step, 'gate': gate, 'detail': detail, 'error_type': error_type},
+                retry_policy=IMPORT_STATUS_HINT_RETRY_POLICY,
+                start_to_close_timeout=STATUS_WRITE_TIMEOUT,
+            )
+        except BaseException as error:  # the report of a report is where this stops
+            if is_cancelled_exception(error):
+                raise
+
+    async def _report_status_write_failure(
+        self,
+        base: dict[str, Any],
+        error: BaseException,
+        verdict: str | None = None,
+    ) -> None:
+        """Route a status write that could not be made through the one failure channel."""
+        try:
+            await workflow.execute_activity_method(
+                Activities.report_import_status_write_failure,
+                {**base, 'detail': str(error), 'verdict': verdict},
+                retry_policy=retry_policy,
+                start_to_close_timeout=STATUS_WRITE_TIMEOUT,
+            )
+        except BaseException as error:  # the report of a report is where this stops
+            if is_cancelled_exception(error):
+                raise
+            # Deliberately silent otherwise: the channel this was trying to reach is the thing that
+            # logs, so there is nowhere left to say it, and an import that worked must not fail
+            # over it.
+
+    # ----------------------------------------------------------- side-effect steps
+
+    @staticmethod
+    async def _ensure_retention(base: dict[str, Any], input_data: dict[str, Any]) -> None:
+        """Ensure the bundle bucket's expiry rule, without making it a reason to fail an import.
+
+        The rule is applied by the importer rather than assumed from a Helm value, so the acceptance
+        criterion is verifiable from the code that depends on it. But a bucket policy the importer
+        cannot set is an operational problem, not a bad import: failing a valid model import because
+        a lifecycle API refused would be the wrong trade, so the failure is dropped here and left
+        where Temporal already records it — the activity's own failure in the event history.
+        """
+        try:
+            await workflow.execute_activity_method(
+                Activities.ensure_import_bundle_retention,
+                {**base, 'bucket': input_data.get('bucket')},
+                retry_policy=retry_policy,
+                start_to_close_timeout=RETENTION_TIMEOUT,
+            )
+        except BaseException as error:  # retention is not a precondition of an import
+            if is_cancelled_exception(error):
+                raise
+            # Dropped, not reported otherwise: the activity's own failure is already in the event
+            # history, and a bucket policy the importer cannot set is an operational problem rather
+            # than a bad import. Failing a valid model import over a lifecycle API would be the
+            # wrong trade.
+
+    @staticmethod
+    async def _cleanup(base: dict[str, Any]) -> dict[str, Any]:
+        """Remove this import's temporary files. Runs on every path, before the terminal write."""
+        try:
+            return await workflow.execute_activity_method(
+                Activities.cleanup_import_files,
+                base,
+                retry_policy=retry_policy,
+                start_to_close_timeout=CLEANUP_TIMEOUT,
+            )
+        except BaseException as error:  # cleanup never replaces the failure that brought us here
+            if is_cancelled_exception(error):
+                raise
+            return {'cleaned': False}

laborious/workflows/minimal_retrain.py

@@ -0,0 +1,142 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.policies import retry_policy
+    from temporalio.exceptions import ApplicationError
+
+    from laborious.activities.activities import Activities
+    from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
+
+
+@workflow.defn(name='minimal_retrain')
+class MinimalRetrain:
+    """
+    Automated model retraining workflow for the Laborious system.
+
+    This workflow implements a complete model retraining pipeline that loads
+    training data, executes model retraining, updates production models,
+    and maintains comprehensive audit trails. It's designed for automated
+    model lifecycle management with minimal manual intervention.
+
+    The workflow provides a robust retraining process with:
+    - Automated data loading from configured data sources
+    - MLFlow model retraining with quality validation
+    - Production model updates with version control
+    - Comprehensive reporting and audit trail maintenance
+    - Error handling and notification integration
+    """
+
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+        """
+        Execute the automated model retraining workflow.
+
+        This method orchestrates the complete model retraining process by:
+        1. Loading training data using the provided custom SQL query
+        2. Executing MLFlow model retraining with the loaded data
+        3. Updating production models with newly trained versions
+        4. Persisting comprehensive retraining reports to database
+
+        The method implements comprehensive error handling and ensures all
+        required parameters are properly configured before proceeding.
+
+        Args:
+            input_data: Complete configuration for the retraining workflow
+                Required keys:
+                    - schedule_name (str): Schedule identifier for the retraining
+                    - model_name (str): Name of the ML model to retrain
+                    - model_id (int): Unique identifier for the model version
+                    - query (str): SQL query for training data loading
+                    - schema (str, optional): Database schema for report storage
+                    - table_name (str, optional): Target table for retraining reports
+                    - datetime_columns (list[str], optional): Columns to treat as datetime
+
+        Returns:
+            None: The workflow completes successfully when all steps finish
+
+        Raises:
+            Exception: If any required parameters are missing or if the workflow fails
+                during data loading, retraining, or model update operations
+        """
+
+        metadata = {
+            'metadata': {
+                'schedule_name': input_data['schedule_name'],
+                'model_name': input_data['model_name'],
+                'model_id': input_data['model_id'],
+                'workflow_name': 'minimal_retrain',
+            }
+        }
+
+        model_name = input_data['model_name']
+        model_config = input_data.get('model_config', {})
+
+        storage_result = await workflow.execute_activity_method(
+            Activities.load_query_with_minio_offload,
+            {
+                **metadata,
+                'query': input_data['query'],
+                'datetime_columns': input_data.get('datetime_columns', []),
+                'model_name': model_name,
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=600),
+        )
+
+        storage_payload = MinioDataFramePayload.from_dict(storage_result)
+        if not storage_payload.has_data():
+            raise ApplicationError(
+                'No data returned from query',
+                type='NoDataReturnedError',
+                non_retryable=True,
+            )
+
+        experiment_response = await workflow.execute_activity_method(
+            Activities.retrain_model,
+            {
+                **metadata,
+                'data': storage_result,
+                'model_name': model_name,
+                'model_config': model_config,
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(hours=1),
+        )
+
+        if experiment_response['success']:
+            update_report = await workflow.execute_activity_method(
+                Activities.update_production_model,
+                {**metadata, 'model_name': model_name, **experiment_response},
+                retry_policy=retry_policy,
+                start_to_close_timeout=timedelta(seconds=60),
+            )
+        else:
+            update_report = {}
+
+        report = await workflow.execute_local_activity_method(
+            Activities.format_retrain_report,
+            {
+                **metadata,
+                'experiment_response': experiment_response,
+                'model_name': model_name,
+                'model_id': input_data['model_id'],
+                'update_report': update_report,
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60),
+        )
+
+        await workflow.execute_activity_method(
+            Activities.export_data_to_postgres,
+            {
+                **metadata,
+                'data': report,
+                'schema': input_data['schema'],
+                'table_name': input_data['table_name'],
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=600),
+        )

laborious/workflows/predictions_batch.py

@@ -1,43 +1,77 @@
 from temporalio import workflow
 
 with workflow.unsafe.imports_passed_through():
-    from laborious.activities.activities import Activities
-    from typing import Any
-    from sientia_do.temporal.utils.policies import retry_policy
     from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.policies import retry_policy
+
+    from laborious.activities.activities import Activities
 
 
-@workflow.defn(name="predictions_batch")
-class PredictionsBatch():
+@workflow.defn(name='predictions_batch')
+class PredictionsBatch:
+    """
+    Main batch prediction workflow for the Laborious system.
+
+    This workflow orchestrates the complete batch prediction process, handling
+    data loading, configuration management, and workflow delegation. It serves
+    as the primary entry point for batch prediction operations and ensures
+    proper data preparation before ML model inference.
+
+    The workflow implements a robust data processing pipeline with:
+    - Custom SQL query execution for data loading
+    - Comprehensive configuration management
+    - Data quality filter application
+    - MLFlow model integration
+    - Workflow delegation to specialized sub-workflows
+
+    Workflow Execution:
+        1. Data Loading: Executes custom SQL query to load prediction data
+        2. Configuration Preparation: Sets up prediction parameters and filters
+        3. Workflow Delegation: Spawns PredictionProcess child workflow
+        4. Error Handling: Implements comprehensive error handling and retry policies
+    """
+
     @workflow.run
     async def run(self, input_data: dict[str, Any]):
         """
-        This workflow runs a batch of predictions based on the input data.
+        Execute the batch prediction workflow.
 
-        The workflow executes in two main steps:
-        1. Prepares the activity with schedule and model information
-        2. Loads data using a custom query and executes the prediction process
+        This method orchestrates the complete batch prediction process by:
+        1. Loading data using the provided custom SQL query
+        2. Preparing prediction configuration and filters
+        3. Delegating to the PredictionProcess workflow for ML operations
+
+        The method implements comprehensive error handling and ensures all
+        required parameters are properly configured before proceeding.
 
         Args:
-            - input_data (dict[str, Any]): The input data for the workflow.
-            Contains the following keys:
-                - schedule_name (str): The name of the schedule.
-                - model_name (str): The name of the model.
-                - model_id (int): The id of the model.
-                - query (str): The SQL query to be executed to load data.
-                - schema (dict, optional): The schema definition for the data.
-                - table_name (str, optional): The name of the table to process.
-                - input_filters (dict, optional): Filters to be applied during prediction.
-                - mlflow_transform_filters (dict, optional): Filters to be applied
-                during prediction.
-                - mlflow_predict_filters (dict, optional): Filters to be applied during prediction.
-                - model_retention (int, optional): The model retention period in minutes.
-                - path_priority (list[str]): The path priority.
+            input_data: Complete configuration for the batch prediction
+                Required keys:
+                    - schedule_name (str): Schedule identifier for the prediction
+                    - model_name (str): Name of the ML model to use
+                    - model_id (int): Unique identifier for the model
+                    - query (str): SQL query for data loading
+                    - schema (dict, optional): Data schema definition
+                    - table_name (str, optional): Target table for predictions
+                    - input_filters (dict, optional): Data quality filters
+                    - mlflow_transform_filters (dict, optional): MLFlow transform filters
+                    - mlflow_predict_filters (dict, optional): MLFlow prediction filters
+                    - model_retention (int, optional): Model retention period in minutes
+                    - path_priority (list[str]): Decision path priority configuration
+                    - opc_output_config (dict, optional): OPC server export configuration
+                    - pi_web_api_output_config (dict, optional): PI Web API export configuration
+                    - datetime_columns (list[str], optional): Columns to treat as datetime
+                    - save_transform (bool, optional): Whether to save transformed data (default: True)
+                    - prediction_store_policy (str, optional): Data retention policy (default: 'lts:1')
+
         Returns:
-            None
+            None: The workflow completes successfully when the child workflow finishes
 
         Raises:
-            Exception: If any of the required parameters are missing or if the workflow fails.
+            Exception: If any required parameters are missing or if the workflow fails
+                during data loading or workflow delegation
         """
 
         metadata = {
@@ -45,18 +79,21 @@ class PredictionsBatch():
                 'schedule_name': input_data['schedule_name'],
                 'model_name': input_data['model_name'],
                 'model_id': input_data['model_id'],
-                'workflow_name': 'predictions_batch'
+                'workflow_name': 'predictions_batch',
             }
         }
 
-        data = await workflow.execute_local_activity_method(
-            Activities.load_custom_query,
+        # Load data using custom query with optional MinIO offload for large frames
+        data = await workflow.execute_activity_method(
+            Activities.load_query_with_minio_offload,
             {
                 **metadata,
                 'query': input_data['query'],
+                'datetime_columns': input_data.get('datetime_columns', []),
+                'model_name': input_data['model_name'],
             },
             retry_policy=retry_policy,
-            start_to_close_timeout=timedelta(seconds=60)
+            start_to_close_timeout=timedelta(seconds=300),
         )
 
         # Prepare input for prediction_process workflow
@@ -65,27 +102,26 @@ class PredictionsBatch():
             'data': data,
             'schema': input_data['schema'],
             'table_name': input_data['table_name'],
+            'transform_table_name': input_data['transform_table_name'],
             'model_id': input_data['model_id'],
             'model_name': input_data['model_name'],
-            'input_filters': input_data.get('input_filters', {
-                'EMPTY_DATA': {
-                    'POLICY': 'STOP'
-                }
-            }),
-            'mlflow_transform_filters': input_data.get('mlflow_transform_filters', {
-                'API_ERROR': {
-                    'POLICY': 'STOP'
-                }
-            }),
-            'mlflow_predict_filters': input_data.get('mlflow_predict_filters', {
-                'API_ERROR': {
-                    'POLICY': 'STOP'
-                }
-            }),
-            'model_retention': input_data.get('model_retention', 60),
+            'input_filters': input_data.get(
+                'input_filters', {'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}}}
+            ),
+            'mlflow_transform_filters': input_data.get(
+                'mlflow_transform_filters', {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}}
+            ),
+            'mlflow_predict_filters': input_data.get(
+                'mlflow_predict_filters', {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}}
+            ),
+            'model_config': input_data.get('model_config', {}),
             'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']),
-            'opc_output_config': input_data.get('opc_output_config', {})
+            'opc_output_config': input_data.get('opc_output_config', {}),
+            'on_conflict': input_data.get('on_conflict', 'error'),
+            'pi_web_api_output_config': input_data.get('pi_web_api_output_config', {}),
+            'prediction_store_policy': input_data.get('prediction_store_policy', 'lts:1'),
+            'save_transform': input_data.get('save_transform', True),
         }
 
-        await workflow.execute_child_workflow(
-            'prediction_process', prediction_input)
+        # Execute prediction process workflow
+        await workflow.execute_child_workflow('subworkflow.prediction_process', prediction_input)

laborious/workflows/simple_metrics.py

@@ -0,0 +1,95 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
+    from sientia_do.temporal.policies import retry_policy
+
+    from laborious.activities.activities import Activities
+
+
+@workflow.defn(name='simple_metrics')
+class SimpleMetrics:
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+        """
+        Execute the simple metrics workflow.
+        """
+        metadata = {
+            'metadata': {
+                'model_id': input_data['model_id'],
+                'model_name': input_data['model_name'],
+                'workflow_name': 'simple_metrics',
+                'schedule_name': input_data['schedule_name'],
+            }
+        }
+
+        model_id = input_data['model_id']
+        interval_minutes = input_data['interval_minutes']
+
+        model_config = input_data['model_config']
+        target_name = model_config['target']
+
+        query = f"""
+            select p."timestamp", p.prediction, ld.value as "target"
+            from "{input_data['schema']}"."{input_data['predictions_table_name']}" p
+            inner join "{input_data['schema']}"."{input_data['data_table_name']}" ld
+                on p."timestamp" = ld."timestamp"
+            where
+                p.model_id = '{model_id}' and
+                p.prediction is not null and
+                ld.variable = '{target_name}' and
+                ld.value is not null and
+                p."timestamp" >= NOW() - INTERVAL '{interval_minutes} minutes'
+            order by
+                p."timestamp" desc;
+        """  # nosec B608 - values come from internal Temporal workflow config, not user input
+
+        target_data = await workflow.execute_activity_method(
+            Activities.load_custom_query,
+            {
+                **metadata,
+                'query': query,
+                'datetime_columns': ['timestamp'],
+                'orient': 'records',
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=300),
+        )
+
+        if not target_data:
+            return
+
+        simple_metrics = await workflow.execute_local_activity_method(
+            Activities.calculate_simple_metrics,
+            {
+                **metadata,
+                'model_id': model_id,
+                'target_data': target_data,
+                'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']),
+                'interval_minutes': interval_minutes,
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=300),
+        )
+
+        if not simple_metrics:
+            return
+
+        await workflow.execute_activity_method(
+            Activities.export_data_to_postgres,
+            {
+                **metadata,
+                'data': simple_metrics,
+                'schema': input_data['schema'],
+                'table_name': input_data['target_table_name'],
+                'timestamp_conversion': {
+                    'column': 'timestamp',
+                    'format': DATETIME_FORMAT_WITH_TZ,
+                },
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=300),
+        )

laborious/workflows/sub_workflows/format_and_export_prediction.py

@@ -1,48 +1,101 @@
 from temporalio import workflow
 
 with workflow.unsafe.imports_passed_through():
-    from laborious.activities.activities import Activities
-    from typing import Any
     from datetime import timedelta
-    from sientia_do.temporal.utils.policies import retry_policy
+    from typing import Any
+
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
+    from sientia_do.temporal.policies import retry_policy
+
+    from laborious.activities.activities import Activities
 
 
-@workflow.defn(name="format_and_export_prediction")
-class FormatAndExportPrediction():
+@workflow.defn(name='subworkflow.format_and_export_prediction')
+class FormatAndExportPrediction:
+    """
+    Data formatting and export workflow for prediction results.
+
+    This workflow handles the final stages of the prediction pipeline, including
+    data formatting, database persistence, OPC server export, and metrics recording.
+    It implements flexible formatting based on prediction quality and provides
+    comprehensive export capabilities to multiple destinations.
+
+    The workflow supports two main prediction paths:
+    1. Normal Prediction: Formats and exports successful prediction results
+    2. Default Prediction: Creates fallback predictions for error conditions
+
+    Export Destinations:
+    - PostgreSQL Database: Persistent storage with timestamp conversion
+    - PI Web API: Real-time industrial system integration for prediction and confidence values
+    - OPC Servers: Real-time industrial system integration
+    - Prometheus Metrics: Performance monitoring and operational visibility
+    """
+
     @workflow.run
     async def run(self, input_data: dict[str, Any]):
         """
-        This workflow formats and exports predictions based on path_flag:
-        - If path_flag is None: formats prediction
-            using input data, timestamp, model_id and confidence
-        - If path_flag exists: creates default prediction
-            with timestamp, model_id, confidence and comment
-        Finally exports formatted prediction to postgres table
+        Execute the prediction formatting and export workflow.
+
+        This method orchestrates the complete data export process by:
+        1. Determining the appropriate formatting strategy based on path_flag
+        2. Formatting prediction data according to quality and requirements
+        3. Exporting data to PI Web API for real-time industrial access (if configured)
+        4. Exporting data to OPC servers for real-time industrial access (if configured)
+        5. Persisting data to PostgreSQL database with comprehensive metadata
+        6. Recording performance metrics for operational monitoring
+
+        The method implements flexible formatting strategies:
+        - Normal predictions: Full data formatting with confidence scores
+        - Error predictions: Default formatting with error indicators
+        - Comprehensive export: Multi-destination data distribution
+
         Args:
-            input_data(dict[str, Any]): The input data for the workflow.
-            Contains the following keys:
-                - path_flag(str): The path flag to determine the type of prediction to format
-                - data(dict[str, Any]): The data to format
-                - prediction_confidence(float): The prediction confidence to be registered
-                - timestamp(str): The timestamp of the prediction, synchronized with the data
-                - model_id(int): The model id of the prediction
-                - model_name(str): The model name of the prediction
-                - model_retention(str): The model retention of the prediction
-                - comment(str): The comment to be registered
-                - schema(str): The schema of the prediction
-                - table_name(str): The table name of the prediction
-                - opc_output_config(dict[str, Any]): The opc output config of the prediction
+            input_data: Complete configuration for the export workflow
+                Required keys:
+                    - metadata (dict): Workflow execution metadata
+                    - path_flag (str | None): Decision path flag for formatting strategy
+                        - None: Normal prediction path with full formatting
+                        - Any other value: Default prediction path for error conditions
+                    - data (dict[str, Any]): Prediction data to format and export
+                    - prediction_confidence (float): Confidence score for the prediction
+                    - timestamp (str): ISO-formatted timestamp for the prediction
+                    - model_id (int): Unique identifier for the ML model
+                    - model_name (str): Name of the ML model
+                    - schema (str): Database schema for data storage
+                    - table_name (str): Target table for data persistence
+                Optional keys:
+                    - opc_output_config (dict[str, Any]): OPC server export configuration
+                    - pi_web_api_output_config (dict[str, Any]): PI Web API export configuration
+                        Contains endpoint, prediction_tags, and confidence_tags mappings
+                    - transformed_data (dict[str, Any]): Transformed data to export separately
+                        Only processed when path_flag is None
+                    - transform_table_name (str): Target table for transformed data export
+                        Required if transformed_data is provided
+                    - prediction_store_policy (str): Data retention policy (e.g., 'lts:1', 'erl:2')
+                        Required when path_flag is None
+                    - comment (str): Operational comment or error description
+                        Required when path_flag is not None
 
         Returns:
-            bool: True if the workflow was successful, False otherwise.
+            None: The workflow completes successfully when all export operations finish
+
+        Note:
+            When transformed_data is provided and path_flag is None, the workflow will:
+            1. Format the transformed data using format_transformed_data
+            2. Export it to a separate table (transform_table_name) asynchronously
+            3. Wait for both prediction and transformed data exports to complete
         """
         metadata = input_data['metadata']
         path_flag = input_data['path_flag']
         data = input_data['data']
+        transformed_data = input_data.get('transformed_data', None)
         prediction_confidence = input_data['prediction_confidence']
 
+        opc_output_config = input_data.get('opc_output_config', None)
+        pi_web_api_output_config = input_data.get('pi_web_api_output_config', None)
+
         if path_flag is None:
-            # proceed with formatting and exporting
+            # Normal prediction path: format prediction data with full metadata
             prediction = await workflow.execute_local_activity_method(
                 Activities.format_prediction,
                 {
@@ -51,13 +104,48 @@ class FormatAndExportPrediction():
                     'timestamp': input_data['timestamp'],
                     'model_id': input_data['model_id'],
                     'prediction_confidence': prediction_confidence,
+                    'prediction_store_policy': input_data['prediction_store_policy'],
+                    'model_name': input_data['model_name'],
                 },
                 retry_policy=retry_policy,
-                start_to_close_timeout=timedelta(seconds=60)
+                start_to_close_timeout=timedelta(seconds=60),
             )
 
+            # Optionally format and export transformed data to separate table
+            if transformed_data is not None:
+                transformed = await workflow.execute_local_activity_method(
+                    Activities.format_transformed_data,
+                    {
+                        **metadata,
+                        'data': transformed_data,
+                        'model_id': input_data['model_id'],
+                        'model_name': input_data['model_name'],
+                    },
+                    retry_policy=retry_policy,
+                    start_to_close_timeout=timedelta(seconds=60),
+                )
+
+                write_transformed_handler = workflow.start_activity_method(
+                    Activities.export_payload_to_postgres,
+                    {
+                        **metadata,
+                        'schema': input_data['schema'],
+                        'table_name': input_data['transform_table_name'],
+                        'data': transformed,
+                        'timestamp_conversion': {
+                            'column': 'timestamp',
+                            'format': DATETIME_FORMAT_WITH_TZ,
+                        },
+                    },
+                    retry_policy=retry_policy,
+                    start_to_close_timeout=timedelta(seconds=60),
+                )
+
+            else:
+                write_transformed_handler = None
+
         else:
-            # create default prediction
+            # Error path: create default prediction with error indicators
             prediction = await workflow.execute_local_activity_method(
                 Activities.format_default_prediction,
                 {
@@ -65,23 +153,41 @@ class FormatAndExportPrediction():
                     'timestamp': input_data['timestamp'],
                     'model_id': input_data['model_id'],
                     'prediction_confidence': prediction_confidence,
-                    'comment': input_data['comment']
+                    'comment': input_data['comment'],
                 },
                 retry_policy=retry_policy,
-                start_to_close_timeout=timedelta(seconds=60)
+                start_to_close_timeout=timedelta(seconds=60),
+            )
+
+            write_transformed_handler = None
+
+        opc_metrics = {}
+
+        # write to pi web api
+        if pi_web_api_output_config:
+            prediction = await workflow.execute_activity_method(
+                Activities.write_pi_web_api_data,
+                {
+                    'pi_web_api_output_config': pi_web_api_output_config,
+                    'data': prediction,
+                    **metadata,
+                },
+                retry_policy=retry_policy,
+                start_to_close_timeout=timedelta(seconds=60),
             )
 
         # write to opc
-        prediction = await workflow.execute_activity_method(
-            Activities.write_opc_data,
-            {
-                **metadata,
-                'opc_output_config': input_data['opc_output_config'],
-                'data': prediction
-            },
-            retry_policy=retry_policy,
-            start_to_close_timeout=timedelta(seconds=60)
-        )
+        if opc_output_config:
+            prediction, opc_metrics = await workflow.execute_activity_method(
+                Activities.write_opc_data,
+                {
+                    'opc_output_config': opc_output_config,
+                    'data': prediction,
+                    **metadata,
+                },
+                retry_policy=retry_policy,
+                start_to_close_timeout=timedelta(seconds=60),
+            )
 
         # write to postgres
         await workflow.execute_activity_method(
@@ -91,7 +197,24 @@ class FormatAndExportPrediction():
                 'schema': input_data['schema'],
                 'table_name': input_data['table_name'],
                 'data': prediction,
+                'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ},
+                'on_conflict': input_data.get('on_conflict', 'error'),
+                'unique_columns': ['model_id', 'timestamp'],
             },
             retry_policy=retry_policy,
-            start_to_close_timeout=timedelta(seconds=60)
+            start_to_close_timeout=timedelta(seconds=180),
+        )
+
+        if write_transformed_handler is not None:
+            await write_transformed_handler
+
+        await workflow.execute_activity_method(
+            Activities.write_metrics,
+            {
+                **metadata,
+                'prediction': prediction,
+                'opc_metrics': opc_metrics,
+            },
+            retry_policy=retry_policy,
+            start_to_close_timeout=timedelta(seconds=60),
         )

laborious/workflows/sub_workflows/prediction_process.py

@@ -1,72 +1,137 @@
 from temporalio import workflow
 
 with workflow.unsafe.imports_passed_through():
-    from laborious.activities.activities import Activities
-    from typing import Any
-    from sientia_do.temporal.utils.policies import retry_policy
     from datetime import timedelta
+    from typing import Any
+
+    from sientia_do.temporal.policies import retry_policy
+
+    from laborious.activities.activities import Activities
 
 
-@workflow.defn(name="prediction_process")
-class PredictionProcess():
+@workflow.defn(name='subworkflow.prediction_process')
+class PredictionProcess:
+    """
+    Core prediction processing workflow for the Laborious system.
+
+    This workflow implements the complete ML model inference pipeline, handling
+    data quality validation, MLFlow model interactions, and prediction processing.
+    It serves as the central orchestrator for all prediction operations and ensures
+    data quality throughout the entire process.
+
+    The workflow implements a robust data processing pipeline with:
+    - Data quality validation using configurable filters
+    - MLFlow model transformation and prediction
+    - Response validation and quality assurance
+    - Flexible decision path handling
+    - Comprehensive error handling and retry policies
+
+    Workflow Execution:
+        1. Timestamp Retrieval: Gets last processed timestamp for incremental processing
+        2. Input Data Gate: Applies data quality filters
+        3. Path Decision: Determines processing path based on filter results
+        4. MLFlow Transform: Requests data transformation using MLFlow models
+        5. Response Validation: Filters transform responses for quality assurance
+        6. MLFlow Prediction: Executes prediction using transformed data
+        7. Content Validation: Filters prediction responses for final quality check
+        8. Export Delegation: Delegates to FormatAndExportPrediction workflow
+    """
+
     @workflow.run
     async def run(self, input_data: dict[str, Any]):
         """
-        This workflow runs a prediction process based on the input data.
+        Execute the prediction process workflow.
 
-        The workflow executes in two main steps:
-        1. Prepares the activity with schedule and model information
-        2. Loads data using a custom query and executes the prediction process
+        This method orchestrates the complete prediction processing pipeline by:
+        1. Retrieving the last processed timestamp for incremental processing
+        2. Applying data quality filters to validate input data
+        3. Executing MLFlow model transformation and prediction
+        4. Validating all responses for quality assurance
+        5. Delegating to export workflow for data persistence
+
+        The method implements comprehensive error handling and ensures all
+        data quality requirements are met before proceeding with ML operations.
 
         Args:
-            - input_data (dict[str, Any]): The input data for the workflow.
-            Contains the following keys:
-                - data (dict[str, Any]): The data to be used for the prediction.
-                - schema (str): The schema of the table.
-                - table_name (str): The name of the table.
-                - model_id (int): The id of the model.
-                - input_filters (dict, optional): Filters to be applied during prediction.
-                - mlflow_transform_filters (dict, optional): Filters to be
-                applied during prediction.
-                - mlflow_predict_filters (dict, optional): Filters to be
-                applied during prediction.
-                - model_name (str): The name of the model.
-                - model_retention (int, optional): The model retention period in minutes.
-                - path_priority (list[str]): The path priority.
-                - opc_output_config (dict[str, Any]): The opc output config of the prediction.
+            input_data: Complete configuration for the prediction process
+                Required keys:
+                    - metadata (dict): Workflow execution metadata
+                    - data (dict): Input data for prediction processing
+                    - schema (dict): Data schema definition
+                    - table_name (str): Target table for predictions
+                    - model_id (str): ML model identifier
+                    - model_name (str): ML model name
+                    - input_filters (dict): Data quality filters
+                    - mlflow_transform_filters (dict): MLFlow transform filters
+                    - mlflow_predict_filters (dict): MLFlow prediction filters
+                    - model_retention (int): Model retention period in minutes
+                    - path_priority (list[str]): Decision path priority configuration
+                    - opc_output_config (dict, optional): OPC server export configuration
+                    - pi_web_api_output_config (dict, optional): PI Web API export configuration
+                    - save_transform (bool, optional): Whether to save transformed data (default: True)
+                    - prediction_store_policy (str, optional): Data retention policy (default: 'lts:1')
+
         Returns:
-            None
+            None: The workflow completes successfully when export workflow finishes
 
         Raises:
-            Exception: If any of the required parameters are missing or if the workflow fails.
+            Exception: If any required parameters are missing or if the workflow fails
+                during data processing, MLFlow operations, or workflow delegation
+
         """
 
         metadata = input_data['metadata']
         data = input_data['data']
         model_id = input_data['model_id']
         model_name = input_data['model_name']
-        model_retention = input_data['model_retention']
+        model_config = input_data.get('model_config', {})
+        save_transform = input_data.get('save_transform', True)
 
-        last_timestamp = await workflow.execute_local_activity_method(
-            Activities.get_last_timestamp,
-            {
-                **metadata,
-                'data': data
-            },
-            retry_policy=retry_policy,
-            start_to_close_timeout=timedelta(minutes=1),
-        )
+        try:
+            await self._run_prediction_pipeline(
+                input_data,
+                metadata,
+                data,
+                model_id,
+                model_name,
+                model_config,
+                save_transform,
+            )
+            await workflow.execute_activity_method(
+                Activities.cleanup_minio_objects_expired,
+                {**metadata, 'data': data},
+                retry_policy=retry_policy,
+                start_to_close_timeout=timedelta(minutes=5),
+            )
+        except Exception as e:
+            await workflow.execute_activity_method(
+                Activities.cleanup_minio_objects_expired,
+                {**metadata, 'data': data},
+                retry_policy=retry_policy,
+                start_to_close_timeout=timedelta(minutes=5),
+            )
+            raise e
 
+    async def _run_prediction_pipeline(
+        self,
+        input_data: dict[str, Any],
+        metadata: dict[str, Any],
+        data: dict[str, Any],
+        model_id: str,
+        model_name: str,
+        model_config: dict[str, Any],
+        save_transform: bool,
+    ) -> None:
+        last_timestamp = data['last_timestamp']
+
+        # Apply input data quality gates
         gate_input = {
             **metadata,
             'filters': input_data['input_filters'],
             'data': data,
-            'path_priority': input_data['path_priority']
+            'path_priority': input_data['path_priority'],
         }
 
-        print(f"Metadata e input atualizadas {gate_input}")
-        print(f"Metadata: {metadata}")
-
         path_flag, confidence, comment = await workflow.execute_local_activity_method(
             Activities.input_gate,
             gate_input,
@@ -74,43 +139,40 @@ class PredictionProcess():
             start_to_close_timeout=timedelta(minutes=1),
         )
 
+        # Handle path decision based on filter results
         if await self.path_flag_handler(
             data, path_flag, input_data, confidence, last_timestamp, comment
         ):
             return
 
-        response_data = await workflow.execute_local_activity_method(
+        # Request MLFlow model transformation
+        transformed_data = await workflow.execute_activity_method(
             Activities.request_transform,
-            {
-                **metadata,
-                'data': data,
-                'model_name': model_name,
-                'model_retention': model_retention
-            },
+            {**metadata, 'data': data, 'model_name': model_name, 'model_config': model_config},
             retry_policy=retry_policy,
-            start_to_close_timeout=timedelta(minutes=1),
+            start_to_close_timeout=timedelta(minutes=5),
         )
 
+        # Validate MLFlow transform response
         path_flag, confidence, comment = await workflow.execute_local_activity_method(
             Activities.mlflow_response_gate,
             {
                 **metadata,
                 'filters': input_data['mlflow_transform_filters'],
-                'data': response_data,
+                'data': transformed_data,
                 'type': 'transform',
-                'path_priority': input_data['path_priority']
+                'path_priority': input_data['path_priority'],
             },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(minutes=1),
         )
 
+        # Handle path decision based on transform validation
         if await self.path_flag_handler(
             data, path_flag, input_data, confidence, last_timestamp, comment
         ):
             return
 
-        transformed_data = response_data['content']
-
         path_flag, confidence, comment = await workflow.execute_local_activity_method(
             Activities.mlflow_content_gate,
             {
@@ -118,7 +180,7 @@ class PredictionProcess():
                 'filters': input_data['mlflow_transform_filters'],
                 'data': transformed_data,
                 'type': 'transform',
-                'path_priority': input_data['path_priority']
+                'path_priority': input_data['path_priority'],
             },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(minutes=1),
@@ -129,93 +191,120 @@ class PredictionProcess():
         ):
             return
 
-        response_data = await workflow.execute_local_activity_method(
+        predicted_data = await workflow.execute_activity_method(
             Activities.request_predict,
             {
                 **metadata,
                 'data': transformed_data,
                 'model_name': model_name,
-                'model_retention': model_retention
+                'model_config': model_config,
             },
             retry_policy=retry_policy,
-            start_to_close_timeout=timedelta(minutes=1),
+            start_to_close_timeout=timedelta(minutes=5),
         )
 
+        # Validate MLFlow prediction response
         path_flag, confidence, comment = await workflow.execute_local_activity_method(
             Activities.mlflow_response_gate,
             {
                 **metadata,
                 'filters': input_data['mlflow_predict_filters'],
-                'data': response_data,
+                'data': predicted_data,
                 'type': 'predict',
-                'path_priority': input_data['path_priority']
+                'path_priority': input_data['path_priority'],
             },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(minutes=1),
         )
 
+        # Handle path decision based on prediction validation
         if await self.path_flag_handler(
             data, path_flag, input_data, confidence, last_timestamp, comment
         ):
             return
 
+        # Delegate to export workflow for data persistence
         await workflow.execute_child_workflow(
-            'format_and_export_prediction',
+            'subworkflow.format_and_export_prediction',
             {
                 'metadata': metadata,
+                'on_conflict': input_data.get('on_conflict', 'error'),
                 'path_flag': path_flag,
-                'data': response_data['content'],
+                'data': predicted_data,
+                'transformed_data': transformed_data if save_transform else None,
                 'prediction_confidence': confidence,
                 'timestamp': last_timestamp,
                 'model_id': model_id,
                 'model_name': model_name,
-                'model_retention': model_retention,
+                'model_config': model_config,
                 'opc_output_config': input_data['opc_output_config'],
+                'pi_web_api_output_config': input_data['pi_web_api_output_config'],
                 'schema': input_data['schema'],
                 'table_name': input_data['table_name'],
-                'comment': comment
-            }
+                'transform_table_name': input_data['transform_table_name'],
+                'comment': comment,
+                'prediction_store_policy': input_data['prediction_store_policy'],
+            },
         )
 
-    async def path_flag_handler(self, data: dict[str, Any], path_flag: str,
-                                input_data: dict[str, Any], confidence: int,
-                                last_timestamp: str, comment: str):
-        """
-        This function handles the path flag and the confidence of the prediction.
-        It returns True if the prediction should be stopped. If path_flag is 'repeat',
-        it repeats the last prediction.
-        If path_flag is 'continue', it calls the write workflow. If path_flag is 'stop',
-        it stops the prediction process.
-        Args:
-            data (dict[str, Any]): The data to be used for the prediction.
-            path_flag (str): The path flag to determine the type of prediction to format
-            confidence (int): The confidence of the prediction
-            schema (str): The schema of the prediction
-            table_name (str): The table name of the prediction
-            model_id (int): The model id of the prediction
-            last_timestamp (str): The timestamp of the last prediction
-            model_name (str): The model name of the prediction
-            model_retention (int): The model retention of the prediction
-            comment (str): The comment of the prediction
-        Returns:
-            bool: True if the prediction should be stopped, False otherwise.
+    async def path_flag_handler(
+        self,
+        data: dict[str, Any],
+        path_flag: str,
+        input_data: dict,
+        confidence: int,
+        last_timestamp: str,
+        comment: str,
+    ) -> bool:
         """
+        Handle path decisions based on filter results and confidence levels.
 
+        This method determines the appropriate action based on the path flag
+        returned by data quality filters. It can stop processing, continue,
+        or repeat operations based on the configured path priority.
+
+        Args:
+            data: Input data for processing
+            path_flag: Path decision from filter (STOP, CONTINUE, REPEAT)
+            input_data: Complete workflow input configuration including:
+                - metadata (dict): Workflow execution metadata
+                - schema (str): Database schema
+                - table_name (str): Target table for predictions
+                - transform_table_name (str): Target table for transformed data
+                - model_id (str): ML model identifier
+                - model_name (str): ML model name
+                - model_config (dict, optional): Model configuration
+                - opc_output_config (dict, optional): OPC server export configuration
+                - pi_web_api_output_config (dict, optional): PI Web API export configuration
+                - prediction_store_policy (str, optional): Data retention policy
+            confidence: Confidence level from filter validation
+            last_timestamp: Last processed timestamp
+            comment: Additional information about the filter result
+
+        Returns:
+            bool: True if processing should stop, False to continue
+
+        Path Handling:
+            - STOP: Terminates workflow execution
+            - CONTINUE: Delegates to FormatAndExportPrediction workflow with current data
+            - REPEAT: Repeats last prediction if available
+        """
         metadata = input_data['metadata']
 
         schema = input_data['schema']
         table_name = input_data['table_name']
+        transform_table_name = input_data['transform_table_name']
         model_id = input_data['model_id']
         model_name = input_data['model_name']
-        model_retention = input_data['model_retention']
+        model_config = input_data.get('model_config', {})
 
         path_flag = path_flag.upper() if path_flag else ''
 
         if path_flag == 'STOP':
+            # Stop processing and exit workflow
             return True
-
         elif path_flag == 'REPEAT':
-            # repeat last prediction
+            # Repeat last prediction if available
             await workflow.execute_activity_method(
                 Activities.repeat_last_prediction,
                 {
@@ -223,17 +312,16 @@ class PredictionProcess():
                     'schema': schema,
                     'table_name': table_name,
                     'model': model_id,
-                    'last_timestamp': last_timestamp
+                    'last_timestamp': last_timestamp,
                 },
                 retry_policy=retry_policy,
                 start_to_close_timeout=timedelta(minutes=1),
             )
             return True
-
         elif path_flag == 'CONTINUE':
             # call write workflow
             await workflow.execute_child_workflow(
-                'format_and_export_prediction',
+                'subworkflow.format_and_export_prediction',
                 {
                     'metadata': metadata,
                     'path_flag': path_flag,
@@ -242,12 +330,16 @@ class PredictionProcess():
                     'timestamp': last_timestamp,
                     'model_id': model_id,
                     'model_name': model_name,
-                    'model_retention': model_retention,
+                    'model_config': model_config,
                     'schema': schema,
                     'table_name': table_name,
+                    'transform_table_name': transform_table_name,
                     'comment': comment,
-                    'opc_output_config': input_data['opc_output_config']
-                }
+                    'opc_output_config': input_data['opc_output_config'],
+                    'pi_web_api_output_config': input_data['pi_web_api_output_config'],
+                    'prediction_store_policy': input_data['prediction_store_policy'],
+                    'on_conflict': input_data.get('on_conflict', 'error'),
+                },
             )
             return True
 

pyproject.toml

@@ -0,0 +1,190 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "laborious"
+version = "0.9.2"
+description = "Sientia DataOps Laborious - ML Model Orchestration System"
+readme = "README.md"
+requires-python = ">=3.11"
+authors = [
+    {name = "Aignosi", email = "dev@aignosi.com"}
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+exclude = [
+    ".git",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "*.pyc",
+    ".pytest_cache",
+    "htmlcov",
+    "tests/laborious/workflows/subworkflows/test_prediction_process.py",
+]
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings
+    "F",   # pyflakes
+    "I",   # isort
+    "B",   # flake8-bugbear
+    "C4",  # flake8-comprehensions
+    "UP",  # pyupgrade
+    "N",   # pep8-naming
+    "YTT", # flake8-2020
+    "S",   # flake8-bandit
+    "BLE", # flake8-blind-except
+    "A",   # flake8-builtins
+    "C90", # mccabe complexity
+]
+
+ignore = [
+    "BLE001",  # ignore blind except, we need to send notifications with any error
+    "E501",  # line too long (handled by formatter)
+    "S101",  # use of assert (needed for tests)
+    "S105",  # possible hardcoded password (false positives)
+    "S106",  # possible hardcoded password (false positives)
+    "S608",  # potential sql injection (false positives)
+    "N802",  # function name should be lowercase (temporal decorators)
+    "N806",  # variable in function should be lowercase
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**/*.py" = [
+    "S101",  # assert allowed in tests
+    "S105",  # hardcoded passwords ok in tests
+    "S106",  # hardcoded passwords ok in tests
+]
+"e2e/conftest.py" = [
+    # Imports below the sientia/ModelAnalysis stub are load-order critical: the stub
+    # must be in sys.modules before Activities is imported, so they cannot move up.
+    "E402",
+]
+"scripts/**/*.py" = [
+    # Notebook-style scripts split into `# %%` cells: each cell imports what it needs
+    # where it needs it, so imports legitimately appear below other statements.
+    "E402",
+]
+
+[tool.ruff.lint.mccabe]
+max-complexity = 15
+
+[tool.ruff.format]
+quote-style = "single"
+indent-style = "space"
+line-ending = "auto"
+
+[tool.mypy]
+python_version = "3.11"
+warn_return_any = false
+warn_unused_configs = true
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+check_untyped_defs = true
+no_implicit_optional = true
+warn_redundant_casts = true
+warn_unused_ignores = false
+warn_no_return = true
+strict_equality = true
+ignore_missing_imports = true
+
+# Ignore missing imports for external packages
+[[tool.mypy.overrides]]
+module = "temporalio.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "sientia_do.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "mlflow.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "prometheus_client.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "sientia.*"
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "pandas.*"
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--strict-markers",
+    # pytest>=9.1 has a known bug where its unraisableexception plugin crashes
+    # (tracemalloc partially-initialized AttributeError) when 2+ unraisable
+    # exceptions land close together — e.g. "coroutine was never awaited" from
+    # AsyncMock-mocked sync methods (metrics_controller, minio_repository) being
+    # GC'd. Harmless mock artifacts turned into a hard ERROR by the plugin itself.
+    "-p", "no:unraisableexception",
+]
+filterwarnings = [
+    # `kafka-python`, pulled in transitively by `sientia_do`, reads its protocol schemas through the
+    # legacy `importlib.resources` API that Python 3.11 deprecated. It emits ~109 warnings per run,
+    # twice each (once at the call site, once inside `importlib`), which is enough noise to bury a
+    # warning worth reading. Nothing here can fix it — the change belongs in `kafka-python`.
+    #
+    # Matched on the message rather than by module so the filter cannot quietly swallow an unrelated
+    # deprecation: any *other* DeprecationWarning, including one from our own code, still shows up.
+    "ignore:read_text is deprecated:DeprecationWarning",
+    "ignore:open_text is deprecated:DeprecationWarning",
+]
+markers = [
+    "asyncio: marks tests as async",
+    "integration: marks tests as integration tests",
+    "unit: marks tests as unit tests",
+    "opc: marks tests that use the in-process OPC UA server (OpcRepository E2E)",
+    # The .sientia import e2e suite. Outside `testpaths`, so `validate --only-tests` and the
+    # coverage gate never collect it; run it on request with `pytest e2e -m import_e2e`.
+    "import_e2e: marks the model-import end-to-end tests (needs Docker and an MLflow server)",
+    "slow: marks tests that run on a real clock and take tens of seconds",
+]
+
+[tool.coverage.run]
+source = ["laborious"]
+omit = [
+    "*/tests/*",
+    "*/venv/*",
+    "*/__pycache__/*",
+    "*/site-packages/*",
+]
+branch = true
+
+[tool.coverage.report]
+precision = 2
+show_missing = true
+skip_covered = false
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "def __str__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+    "@overload",
+]
+
+[tool.coverage.html]
+directory = "htmlcov"
+
+[tool.bandit]
+exclude_dirs = ["tests", "venv", ".venv"]
+skips = ["B101", "B601", "B608"]  # Skip assert, shell injection, and SQL injection (false positives)

requirements-dev.txt

@@ -0,0 +1,25 @@
+# Development and Testing Dependencies
+# These packages are only needed for development, testing, and code quality checks
+# Install with: pip install -r requirements-dev.txt
+
+# Code Quality & Linting
+ruff>=0.1.0              # Fast Python linter and formatter (replaces flake8, black, isort)
+mypy>=1.7.0              # Static type checker
+bandit>=1.7.5            # Security vulnerability scanner
+pandas-stubs>=2.0.0      # Type stubs for pandas
+types-requests>=2.31.0   # Type stubs for requests
+
+# Testing
+pytest>=7.4.0            # Testing framework
+pytest-cov>=4.1.0        # Coverage plugin for pytest
+pytest-asyncio>=0.21.0   # Async test support (already in main requirements)
+testcontainers[postgres,minio,mongodb]>=4.15.0 # PostgreSQL, MinIO and MongoDB containers for E2E tests
+                         # Floor is the `testcontainers.community.*` namespace: the flat
+                         # `testcontainers.minio` / `.postgres` / `.mongodb` paths the e2e suite used
+                         # to import are deprecated shims that warn on import.
+
+# Development Tools
+ipython>=8.12.0          # Enhanced Python shell
+ipdb>=0.13.13            # IPython debugger
+ipykernel==6.30.1        # IPython kernel for Jupyter notebooks
+python-dotenv>=1.0.0     # .env loading for scripts/import_model_manual_run.py

requirements-light.txt

@@ -0,0 +1,21 @@
+temporalio
+psycopg2-binary
+sqlalchemy<2.1
+asyncua==1.0.6
+redis
+sientia_do>=1.12.2
+mlflow==2.10.1
+prometheus-client
+botocore
+boto3
+s3fs
+pyarrow
+kaleido
+hyperopt
+shap
+pycurl
+scipy<1.14.0
+scikit-learn==1.4.2
+pycaret==3.3.2
+joblib
+pynacl==1.5.0

requirements-local.txt

@@ -0,0 +1,26 @@
+temporalio
+psycopg2-binary
+sqlalchemy<2.1
+asyncua==1.0.6
+redis
+git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.12.2
+git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.41.1
+prometheus-client   
+# fsspec/s3fs/gcsfs pinned as a set: nannyml (via the mlops library) pulls both s3fs and
+# gcsfs, and each pins fsspec exactly, so a free `s3fs` makes the resolver fail against
+# evidently's fsspec>=2024.2.0. Versions match the library-distribution-controller freeze.
+fsspec==2026.7.0
+s3fs==2026.7.0
+gcsfs==2026.8.0
+# aiobotocore 3.9.0 (pulled by s3fs) requires botocore<1.43.57,>=1.43.3.
+botocore==1.43.56
+boto3==1.43.56
+pyarrow
+kaleido
+hyperopt
+shap
+pycurl
+scipy<1.14.0
+scikit-learn==1.4.2
+pycaret==3.3.2
+joblib

requirements.txt

@@ -1,7 +1,33 @@
 temporalio
 psycopg2-binary
-sqlalchemy
-asyncua
+sqlalchemy<2.1
+asyncua==1.0.6
 redis
-git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.2.0
-git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.1
+sientia_do>=1.12.2
+sientia>=0.41.1
+prometheus-client
+# fsspec/s3fs/gcsfs pinned as a set: nannyml (via sientia) pulls both s3fs and gcsfs, and
+# each pins fsspec exactly, so a free `s3fs` makes the resolver fail against evidently's
+# fsspec>=2024.2.0. Versions match the library-distribution-controller freeze.
+fsspec==2026.7.0
+s3fs==2026.7.0
+gcsfs==2026.8.0
+# aiobotocore 3.9.0 (pulled by s3fs) requires botocore<1.43.57,>=1.43.3.
+botocore==1.43.56
+boto3==1.43.56
+pyarrow
+kaleido
+hyperopt
+shap
+pycurl
+scipy<1.14.0
+scikit-learn==1.4.2
+pycaret==3.3.2
+joblib
+# .sientia bundle import (Argon2id + XChaCha20-Poly1305 secretstream): same pin as the
+# producer in sientia-projects-templates, so both sides open the same bytes.
+pynacl==1.5.0
+# AES-256-GCM password envelope; already present transitively, declared explicitly.
+cryptography
+# MongoDB models collection (the model listing); transitive via sientia_do, declared explicitly.
+pymongo

scripts/import_model_manual_run.py

@@ -0,0 +1,586 @@
+"""Run one `.sientia` model import by hand, standing in for the frontend that does not exist yet.
+
+**What this is.** The Angular screen plus the Spring Boot BFF will, between them, pick a `.sientia`
+file, hash it, wrap the bundle password for the worker, upload the object and insert one `PENDING`
+row into `public.experiment_run` before starting `import_model`. None of that exists yet. This
+script does exactly those four things, against whatever environment `.env` points at, so the import
+path can be exercised for real and so whoever writes the frontend has a working reference for the
+wire shapes (the digest, the envelope, the object key, the row).
+
+**What it writes.** Two things, and it can undo both: one object under
+`{IMPORT_BUNDLE_BUCKET}/{IMPORT_BUNDLE_PREFIX}` in MinIO, and one row in `public.experiment_run` in
+the BFF Postgres database. Everything else it does is a read. If the workflow runs, *it* creates the
+MLflow experiment, run and registered version and the MongoDB model document — cell 10 can remove
+those too, behind its own flag.
+
+**The safety rule.** Every target comes from the environment, and this file contains no host, user,
+password or key literal. A `.env` inherited from a cluster deployment can perfectly well name a
+production database and a production bucket, so cell 5 and cell 6 — the only cells that write — are
+gated on cell 2, where the operator has to restate the Postgres host that cell 1 printed. This is a
+development tool. It is not for production, and the guard is a mitigation, not a guarantee.
+
+**Cells.** Run them one at a time in the VS Code interactive window, or run the module top to
+bottom. Cell order is the contract:
+
+    1  resolve every target and print it          6  insert the PENDING row -> import_run_id
+    2  confirmation guard (first write is after)  7  print the workflow input
+    3  pick the bundle, compute expected_digest   8  optionally start the workflow and wait
+    4  build the password envelope                9  read the outcome back
+    5  upload the object, verify the stored bytes 10 undo
+
+**Boundaries.** This script imports nothing from `laborious.activities` or `laborious.workflows` —
+it stands in for the frontend, so it must not borrow the consumer's code. The one exception is
+`laborious.utils.bundle.format`, which is pure `struct`/`hashlib` over bytes and lets cell 3 print
+the header of the file it is about to upload. The envelope is built here in four visible lines
+rather than imported, because those four lines are what the frontend has to transliterate to
+WebCrypto; `laborious/utils/import_password.py` is the decrypt counterpart.
+"""
+
+# %%
+# Cell 1 — resolve every target from the environment and print what we are pointed at.
+# Reads only. No secret is printed here or anywhere else in this script.
+from __future__ import annotations
+
+import base64
+import hashlib
+import io
+import json
+import os
+import secrets
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+import mlflow
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from dotenv import load_dotenv
+from minio import Minio
+from pymongo import MongoClient
+from sqlalchemy import create_engine, text
+from sqlalchemy.engine import URL
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+load_dotenv(REPO_ROOT / '.env')
+
+# Below the `sys.path` insert on purpose: `laborious` is only importable once the repository root is
+# on the path, which is what those three lines above just arranged. Hoisting this to the top of the
+# cell is a `ModuleNotFoundError` when the script runs from anywhere but the repository root.
+from laborious.utils.bundle.format import DIGEST_OFFSET, HEADER_SIZE, parse_header  # noqa: E402
+
+
+def env(name: str, default: str = '') -> str:
+    """The environment variable, stripped, or `default`."""
+    return os.getenv(name, default).strip()
+
+
+def env_required(name: str) -> str:
+    """The environment variable, or a `RuntimeError` naming it."""
+    value = env(name)
+    if not value:
+        raise RuntimeError(f'{name} is not set — see .env.example')
+    return value
+
+
+def env_secret_required(name: str) -> str:
+    """The environment variable *verbatim*, or a `RuntimeError` naming it.
+
+    Unlike `env_required` this does not strip, and the difference is not cosmetic. A key and a
+    password are byte strings whose surrounding whitespace is part of the secret. The worker reads
+    the same variables with a plain `os.getenv`, so stripping here would seal the envelope under
+    bytes the worker never derives, and the import would fail at a gate with nothing in the message
+    to say why.
+    """
+    value = os.getenv(name, '')
+    if not value:
+        raise RuntimeError(f'{name} is not set — see .env.example')
+    return value
+
+
+def _line(label: str, value: object) -> str:
+    """One aligned `label  value` line, for output that stays readable in a terminal."""
+    return f'  {label:<24} {value}'
+
+
+# Every connection target below is *required* rather than defaulted. The worker's own config falls
+# back to localhost, which is right for a service that must boot; it is wrong for a script that
+# writes, because a silent fallback is a write to somewhere nobody read out loud.
+#
+# The import log database: the POSTGRES_* server from .env, with only the database name of its own,
+# because the log lives in the Spring Boot BFF's database rather than in `sientia`. The schema and
+# table are constants of the worker's code, never configuration.
+#
+# `build_import_status_config` also honours IMPORT_STATUS_DB_{HOST,PORT,USER,PASSWORD} as overrides,
+# for a deployment whose log sits on another server. This script does not: it is a development tool
+# pointed at a real environment, and the override exists for the e2e suite's two containers.
+STATUS_DB_HOST = env_required('POSTGRES_HOST')
+STATUS_DB_PORT = int(env_required('POSTGRES_PORT'))
+STATUS_DB_NAME = env_required('IMPORT_STATUS_DB_NAME')
+STATUS_DB_USER = env_required('POSTGRES_USER')
+STATUS_DB_PASSWORD = os.getenv('POSTGRES_PASSWORD', '')
+STATUS_TABLE = 'public.experiment_run'
+
+# Where the frontend uploads the bundle. The prefix keeps the worker's own default, which is a
+# format decision rather than a target.
+MINIO_ENDPOINT_URL = env_required('MINIO_ENDPOINT_URL')
+MINIO_ACCESS_KEY = env_required('MINIO_ACCESS_KEY')
+MINIO_SECRET_KEY = env_required('MINIO_SECRET_KEY')
+BUNDLE_BUCKET = env_required('IMPORT_BUNDLE_BUCKET')
+_raw_prefix = env('IMPORT_BUNDLE_PREFIX', 'imported_models/')
+BUNDLE_PREFIX = _raw_prefix if _raw_prefix.endswith('/') else f'{_raw_prefix}/'
+
+# Read-back targets. This script never writes to these; the workflow does.
+MONGODB_URL = env_required('MONGODB_URL')
+MONGODB_USERNAME = env('MONGODB_USERNAME')
+MONGODB_PASSWORD = os.getenv('MONGODB_PASSWORD', '')
+MONGODB_DATABASE = env_required('MONGODB_DATABASE')
+MODELS_COLLECTION = env('IMPORT_MODELS_COLLECTION', 'models')
+
+MLFLOW_HOST = env_required('MLFLOW_HOST')
+MLFLOW_PORT = env_required('MLFLOW_PORT')
+MLFLOW_TRACKING_URI = f'{MLFLOW_HOST}:{MLFLOW_PORT}'
+
+TEMPORAL_HOST = env_required('TEMPORAL_HOST')
+TEMPORAL_NAMESPACE = env_required('TEMPORAL_NAMESPACE')
+RUNTIME = env_required('RUNTIME')
+IMPORT_TASK_QUEUE = 'import_model-legacy-queue'
+
+# This script's own inputs.
+BUNDLE_PATH = Path(
+    env('IMPORT_SCRIPT_BUNDLE_PATH', 'tests/fixtures/bundle/golden_model_v3.sientia')
+)
+if not BUNDLE_PATH.is_absolute():
+    BUNDLE_PATH = REPO_ROOT / BUNDLE_PATH
+IMPORT_USERNAME = env_required('IMPORT_SCRIPT_USERNAME')
+
+print('resolved targets — nothing has been written yet')
+print(_line('status postgres', f'{STATUS_DB_HOST}:{STATUS_DB_PORT}/{STATUS_DB_NAME}'))
+print(_line('status user', STATUS_DB_USER))
+print(_line('status table', STATUS_TABLE))
+print(_line('minio endpoint', MINIO_ENDPOINT_URL))
+print(_line('minio bucket/prefix', f'{BUNDLE_BUCKET}/{BUNDLE_PREFIX}'))
+print(_line('mongo', f'{MONGODB_URL}/{MONGODB_DATABASE}.{MODELS_COLLECTION}'))
+print(_line('mlflow', MLFLOW_TRACKING_URI))
+print(_line('temporal', f'{TEMPORAL_HOST} ns={TEMPORAL_NAMESPACE}'))
+print(_line('task queue', IMPORT_TASK_QUEUE))
+print(_line('bundle', BUNDLE_PATH))
+print(_line('username', IMPORT_USERNAME))
+print()
+print(f'to continue, set the cell 2 confirmation to: {STATUS_DB_HOST}')
+
+# %%
+# Cell 2 — the confirmation guard. Cells 5 and 6 write; this is the last cell before them.
+#
+# Restate the Postgres host cell 1 printed. Either set IMPORT_SCRIPT_CONFIRM_TARGET in .env, or
+# replace the right-hand side below with the value cell 1 told you. It is a read-then-restate on
+# purpose: a boolean flag can be flipped without ever looking at what it is pointed at.
+CONFIRM_TARGET = 'localhost'
+
+if CONFIRM_TARGET != STATUS_DB_HOST:
+    raise RuntimeError(
+        'refusing to write: the confirmation does not match the resolved status database host. '
+        f'confirmation={CONFIRM_TARGET!r}, resolved POSTGRES_HOST={STATUS_DB_HOST!r}. '
+        'Read the targets cell 1 printed, then restate the host here.'
+    )
+
+print(f'target confirmed: {STATUS_DB_HOST} — the write cells may run')
+
+# %%
+# Cell 3 — pick the bundle, compute the digest the frontend must send, print its header.
+#
+# `expected_digest` is the SHA-256 of the exact bytes that get uploaded, computed the way the
+# frontend computes it: over the file, not over anything derived from it. The header fields printed
+# here are plaintext at the front of the file — reading them decrypts nothing.
+
+
+bundle_bytes = BUNDLE_PATH.read_bytes()
+expected_digest = hashlib.sha256(bundle_bytes).hexdigest()
+object_name = BUNDLE_PATH.name
+object_key = f'{BUNDLE_PREFIX}{object_name}'
+
+header = parse_header(bundle_bytes[:HEADER_SIZE])
+
+print(_line('file', BUNDLE_PATH))
+print(_line('size', f'{len(bundle_bytes)} bytes'))
+print(_line('expected_digest', expected_digest))
+print(_line('object key', f'{BUNDLE_BUCKET}/{object_key}'))
+print(_line('format version', header.format_version))
+print(_line('aead / kdf id', f'{header.aead_id} / {header.kdf_id}'))
+print(_line('kdf memlimit', f'{header.kdf_memlimit_bytes} bytes'))
+print(_line('kdf opslimit', header.kdf_opslimit))
+print(_line('kdf parallelism', header.kdf_parallelism))
+print(_line('salt', header.salt.hex()))
+print(_line('ciphertext digest', header.ciphertext_digest.hex()))
+print(_line('chunk stream from', f'offset {DIGEST_OFFSET}'))
+
+# %%
+# Cell 4 — build the password envelope, exactly as the frontend must build it.
+#
+# Wire shape: base64( nonce(12) || ciphertext || GCM tag(16) ) under AES-256-GCM with the 32-byte
+# key that `IMPORT_PASSWORD_KEY` carries base64-encoded. The four lines below are the whole of it,
+# written out rather than imported so the Angular side can transliterate them to WebCrypto:
+#
+#   const k = await crypto.subtle.importKey('raw', keyBytes, 'AES-GCM', false, ['encrypt']);
+#   const nonce = crypto.getRandomValues(new Uint8Array(12));
+#   const sealed = await crypto.subtle.encrypt({name: 'AES-GCM', iv: nonce}, k, passwordBytes);
+#   const envelope = base64(concat(nonce, new Uint8Array(sealed)));
+#
+# WebCrypto appends the GCM tag to the ciphertext, and so does `cryptography`'s AESGCM, so the two
+# produce byte-identical payloads. The decrypt counterpart the worker runs is
+# `laborious/utils/import_password.py` (`decrypt_password_envelope`).
+#
+# `IMPORT_PASSWORD_KEY` must be base64 of exactly 32 bytes — `openssl rand -base64 32`, or the
+# value the deployment's Kubernetes Secret carries. Anything else fails here, which is the point:
+# the worker resolves the same variable the same way, so a value this cell cannot decode is a value
+# the worker could not have decoded either.
+#
+# The bundle password is read verbatim — `env_secret_required`, not `env_required` — because a
+# trailing space is part of the password, and stripping it here would seal an envelope whose
+# plaintext the bundle reader rejects at gate 3.
+#
+# Neither the key nor the password nor the plaintext is printed — only the envelope's length.
+
+
+envelope_key = base64.b64decode(env_required('IMPORT_PASSWORD_KEY'), validate=True)
+nonce = secrets.token_bytes(12)
+sealed = AESGCM(envelope_key).encrypt(
+    nonce, env_secret_required('IMPORT_SCRIPT_BUNDLE_PASSWORD').encode('utf-8'), None
+)
+password_envelope = base64.b64encode(nonce + sealed).decode('ascii')
+
+print(_line('envelope key', f'{len(envelope_key)} bytes (not printed)'))
+print(_line('envelope', f'{len(password_envelope)} base64 characters (not printed)'))
+
+# %%
+# Cell 5 — FIRST WRITE. Upload the object, then prove the stored bytes are the ones we hashed.
+#
+# The re-read is the point: `expected_digest` is a promise about what is in the bucket, and the
+# worker's gate 1 holds the upload to it. If the digest of the stored object differs, the frontend
+# contract is already broken and there is no reason to insert a row.
+
+
+_endpoint = urlparse(MINIO_ENDPOINT_URL)
+minio_client = Minio(
+    _endpoint.netloc or MINIO_ENDPOINT_URL,
+    access_key=MINIO_ACCESS_KEY,
+    secret_key=MINIO_SECRET_KEY,
+    secure=_endpoint.scheme == 'https',
+)
+
+minio_client.put_object(
+    BUNDLE_BUCKET,
+    object_key,
+    io.BytesIO(bundle_bytes),
+    length=len(bundle_bytes),
+    content_type='application/octet-stream',
+)
+
+stat = minio_client.stat_object(BUNDLE_BUCKET, object_key)
+_response = minio_client.get_object(BUNDLE_BUCKET, object_key)
+try:
+    stored_bytes = _response.read()
+finally:
+    _response.close()
+    _response.release_conn()
+
+stored_digest = hashlib.sha256(stored_bytes).hexdigest()
+if stored_digest != expected_digest:
+    raise RuntimeError(
+        f'the stored object does not hash to the digest we computed: stored {stored_digest}, '
+        f'expected {expected_digest} — do not insert a row for this upload'
+    )
+
+print(_line('uploaded', f'{BUNDLE_BUCKET}/{object_key}'))
+print(_line('reported size', f'{stat.size} bytes'))
+print(_line('etag', stat.etag))
+print(_line('re-read digest', f'{stored_digest} (matches expected_digest)'))
+
+# %%
+# Cell 6 — SECOND WRITE. Insert the import log row: the BFF's half of the contract.
+#
+# `URL.create` rather than a URL string, so a password containing `@ : /` survives. `experiment_name`
+# is a placeholder: the column is NOT NULL and the frontend cannot see inside an encrypted bundle,
+# so the worker overwrites it with the bundle's own experiment name as soon as the bundle is open
+# (`record_import_names`). `file_name` is the object key, because the worker compares the two and
+# refuses the import at the `received` step when they disagree.
+
+
+status_engine = create_engine(
+    URL.create(
+        'postgresql',
+        username=STATUS_DB_USER,
+        password=STATUS_DB_PASSWORD,
+        host=STATUS_DB_HOST,
+        port=STATUS_DB_PORT,
+        database=STATUS_DB_NAME,
+    ),
+    pool_pre_ping=True,
+)
+
+request_data = {
+    'expected_digest': expected_digest,
+    'bucket': BUNDLE_BUCKET,
+    'object_key': object_key,
+    'file_name': object_name,
+    'source': 'scripts/import_model_manual_run.py',
+}
+
+_now = datetime.now(UTC).replace(tzinfo=None)
+_placeholder_experiment = f'import-{Path(object_name).stem}'[:50]
+
+INSERT_ROW_SQL = text(
+    'INSERT INTO public.experiment_run '
+    '(experiment_name, username, status, created_at, updated_at, '
+    ' bucket_name, file_name, request_data, run_type) '
+    'VALUES (:experiment_name, :username, :status, :created_at, :updated_at, '
+    ' :bucket_name, :file_name, CAST(:request_data AS JSONB), :run_type) '
+    'RETURNING id'
+)
+
+with status_engine.begin() as connection:
+    import_run_id = connection.execute(
+        INSERT_ROW_SQL,
+        {
+            'experiment_name': _placeholder_experiment,
+            'username': IMPORT_USERNAME,
+            'status': 'PENDING',
+            'created_at': _now,
+            'updated_at': _now,
+            'bucket_name': BUNDLE_BUCKET,
+            'file_name': object_key,
+            'request_data': json.dumps(request_data),
+            'run_type': 'IMPORT',
+        },
+    ).scalar_one()
+
+print(_line('import_run_id', import_run_id))
+print(_line('status / run_type', 'PENDING / IMPORT'))
+print(_line('experiment_name', f'{_placeholder_experiment} (the worker overwrites this)'))
+print(_line('file_name', object_key))
+
+# %%
+# Cell 7 — the workflow input, ready to paste into the Temporal UI.
+#
+# These five keys are the whole contract. `bucket` is optional — the worker falls back to
+# IMPORT_BUNDLE_BUCKET — and is sent explicitly so the printed input is self-describing. No model
+# name is sent: the worker takes it from the bundle's own metadata, verbatim.
+workflow_input = {
+    'import_run_id': import_run_id,
+    'bucket': BUNDLE_BUCKET,
+    'object_key': object_key,
+    'expected_digest': expected_digest,
+    'password_envelope': password_envelope,
+}
+
+print(f'workflow: import_model    task queue: {IMPORT_TASK_QUEUE}')
+print(json.dumps(workflow_input, indent=2))
+
+# %%
+# Cell 8 — optionally start the workflow and wait for it.
+#
+# Skipping this cell is normal: the operator may prefer to start the run from the Temporal UI with
+# the JSON cell 7 printed. Cell 9 works either way, because it looks the outcome up by row id.
+#
+# The coroutine runs on a loop of its own in a worker thread rather than through `asyncio.run`, so
+# this cell behaves the same in the interactive window (which already has a running loop) as it
+# does when the module is executed top to bottom (which does not).
+START_THE_WORKFLOW = True
+
+if START_THE_WORKFLOW:
+    import asyncio
+    from concurrent.futures import ThreadPoolExecutor
+
+    from temporalio.client import Client
+
+    async def start_and_wait() -> Any:
+        """Start `import_model` on the import queue and return its result."""
+        client = await Client.connect(TEMPORAL_HOST, namespace=TEMPORAL_NAMESPACE)
+        handle = await client.start_workflow(
+            'import_model',
+            workflow_input,
+            id=f'manual-import-{import_run_id}',
+            task_queue=IMPORT_TASK_QUEUE,
+        )
+        print(_line('workflow id', handle.id))
+        print(_line('run id', handle.result_run_id))
+        return await handle.result()
+
+    with ThreadPoolExecutor(max_workers=1) as _pool:
+        workflow_result = _pool.submit(asyncio.run, start_and_wait()).result()
+    print(json.dumps(workflow_result, indent=2, default=str))
+else:
+    print('not started — paste the input from cell 7 into the Temporal UI, or set')
+    print(f'START_THE_WORKFLOW = True and re-run this cell. Task queue: {IMPORT_TASK_QUEUE}')
+
+# %%
+# Cell 9 — read the outcome back from all three places it lands.
+#
+# The row is authoritative for the verdict; MLflow and MongoDB are what the verdict is about. Each
+# block is labelled, so the output can be pasted into a ticket as it is.
+print('--- import log row ---')
+READ_ROW_SQL = text(
+    'SELECT id, experiment_name, run_name, status, error_message, run_type, '
+    '       bucket_name, file_name, created_at, updated_at, '
+    '       request_data, orchestrator_response_data '
+    'FROM public.experiment_run WHERE id = :id'
+)
+
+with status_engine.begin() as connection:
+    _result = connection.execute(READ_ROW_SQL, {'id': import_run_id})
+    row = _result.mappings().one_or_none()
+
+if row is None:
+    print(f'  no row with id {import_run_id}')
+else:
+    for _column, _value in row.items():
+        print(_line(_column, _value))
+
+print()
+print('--- mlflow ---')
+
+os.environ['MLFLOW_TRACKING_USERNAME'] = env('MLFLOW_USERNAME')
+os.environ['MLFLOW_TRACKING_PASSWORD'] = os.getenv('MLFLOW_PASSWORD', '')
+mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)
+mlflow_client = mlflow.tracking.MlflowClient()
+
+_experiment_name = row['experiment_name'] if row is not None else ''
+_experiment = mlflow_client.get_experiment_by_name(_experiment_name) if _experiment_name else None
+# The experiment is get-or-create, so it can perfectly well be the model's own experiment with
+# months of training runs in it. `_run_ids` is everything in it — printed, and used to decide
+# whether the experiment is this import's alone — while `_import_run_ids` is only the run this
+# import created, matched on the run name the record carries. The undo acts on the second list.
+_row_run_name = (row['run_name'] if row is not None else '') or ''
+_run_ids: list[str] = []
+_import_run_ids: list[str] = []
+if _experiment is None:
+    print(f'  no experiment named {_experiment_name!r}')
+else:
+    print(_line('experiment', f'{_experiment.name} ({_experiment.experiment_id})'))
+    for _run in mlflow_client.search_runs([_experiment.experiment_id]):
+        _mine = bool(_row_run_name) and _run.info.run_name == _row_run_name
+        print(
+            _line(
+                'run',
+                f'{_run.info.run_name} ({_run.info.run_id}) {_run.info.status}'
+                f'{"  <- this import" if _mine else ""}',
+            )
+        )
+        _run_ids.append(_run.info.run_id)
+        if _mine:
+            _import_run_ids.append(_run.info.run_id)
+        for _artifact in mlflow_client.list_artifacts(_run.info.run_id):
+            print(_line('  artifact', _artifact.path))
+
+# The registered model is looked up by `run_id`, not by a name reconstructed from `run_name`. The
+# run name is `import-{model_name}-v{model_version}` truncated to 50 characters (`derive_run_name`),
+# and a 32-character `model_version` leaves ten characters for the name — so `Quartz-demo` comes
+# back as `Quartz-dem` and the lookup finds nothing. The version's `run_id` is exact, and it is what
+# ties the registered model to the run this import created whatever the name was truncated to.
+_registered_names: set[str] = set()
+_registered_versions: list[tuple[str, str]] = []
+for _run_id in _import_run_ids:
+    try:
+        for _version in mlflow_client.search_model_versions(f"run_id='{_run_id}'"):
+            _registered_names.add(_version.name)
+            _registered_versions.append((_version.name, str(_version.version)))
+            print(
+                _line(
+                    'version',
+                    f'{_version.name} v{_version.version} stage={_version.current_stage} '
+                    f'run={_version.run_id}',
+                )
+            )
+            print(_line('  source', _version.source))
+    except Exception as error:
+        # A read-back cell reports what it could not read; it does not fail the inspection.
+        print(f'  registry read failed: {type(error).__name__}: {error}')
+if not _import_run_ids:
+    print('  no run of this import in the experiment yet, so no registered model to look up')
+elif not _registered_names:
+    print("  no registered version points at this import's run")
+
+print()
+print('--- mongodb model document ---')
+
+_credentials = f'{MONGODB_USERNAME}:{MONGODB_PASSWORD}@' if MONGODB_USERNAME else ''
+mongo_client: MongoClient = MongoClient(f'mongodb://{_credentials}{MONGODB_URL}')
+_collection = mongo_client[MONGODB_DATABASE][MODELS_COLLECTION]
+for _document in _collection.find({}, {'_id': False}).sort('id', 1):
+    print(_line('document', _document))
+
+# %%
+# Cell 10 — undo.
+#
+# The object and the row always: they are what this script created, and leaving them behind leaves a
+# PENDING import nobody will ever run. What MLflow and MongoDB hold is left alone unless
+# DELETE_PROVISIONING is set, because after a successful run that is the thing you wanted to look at.
+#
+# When it *is* set, the undo removes only what this import added. An import of a model the platform
+# already had lands as a new *version* of that registered model and writes no document at all, so
+# deleting the registered model by name would take versions trained months ago with it. The version
+# this import created is deleted by number; the registered model and the document go only when
+# nothing else is left pointing at that name — which is exactly the case where this import was the
+# first one.
+DELETE_PROVISIONING = False
+
+try:
+    minio_client.remove_object(BUNDLE_BUCKET, object_key)
+    print(_line('deleted object', f'{BUNDLE_BUCKET}/{object_key}'))
+except Exception as error:
+    # Undo reports what it could not remove and carries on to the row.
+    print(f'  could not delete the object: {type(error).__name__}: {error}')
+
+with status_engine.begin() as connection:
+    _deleted = connection.execute(
+        text('DELETE FROM public.experiment_run WHERE id = :id'),
+        {'id': import_run_id},
+    ).rowcount
+print(_line('deleted rows', _deleted))
+
+if DELETE_PROVISIONING:
+    for _name, _number in _registered_versions:
+        try:
+            mlflow_client.delete_model_version(_name, _number)
+            print(_line('deleted version', f'{_name} v{_number}'))
+        except Exception as error:
+            # Same rule as above: report and carry on to the next one.
+            print(f'  could not delete the model version: {type(error).__name__}: {error}')
+
+    for _registered in sorted(_registered_names):
+        try:
+            _remaining = list(mlflow_client.search_model_versions(f"name='{_registered}'"))
+        except Exception as error:
+            print(f'  could not count the remaining versions: {type(error).__name__}: {error}')
+            continue
+        if _remaining:
+            # Versions this import did not create: the model was on the platform before it ran, so
+            # the registered model and its document are somebody else's and stay exactly as they are.
+            print(_line('kept model', f'{_registered} ({len(_remaining)} other versions)'))
+            continue
+        try:
+            mlflow_client.delete_registered_model(_registered)
+            print(_line('deleted model', _registered))
+        except Exception as error:
+            print(f'  could not delete the registered model: {type(error).__name__}: {error}')
+        _removed = _collection.delete_many({'name': _registered}).deleted_count
+        print(_line('deleted documents', f'{_removed} named {_registered}'))
+
+    # Last, and only when the experiment holds nothing but this import's own run: the experiment is
+    # get-or-create, so an import into a model the platform already had reuses the experiment its
+    # training runs live in, and deleting it would take them all.
+    if _experiment is not None:
+        _others = len(_run_ids) - len(_import_run_ids)
+        if _others:
+            print(_line('kept experiment', f'{_experiment.name} ({_others} other runs)'))
+        else:
+            mlflow_client.delete_experiment(_experiment.experiment_id)
+            print(_line('deleted experiment', _experiment.experiment_id))
+else:
+    print('  provisioning kept — set DELETE_PROVISIONING = True to remove it too')
+
+status_engine.dispose()
+mongo_client.close()

scripts/run_coverage.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Exit on any error
+set -e
+
+echo "Activating virtual environment..."
+source ./venv/bin/activate
+
+pytest --cov=laborious --cov-report=html
+
+xdg-open htmlcov/index.html

scripts/run_local.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Exit on any error
+set -e
+
+echo "Activating virtual environment..."
+source ./venv/bin/activate
+
+echo "Loading environment variables from .env..."
+if [ -f .env ]; then
+    export $(cat .env | grep -v '^#' | xargs)
+    echo "Environment variables loaded from .env"
+else
+    echo "Warning: .env file not found. Continuing without environment variables."
+fi
+
+echo "Starting ingestor application..."
+python -m laborious.worker.worker

simulator/Dockerfile

@@ -1,30 +0,0 @@
-# syntax=docker/dockerfile:1.4
-
-FROM python:3.11-slim
-
-# Enable use of SSH agent/socket
-# This line enables SSH during build
-# (don't forget the syntax header above)
-RUN apt-get update && apt-get install -y git openssh-client && rm -rf /var/lib/apt/lists/*
-
-# Use build-time SSH mount for Git clone
-# The SSH key will NOT remain in the image
-# IMPORTANT: this block requires BuildKit
-# and the --ssh flag during docker build
-
-# SSH config to skip host key check (safe in CI/local dev)
-RUN mkdir -p /root/.ssh && echo "StrictHostKeyChecking no" > /root/.ssh/config
-
-WORKDIR /app
-
-# Clone using SSH
-ARG GIT_REPO
-ARG GIT_BRANCH=main
-
-# Mount SSH key just for this RUN
-RUN --mount=type=ssh git clone --branch ${GIT_BRANCH} ${GIT_REPO} .
-
-# Install requirements if exists
-RUN if [ -f requirements.txt ]; then pip install --no-cache-dir -r requirements.txt; fi
-
-CMD ["python", "server.py"]

sonar-project.properties

@@ -1,11 +1,11 @@
-sonar.projectKey=Aignosi_sientia-dataops-laborious_temporal_beaec423-6c42-4f26-8134-b676287b499d
-sonar.projectName=sientia-dataops-laborious_temporal
+sonar.projectKey=Aignosi_sientia-dataops-laborious_temporal_ca1a7039-6db9-49e5-be78-54d29bc93e4f
+sonar.projectName=sientia-dataops-laborious-legacy
 sonar.sources=laborious
 sonar.tests=tests
 sonar.projectVersion=1.0.0
-sonar.coverage.exclusions=laborious/worker/worker.py
+sonar.coverage.exclusions=laborious/worker/*
 sonar.qualitygate.wait=true
 sonar.qualitygate.timeout=300
 sonar.python.coverage.reportPaths=coverage.xml
 sonar.python.xunit.reportPath=pytest.xml
-sonar.python.version=3.11
+sonar.python.version=3.11

tests/conftest.py

@@ -0,0 +1,59 @@
+import os
+import sys
+from unittest.mock import MagicMock
+
+# The production code converts SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES to int at import-time.
+# Tests must set it to a valid integer string to avoid import errors.
+os.environ.setdefault('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1')
+
+
+class DummyMinioDataFramePayload:
+    """
+    Minimal payload double used by unit tests.
+
+    The production workflow/gates expect a MinioDataFramePayload-like object with:
+    - async retrieve(minio_repo, workflow_metadata) -> DataFrame | dict
+    - has_data() -> bool
+    - cleanup_prefix() -> str | None
+    - last_timestamp: attribute
+    - status: attribute
+    """
+
+    def __init__(
+        self,
+        *,
+        retrieve_return=None,
+        has_data: bool = True,
+        cleanup_prefix: str | None = None,
+        last_timestamp: str = '2024-01-01',
+        status: dict | None = None,
+    ):
+        self._retrieve_return = retrieve_return
+        self._has_data = has_data
+        self._cleanup_prefix = cleanup_prefix
+        self.last_timestamp = last_timestamp
+        self.status = status
+
+    async def retrieve(self, _minio_repo, _workflow_metadata=None):
+        return self._retrieve_return
+
+    def has_data(self) -> bool:
+        return self._has_data
+
+    def cleanup_prefix(self) -> str | None:
+        return self._cleanup_prefix
+
+
+"""
+Pytest configuration file with global mocks for external dependencies.
+
+This module mocks the 'sientia' module to avoid requiring its installation
+during unit tests. The mock is registered in sys.modules before any test
+imports are executed.
+"""
+
+# Mock sientia module
+sientia_mock = MagicMock()
+sientia_mock.ModelAnalysis = MagicMock
+sys.modules['sientia'] = sientia_mock
+sys.modules['sientia.ModelAnalysis'] = MagicMock()

tests/fixtures/bundle/README.md

@@ -0,0 +1,86 @@
+# Golden `.sientia` fixtures
+
+Binary conformance vectors for `laborious/utils/bundle/`. They are **bytes the producer's own code
+emitted**, not bytes this repository knows how to write: there is no production writer here (change
+`import-model-workflow`, design D1). If the producer's layout moves, the reader's tests fail against
+these files instead of failing in production.
+
+| File | What it is |
+|---|---|
+| `golden_model_v3.sientia` | `format_version` 1 bundle, 110767 bytes, plaintext zip 110569 bytes — **two chunks**, so `MESSAGE`/`FINAL` framing is exercised by the golden fixture itself |
+| `golden_model_v3.metadata.json` | the `metadata.json` that went into that bundle, in plaintext, so a test can assert the recovered value without decrypting first |
+| `regenerate.py` | the script that produced them, importing the producer's `bundle.py`/`crypto.py` by path |
+
+## Password
+
+```
+Import1Model!Fixture
+```
+
+Recorded deliberately: the fixture protects nothing. It satisfies the producer's export-side
+strength rule (≥ 12 characters, upper, lower, digit, symbol), which is checked by
+`crypto.validate_password_strength` at generation time.
+
+## Provenance
+
+- Producer repository: `sientia-projects-templates`
+- Producer commit: `8d3c1fd117c697e8b352c34f11c732809a4a129b`
+  (`8d3c1fd QTZPOC-7: adapt to lint rules picked up from merging main`)
+- Producer modules used: `app/src/operations/model_export/bundle.py`,
+  `app/src/operations/model_export/crypto.py` — loaded by path, so every constant, the `struct`
+  layout, the zip layout, the KDF parameters and the chunk framing come from the producer and none of
+  them is re-implemented here
+- Producer pin honoured: `pynacl==1.5.0` (this repository pins the same version)
+
+One deviation from task 0.3, recorded rather than hidden: the bytes were generated with the
+producer's **code** under *this* repository's venv, not inside the producer's own venv, because that
+venv has neither `pynacl` nor `mlflow` installed. The dependency that matters for the bytes is
+`pynacl`, pinned identically on both sides, and `mlflow` is not reached — `regenerate.py` builds the
+artifact tree itself instead of calling `client.download_artifacts`.
+
+## Contents of the bundle
+
+```
+metadata.json                          (written first, no indent, by bundle.build_zip)
+artifacts/prediction_model/MLmodel
+artifacts/prediction_model/model.pkl            (90 KiB of incompressible filler)
+artifacts/prediction_model/conda.yaml
+artifacts/prediction_model/python_env.yaml
+artifacts/prediction_model/requirements.txt
+artifacts/data_model/MLmodel
+artifacts/data_model/model.pkl                  (8 KiB)
+artifacts/data_model/conda.yaml
+artifacts/data_model/python_env.yaml
+artifacts/data_model/requirements.txt
+artifacts/model_card/model_card.json
+artifacts/train_data.csv
+artifacts/test_data.csv
+artifacts/raw_data.csv
+```
+
+No directory entries are stored — the producer only adds paths for which `is_file()` is true.
+
+`parameters` carries `target_variable: "SE"`, which gate 6 requires and which becomes
+`model_config.target` in the MongoDB model document.
+
+## Digest boundary — the reason this fixture exists
+
+The header's `ciphertext_digest` is `sha256(file[164:])`: the producer computes
+`hashlib.sha256(ciphertext)` over the joined chunks and only then writes
+`header + stream_header + ciphertext`, so the 24-byte `secretstream` header is **not** covered.
+Verified against this fixture and against a freshly produced bundle. Earlier prose (this change's own
+first draft, and `aig-docs/03-sientia-bundle-format.md`) said "everything after the plaintext
+header", i.e. from offset 140 — that is wrong, and `tests/laborious/utils/bundle/test_format.py`
+asserts both directions so it cannot come back.
+
+## Regenerating
+
+```bash
+source ./venv/bin/activate
+PRODUCER_REPO=../sientia-projects-templates python tests/fixtures/bundle/regenerate.py
+```
+
+Two fields change on every run and are expected to: the 16-byte salt (`os.urandom`) and
+`metadata.metadata.export_timestamp` (`datetime.now(UTC)`). Everything else is deterministic —
+the filler bytes come from a seeded PRNG. After regenerating, update the producer commit above and
+the byte counts in the table.

tests/fixtures/bundle/golden_model_v3.metadata.json

@@ -0,0 +1,24 @@
+{
+  "parameters": {
+    "model_name": "fixture_model",
+    "target_variable": "SE",
+    "input_variables": "['feature_a', 'feature_b']",
+    "models_params": "{'alpha': 0.1}",
+    "nan_treatment": "drop",
+    "scaler_name": "StandardScaler",
+    "train_size": "0.8",
+    "test_size": "0.2",
+    "include_ar": "False",
+    "retrain": "False",
+    "start_date": "2026-01-01 00:00:00",
+    "end_date": "2026-08-01 00:00:00"
+  },
+  "metadata": {
+    "model_name": "fixture_model",
+    "experiment_name": "fixture_experiment",
+    "run_id": "8f14e45fceea167a5a36dedd4bea2543",
+    "model_version": "3",
+    "model_project": "fixture_project",
+    "export_timestamp": "2026-08-25T12:08:59.533441+00:00"
+  }
+}

tests/fixtures/bundle/regenerate.py

@@ -0,0 +1,185 @@
+"""Regenerate the golden `.sientia` fixtures using the producer's own source.
+
+This script is the auditable half of the anti-drift contract described in the change
+`import-model-workflow` (design D1): the committed fixtures are bytes produced by
+`sientia-projects-templates/app/src/operations/model_export/{bundle,crypto}.py`, loaded here by
+path so no constant is re-implemented. Regenerating is a deliberate act — see the README next to
+this file for the recorded producer commit and the fixture password.
+
+Usage (from the repository root, inside the venv):
+
+    PRODUCER_REPO=../sientia-projects-templates ./venv/bin/python tests/fixtures/bundle/regenerate.py
+
+The producer's own venv is not used: it does not have `pynacl` installed, while this repository
+pins the producer's exact version (`pynacl==1.5.0`). The bytes are therefore produced by the
+producer's code on the producer's dependency pin, which is what the fixture is for.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import random
+import shutil
+import subprocess
+import sys
+import tempfile
+import types
+from pathlib import Path
+
+FIXTURE_DIR = Path(__file__).resolve().parent
+DEFAULT_PRODUCER_REPO = FIXTURE_DIR.parents[2].parent / 'sientia-projects-templates'
+
+# Recorded in the README alongside the fixtures; the password satisfies the producer's own
+# export-side strength rule (>= 12 chars, upper, lower, digit, symbol).
+FIXTURE_PASSWORD = 'Import1Model!Fixture'
+
+# Deterministic filler so a regeneration differs only in the salt and the export timestamp.
+FILLER_SEED = 20260825
+
+
+def _load_producer_module(repo: Path, name: str) -> types.ModuleType:
+    """Import one producer module by path, without importing the Streamlit app."""
+    path = repo / 'app' / 'src' / 'operations' / 'model_export' / f'{name}.py'
+    if not path.is_file():
+        raise SystemExit(f'producer module not found: {path}')
+    spec = importlib.util.spec_from_file_location(f'producer_{name}', path)
+    if spec is None or spec.loader is None:
+        raise SystemExit(f'cannot load producer module: {path}')
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _producer_commit(repo: Path) -> str:
+    result = subprocess.run(  # noqa: S603 - fixed argv, no shell
+        ['git', '-C', str(repo), 'rev-parse', 'HEAD'],  # noqa: S607 - git from PATH is intended
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    return result.stdout.strip() or 'unknown'
+
+
+def _write_model_dir(root: Path, flavor: str, pickle_bytes: bytes) -> None:
+    """Write the files an MLflow 2.x model directory carries for this platform."""
+    root.mkdir(parents=True, exist_ok=True)
+    (root / 'MLmodel').write_text(
+        f'artifact_path: {flavor}\n'
+        'flavors:\n'
+        '  python_function:\n'
+        '    loader_module: mlflow.sklearn\n'
+        '    model_path: model.pkl\n'
+        '  sklearn:\n'
+        '    pickled_model: model.pkl\n'
+        '    serialization_format: cloudpickle\n'
+        '    sklearn_version: 1.4.2\n'
+        'mlflow_version: 2.10.1\n'
+        'model_uuid: 0f9c1c1f9c1f4c0f9c1c1f9c1f4c0f9c\n'
+        'utc_time_created: "2026-08-25 12:00:00.000000"\n',
+        encoding='utf-8',
+    )
+    (root / 'model.pkl').write_bytes(pickle_bytes)
+    (root / 'conda.yaml').write_text(
+        'channels:\n- conda-forge\ndependencies:\n- python=3.11\n- pip\n- pip:\n'
+        '  - mlflow==2.10.1\n  - scikit-learn==1.4.2\nname: mlflow-env\n',
+        encoding='utf-8',
+    )
+    (root / 'python_env.yaml').write_text(
+        'python: 3.11.15\nbuild_dependencies:\n- pip==24.0\ndependencies:\n- -r requirements.txt\n',
+        encoding='utf-8',
+    )
+    (root / 'requirements.txt').write_text(
+        'mlflow==2.10.1\nscikit-learn==1.4.2\ncloudpickle==3.0.0\n', encoding='utf-8'
+    )
+
+
+def _write_csv(path: Path, rows: int, rng: random.Random) -> None:
+    lines = ['datetime,feature_a,feature_b,SE']
+    for i in range(rows):
+        lines.append(
+            f'2026-08-{(i % 27) + 1:02d}T00:00:00,'
+            f'{rng.uniform(0, 100):.6f},{rng.uniform(0, 100):.6f},{rng.uniform(0, 100):.6f}'
+        )
+    path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
+
+
+def _build_artifact_tree(root: Path) -> None:
+    rng = random.Random(FILLER_SEED)  # noqa: S311 - test filler, not key material
+    # Incompressible filler keeps the zip above the 64 KiB chunk size, so the golden fixture
+    # exercises multi-chunk `MESSAGE`/`FINAL` framing with real producer bytes.
+    big_pickle = bytes(rng.getrandbits(8) for _ in range(90 * 1024))
+    small_pickle = bytes(rng.getrandbits(8) for _ in range(8 * 1024))
+    _write_model_dir(root / 'prediction_model', 'prediction_model', big_pickle)
+    _write_model_dir(root / 'data_model', 'data_model', small_pickle)
+    (root / 'model_card').mkdir(parents=True, exist_ok=True)
+    (root / 'model_card' / 'model_card.json').write_text(
+        json.dumps(
+            {
+                'model_name': 'fixture_model',
+                'target': 'SE',
+                'created_by': 'fixture',
+                'creation_method': 'time_series_template',
+            }
+        ),
+        encoding='utf-8',
+    )
+    _write_csv(root / 'train_data.csv', 120, rng)
+    _write_csv(root / 'test_data.csv', 40, rng)
+    _write_csv(root / 'raw_data.csv', 200, rng)
+
+
+RUN_PARAMETERS = {
+    'model_name': 'fixture_model',
+    'target_variable': 'SE',
+    'input_variables': "['feature_a', 'feature_b']",
+    'models_params': "{'alpha': 0.1}",
+    'nan_treatment': 'drop',
+    'scaler_name': 'StandardScaler',
+    'train_size': '0.8',
+    'test_size': '0.2',
+    'include_ar': 'False',
+    'retrain': 'False',
+    'start_date': '2026-01-01 00:00:00',
+    'end_date': '2026-08-01 00:00:00',
+}
+
+METADATA_FIELDS = {
+    'model_name': 'fixture_model',
+    'experiment_name': 'fixture_experiment',
+    'run_id': '8f14e45fceea167a5a36dedd4bea2543',
+    'model_version': '3',
+    'model_project': 'fixture_project',
+}
+
+
+def main() -> int:
+    repo = Path(os.getenv('PRODUCER_REPO', str(DEFAULT_PRODUCER_REPO))).resolve()
+    bundle = _load_producer_module(repo, 'bundle')
+    crypto = _load_producer_module(repo, 'crypto')
+
+    crypto.validate_password_strength(FIXTURE_PASSWORD)
+
+    staging = Path(tempfile.mkdtemp(prefix='sientia_fixture_'))
+    try:
+        _build_artifact_tree(staging)
+        metadata = bundle.build_metadata_json(run_parameters=RUN_PARAMETERS, **METADATA_FIELDS)
+        zip_bytes = bundle.build_zip(staging, metadata)
+        sientia_bytes = crypto.encrypt_archive(zip_bytes, FIXTURE_PASSWORD)
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)
+
+    target = FIXTURE_DIR / 'golden_model_v3.sientia'
+    target.write_bytes(sientia_bytes)
+    (FIXTURE_DIR / 'golden_model_v3.metadata.json').write_text(
+        json.dumps(metadata, indent=2) + '\n', encoding='utf-8'
+    )
+    print(f'wrote {target} ({len(sientia_bytes)} bytes, zip {len(zip_bytes)} bytes)')
+    print(f'producer commit: {_producer_commit(repo)}')
+    print(f'password: {FIXTURE_PASSWORD}')
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tests/helpers/bundle_factory.py

@@ -0,0 +1,295 @@
+"""Test-only `.sientia` writer, mirrored field for field from the producer.
+
+There is no production writer in this repository (change `import-model-workflow`, design D1): the
+producer is the Streamlit app in `sientia-projects-templates`. This factory exists for one reason —
+building the **hostile** variants the gate tests need, which the producer cannot emit. Its output is
+pinned against the golden fixture by `test_format.py::test_factory_header_matches_fixture_layout`,
+so a drift in either direction shows up as a failing test.
+
+Mirrored from:
+- `app/src/operations/model_export/crypto.py` — header layout, KDF, `secretstream` framing
+- `app/src/operations/model_export/bundle.py` — zip layout and the two-top-level-key JSON
+
+Deliberate deviation: the default KDF cost is a fraction of the producer's (8 MiB / 1 pass instead
+of 256 MiB / 3), because the reader honours whatever the header declares and a test suite that
+derives thirty 256 MiB Argon2id keys is a test suite nobody runs. The golden fixture is what covers
+the production cost parameters.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import io
+import json
+import os
+import stat
+import struct
+import unicodedata
+import zipfile
+from dataclasses import dataclass, field
+from typing import Any
+
+import nacl.bindings as sodium
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+# --- mirrored from the producer's crypto.py ---
+MAGIC = b'SIENTIA1'
+FORMAT_VERSION = 1
+AEAD_ID = 1
+KDF_ID = 1
+SALT_SIZE = 16
+RESERVED_SIGNATURE_SIZE = 64
+CHUNK_SIZE = 64 * 1024
+HEADER_STRUCT = struct.Struct('>8sBBB16sQQB32s64s')
+HEADER_SIZE = HEADER_STRUCT.size
+PRODUCER_KDF_MEMLIMIT_BYTES = 256 * 1024 * 1024
+PRODUCER_KDF_OPSLIMIT = 3
+
+# Cheap-but-valid cost parameters for the suite; see the module docstring.
+TEST_KDF_MEMLIMIT_BYTES = 8 * 1024 * 1024
+TEST_KDF_OPSLIMIT = 1
+
+DEFAULT_PASSWORD = 'Factory1Model!Test'
+DEFAULT_ENVELOPE_KEY = bytes(range(32))
+
+MODEL_DIRS = ('prediction_model', 'data_model')
+MODEL_FILES = ('MLmodel', 'model.pkl', 'conda.yaml', 'python_env.yaml', 'requirements.txt')
+CSV_FILES = ('train_data.csv', 'test_data.csv', 'raw_data.csv')
+
+DEFAULT_METADATA = {
+    'model_name': 'factory_model',
+    'experiment_name': 'factory_experiment',
+    'run_id': 'aa11bb22cc33dd44ee55ff6677889900',
+    'model_version': '2',
+    'model_project': 'factory_project',
+    'export_timestamp': '2026-08-25T12:00:00+00:00',
+}
+
+DEFAULT_PARAMETERS = {
+    'model_name': 'factory_model',
+    'target_variable': 'SE',
+    'input_variables': "['feature_a', 'feature_b']",
+    'retrain': 'False',
+}
+
+
+def default_tree() -> dict[str, bytes]:
+    """The artifact tree a well-formed bundle carries, as archive-name -> bytes."""
+    files: dict[str, bytes] = {}
+    for directory in MODEL_DIRS:
+        for name in MODEL_FILES:
+            files[f'artifacts/{directory}/{name}'] = f'{directory}/{name}\n'.encode()
+    files['artifacts/model_card/model_card.json'] = json.dumps({'target': 'SE'}).encode()
+    for name in CSV_FILES:
+        files[f'artifacts/{name}'] = b'datetime,feature_a,SE\n2026-08-25T00:00:00,1.0,2.0\n'
+    return files
+
+
+def build_metadata_document(
+    *,
+    metadata: dict[str, Any] | None = None,
+    parameters: dict[str, Any] | None = None,
+    drop_metadata_field: str | None = None,
+    drop_top_level_key: str | None = None,
+    drop_parameter: str | None = None,
+) -> dict[str, Any]:
+    """Build `metadata.json`, with hooks for the malformed variants gate 6 must reject."""
+    meta = {**DEFAULT_METADATA, **(metadata or {})}
+    params = {**DEFAULT_PARAMETERS, **(parameters or {})}
+    if drop_metadata_field:
+        meta.pop(drop_metadata_field, None)
+    if drop_parameter:
+        params.pop(drop_parameter, None)
+    document: dict[str, Any] = {'parameters': params, 'metadata': meta}
+    if drop_top_level_key:
+        document.pop(drop_top_level_key, None)
+    return document
+
+
+@dataclass
+class ZipSpec:
+    """How to lay out the zip inside the bundle."""
+
+    files: dict[str, bytes] = field(default_factory=default_tree)
+    document: dict[str, Any] | None = None
+    include_metadata: bool = True
+    symlinks: dict[str, str] = field(default_factory=dict)
+    absolute_entry: str | None = None
+    traversal_entry: str | None = None
+    bomb_bytes: int = 0
+    extra_files: dict[str, bytes] = field(default_factory=dict)
+    # name -> a raw `stat.S_IF*` mode, for an entry that is neither a symlink, a regular file nor a
+    # directory (e.g. `stat.S_IFIFO`) — gate 4 must reject these on their file-type bits alone.
+    special_mode_entries: dict[str, int] = field(default_factory=dict)
+    # Directory entries the producer never writes (`build_zip` stores none), but a gate 5 defense
+    # this repository still carries has to be exercised by something.
+    directory_entries: list[str] = field(default_factory=list)
+    # A file name already in `files`/`extra_files`, written a second time under the same name — two
+    # archive entries claiming the same path, which only gate 5's extraction can catch.
+    repeat_file: str | None = None
+
+
+def build_zip(spec: ZipSpec | None = None) -> bytes:
+    """Zip `artifacts/` plus `metadata.json`, mirroring the producer's `build_zip`.
+
+    `metadata.json` is written first with `json.dumps` and no indent, entries use `/` separators,
+    and no directory entries are stored — all three matter to the reader.
+    """
+    spec = spec or ZipSpec()
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as archive:
+        if spec.include_metadata:
+            document = spec.document if spec.document is not None else build_metadata_document()
+            archive.writestr('metadata.json', json.dumps(document))
+        for name, content in {**spec.files, **spec.extra_files}.items():
+            archive.writestr(_regular_file_info(name), content)
+        for name, target in spec.symlinks.items():
+            info = zipfile.ZipInfo(name)
+            info.create_system = 3
+            info.external_attr = (0o120777 << 16) | 0o200000
+            archive.writestr(info, target)
+        if spec.absolute_entry:
+            archive.writestr(_regular_file_info(spec.absolute_entry), b'absolute\n')
+        if spec.traversal_entry:
+            archive.writestr(_regular_file_info(spec.traversal_entry), b'traversal\n')
+        if spec.bomb_bytes:
+            archive.writestr(_regular_file_info('artifacts/bomb.csv'), b'\0' * spec.bomb_bytes)
+        for name, mode in spec.special_mode_entries.items():
+            info = zipfile.ZipInfo(name)
+            info.create_system = 3
+            info.external_attr = mode << 16
+            archive.writestr(info, b'')
+        for name in spec.directory_entries:
+            info = zipfile.ZipInfo(name if name.endswith('/') else f'{name}/')
+            info.create_system = 3
+            info.external_attr = (stat.S_IFDIR | 0o755) << 16
+            archive.writestr(info, b'')
+        if spec.repeat_file:
+            content = {**spec.files, **spec.extra_files}[spec.repeat_file]
+            archive.writestr(_regular_file_info(spec.repeat_file), content)
+    return buffer.getvalue()
+
+
+def _regular_file_info(name: str) -> zipfile.ZipInfo:
+    """A zip entry carrying a Unix regular-file mode, as `ZipFile.write` would store it."""
+    info = zipfile.ZipInfo(name)
+    info.create_system = 3
+    info.compress_type = zipfile.ZIP_DEFLATED
+    info.external_attr = 0o100644 << 16
+    return info
+
+
+def derive_key(password: str, salt: bytes, opslimit: int, memlimit: int) -> bytes:
+    """Argon2id over the NFC-normalised password — the producer's `_derive_key`."""
+    normalized = unicodedata.normalize('NFC', password)
+    return sodium.crypto_pwhash_alg(
+        sodium.crypto_secretstream_xchacha20poly1305_KEYBYTES,
+        normalized.encode('utf-8'),
+        salt,
+        opslimit,
+        memlimit,
+        sodium.crypto_pwhash_ALG_ARGON2ID13,
+    )
+
+
+def encrypt_archive(  # noqa: PLR0913 - every knob here exists to build one hostile variant
+    plaintext: bytes,
+    password: str = DEFAULT_PASSWORD,
+    *,
+    format_version: int = FORMAT_VERSION,
+    aead_id: int = AEAD_ID,
+    kdf_id: int = KDF_ID,
+    memlimit: int = TEST_KDF_MEMLIMIT_BYTES,
+    opslimit: int = TEST_KDF_OPSLIMIT,
+    declared_memlimit: int | None = None,
+    declared_opslimit: int | None = None,
+    parallelism: int = 1,
+    reserved_signature: bytes | None = None,
+    magic: bytes = MAGIC,
+    digest_override: bytes | None = None,
+    salt: bytes | None = None,
+    drop_final_tag: bool = False,
+) -> bytes:
+    """Produce a full `.sientia` byte stream, mirroring the producer's `encrypt_archive`.
+
+    The `declared_*` arguments write cost parameters into the header that differ from the ones
+    actually used, which is how a "512 MiB header" is built without allocating 512 MiB.
+    `drop_final_tag` pushes every chunk as `MESSAGE`, so the stream is framed as if more were
+    coming — the truncation the digest cannot see.
+    """
+    salt = salt if salt is not None else os.urandom(SALT_SIZE)
+    key = derive_key(password, salt, opslimit, memlimit)
+
+    state = sodium.crypto_secretstream_xchacha20poly1305_state()
+    stream_header = sodium.crypto_secretstream_xchacha20poly1305_init_push(state, key)
+
+    chunks = []
+    offset = 0
+    total = len(plaintext)
+    while offset < total:
+        chunk = plaintext[offset : offset + CHUNK_SIZE]
+        offset += len(chunk)
+        is_last = offset >= total and not drop_final_tag
+        tag = (
+            sodium.crypto_secretstream_xchacha20poly1305_TAG_FINAL
+            if is_last
+            else sodium.crypto_secretstream_xchacha20poly1305_TAG_MESSAGE
+        )
+        chunks.append(sodium.crypto_secretstream_xchacha20poly1305_push(state, chunk, None, tag))
+    ciphertext = b''.join(chunks)
+
+    digest = digest_override if digest_override is not None else hashlib.sha256(ciphertext).digest()
+    header = HEADER_STRUCT.pack(
+        magic,
+        format_version,
+        aead_id,
+        kdf_id,
+        salt,
+        declared_memlimit if declared_memlimit is not None else memlimit,
+        declared_opslimit if declared_opslimit is not None else opslimit,
+        parallelism,
+        digest,
+        reserved_signature if reserved_signature is not None else bytes(RESERVED_SIGNATURE_SIZE),
+    )
+    return header + stream_header + ciphertext
+
+
+def make_bundle(
+    zip_spec: ZipSpec | None = None,
+    password: str = DEFAULT_PASSWORD,
+    **header_kwargs: Any,
+) -> bytes:
+    """The common case: a well-formed tree, encrypted with the test cost parameters."""
+    return encrypt_archive(build_zip(zip_spec), password, **header_kwargs)
+
+
+def tamper_ciphertext(bundle: bytes, offset_from_stream: int = 8) -> bytes:
+    """Flip one byte inside the chunk stream **and** repair the header digest.
+
+    Repairing the digest is what makes this a gate-3 test rather than a gate-2 one: the file is
+    self-consistent and only the AEAD can tell it was altered.
+    """
+    position = HEADER_SIZE + 24 + offset_from_stream
+    altered = bytearray(bundle)
+    altered[position] ^= 0xFF
+    repaired = hashlib.sha256(bytes(altered[HEADER_SIZE + 24 :])).digest()
+    altered[44:76] = repaired
+    return bytes(altered)
+
+
+def truncate_bundle(bundle: bytes, drop_bytes: int = 32) -> bytes:
+    """Cut the tail off a bundle, leaving the header's digest describing bytes that are gone."""
+    return bundle[:-drop_bytes]
+
+
+def build_envelope(password: str = DEFAULT_PASSWORD, key: bytes = DEFAULT_ENVELOPE_KEY) -> str:
+    """`base64(nonce || ciphertext || tag)` under AES-256-GCM, as the frontend will send it."""
+    nonce = os.urandom(12)
+    sealed = AESGCM(key).encrypt(nonce, password.encode('utf-8'), None)
+    return base64.b64encode(nonce + sealed).decode('ascii')
+
+
+def envelope_key_b64(key: bytes = DEFAULT_ENVELOPE_KEY) -> str:
+    """The key as `IMPORT_PASSWORD_KEY` carries it."""
+    return base64.b64encode(key).decode('ascii')

tests/helpers/status_engine.py

@@ -0,0 +1,174 @@
+"""A recording stand-in for the import log's SQLAlchemy engine.
+
+The import's status writes are the part of the feature with the strictest contract: which table they
+name, which predicate they carry, which columns they touch, how many transactions they open and
+which of them may fail without failing the import. Asserting that against a mock whose `execute`
+returns a `MagicMock` proves nothing — the assertions need the statement text and the parameters.
+
+So this double records every statement it is handed, answers reads from a queue the test fills, and
+lets a test make a write fail on a chosen attempt. It deliberately understands nothing about SQL
+beyond the first word of the statement.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class RecordedStatement:
+    """One statement as the engine saw it: its normalised text and its bound parameters."""
+
+    def __init__(self, statement: Any, parameters: dict[str, Any] | None) -> None:
+        self.sql = ' '.join(str(statement).split())
+        self.parameters: dict[str, Any] = dict(parameters or {})
+
+    @property
+    def verb(self) -> str:
+        return self.sql.split(' ', 1)[0].upper()
+
+    @property
+    def is_write(self) -> bool:
+        return self.verb in ('UPDATE', 'INSERT', 'DELETE')
+
+    @property
+    def table(self) -> str:
+        """The relation the statement names, for the "one table only" assertions."""
+        words = self.sql.split()
+        if self.verb == 'UPDATE':
+            return words[1]
+        for index, word in enumerate(words):
+            if word.upper() in ('FROM', 'INTO') and index + 1 < len(words):
+                return words[index + 1]
+        return ''
+
+    @property
+    def assignments(self) -> set[str]:
+        """The column names a `SET` clause assigns, so a test can assert what is never written."""
+        upper = self.sql.upper()
+        if ' SET ' not in upper:
+            return set()
+        body = self.sql[upper.index(' SET ') + 5 :]
+        where = body.upper().find(' WHERE ')
+        if where != -1:
+            body = body[:where]
+        return {part.split('=')[0].strip() for part in body.split(',') if '=' in part}
+
+    def __repr__(self) -> str:  # pragma: no cover - debugging aid
+        return f'RecordedStatement({self.sql!r}, {self.parameters!r})'
+
+
+class FakeResult:
+    """What `Connection.execute` returns: a rowcount for writes, mappings for reads."""
+
+    def __init__(self, rowcount: int = 1, rows: list[dict[str, Any]] | None = None) -> None:
+        self.rowcount = rowcount
+        self._rows = rows or []
+
+    def mappings(self) -> FakeResult:
+        return self
+
+    def first(self) -> dict[str, Any] | None:
+        return self._rows[0] if self._rows else None
+
+    def all(self) -> list[dict[str, Any]]:
+        return list(self._rows)
+
+
+class FakeConnection:
+    """A connection that records and then answers from the engine's queues."""
+
+    def __init__(self, engine: FakeStatusEngine) -> None:
+        self._engine = engine
+
+    def execute(self, statement: Any, parameters: dict[str, Any] | None = None) -> FakeResult:
+        recorded = RecordedStatement(statement, parameters)
+        self._engine.statements.append(recorded)
+        if recorded.is_write:
+            return self._engine._answer_write()
+        return self._engine._answer_read()
+
+
+class _Scope:
+    """The context manager `engine.begin()` and `engine.connect()` both hand out."""
+
+    def __init__(self, engine: FakeStatusEngine, transactional: bool) -> None:
+        self._engine = engine
+        self._transactional = transactional
+
+    def __enter__(self) -> FakeConnection:
+        self._engine.open_scopes += 1
+        if self._transactional:
+            self._engine.transactions += 1
+        else:
+            self._engine.connections += 1
+        return FakeConnection(self._engine)
+
+    def __exit__(self, *_exc: object) -> bool:
+        self._engine.open_scopes -= 1
+        return False
+
+
+class FakeStatusEngine:
+    """An engine double that records statements and can be told to fail a write.
+
+    Attributes:
+        statements: every statement executed, in order.
+        rowcounts: rowcounts for writes, consumed in order; the last one repeats.
+        rows: rows for reads, consumed in order; the last one repeats. `None` means "no such row".
+        write_errors: exceptions to raise instead of executing a write, consumed in order. A `None`
+            entry means "this attempt succeeds", which is how a retry is modelled.
+        transactions: how many `begin()` scopes were opened — one per write, or the writes are
+            sharing a transaction.
+        open_scopes: scopes currently open, so a test can assert none is left open.
+    """
+
+    def __init__(
+        self,
+        rowcounts: list[int] | None = None,
+        rows: list[dict[str, Any] | None] | None = None,
+        write_errors: list[BaseException | None] | None = None,
+    ) -> None:
+        self.statements: list[RecordedStatement] = []
+        self.rowcounts: list[int] = list(rowcounts or [])
+        self.rows: list[dict[str, Any] | None] = list(rows or [])
+        self.write_errors: list[BaseException | None] = list(write_errors or [])
+        self.transactions = 0
+        self.connections = 0
+        self.open_scopes = 0
+        self.disposed = False
+
+    # ------------------------------------------------------------------ engine API
+    def begin(self) -> _Scope:
+        return _Scope(self, transactional=True)
+
+    def connect(self) -> _Scope:
+        return _Scope(self, transactional=False)
+
+    def dispose(self) -> None:
+        self.disposed = True
+
+    # ------------------------------------------------------------------- answering
+    def _answer_write(self) -> FakeResult:
+        if self.write_errors:
+            error = self.write_errors.pop(0)
+            if error is not None:
+                raise error
+        rowcount = self.rowcounts.pop(0) if len(self.rowcounts) > 1 else (self.rowcounts or [1])[0]
+        return FakeResult(rowcount=rowcount)
+
+    def _answer_read(self) -> FakeResult:
+        row = self.rows.pop(0) if len(self.rows) > 1 else (self.rows or [None])[0]
+        return FakeResult(rows=[row] if row is not None else [])
+
+    # ---------------------------------------------------------------- convenience
+    @property
+    def writes(self) -> list[RecordedStatement]:
+        return [statement for statement in self.statements if statement.is_write]
+
+    @property
+    def reads(self) -> list[RecordedStatement]:
+        return [statement for statement in self.statements if not statement.is_write]
+
+    @property
+    def tables(self) -> set[str]:
+        return {statement.table for statement in self.statements if statement.table}

tests/laborious/activities/test_activities.py

@@ -1,18 +1,36 @@
+from unittest.mock import ANY, AsyncMock, MagicMock, patch
+
 from pytest import mark
-from unittest.mock import patch, MagicMock, ANY
-from sientia_do.temporal.activities.postgres import Postgres
+
 from laborious.activities.activities import Activities
-from laborious.activities.mlflow import MLFlow
+from laborious.activities.api import API
 from laborious.activities.gates import Gates
+from laborious.activities.mlflow import MLFlow
+from laborious.activities.model_metrics import ModelMetrics
 from laborious.activities.opc import OPC
+from laborious.activities.storage import Storage
 
 
-@patch('laborious.activities.activities.Postgres.__init__')
+@patch('laborious.activities.activities.Storage.__init__')
 @patch('laborious.activities.activities.MLFlow.__init__')
 @patch('laborious.activities.activities.OPC.__init__')
 @patch('laborious.activities.activities.Gates.__init__')
-def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgres_init):
-
+@patch('laborious.activities.activities.ModelMetrics.__init__')
+@patch('laborious.activities.activities.API.__init__')
+@patch('laborious.activities.activities.MLFlowRepository')
+@patch('laborious.activities.activities.MinioRepository')
+@patch('laborious.activities.activities.MetricsController')
+def test___init__(
+    mock_metrics_controller,
+    mock_minio_repository,
+    mock_mlflow_repository,
+    mock_api_init,
+    mock_model_metrics_init,
+    mock_gates_init,
+    mock_opc_init,
+    mock_mlflow_init,
+    mock_storage_init,
+):
     postgres_config = {
         'host': 'localhost',
         'port': 5432,
@@ -20,20 +38,30 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre
         'password': 'postgres',
         'dbname': 'postgres',
         'min_connections': 1,
-        'max_connections': 10
+        'max_connections': 10,
     }
 
-    mlflow_config = {
-        'host': 'localhost',
-        'port': 5000,
-        'username': 'mlflow',
-        'password': 'mlflow'
+    minio_config = {
+        'endpoint_url': 'localhost:9000',
+        'access_key': 'minio',
+        'secret_key': 'minio123',
+        'default_bucket': 'test',
+        'retention_hours': 24,
+        'secure': False,
     }
 
+    mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'}
+
     opc_config = {
         'bootstrap_servers': 'localhost:9092',
         'polling_time': 1000,
-        'group_id': 'test-group'
+        'group_id': 'test-group',
+    }
+
+    pi_web_api_config = {
+        'base_url': 'https://test-pi-server.com',
+        'auth_type': 'bearer',
+        'auth_token': 'test_token',
     }
 
     logger = MagicMock()
@@ -42,18 +70,22 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre
     activities = Activities(
         postgres_config=postgres_config,
         mlflow_config=mlflow_config,
+        minio_config=minio_config,
         opc_config=opc_config,
+        pi_web_api_config=pi_web_api_config,
         logger=logger,
-        notification_handler=notification_handler
+        notification_handler=notification_handler,
     )
 
     assert isinstance(activities, Activities)
-    assert isinstance(activities, Postgres)
+    assert isinstance(activities, Storage)
     assert isinstance(activities, MLFlow)
     assert isinstance(activities, OPC)
     assert isinstance(activities, Gates)
+    assert isinstance(activities, ModelMetrics)
+    assert isinstance(activities, API)
 
-    mock_postgres_init.assert_called_once_with(
+    mock_storage_init.assert_called_once_with(
         ANY,
         host=postgres_config['host'],
         port=postgres_config['port'],
@@ -62,40 +94,251 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre
         dbname=postgres_config['dbname'],
         min_connections=postgres_config['min_connections'],
         max_connections=postgres_config['max_connections'],
+        retention_hours=minio_config['retention_hours'],
+        minio_repository=mock_minio_repository.return_value,
         logger=logger,
-        notification_handler=notification_handler
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
     )
 
     mock_mlflow_init.assert_called_once_with(
         ANY,
-        mlflow_host=mlflow_config['host'],
-        mlflow_port=mlflow_config['port'],
-        mlflow_username=mlflow_config['username'],
-        mlflow_password=mlflow_config['password'],
+        mlflow_repository=mock_mlflow_repository.return_value,
+        minio_repository=mock_minio_repository.return_value,
         logger=logger,
-        notification_handler=notification_handler
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_mlflow_repository.assert_called_once_with(
+        f'{mlflow_config["host"]}:{mlflow_config["port"]}',
+        mlflow_config['username'],
+        mlflow_config['password'],
+        logger,
+        notification_handler,
+        mock_metrics_controller.return_value,
     )
 
     mock_opc_init.assert_called_once_with(
         ANY,
         opc_servers=opc_config,
         logger=logger,
-        notification_handler=notification_handler
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
     )
 
     mock_gates_init.assert_called_once_with(
         ANY,
+        minio_repository=mock_minio_repository.return_value,
         logger=logger,
-        notification_handler=notification_handler
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
     )
 
+    mock_model_metrics_init.assert_called_once_with(
+        ANY,
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_api_init.assert_called_once_with(
+        ANY,
+        base_url=pi_web_api_config['base_url'],
+        auth_type=pi_web_api_config['auth_type'],
+        auth_token=pi_web_api_config['auth_token'],
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_minio_repository.assert_called_once_with(
+        endpoint=minio_config['endpoint_url'],
+        access_key=minio_config['access_key'],
+        secret_key=minio_config['secret_key'],
+        bucket=minio_config['default_bucket'],
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+        secure=minio_config['secure'],
+    )
+
+
+BASE_CONFIGS = {
+    'postgres_config': {
+        'host': 'localhost',
+        'port': 5432,
+        'user': 'postgres',
+        'password': 'postgres',
+        'dbname': 'postgres',
+        'min_connections': 1,
+        'max_connections': 10,
+    },
+    'minio_config': {
+        'endpoint_url': 'localhost:9000',
+        'access_key': 'minio',
+        'secret_key': 'minio123',
+        'default_bucket': 'test',
+        'retention_hours': 24,
+        'secure': False,
+    },
+    'mlflow_config': {'host': 'localhost', 'port': 5000, 'username': 'm', 'password': 'm'},
+    'opc_config': {},
+    'pi_web_api_config': {'base_url': 'https://pi', 'auth_type': 'bearer', 'auth_token': 't'},
+}
+
+IMPORT_CONFIG = {'bucket': 'sientia', 'prefix': 'imported_models/'}
+IMPORT_STATUS_CONFIG = {
+    'host': 'bff-db',
+    'port': 5432,
+    'dbname': 'sientia-core-mlops-bff',
+    'user': 'u',
+    'password': 'p',
+    'schema': 'public',
+}
+
+
+@patch('laborious.activities.activities.ModelImport.__init__')
+@patch('laborious.activities.activities.Storage.__init__')
+@patch('laborious.activities.activities.MLFlow.__init__')
+@patch('laborious.activities.activities.OPC.__init__')
+@patch('laborious.activities.activities.Gates.__init__')
+@patch('laborious.activities.activities.ModelMetrics.__init__')
+@patch('laborious.activities.activities.API.__init__')
+@patch('laborious.activities.activities.MongoDBRepository')
+@patch('laborious.activities.activities.MLFlowRepository')
+@patch('laborious.activities.activities.MinioRepository')
+@patch('laborious.activities.activities.MetricsController')
+def test_mlflow_and_import_activities_receive_the_same_mlflow_repository_instance(  # noqa: PLR0913
+    _mock_metrics_controller,
+    mock_minio_repository,
+    mock_mlflow_repository,
+    mock_mongo_repository,
+    _mock_api_init,
+    _mock_model_metrics_init,
+    _mock_gates_init,
+    _mock_opc_init,
+    mock_mlflow_init,
+    _mock_storage_init,
+    mock_model_import_init,
+):
+    """Setting the MLflow tracking URI is a process-wide side effect, so two repository instances
+    would be two chances to disagree about where MLflow is."""
+    Activities(
+        **BASE_CONFIGS,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        mongo_config={'connection_string': 'mongodb://mongo', 'database_name': 'sientia'},
+        import_config=IMPORT_CONFIG,
+        import_status_config=IMPORT_STATUS_CONFIG,
+    )
+
+    repository = mock_mlflow_repository.return_value
+    assert mock_mlflow_init.call_args.kwargs['mlflow_repository'] is repository
+
+    passed = mock_model_import_init.call_args.kwargs
+    assert passed['mlflow_repository'] is repository
+    assert passed['minio_repository'] is mock_minio_repository.return_value
+    assert passed['mongo_repository'] is mock_mongo_repository.return_value
+    assert passed['import_config'] is IMPORT_CONFIG
+    assert passed['status_config'] is IMPORT_STATUS_CONFIG
+
+
+@patch('laborious.activities.activities.ModelImport.__init__')
+@patch('laborious.activities.activities.Storage.__init__')
+@patch('laborious.activities.activities.MLFlow.__init__')
+@patch('laborious.activities.activities.OPC.__init__')
+@patch('laborious.activities.activities.Gates.__init__')
+@patch('laborious.activities.activities.ModelMetrics.__init__')
+@patch('laborious.activities.activities.API.__init__')
+@patch('laborious.activities.activities.MongoDBRepository')
+@patch('laborious.activities.activities.MLFlowRepository')
+@patch('laborious.activities.activities.MinioRepository')
+@patch('laborious.activities.activities.MetricsController')
+def test_without_a_mongo_config_the_import_gets_no_repository(  # noqa: PLR0913
+    _mock_metrics_controller,
+    _mock_minio_repository,
+    _mock_mlflow_repository,
+    mock_mongo_repository,
+    _mock_api_init,
+    _mock_model_metrics_init,
+    _mock_gates_init,
+    _mock_opc_init,
+    _mock_mlflow_init,
+    _mock_storage_init,
+    mock_model_import_init,
+):
+    """It refuses rather than guessing: the notification handler keeps its own client either way."""
+    Activities(
+        **BASE_CONFIGS,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        import_config=IMPORT_CONFIG,
+        import_status_config=IMPORT_STATUS_CONFIG,
+    )
+
+    assert mock_model_import_init.call_args.kwargs['mongo_repository'] is None
+    mock_mongo_repository.assert_not_called()
+
+
+@patch('laborious.activities.activities.ModelImport.__init__')
+@patch('laborious.activities.activities.Storage.__init__')
+@patch('laborious.activities.activities.MLFlow.__init__')
+@patch('laborious.activities.activities.OPC.__init__')
+@patch('laborious.activities.activities.Gates.__init__')
+@patch('laborious.activities.activities.ModelMetrics.__init__')
+@patch('laborious.activities.activities.API.__init__')
+@patch('laborious.activities.activities.MongoDBRepository')
+@patch('laborious.activities.activities.MLFlowRepository')
+@patch('laborious.activities.activities.MinioRepository')
+@patch('laborious.activities.activities.MetricsController')
+def test_exactly_one_mlflow_repository_is_built_even_with_import_active(  # noqa: PLR0913
+    _mock_metrics_controller,
+    _mock_minio_repository,
+    mock_mlflow_repository,
+    _mock_mongo_repository,
+    _mock_api_init,
+    _mock_model_metrics_init,
+    _mock_gates_init,
+    _mock_opc_init,
+    _mock_mlflow_init,
+    _mock_storage_init,
+    _mock_model_import_init,
+):
+    """No second `MLFlowRepository` is ever created on any path: `MLFlow` and `ModelImport` only
+    receive the one instance built here, they never build their own."""
+    Activities(
+        **BASE_CONFIGS,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        mongo_config={'connection_string': 'mongodb://mongo', 'database_name': 'sientia'},
+        import_config=IMPORT_CONFIG,
+        import_status_config=IMPORT_STATUS_CONFIG,
+    )
+
+    mock_mlflow_repository.assert_called_once()
+
 
 @mark.asyncio
-@patch('laborious.activities.activities.Postgres.__init__')
-@patch('laborious.activities.activities.MLFlow.__init__')
-@patch('laborious.activities.activities.OPC.__init__')
-async def test_prepare_activity(_mock_opc_init,
-                                _mock_mlflow_init, _mock_postgres_init):
+@patch('laborious.activities.activities.Storage')
+@patch('laborious.activities.activities.MLFlow')
+@patch('laborious.activities.activities.OPC')
+@patch('laborious.activities.activities.Gates')
+@patch('laborious.activities.activities.ModelMetrics')
+@patch('laborious.activities.activities.API')
+@patch('laborious.activities.activities.MLFlowRepository')
+@patch('laborious.activities.activities.MinioRepository')
+async def test_shutdown(
+    _mock_minio_repository,
+    _mock_mlflow_repository,
+    mock_api_init,
+    mock_model_metrics_init,
+    mock_gates_init,
+    mock_opc_init,
+    mock_mlflow_init,
+    mock_storage_init,
+):
+    mock_opc_init.aclose = AsyncMock()
     postgres_config = {
         'host': 'localhost',
         'port': 5432,
@@ -103,20 +346,30 @@ async def test_prepare_activity(_mock_opc_init,
         'password': 'postgres',
         'dbname': 'postgres',
         'min_connections': 1,
-        'max_connections': 10
+        'max_connections': 10,
     }
 
-    mlflow_config = {
-        'host': 'localhost',
-        'port': 5000,
-        'username': 'mlflow',
-        'password': 'mlflow'
+    minio_config = {
+        'endpoint_url': 'localhost:9000',
+        'access_key': 'minio',
+        'secret_key': 'minio123',
+        'default_bucket': 'test',
+        'retention_hours': 24,
+        'secure': False,
     }
 
+    mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'}
+
     opc_config = {
         'bootstrap_servers': 'localhost:9092',
         'polling_time': 1000,
-        'group_id': 'test-group'
+        'group_id': 'test-group',
+    }
+
+    pi_web_api_config = {
+        'base_url': 'https://test-pi-server.com',
+        'auth_type': 'bearer',
+        'auth_token': 'test_token',
     }
 
     logger = MagicMock()
@@ -125,69 +378,17 @@ async def test_prepare_activity(_mock_opc_init,
     activities = Activities(
         postgres_config=postgres_config,
         mlflow_config=mlflow_config,
+        minio_config=minio_config,
         opc_config=opc_config,
+        pi_web_api_config=pi_web_api_config,
         logger=logger,
-        notification_handler=notification_handler
+        notification_handler=notification_handler,
     )
 
-    input_data = {
-        'workflow_name': 'test-workflow-name',
-        'schedule_name': 'test-schedule-name',
-        'model_name': 'test-model-name',
-        'model_id': 'test-model-id'
-    }
-
-    await activities.prepare_activity(input_data)
-
-    assert activities.notification_handler.base_notification.pipeline == input_data[
-        'workflow_name']
-    assert activities.notification_handler.base_notification.trigger == input_data[
-        'schedule_name']
-    assert activities.notification_handler.base_notification.model_name == input_data[
-        'model_name']
-    assert activities.notification_handler.base_notification.model_id == input_data[
-        'model_id']
-
-
-@patch('laborious.activities.activities.Postgres', return_value=MagicMock())
-@patch('laborious.activities.activities.MLFlow', return_value=MagicMock())
-@patch('laborious.activities.activities.OPC', return_value=MagicMock())
-def test_shutdown(mock_opc_init,
-                  _mock_mlflow_init, mock_postgres_init):
-    postgres_config = {
-        'host': 'localhost',
-        'port': 5432,
-        'user': 'postgres',
-        'password': 'postgres',
-        'dbname': 'postgres',
-        'min_connections': 1,
-        'max_connections': 10
-    }
-
-    mlflow_config = {
-        'host': 'localhost',
-        'port': 5000,
-        'username': 'mlflow',
-        'password': 'mlflow'
-    }
-
-    opc_config = {
-        'bootstrap_servers': 'localhost:9092',
-        'polling_time': 1000,
-        'group_id': 'test-group'
-    }
-
-    logger = MagicMock()
-    notification_handler = MagicMock()
-
-    activities = Activities(
-        postgres_config=postgres_config,
-        mlflow_config=mlflow_config,
-        opc_config=opc_config,
-        logger=logger,
-        notification_handler=notification_handler
-    )
-
-    activities.shutdown()
-    mock_opc_init.shutdown.assert_called_once()
-    mock_postgres_init.close.assert_called_once()
+    await activities.shutdown()
+    mock_opc_init.aclose.assert_called_once()
+    mock_storage_init.close.assert_called_once()
+    mock_mlflow_init.close.assert_called_once()
+    mock_gates_init.close.assert_called_once()
+    mock_model_metrics_init.close.assert_called_once()
+    mock_api_init.close.assert_called_once()

tests/laborious/activities/test_api.py

@@ -0,0 +1,530 @@
+from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
+
+import pytest_asyncio
+from pytest import fixture, mark
+from sientia_do.notifications.models import NotificationLevel
+
+from laborious.activities.api import API, PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+
+metadata = {
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
+    },
+}
+
+
+def _create_mock_dataframe(to_dict_return=None):
+    """Helper function to create a mocked DataFrame for testing."""
+    mock_df = MagicMock()
+    mock_head = MagicMock()
+
+    def get_column_values(key):
+        if key == 'prediction':
+            return MagicMock(values=[0.75])
+        elif key == 'prediction_confidence':
+            return MagicMock(values=[0.95])
+        else:
+            return MagicMock(values=['2024-01-01T00:00:00+00:00'])
+
+    mock_head.__getitem__.side_effect = get_column_values
+    mock_df.head.return_value = mock_head
+
+    if to_dict_return is None:
+        to_dict_return = {
+            'prediction': [0.75],
+            'prediction_confidence': [0.95],
+            'timestamp': ['2024-01-01T00:00:00+00:00'],
+        }
+    mock_df.to_dict.return_value = to_dict_return
+
+    return mock_df
+
+
+@fixture
+def base_input_data():
+    """Base input data for PI Web API tests."""
+    return {
+        **metadata,
+        'data': {
+            'prediction': [0.75],
+            'prediction_confidence': [0.95],
+            'timestamp': ['2024-01-01T00:00:00+00:00'],
+        },
+        'pi_web_api_output_config': {
+            'endpoint': 'https://test-pi-server.com/piwebapi',
+            'prediction_tags': {'tag1': 'web_id_1'},
+            'confidence_tags': {'tag2': 'web_id_2'},
+        },
+    }
+
+
+@patch('laborious.activities.api.PIWebAPIClient')
+def test_get_pi_web_api_core_labels_without_operation_type(mock_pi_web_api_client):
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+
+    api_instance = API(
+        base_url='https://test-pi-server.com',
+        auth_type='bearer',
+        auth_token='test_token',
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+    with patch.object(
+        SientiaMonitoring,
+        'get_core_labels',
+        return_value={
+            'pod_id': 'test_pod',
+            'runtime': 'local',
+            'model_name': 'test_model',
+            'workflow_name': 'test_workflow',
+            'operation_type': 'write_pi_web_api_data',
+        },
+    ):
+        labels = api_instance.get_pi_web_api_core_labels(metadata=metadata['metadata'])
+    assert labels['operation_type'] == 'write_pi_web_api_data'
+    assert labels == {
+        'pod_id': 'test_pod',
+        'runtime': 'local',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'operation_type': 'write_pi_web_api_data',
+    }
+
+
+@patch('laborious.activities.api.PIWebAPIClient')
+def test_get_pi_web_api_core_labels_with_operation_type(mock_pi_web_api_client):
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+
+    api_instance = API(
+        base_url='https://test-pi-server.com',
+        auth_type='bearer',
+        auth_token='test_token',
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+    with patch.object(
+        SientiaMonitoring,
+        'get_core_labels',
+        return_value={
+            'pod_id': 'test_pod',
+            'runtime': 'k8s',
+            'model_name': 'test_model',
+            'workflow_name': 'test_workflow',
+            'operation_type': 'write',
+        },
+    ):
+        labels = api_instance.get_pi_web_api_core_labels(
+            metadata=metadata['metadata'], operation_type='write'
+        )
+    assert labels['operation_type'] == 'write'
+    assert labels['runtime'] == 'k8s'
+
+
+def test__init__():
+    api = API(
+        base_url='https://test-pi-server.com',
+        auth_type='bearer',
+        auth_token='test_token',
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+
+    assert api.pi_web_api_client is not None
+
+
+@pytest_asyncio.fixture
+@patch('laborious.activities.api.PIWebAPIClient')
+def api(mock_pi_web_api_client):
+    mock_client = MagicMock()
+    mock_client.write_value = AsyncMock()
+    mock_client.close = MagicMock()
+    mock_client.base_url = 'https://test-pi-server.com'
+    mock_pi_web_api_client.return_value = mock_client
+
+    api_instance = API(
+        base_url='https://test-pi-server.com',
+        auth_type='bearer',
+        auth_token='test_token',
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
+    )
+    api_instance.send_notification_async = AsyncMock()
+    api_instance.info = MagicMock()
+    api_instance.error = MagicMock()
+    api_instance.emit_metric = AsyncMock()
+    api_instance.get_core_labels = MagicMock(
+        return_value={
+            'pod_id': 'test_pod',
+            'runtime': 'local',
+            'model_name': 'test_model',
+            'workflow_name': 'test_workflow',
+        }
+    )
+    return api_instance
+
+
+@mark.asyncio
+@patch('laborious.activities.api.DataFrame')
+async def test_write_pi_web_api_data_success(mock_dataframe, api, base_input_data):
+    input_data = {
+        **base_input_data,
+        'pi_web_api_output_config': {
+            'endpoint': 'https://test-pi-server.com/piwebapi',
+            'prediction_tags': {'tag1': 'web_id_1', 'tag2': 'web_id_2'},
+            'confidence_tags': {'tag3': 'web_id_3', 'tag4': 'web_id_4'},
+        },
+    }
+
+    mock_dataframe.return_value = _create_mock_dataframe()
+
+    # Mock successful responses
+    api.pi_web_api_client.write_value.side_effect = [
+        [{'WebId': 'web_id_1', 'Errors': []}, {'WebId': 'web_id_2', 'Errors': []}],
+        [{'WebId': 'web_id_3', 'Errors': []}, {'WebId': 'web_id_4', 'Errors': []}],
+    ]
+
+    result = await api.write_pi_web_api_data(input_data)
+
+    api.pi_web_api_client.write_value.assert_has_calls(
+        [
+            call(
+                web_ids=['web_id_1', 'web_id_2'],
+                value={
+                    'Timestamp': '2024-01-01T00:00:00+00:00',
+                    'Value': 0.75,
+                },
+                metadata=metadata['metadata'],
+            ),
+            call(
+                web_ids=['web_id_3', 'web_id_4'],
+                value={
+                    'Timestamp': '2024-01-01T00:00:00+00:00',
+                    'Value': 0.95,
+                },
+                metadata=metadata['metadata'],
+            ),
+        ]
+    )
+
+    assert result == {
+        'prediction': [0.75],
+        'prediction_confidence': [0.95],
+        'timestamp': ['2024-01-01T00:00:00+00:00'],
+    }
+
+
+@mark.asyncio
+@patch('laborious.activities.api.DataFrame')
+async def test_write_pi_web_api_data_partial_prediction_mismatch_downgrades_confidence(
+    mock_dataframe, api, base_input_data
+):
+    """A partial write (some tags report Errors) doesn't raise — it's caught by the tag-count
+    mismatch check, which still must downgrade confidence and attach the mismatch message."""
+    input_data = {
+        **base_input_data,
+        'pi_web_api_output_config': {
+            'endpoint': 'https://test-pi-server.com/piwebapi',
+            'prediction_tags': {'tag1': 'web_id_1', 'tag2': 'web_id_2'},
+            'confidence_tags': {'tag3': 'web_id_3'},
+        },
+    }
+    mock_df = _create_mock_dataframe()
+    mock_dataframe.return_value = mock_df
+
+    api.pi_web_api_client.write_value.side_effect = [
+        [{'WebId': 'web_id_1', 'Errors': []}, {'WebId': 'web_id_2', 'Errors': ['bad tag']}],
+        [{'WebId': 'web_id_3', 'Errors': []}],
+    ]
+
+    await api.write_pi_web_api_data(input_data)
+
+    mock_df.__setitem__.assert_any_call(
+        'prediction_confidence', PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+    )
+    mock_df.__setitem__.assert_any_call('comments', ANY)
+    api.send_notification_async.assert_called_once_with(
+        metadata=metadata['metadata'],
+        notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
+        message=ANY,
+        block='write_pi_web_api_data',
+        level=NotificationLevel.ERROR,
+    )
+
+
+@mark.asyncio
+@patch('laborious.activities.api.DataFrame')
+async def test_write_pi_web_api_data_prediction_error(mock_dataframe, api, base_input_data):
+    mock_dataframe.return_value = _create_mock_dataframe(
+        {
+            'prediction': [0.75],
+            'prediction_confidence': [PI_WEB_API_PREDICTION_ERROR_CONFIDENCE],
+            'timestamp': ['2024-01-01T00:00:00+00:00'],
+        }
+    )
+
+    api.pi_web_api_client.write_value.side_effect = Exception('Prediction write failed')
+
+    result = await api.write_pi_web_api_data(base_input_data)
+
+    api.send_notification_async.assert_called_once_with(
+        metadata=metadata['metadata'],
+        notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
+        message="Error writing prediction data to PI Web API: Prediction write failed\n Tags: {'tag1': 'web_id_1'}",
+        block='write_pi_web_api_data',
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY,
+    )
+
+    assert result['prediction_confidence'][0] == PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+    assert api.pi_web_api_client.write_value.call_count == 1
+
+
+@mark.asyncio
+@patch('laborious.activities.api.DataFrame')
+async def test_write_pi_web_api_data_confidence_error(mock_dataframe, api, base_input_data):
+    mock_dataframe.return_value = _create_mock_dataframe()
+
+    # First call succeeds, second fails
+    api.pi_web_api_client.write_value.side_effect = [
+        [{'WebId': 'web_id_1', 'Errors': []}],
+        Exception('Confidence write failed'),
+    ]
+
+    result = await api.write_pi_web_api_data(base_input_data)
+
+    api.send_notification_async.assert_called_once_with(
+        metadata=metadata['metadata'],
+        notification_id='WRITE_PI_WEB_API_CONFIDENCE_ERROR',
+        message="Error writing confidence data to PI Web API: Confidence write failed\n Tags: {'tag2': 'web_id_2'}",
+        block='write_pi_web_api_data',
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY,
+    )
+
+    assert result == {
+        'prediction': [0.75],
+        'prediction_confidence': [0.95],
+        'timestamp': ['2024-01-01T00:00:00+00:00'],
+    }
+    assert api.pi_web_api_client.write_value.call_count == 2
+
+
+@mark.asyncio
+@patch('laborious.activities.api.DataFrame')
+async def test_write_pi_web_api_data_empty_tags(mock_dataframe, api, base_input_data):
+    input_data = {
+        **base_input_data,
+        'pi_web_api_output_config': {
+            'endpoint': 'https://test-pi-server.com/piwebapi',
+            'prediction_tags': {},
+            'confidence_tags': {},
+        },
+    }
+
+    mock_dataframe.return_value = _create_mock_dataframe()
+
+    # Mock empty responses
+    api.pi_web_api_client.write_value.side_effect = [
+        [],
+        [],
+    ]
+
+    result = await api.write_pi_web_api_data(input_data)
+
+    api.pi_web_api_client.write_value.assert_has_calls(
+        [
+            call(
+                web_ids=[],
+                value={
+                    'Timestamp': '2024-01-01T00:00:00+00:00',
+                    'Value': 0.75,
+                },
+                metadata=metadata['metadata'],
+            ),
+            call(
+                web_ids=[],
+                value={
+                    'Timestamp': '2024-01-01T00:00:00+00:00',
+                    'Value': 0.95,
+                },
+                metadata=metadata['metadata'],
+            ),
+        ]
+    )
+
+    assert result == {
+        'prediction': [0.75],
+        'prediction_confidence': [0.95],
+        'timestamp': ['2024-01-01T00:00:00+00:00'],
+    }
+
+
+@mark.asyncio
+async def test_close(api):
+    api.close()
+
+    api.pi_web_api_client.close.assert_called_once()
+
+
+@mark.asyncio
+async def test_process_pi_web_api_response_success(api):
+    """Test successful processing of PI Web API response with all tags written."""
+    response_data = [
+        {'WebId': 'web_id_1', 'Errors': []},
+        {'WebId': 'web_id_2', 'Errors': []},
+    ]
+    tags = {'tag1': 'web_id_1', 'tag2': 'web_id_2'}
+    core_labels = {
+        'pod_id': 'test_pod',
+        'runtime': 'local',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+    }
+
+    confidence, message = await api.process_pi_web_api_response(
+        response_data=response_data,
+        tags=tags,
+        core_labels=core_labels,
+        metadata=metadata['metadata'],
+    )
+
+    assert confidence == 0
+    assert message == ''
+    assert api.emit_metric.call_count == 2
+    # Verify that emit_metric was called with correct tags structure
+    call_args_list = api.emit_metric.call_args_list
+    assert len(call_args_list) == 2
+    # Check that all calls include core_labels and tag_name
+    for call_args in call_args_list:
+        assert 'tag_name' in call_args.kwargs['tags']
+        assert call_args.kwargs['tags']['tag_name'] in ['tag1', 'tag2']
+
+
+@mark.asyncio
+async def test_process_pi_web_api_response_with_errors(api):
+    """Test processing response with errors in some tags."""
+    response_data = [
+        {'WebId': 'web_id_1', 'Errors': ['Error writing tag']},
+        {'WebId': 'web_id_2', 'Errors': []},
+    ]
+    tags = {'tag1': 'web_id_1', 'tag2': 'web_id_2'}
+    core_labels = {
+        'pod_id': 'test_pod',
+        'runtime': 'local',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+    }
+
+    confidence, message = await api.process_pi_web_api_response(
+        response_data=response_data,
+        tags=tags,
+        core_labels=core_labels,
+        metadata=metadata['metadata'],
+    )
+
+    assert confidence == PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+    assert (
+        message
+        == "The number of written tags does not match the number of tag names: Expected ['tag1', 'tag2'] tags, but ['tag2'] tags were written."
+    )
+    assert api.emit_metric.call_count == 2
+
+
+@mark.asyncio
+async def test_process_pi_web_api_response_missing_tags(api):
+    """Test processing response when number of written tags doesn't match expected."""
+    response_data = [
+        {'WebId': 'web_id_1', 'Errors': []},
+    ]
+    tags = {'tag1': 'web_id_1', 'tag2': 'web_id_2'}
+    core_labels = {
+        'pod_id': 'test_pod',
+        'runtime': 'local',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+    }
+
+    confidence, message = await api.process_pi_web_api_response(
+        response_data=response_data,
+        tags=tags,
+        core_labels=core_labels,
+        metadata=metadata['metadata'],
+    )
+
+    assert confidence == PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+    assert (
+        message
+        == "The number of written tags does not match the number of tag names: Expected ['tag1', 'tag2'] tags, but ['tag1'] tags were written."
+    )
+    api.send_notification_async.assert_called_once()
+    call_args = api.send_notification_async.call_args
+    assert call_args.kwargs['notification_id'] == 'WRITE_PI_WEB_API_PREDICTION_ERROR'
+    assert call_args.kwargs['level'] == NotificationLevel.ERROR
+
+
+@mark.asyncio
+async def test_process_pi_web_api_response_missing_webid(api):
+    """Test processing response when WebId is missing in response item."""
+    response_data = [
+        {'Errors': []},
+        {'WebId': 'web_id_2', 'Errors': []},
+    ]
+    tags = {'tag1': 'web_id_1', 'tag2': 'web_id_2'}
+    core_labels = {
+        'pod_id': 'test_pod',
+        'runtime': 'local',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+    }
+
+    confidence, message = await api.process_pi_web_api_response(
+        response_data=response_data,
+        tags=tags,
+        core_labels=core_labels,
+        metadata=metadata['metadata'],
+    )
+
+    assert confidence == PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+    assert (
+        message
+        == "The number of written tags does not match the number of tag names: Expected ['tag1', 'tag2'] tags, but ['tag2'] tags were written."
+    )
+    api.error.assert_any_call('The response did not contain some WebIds', metadata['metadata'])
+
+
+@mark.asyncio
+async def test_process_pi_web_api_response_missing_tag_name(api):
+    """Test processing response when tag name is not found for WebId."""
+    response_data = [
+        {'WebId': 'unknown_web_id', 'Errors': []},
+    ]
+    tags = {'tag1': 'web_id_1'}
+    core_labels = {
+        'pod_id': 'test_pod',
+        'runtime': 'local',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+    }
+
+    confidence, message = await api.process_pi_web_api_response(
+        response_data=response_data,
+        tags=tags,
+        core_labels=core_labels,
+        metadata=metadata['metadata'],
+    )
+
+    assert confidence == PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
+    assert (
+        message
+        == "The number of written tags does not match the number of tag names: Expected ['tag1'] tags, but [] tags were written."
+    )
+    api.error.assert_any_call(
+        'The response did not contain the tag name for WebId unknown_web_id', metadata['metadata']
+    )

tests/laborious/activities/test_mlflow.py

@@ -1,131 +1,657 @@
-from unittest.mock import MagicMock, patch
+from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
 
 import numpy as np
-from pytest import fixture, mark
+from pytest import fixture, mark, raises
+from sientia_do.notifications.models import NotificationLevel
+from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
+
 from laborious.activities.mlflow import MLFlow
 
 
-@patch("laborious.activities.mlflow.MLFlowRepository")
-def test___init__(mock_mlflow_repository):
+@fixture(autouse=True)
+def _passthrough_from_dict():
+    with patch(
+        'laborious.activities.mlflow.MinioDataFramePayload.from_dict', side_effect=lambda x: x
+    ):
+        yield
+
+
+@patch('laborious.activities.mlflow.MinioRepository')
+def test___init__(mock_minio_repository):
+    logger = MagicMock()
+    notification_handler = MagicMock()
+    metrics_controller = AsyncMock()
+    mock_mlflow_repository = MagicMock()
+    minio_repo = mock_minio_repository(
+        endpoint='localhost:9000',
+        access_key='minio',
+        secret_key='minio123',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+        bucket='test',
+    )
     mlflow = MLFlow(
-        mlflow_host="http://localhost",
-        mlflow_port=5000,
-        mlflow_username="admin",
-        mlflow_password="admin",
-        logger=MagicMock(),
-        notification_handler=MagicMock()
+        mlflow_repository=mock_mlflow_repository,
+        minio_repository=minio_repo,
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
     )
 
-    assert mlflow.mlflow_host == "http://localhost"
-    assert mlflow.mlflow_port == 5000
-    assert mlflow.mlflow_username == "admin"
-    assert mlflow.mlflow_password == "admin"
+    assert mlflow.model_monitoring_repository is mock_mlflow_repository
 
-    mock_mlflow_repository.assert_called_once_with(
-        "http://localhost:5000", "admin", "admin"
+    mock_minio_repository.assert_called_once_with(
+        endpoint='localhost:9000',
+        access_key='minio',
+        secret_key='minio123',
+        logger=ANY,
+        notification_handler=ANY,
+        metrics_controller=ANY,
+        bucket='test',
     )
 
 
+def test___init___requires_mlflow_repository():
+    """The failure moves to the signature, at construction time, instead of a `None` that only
+    surfaces silently at first use."""
+    minio_repo = MagicMock()
+    logger = MagicMock()
+    with raises(TypeError):
+        MLFlow(minio_repository=minio_repo, logger=logger)
+
+
 @fixture
-@patch("laborious.activities.mlflow.MLFlowRepository")
-def mlflow(mock_mlflow_repository):
-    return MLFlow(
-        mlflow_host="http://localhost:5000",
-        mlflow_port=5000,
-        mlflow_username="admin",
-        mlflow_password="admin",
-        logger=MagicMock(),
-        notification_handler=MagicMock()
+@patch('laborious.activities.mlflow.MinioRepository')
+def mlflow(mock_minio_repository):
+    logger = MagicMock()
+    notification_handler = MagicMock()
+    metrics_controller = AsyncMock()
+    minio_repo = mock_minio_repository(
+        endpoint='localhost:9000',
+        access_key='minio',
+        secret_key='minio123',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+        bucket='test',
     )
+    mlflow = MLFlow(
+        mlflow_repository=MagicMock(),
+        minio_repository=minio_repo,
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+
+    mlflow.model_monitoring_repository = AsyncMock()
+    mlflow.minio_repository = AsyncMock()
+
+    mlflow.send_notification = MagicMock()
+    mlflow.emit_metric = AsyncMock()
+    mlflow.send_notification_async = AsyncMock()
+    mlflow.error = MagicMock()
+    mlflow.debug = MagicMock()
+    mlflow.info = MagicMock()
+    mlflow.warning = MagicMock()
+    mlflow.critical = MagicMock()
+
+    return mlflow
+
+
+metadata = {
+    'metadata': {
+        'model_id': 'test_model',
+        'model_name': 'test_model',
+        'workflow_name': 'test_workflow',
+        'schema_name': 'test_schedule',
+    },
+}
 
 
 @mark.asyncio
-@patch("laborious.activities.mlflow.DataFrame")
-@patch("laborious.activities.mlflow.max")
-async def test_request_transform(mock_max, mock_dataframe, mlflow):
-    mock_max.return_value = '2024-01-02'
-    # Mock input data
+@patch(
+    'laborious.activities.mlflow.MinioDataFramePayload.from_dataframe',
+    new_callable=AsyncMock,
+)
+async def test_request_transform_success(mock_from_dataframe, mlflow):
+    data_mock = MagicMock()
+    payload = AsyncMock()
+    payload.retrieve = AsyncMock(return_value=data_mock)
+
     input_data = {
-        'data': [
-            {'timestamp': '2024-01-01', 'variable': 'var1',
-                'value': 1.0, 'created_at': '2024-01-01 12:00:00'},
-            {'timestamp': '2024-01-01', 'variable': 'var2',
-                'value': 2.0, 'created_at': '2024-01-01 12:00:00'},
-            {'timestamp': '2024-01-02', 'variable': 'var1',
-                'value': 3.0, 'created_at': '2024-01-02 12:00:00'},
-            {'timestamp': '2024-01-02', 'variable': 'var2',
-                'value': 4.0, 'created_at': '2024-01-02 12:00:00'},
-            {'timestamp': '2024-01-02', 'variable': 'var1',
-                'value': 1.0, 'created_at': '2024-01-01 12:00:00'},
-            {'timestamp': '2024-01-02', 'variable': 'var2',
-                'value': 1.0, 'created_at': '2024-01-01 12:00:00'}
-        ],
+        **metadata,
+        'data': payload,
         'model_name': 'test_model',
-        'model_retention': 30
+        'model_config': {},
     }
 
-    # Mock the transform response
-    expected_response = {'prediction': [0.5, 0.6]}
-    mlflow.model_monitoring_repository.transform.return_value = expected_response
+    transform_response = {'success': True, 'content': MagicMock()}
+    mlflow.model_monitoring_repository.transform.return_value = transform_response
 
-    mock_dataframe.return_value.sort_values.return_value = mock_dataframe.return_value
-    mock_dataframe.return_value.drop_duplicates.return_value = mock_dataframe.return_value
+    data_mock.sort_values.return_value = data_mock
+    data_mock.drop_duplicates.return_value = data_mock
+    data_mock.pivot.return_value = data_mock
 
-    # Call the method
     response_data = await mlflow.request_transform(input_data)
 
-    # Verify the data was correctly transformed
-    mock_dataframe.assert_called_once_with(input_data['data'])
-    mock_dataframe.return_value.pivot.assert_called_once_with(
-        index='timestamp', columns='variable', values='value'
-    )
-    mock_dataframe = mock_dataframe.return_value.pivot.return_value
-    mock_dataframe.fillna.assert_called_once_with(np.nan, inplace=True)
-    mock_dataframe.reset_index.assert_called_once()
-    mock_dataframe.columns.name = None
-
-    # Verify the response
-    assert response_data == expected_response
-
-    # Verify the repository was called with correct arguments
     mlflow.model_monitoring_repository.transform.assert_called_once_with(
-        'test_model', mock_dataframe, 30
+        'test_model', data_mock, {}, metadata['metadata']
+    )
+    mock_from_dataframe.assert_called_once()
+    assert response_data == mock_from_dataframe.return_value
+
+
+@mark.asyncio
+@patch(
+    'laborious.activities.mlflow.MinioDataFramePayload.from_dataframe',
+    new_callable=AsyncMock,
+)
+async def test_request_transform_failure(mock_from_dataframe, mlflow):
+    data_mock = MagicMock()
+    payload = AsyncMock()
+    payload.retrieve = AsyncMock(return_value=data_mock)
+
+    input_data = {
+        **metadata,
+        'data': payload,
+        'model_name': 'test_model',
+        'model_config': {},
+    }
+
+    transform_response = {'success': False, 'message': 'Transform failed'}
+    mlflow.model_monitoring_repository.transform.return_value = transform_response
+
+    data_mock.sort_values.return_value = data_mock
+    data_mock.drop_duplicates.return_value = data_mock
+    data_mock.pivot.return_value = data_mock
+
+    response_data = await mlflow.request_transform(input_data)
+
+    mock_from_dataframe.assert_called_once_with(
+        dataframe=None,
+        minio_repo=mlflow.minio_repository,
+        model_name='test_model',
+        operation='transform',
+        status=transform_response,
+        workflow_metadata=metadata['metadata'],
+        last_timestamp=payload.last_timestamp,
+        logger=mlflow.logger,
+    )
+    assert response_data == mock_from_dataframe.return_value
+
+
+@mark.asyncio
+@patch(
+    'laborious.activities.mlflow.MinioDataFramePayload.from_dataframe',
+    new_callable=AsyncMock,
+)
+@patch('laborious.activities.mlflow.to_datetime')
+async def test_request_predict(mock_to_datetime, mock_from_dataframe, mlflow):
+    data_mock = MagicMock()
+    payload = AsyncMock()
+    payload.retrieve = AsyncMock(return_value=data_mock)
+
+    input_data = {
+        **metadata,
+        'data': payload,
+        'model_name': 'test_model',
+        'model_config': {},
+    }
+
+    predict_response = {'success': True, 'content': MagicMock()}
+    mlflow.model_monitoring_repository.predict.return_value = predict_response
+
+    response_data = await mlflow.request_predict(input_data)
+
+    data_mock.replace.assert_called_once_with(np.nan, None, inplace=True)
+    mock_to_datetime.assert_called_once_with(
+        data_mock.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ
+    )
+    mock_to_datetime.return_value.dt.strftime.assert_called_once_with(DATETIME_FORMAT)
+
+    mlflow.model_monitoring_repository.predict.assert_called_once_with(
+        'test_model', data_mock, {}, metadata['metadata']
+    )
+    mock_from_dataframe.assert_called_once()
+    assert response_data == mock_from_dataframe.return_value
+
+
+@mark.asyncio
+@patch(
+    'laborious.activities.mlflow.MinioDataFramePayload.from_dataframe',
+    new_callable=AsyncMock,
+)
+@patch('laborious.activities.mlflow.to_datetime')
+async def test_request_predict_skips_timestamp_reinsertion_when_configured(
+    mock_to_datetime, mock_from_dataframe, mlflow
+):
+    """skip_predict_timestamp=True leaves the caller's index alone — no reinsertion round-trip."""
+    data_mock = MagicMock()
+    payload = AsyncMock()
+    payload.retrieve = AsyncMock(return_value=data_mock)
+
+    input_data = {
+        **metadata,
+        'data': payload,
+        'model_name': 'test_model',
+        'model_config': {'skip_predict_timestamp': True},
+    }
+
+    predict_response = {'success': True, 'content': MagicMock()}
+    mlflow.model_monitoring_repository.predict.return_value = predict_response
+
+    await mlflow.request_predict(input_data)
+
+    data_mock.replace.assert_called_once_with(np.nan, None, inplace=True)
+    mock_to_datetime.assert_not_called()
+    mlflow.model_monitoring_repository.predict.assert_called_once_with(
+        'test_model', data_mock, {'skip_predict_timestamp': True}, metadata['metadata']
     )
 
 
 @mark.asyncio
-@patch("laborious.activities.mlflow.DataFrame")
-@patch("laborious.activities.mlflow.max")
-async def test_request_predict(mock_max, mock_dataframe, mlflow):
-    mock_max.return_value = '2024-01-02'
-    # Mock input data
+@patch(
+    'laborious.activities.mlflow.MinioDataFramePayload.from_dataframe',
+    new_callable=AsyncMock,
+)
+@patch('laborious.activities.mlflow.to_datetime')
+async def test_request_predict_failure(mock_to_datetime, mock_from_dataframe, mlflow):
+    data_mock = MagicMock()
+    payload = AsyncMock()
+    payload.retrieve = AsyncMock(return_value=data_mock)
+
     input_data = {
-        'data': [
-            {'timestamp': '2024-01-01', 'variable': 'var1', 'value': 1.0},
-            {'timestamp': '2024-01-01', 'variable': 'var2', 'value': 2.0},
-            {'timestamp': '2024-01-02', 'variable': 'var1', 'value': 3.0},
-            {'timestamp': '2024-01-02', 'variable': 'var2', 'value': 4.0}
-        ],
+        **metadata,
+        'data': payload,
         'model_name': 'test_model',
-        'model_retention': 30
+        'model_config': {},
     }
 
-    # Mock the predict response
-    expected_response = {'prediction': [0.5, 0.6]}
-    mlflow.model_monitoring_repository.predict.return_value = expected_response
+    predict_response = {'success': False, 'message': 'Predict failed'}
+    mlflow.model_monitoring_repository.predict.return_value = predict_response
 
-    # Call the method
     response_data = await mlflow.request_predict(input_data)
 
-    mock_dataframe.assert_called_once_with(input_data['data'])
-    mock_dataframe.return_value.replace.assert_called_once_with(
-        np.nan, None, inplace=True
+    mock_from_dataframe.assert_called_once_with(
+        dataframe=None,
+        minio_repo=mlflow.minio_repository,
+        model_name='test_model',
+        operation='predict',
+        status=predict_response,
+        workflow_metadata=metadata['metadata'],
+        last_timestamp=payload.last_timestamp,
+        logger=mlflow.logger,
+    )
+    assert response_data == mock_from_dataframe.return_value
+
+
+@mark.asyncio
+@patch('laborious.activities.mlflow.to_datetime')
+async def test_retrain_model_success_data_success_retrain(mock_to_datetime, mlflow):
+    mlflow.model_monitoring_repository.retrain_model.return_value = {
+        'success': True,
+        'experiment': 'test_experiment',
+        'message': 'Model retrained successfully.',
+    }
+
+    raw_data = MagicMock(columns=['variable', 'timestamp', 'value'])
+    payload = AsyncMock()
+    payload.retrieve = AsyncMock(return_value=raw_data)
+
+    response = await mlflow.retrain_model(
+        {
+            **metadata,
+            'data': payload,
+            'model_name': 'test_model',
+            'model_config': {
+                'target': 'target',
+                'transform_flavor': 'sklearn',
+                'predict_flavor': 'pyfunc',
+            },
+        }
     )
 
-    # Verify the response
-    assert response_data == expected_response
+    timestamp = raw_data.__getitem__.return_value.max.return_value
 
-    # Verify the repository was called with correct arguments
-    mlflow.model_monitoring_repository.predict.assert_called_once_with(
-        'test_model', mock_dataframe.return_value, 30
+    raw_data.sort_values.assert_not_called()
+    raw_data.drop_duplicates.assert_called_once_with(subset=['variable', 'timestamp'], keep='first')
+    raw_data = raw_data.drop_duplicates.return_value
+
+    raw_data.drop.assert_has_calls(
+        [
+            call(columns=['model_id'], inplace=True, errors='ignore'),
+            call(columns=['created_at'], inplace=True, errors='ignore'),
+        ]
+    )
+    raw_data.pivot.assert_called_once_with(index='timestamp', columns='variable', values='value')
+    raw_data.pivot.return_value.fillna.assert_called_once_with(np.nan, inplace=True)
+
+    raw_data = raw_data.pivot.return_value
+
+    raw_data.__setitem__.assert_has_calls(
+        [
+            call('timestamp', raw_data.index),
+            call('timestamp', mock_to_datetime.return_value.dt.strftime.return_value),
+            call('timestamp', mock_to_datetime.return_value),
+        ]
+    )
+    mock_to_datetime.assert_has_calls(
+        [call(raw_data.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ)]
+    )
+    mock_to_datetime.assert_has_calls(
+        [call(raw_data.__getitem__.return_value, format=DATETIME_FORMAT)]
+    )
+
+    mlflow.model_monitoring_repository.retrain_model.assert_called_once_with(
+        data=raw_data,
+        model_name='test_model',
+        model_config={
+            'target': 'target',
+            'transform_flavor': 'sklearn',
+            'predict_flavor': 'pyfunc',
+        },
+        metadata=metadata['metadata'],
+    )
+
+    assert response == {
+        'success': True,
+        'experiment': 'test_experiment',
+        'message': 'Model retrained successfully.',
+        'timestamp': timestamp,
+    }
+
+
+@mark.asyncio
+@patch('laborious.activities.mlflow.to_datetime')
+async def test_retrain_model_success_with_payload_data(mock_to_datetime, mlflow):
+    mlflow.model_monitoring_repository.retrain_model.return_value = {
+        'success': True,
+        'experiment': 'test_experiment',
+        'message': 'Model retrained successfully.',
+    }
+
+    raw_data = MagicMock(columns=['variable', 'timestamp', 'value', 'created_at'])
+    payload = AsyncMock()
+    payload.retrieve = AsyncMock(return_value=raw_data)
+
+    response = await mlflow.retrain_model(
+        {
+            **metadata,
+            'data': payload,
+            'model_name': 'test_model',
+            'model_config': {
+                'target': 'target',
+                'transform_flavor': 'sklearn',
+                'predict_flavor': 'pyfunc',
+            },
+        }
+    )
+
+    assert response['success'] is True
+    mlflow.minio_repository.download_file.assert_not_called()
+
+
+@mark.asyncio
+@patch('laborious.activities.mlflow.to_datetime')
+async def test_retrain_model_success_data_fail_retrain(mock_to_datetime, mlflow):
+    mlflow.model_monitoring_repository.retrain_model.return_value = {
+        'success': False,
+        'traceback': 'test_traceback',
+        'message': 'Model retrained failed.',
+    }
+
+    raw_data = MagicMock(columns=['variable', 'timestamp', 'value', 'created_at'])
+    payload = AsyncMock()
+    payload.retrieve = AsyncMock(return_value=raw_data)
+
+    response = await mlflow.retrain_model(
+        {
+            **metadata,
+            'data': payload,
+            'model_name': 'test_model',
+            'model_config': {
+                'target': 'target',
+                'transform_flavor': 'sklearn',
+                'predict_flavor': 'pyfunc',
+            },
+        }
+    )
+
+    timestamp = raw_data.__getitem__.return_value.max.return_value
+
+    raw_data.sort_values.assert_called_once_with('created_at', ascending=False)
+    raw_data.sort_values.return_value.drop_duplicates.assert_called_once_with(
+        subset=['variable', 'timestamp'], keep='first'
+    )
+    raw_data = raw_data.sort_values.return_value.drop_duplicates.return_value
+
+    raw_data.drop.assert_has_calls(
+        [
+            call(columns=['model_id'], inplace=True, errors='ignore'),
+            call(columns=['created_at'], inplace=True, errors='ignore'),
+        ]
+    )
+    raw_data.pivot.assert_called_once_with(index='timestamp', columns='variable', values='value')
+    raw_data.pivot.return_value.fillna.assert_called_once_with(np.nan, inplace=True)
+
+    raw_data = raw_data.pivot.return_value
+
+    raw_data.__setitem__.assert_has_calls(
+        [
+            call('timestamp', raw_data.index),
+            call('timestamp', mock_to_datetime.return_value.dt.strftime.return_value),
+            call('timestamp', mock_to_datetime.return_value),
+        ]
+    )
+    mock_to_datetime.assert_has_calls(
+        [call(raw_data.__getitem__.return_value, format=DATETIME_FORMAT_WITH_TZ)]
+    )
+    mock_to_datetime.assert_has_calls(
+        [call(raw_data.__getitem__.return_value, format=DATETIME_FORMAT)]
+    )
+
+    mlflow.model_monitoring_repository.retrain_model.assert_called_once_with(
+        data=raw_data,
+        model_name='test_model',
+        model_config={
+            'target': 'target',
+            'transform_flavor': 'sklearn',
+            'predict_flavor': 'pyfunc',
+        },
+        metadata=metadata['metadata'],
+    )
+
+    mlflow.send_notification_async.assert_called_once_with(
+        metadata=metadata['metadata'],
+        notification_id='RETRAIN_MODEL_ERROR',
+        message='Error retraining model test_model: Model retrained failed.',
+        block='retrain_model',
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY,
+    )
+
+    assert response == {
+        'success': False,
+        'traceback': 'test_traceback',
+        'message': 'Model retrained failed.',
+        'timestamp': timestamp,
+    }
+
+
+@mark.asyncio
+async def test_retrain_model_data_error(mlflow):
+    response = await mlflow.retrain_model(
+        {
+            **metadata,
+            'model_name': 'test_model',
+            'model_config': {
+                'target': 'target',
+                'transform_flavor': 'sklearn',
+                'predict_flavor': 'pyfunc',
+            },
+        }
+    )
+
+    assert response == {
+        'success': False,
+        'message': "Error loading retrain data: 'data'",
+        'traceback': ANY,
+        'timestamp': ANY,
+    }
+
+
+@mark.asyncio
+async def test_retrain_model_data_error_no_minio_repository(mlflow):
+    mlflow.minio_repository = None
+
+    with raises(ValueError) as e:
+        await mlflow.retrain_model(
+            {
+                **metadata,
+                'object_key': 'test_object_key',
+                'model_name': 'test_model',
+                'model_config': {
+                    'target': 'target',
+                    'transform_flavor': 'sklearn',
+                    'predict_flavor': 'pyfunc',
+                },
+            }
+        )
+
+    assert str(e.value) == 'Minio repository not initialized'
+
+
+@mark.asyncio
+async def test_update_production_model(mlflow):
+    input_data = {
+        **metadata,
+        'model_name': 'test_model',
+        'model_id': 1,
+        'experiment': 'test',
+        'timestamp': 2,
+        'status': 'success',
+    }
+
+    response = await mlflow.update_production_model(input_data)
+
+    mlflow.model_monitoring_repository.update_production_model.assert_called_once_with(
+        experiment='test', model_name='test_model', metadata=metadata['metadata']
+    )
+
+    assert response == mlflow.model_monitoring_repository.update_production_model.return_value
+
+
+@mark.asyncio
+async def test_update_production_model_error(mlflow):
+    mlflow.model_monitoring_repository.update_production_model.side_effect = Exception(
+        'Error updating production model'
+    )
+
+    input_data = {
+        **metadata,
+        'model_name': 'test_model',
+        'model_id': 1,
+        'experiment': 'test',
+        'timestamp': 2,
+        'status': 'success',
+    }
+
+    try:
+        await mlflow.update_production_model(input_data)
+    except Exception as e:
+        assert str(e) == 'Error updating production model'
+        mlflow.send_notification_async.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='UPDATE_PRODUCTION_MODEL_ERROR',
+            message='Error updating production model test_model: Error updating production model',
+            block='update_production_model',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+    else:
+        raise AssertionError('No exception raised')
+
+
+@mark.asyncio
+@patch('laborious.activities.mlflow.to_datetime')
+async def test_get_reference_data_success(mock_to_datetime, mlflow):
+    # Arrange
+    input_data = {
+        **metadata,
+        'model_name': 'test_model',
+    }
+
+    # Mock reference data DataFrame
+    mock_reference_data = MagicMock()
+    mock_reference_data.__getitem__.return_value = MagicMock()
+    mock_to_datetime.return_value.dt.strftime.return_value = MagicMock()
+    mock_reference_data.to_dict.return_value = [
+        {'timestamp': '2023-05-26 11:12:27', 'value': 1.0},
+        {'timestamp': '2023-05-26 11:12:28', 'value': 2.0},
+    ]
+
+    mlflow.model_monitoring_repository.load_artifact_dataframe.return_value = mock_reference_data
+
+    # Act
+    result = await mlflow.get_reference_data(input_data)
+
+    # Assert
+    mlflow.model_monitoring_repository.load_artifact_dataframe.assert_called_once_with(
+        model_name='test_model',
+        artifact_path='evaluation_data.csv',
+        metadata=metadata['metadata'],
+    )
+    mock_to_datetime.assert_called_once_with(mock_reference_data.__getitem__.return_value)
+
+    mock_reference_data.to_dict.assert_called_once_with(orient='records')
+    assert result == mock_reference_data.to_dict.return_value
+
+
+@mark.asyncio
+async def test_get_reference_data_not_found(mlflow):
+    # Arrange
+    input_data = {
+        **metadata,
+        'model_name': 'test_model',
+    }
+
+    mlflow.model_monitoring_repository.load_artifact_dataframe.return_value = None
+
+    # Act
+    result = await mlflow.get_reference_data(input_data)
+
+    # Assert
+    mlflow.model_monitoring_repository.load_artifact_dataframe.assert_called_once_with(
+        model_name='test_model',
+        artifact_path='evaluation_data.csv',
+        metadata=metadata['metadata'],
+    )
+    mlflow.warning.assert_called_once_with(
+        'Reference data not found for model test_model', metadata['metadata']
+    )
+    assert result is None
+
+
+@mark.asyncio
+async def test_get_reference_data_exception(mlflow):
+    # Arrange
+    input_data = {
+        **metadata,
+        'model_name': 'test_model',
+    }
+
+    mlflow.model_monitoring_repository.load_artifact_dataframe.side_effect = RuntimeError(
+        'Error loading artifact'
+    )
+
+    # Act & Assert
+    with raises(RuntimeError) as e:
+        await mlflow.get_reference_data(input_data)
+
+    assert str(e.value) == 'Error loading artifact'
+    mlflow.model_monitoring_repository.load_artifact_dataframe.assert_called_once_with(
+        model_name='test_model',
+        artifact_path='evaluation_data.csv',
+        metadata=metadata['metadata'],
     )