Code import - branch release/SIENTIAPDE-1646

e2e/__init__.py

@@ -0,0 +1,3 @@
+"""
+End-to-end tests for laborious temporal workflows.
+"""

e2e/conftest.py

@@ -0,0 +1,603 @@
+"""Pytest configuration and fixtures for E2E tests."""
+
+import asyncio
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pandas as pd
+import pytest
+import pytest_asyncio
+from sqlalchemy import create_engine
+from testcontainers.core.container import DockerContainer
+from testcontainers.minio import MinioContainer
+from testcontainers.postgres import PostgresContainer
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.opc_test_server import OpcE2ETestServer
+from laborious.activities.activities import Activities
+from laborious.workflows.drift import Drift
+from laborious.workflows.minimal_retrain import MinimalRetrain
+from laborious.workflows.predictions_batch import PredictionsBatch
+from laborious.workflows.simple_metrics import SimpleMetrics
+from laborious.workflows.sub_workflows.format_and_export_prediction import (
+    FormatAndExportPrediction,
+)
+from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
+from sientia_do.notifications.handlers import CoreNotificationHandler
+from sientia_do.observability.logger import Logger
+from sientia_do.observability.metrics_controller import MetricsController
+
+# Single source of truth for the test database schema. Mirrors the production
+# DDL for ``sientia_data`` so any production change can be pasted directly into
+# this file (see ``e2e/db_schema.sql``) without touching Python.
+DB_SCHEMA_SQL_PATH = Path(__file__).parent / 'db_schema.sql'
+
+
+@pytest_asyncio.fixture(scope='session')
+def postgres_container():
+    """PostgreSQL testcontainer used by all E2E tests."""
+    postgres = PostgresContainer('postgres:15')
+    postgres.start()
+    yield postgres
+    postgres.stop()
+
+
+@pytest_asyncio.fixture(scope='session')
+def minio_container():
+    """MinIO testcontainer used by E2E offload and payload retrieval paths."""
+    minio = MinioContainer()
+    minio.start()
+    yield minio
+    minio.stop()
+
+
+@pytest_asyncio.fixture(scope='session')
+def mongo_container():
+    """MongoDB testcontainer used by real CoreNotificationHandler."""
+    mongo = DockerContainer('mongo:7').with_exposed_ports(27017)
+    mongo.start()
+    yield mongo
+    mongo.stop()
+
+
+@pytest_asyncio.fixture
+def postgres_engine(postgres_container):
+    """SQLAlchemy engine bound to the PostgreSQL testcontainer."""
+    engine = create_engine(postgres_container.get_connection_url())
+    yield engine
+    engine.dispose()
+
+
+def _create_schema_and_tables(engine):
+    """
+    Create all schemas/tables required by workflow and activity paths.
+
+    Loads the DDL from ``e2e/db_schema.sql`` (single source of truth that
+    mirrors the production schema). The SQL file is executed via the raw
+    DBAPI cursor so multi-statement DDL is supported.
+    """
+    sql_text = DB_SCHEMA_SQL_PATH.read_text(encoding='utf-8')
+    with engine.begin() as conn:
+        conn.exec_driver_sql(sql_text)
+
+
+@pytest_asyncio.fixture(autouse=True)
+def setup_postgres_schema_and_tables(postgres_engine):
+    """Ensure required schema and tables exist before each E2E test."""
+    _create_schema_and_tables(postgres_engine)
+    yield
+
+
+@pytest_asyncio.fixture
+def mock_logger():
+    """Logger double with readable console output for E2E runs."""
+    logger = MagicMock(spec=Logger)
+    logger.info = MagicMock(side_effect=lambda msg: print(f'[LOG] {msg}'))
+    logger.debug = MagicMock(side_effect=lambda msg: print(f'[LOG] {msg}'))
+    logger.error = MagicMock(side_effect=lambda msg: print(f'[LOG] {msg}'))
+    logger.warning = MagicMock(side_effect=lambda msg: print(f'[LOG] {msg}'))
+    logger.custom_info = MagicMock(side_effect=lambda msg, _meta=None: print(f'[LOG] {msg}'))
+    logger.custom_debug = MagicMock(side_effect=lambda msg, _meta=None: print(f'[LOG] {msg}'))
+    logger.custom_error = MagicMock(side_effect=lambda msg, _meta=None: print(f'[LOG] {msg}'))
+    logger.custom_warning = MagicMock(side_effect=lambda msg, _meta=None: print(f'[LOG] {msg}'))
+    return logger
+
+
+@pytest_asyncio.fixture
+def metrics_controller(mock_logger):
+    """Real metrics controller for E2E observability paths."""
+    return MetricsController(logger=mock_logger)
+
+
+@pytest_asyncio.fixture
+def notification_handler(mock_logger, mongo_container):
+    """Real notification handler using MongoDB testcontainer."""
+    mongo_port = mongo_container.get_exposed_port(27017)
+    handler = CoreNotificationHandler(
+        connection_string=f'mongodb://localhost:{mongo_port}',
+        database='test_db',
+        logger=mock_logger,
+        project_name='laborious',
+    )
+    try:
+        yield handler
+    finally:
+        handler.shutdown()
+
+
+@pytest.fixture
+def notification_inserts(notification_handler):
+    """Spy on real Mongo insert calls issued by notification handler."""
+    collection = notification_handler.mongo_collection
+    original_insert_one = collection.insert_one
+    spy = MagicMock(wraps=original_insert_one)
+    collection.insert_one = spy
+    try:
+        yield spy
+    finally:
+        collection.insert_one = original_insert_one
+
+
+class _FakeModelWrapper:
+    """External MLflow wrapper double used by repository stub."""
+
+    def __init__(self):
+        self.transform = MagicMock(side_effect=self._default_transform)
+        self.predict = MagicMock(side_effect=self._default_predict)
+
+    @staticmethod
+    def _default_transform(data: pd.DataFrame):
+        result = pd.DataFrame(
+            {
+                'feature_1': [0.234] * len(data),
+                'feature_2': [0.783] * len(data),
+            }
+        )
+        result.index = data.index
+        return result, {}
+
+    @staticmethod
+    def _default_predict(_params: dict, data: pd.DataFrame):
+        pred = pd.DataFrame([0.5] * len(data), columns=['placeholder'])
+        pred.index = data.index
+        return pred, {}
+
+
+@pytest_asyncio.fixture
+def mlflow_repository_stub():
+    """External MLflow repository stub."""
+    repo = MagicMock()
+    wrapper = _FakeModelWrapper()
+    repo.stub_wrapper = wrapper
+    repo.get_cached_model = MagicMock(return_value=wrapper)
+    repo._client = MagicMock()
+    return repo
+
+
+class _FakePIWebAPIClient:
+    """External PI Web API client stub with deterministic responses."""
+
+    def __init__(self):
+        self._responses = None
+        self.write_value = MagicMock(side_effect=self._write_value)
+        self.close = MagicMock()
+
+    def set_side_effect(self, side_effect):
+        self._responses = side_effect
+
+    def _write_value(self, web_ids, value, metadata=None, **kwargs):
+        if isinstance(self._responses, Exception):
+            raise self._responses
+        if isinstance(self._responses, list):
+            item = self._responses.pop(0)
+            if isinstance(item, Exception):
+                raise item
+            return item
+        if callable(self._responses):
+            return self._responses(web_ids=web_ids, value=value, metadata=metadata, **kwargs)
+        return [{'WebId': wid, 'Errors': []} for wid in web_ids]
+
+
+@pytest_asyncio.fixture
+def pi_web_api_client_stub():
+    """PI Web API stub fixture."""
+    return _FakePIWebAPIClient()
+
+
+@pytest_asyncio.fixture
+def opc_repository_stub():
+    """OPC external dependency stub."""
+    repo = MagicMock()
+    repo.write_data = MagicMock(return_value=(True, {'response_time': 0.1}))
+    repo.disconnect = MagicMock()
+    return repo
+
+
+@pytest_asyncio.fixture
+def plugin_store_stub():
+    """Plugin store external dependency stub."""
+    return MagicMock()
+
+
+@pytest_asyncio.fixture(scope='function')
+async def test_activities(
+    postgres_container,
+    minio_container,
+    mock_logger,
+    notification_handler,
+    metrics_controller,
+    mlflow_repository_stub,
+    plugin_store_stub,
+    pi_web_api_client_stub,
+    opc_repository_stub,
+):
+    """Activities with real infra and external-system stubs only."""
+    minio_client = minio_container.get_client()
+    if not minio_client.bucket_exists('test-bucket'):
+        minio_client.make_bucket('test-bucket')
+    minio_port = minio_container.get_exposed_port(9000)
+
+    activities = Activities(
+        postgres_config={
+            'host': 'localhost',
+            'port': int(postgres_container.get_exposed_port(5432)),
+            'user': postgres_container.username,
+            'password': postgres_container.password,
+            'dbname': postgres_container.dbname,
+            'min_connections': 1,
+            'max_connections': 5,
+        },
+        plugin_store=plugin_store_stub,
+        minio_config={
+            'endpoint_url': f'localhost:{minio_port}',
+            'access_key': 'minioadmin',
+            'secret_key': 'minioadmin',
+            'default_bucket': 'test-bucket',
+            'retention_hours': 24,
+            'secure': False,
+        },
+        opc_config={},
+        pi_web_api_config={
+            'base_url': 'http://localhost:8080',
+            'auth_type': 'bearer',
+            'auth_token': 'test_token',
+        },
+        logger=mock_logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+        mlflow_repository=mlflow_repository_stub,
+    )
+    activities.pi_web_api_client = pi_web_api_client_stub
+    activities.opc_repository = {'1': opc_repository_stub}
+    try:
+        yield activities
+    finally:
+        activities.shutdown()
+
+
+@pytest_asyncio.fixture(scope='function')
+async def test_activities_real_minio(
+    postgres_container,
+    minio_container,
+    mock_logger,
+    notification_handler,
+    metrics_controller,
+    mlflow_repository_stub,
+    plugin_store_stub,
+    pi_web_api_client_stub,
+    opc_repository_stub,
+):
+    """Compatibility alias for offload tests."""
+    minio_client = minio_container.get_client()
+    if not minio_client.bucket_exists('test-bucket'):
+        minio_client.make_bucket('test-bucket')
+    minio_port = minio_container.get_exposed_port(9000)
+
+    activities = Activities(
+        postgres_config={
+            'host': 'localhost',
+            'port': int(postgres_container.get_exposed_port(5432)),
+            'user': postgres_container.username,
+            'password': postgres_container.password,
+            'dbname': postgres_container.dbname,
+            'min_connections': 1,
+            'max_connections': 5,
+        },
+        plugin_store=plugin_store_stub,
+        minio_config={
+            'endpoint_url': f'localhost:{minio_port}',
+            'access_key': 'minioadmin',
+            'secret_key': 'minioadmin',
+            'default_bucket': 'test-bucket',
+            'retention_hours': 24,
+            'secure': False,
+        },
+        opc_config={},
+        pi_web_api_config={
+            'base_url': 'http://localhost:8080',
+            'auth_type': 'bearer',
+            'auth_token': 'test_token',
+        },
+        logger=mock_logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+        mlflow_repository=mlflow_repository_stub,
+    )
+    activities.pi_web_api_client = pi_web_api_client_stub
+    activities.opc_repository = {'1': opc_repository_stub}
+    try:
+        yield activities
+    finally:
+        activities.shutdown()
+
+
+def _worker_activity_list(test_activities: Activities):
+    """List of registered activity callables used by Temporal worker in E2E."""
+    return [
+        test_activities.load_query_with_minio_offload,
+        test_activities.cleanup_minio_objects_expired,
+        test_activities.input_gate,
+        test_activities.request_transform,
+        test_activities.mlflow_response_gate,
+        test_activities.mlflow_content_gate,
+        test_activities.request_predict,
+        test_activities.repeat_last_prediction,
+        test_activities.format_prediction,
+        test_activities.format_transformed_data,
+        test_activities.format_default_prediction,
+        test_activities.write_pi_web_api_data,
+        test_activities.write_opc_data,
+        test_activities.export_data_to_postgres,
+        test_activities.export_payload_to_postgres,
+        test_activities.write_metrics,
+    ]
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_test_env():
+    """Temporal test environment with time-skipping."""
+    env = await WorkflowEnvironment.start_time_skipping()
+    async with env:
+        yield env
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker(temporal_test_env, test_activities):
+    """Temporal worker for full predictions-batch and child workflows."""
+    with ThreadPoolExecutor(max_workers=32) as activity_executor:
+        async with Worker(
+            temporal_test_env.client,
+            task_queue='test-queue',
+            workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
+            activities=_worker_activity_list(test_activities),
+            activity_executor=activity_executor,
+        ) as worker:
+            yield worker
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker_real_minio(temporal_test_env, test_activities_real_minio):
+    """Temporal worker alias for tests that emphasize MinIO behavior."""
+    with ThreadPoolExecutor(max_workers=32) as activity_executor:
+        async with Worker(
+            temporal_test_env.client,
+            task_queue='test-queue',
+            workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
+            activities=_worker_activity_list(test_activities_real_minio),
+            activity_executor=activity_executor,
+        ) as worker:
+            yield worker
+
+
+def _drift_worker_activity_list(test_activities: Activities):
+    """Activity callables registered on the drift Temporal worker."""
+    return [
+        test_activities.load_custom_query,
+        test_activities.get_reference_data,
+        test_activities.calculate_drift,
+        test_activities.export_data_to_postgres,
+    ]
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker_drift(temporal_test_env, test_activities):
+    """Temporal worker registered with the Drift workflow and its activities."""
+    with ThreadPoolExecutor(max_workers=32) as activity_executor:
+        async with Worker(
+            temporal_test_env.client,
+            task_queue='test-queue',
+            workflows=[Drift],
+            activities=_drift_worker_activity_list(test_activities),
+            activity_executor=activity_executor,
+        ) as worker:
+            yield worker
+
+
+def _simple_metrics_worker_activity_list(test_activities: Activities):
+    """Activity callables registered on the simple-metrics Temporal worker."""
+    return [
+        test_activities.load_custom_query,
+        test_activities.calculate_simple_metrics,
+        test_activities.export_data_to_postgres,
+    ]
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker_simple_metrics(temporal_test_env, test_activities):
+    """Temporal worker registered with the SimpleMetrics workflow and its activities."""
+    with ThreadPoolExecutor(max_workers=32) as activity_executor:
+        async with Worker(
+            temporal_test_env.client,
+            task_queue='test-queue',
+            workflows=[SimpleMetrics],
+            activities=_simple_metrics_worker_activity_list(test_activities),
+            activity_executor=activity_executor,
+        ) as worker:
+            yield worker
+
+
+def _minimal_retrain_worker_activity_list(test_activities: Activities):
+    """Activity callables registered on the minimal-retrain Temporal worker."""
+    return [
+        test_activities.load_query_with_minio_offload,
+        test_activities.retrain_model,
+        test_activities.update_production_model,
+        test_activities.format_retrain_report,
+        test_activities.export_data_to_postgres,
+    ]
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker_minimal_retrain(temporal_test_env, test_activities):
+    """Temporal worker registered with the MinimalRetrain workflow and its activities."""
+    with ThreadPoolExecutor(max_workers=32) as activity_executor:
+        async with Worker(
+            temporal_test_env.client,
+            task_queue='test-queue',
+            workflows=[MinimalRetrain],
+            activities=_minimal_retrain_worker_activity_list(test_activities),
+            activity_executor=activity_executor,
+        ) as worker:
+            yield worker
+
+
+def _connect_activities_to_opc_server(activities: Activities, server_url: str) -> None:
+    """
+    Initialize OPC repositories and block until the E2E server session is ready.
+
+    Runs synchronously (typically via ``asyncio.to_thread``) so the asyncua test
+    server event loop is not blocked during ``Client.connect()``.
+
+    Args:
+        activities (Activities): Worker activities under test.
+        server_url (str): ``opc.tcp://`` URL from ``OpcE2ETestServer``.
+    """
+    activities.init_opc()
+    repo = activities.opc_repository['1']
+    deadline = time.monotonic() + 30.0
+    while time.monotonic() < deadline:
+        if repo._session_ready.is_set():
+            return
+        connected, _ = repo.connect()
+        if connected:
+            return
+        time.sleep(0.5)
+    raise RuntimeError(f'Could not connect OpcRepository to OPC E2E server at {server_url}')
+
+
+def _build_e2e_opc_config(server_url: str) -> dict[str, dict]:
+    """
+    OPC server config for E2E Activities pointing at an in-process asyncua server.
+
+    Args:
+        server_url (str): ``opc.tcp://`` endpoint from ``OpcE2ETestServer``.
+
+    Return:
+        dict: ``opc_config`` payload for ``Activities`` (server id ``1``).
+    """
+    return {
+        '1': {
+            'id': '1',
+            'server_name': 'e2e_opcua',
+            'url': server_url,
+            'server_uri': server_url,
+            'cert_path': None,
+            'private_key_path': None,
+            'server_cert_path': None,
+            'reconnection_interval': 0,
+        }
+    }
+
+
+@pytest_asyncio.fixture
+async def opc_e2e_server():
+    """In-process asyncua server with writable prediction/confidence nodes."""
+    server = OpcE2ETestServer()
+    await server.start()
+    await asyncio.sleep(0.5)
+    try:
+        yield server
+    finally:
+        await server.stop()
+
+
+@pytest_asyncio.fixture(scope='function')
+async def test_activities_real_opc(
+    postgres_container,
+    minio_container,
+    mock_logger,
+    notification_handler,
+    metrics_controller,
+    mlflow_repository_stub,
+    plugin_store_stub,
+    pi_web_api_client_stub,
+    opc_e2e_server: OpcE2ETestServer,
+):
+    """Activities with real OpcRepository connected to the in-process OPC UA server."""
+    minio_client = minio_container.get_client()
+    if not minio_client.bucket_exists('test-bucket'):
+        minio_client.make_bucket('test-bucket')
+    minio_port = minio_container.get_exposed_port(9000)
+
+    activities = Activities(
+        postgres_config={
+            'host': 'localhost',
+            'port': int(postgres_container.get_exposed_port(5432)),
+            'user': postgres_container.username,
+            'password': postgres_container.password,
+            'dbname': postgres_container.dbname,
+            'min_connections': 1,
+            'max_connections': 5,
+        },
+        plugin_store=plugin_store_stub,
+        minio_config={
+            'endpoint_url': f'localhost:{minio_port}',
+            'access_key': 'minioadmin',
+            'secret_key': 'minioadmin',
+            'default_bucket': 'test-bucket',
+            'retention_hours': 24,
+            'secure': False,
+        },
+        opc_config=_build_e2e_opc_config(opc_e2e_server.url),
+        pi_web_api_config={
+            'base_url': 'http://localhost:8080',
+            'auth_type': 'bearer',
+            'auth_token': 'test_token',
+        },
+        logger=mock_logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+        mlflow_repository=mlflow_repository_stub,
+    )
+    activities.pi_web_api_client = pi_web_api_client_stub
+    await asyncio.to_thread(_connect_activities_to_opc_server, activities, opc_e2e_server.url)
+    try:
+        yield activities
+    finally:
+        await asyncio.to_thread(_teardown_real_opc_activities, activities)
+
+
+def _teardown_real_opc_activities(activities: Activities) -> None:
+    """Disconnect OPC sessions and shut down activities (sync, for asyncio.to_thread)."""
+    for opc_repo in activities.opc_repository.values():
+        opc_repo.disconnect()
+    activities.shutdown()
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker_real_opc(temporal_test_env, test_activities_real_opc):
+    """Temporal worker using real OpcRepository against the in-process OPC UA server."""
+    with ThreadPoolExecutor(max_workers=32) as activity_executor:
+        async with Worker(
+            temporal_test_env.client,
+            task_queue='test-queue',
+            workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
+            activities=_worker_activity_list(test_activities_real_opc),
+            activity_executor=activity_executor,
+        ) as worker:
+            yield worker
+
+

e2e/db_schema.sql

@@ -0,0 +1,109 @@
+-- =============================================================================
+-- E2E test database schema for the ``sientia_data`` namespace.
+--
+-- Mirrors the production DDL one-to-one so any change in production can be
+-- pasted directly into this file. The conftest fixture loads this SQL into the
+-- testcontainers Postgres before each test run.
+--
+-- Notes on differences from production:
+--  * Tables that are partitioned in production (e.g. ``simple_metrics``,
+--    ``transformed_data``, ``drift_metrics``) are created as plain tables
+--    here because the test suite does not exercise partition pruning.
+--  * Indexes are intentionally omitted; tests rely on functional behavior,
+--    not query plans.
+-- =============================================================================
+
+CREATE SCHEMA IF NOT EXISTS sientia_data;
+
+-- -----------------------------------------------------------------------------
+-- sientia_data.laborious_data
+-- -----------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS sientia_data.laborious_data (
+    model_id    int4 NOT NULL,
+    variable    text NOT NULL,
+    value       numeric NULL,
+    "timestamp" timestamptz NOT NULL,
+    created_at  timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    CONSTRAINT unique_timestamp_variable
+        UNIQUE (model_id, "timestamp", variable)
+);
+
+-- -----------------------------------------------------------------------------
+-- sientia_data.predictions
+-- -----------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS sientia_data.predictions (
+    model_id              int4 NOT NULL,
+    prediction            numeric NULL,
+    prediction_confidence numeric NOT NULL,
+    response_time         numeric NOT NULL,
+    prediction_status     text NOT NULL,
+    "timestamp"           timestamptz NOT NULL,
+    created_at            timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    "comments"            text NULL,
+    CONSTRAINT unique_model_id_timestamp
+        UNIQUE (model_id, "timestamp")
+);
+
+-- -----------------------------------------------------------------------------
+-- sientia_data.transformed_data
+--   Production: PARTITION BY RANGE (created_at). Tests use a plain table.
+-- -----------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS sientia_data.transformed_data (
+    id          SERIAL NOT NULL,
+    model_id    int4 NOT NULL,
+    variable    text NOT NULL,
+    value       numeric NULL,
+    "timestamp" timestamptz NOT NULL,
+    created_at  timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    PRIMARY KEY (id, created_at)
+);
+
+-- -----------------------------------------------------------------------------
+-- sientia_data.drift_metrics
+--   Production: PARTITION BY RANGE (created_at). Tests use a plain table.
+-- -----------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS sientia_data.drift_metrics (
+    id               SERIAL NOT NULL,
+    model_id         text NOT NULL,
+    feature          text NULL,
+    method           text NOT NULL,
+    value            numeric NOT NULL,
+    alert            bool NOT NULL,
+    chunk_index      int4 NOT NULL,
+    chunk_start_date text NOT NULL,
+    chunk_end_date   text NOT NULL,
+    accurate         bool NOT NULL,
+    "timestamp"      timestamptz NULL,
+    created_at       timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    PRIMARY KEY (id, created_at)
+);
+
+-- -----------------------------------------------------------------------------
+-- sientia_data.simple_metrics
+--   Production: PARTITION BY RANGE (created_at). Tests use a plain table.
+-- -----------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS sientia_data.simple_metrics (
+    id               SERIAL NOT NULL,
+    model_id         text NOT NULL,
+    metric           text NOT NULL,
+    value            numeric NOT NULL,
+    "timestamp"      timestamptz NULL,
+    data_size        int4 NOT NULL,
+    interval_minutes int4 NOT NULL,
+    created_at       timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    PRIMARY KEY (id, created_at)
+);
+
+-- -----------------------------------------------------------------------------
+-- sientia_data.log_retrain
+--   No primary key in production; all columns nullable.
+-- -----------------------------------------------------------------------------
+CREATE TABLE IF NOT EXISTS sientia_data.log_retrain (
+    mlflow_experiment_id int8 NULL,
+    mlflow_run_id        text NULL,
+    model_id             text NULL,
+    model_name           text NULL,
+    status               text NULL,
+    "timestamp"          timestamptz NULL,
+    "version"            text NULL
+);

e2e/helpers.py

@@ -0,0 +1,368 @@
+"""
+Shared helpers for E2E tests (Temporal workflows + PostgreSQL).
+"""
+
+import asyncio
+import json
+from datetime import datetime
+from decimal import Decimal
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import text
+from sqlalchemy.engine import Engine
+
+SCENARIO_INPUTS_DIR = Path(__file__).parent / 'scenario_inputs'
+
+
+def _replace_template_values(payload: Any, model_id: int) -> Any:
+    """
+    Replace string placeholders in scenario payloads with the concrete model id.
+
+    Args:
+        payload: JSON-like structure loaded from scenario input file.
+        model_id: Model id used to render template placeholders.
+
+    Return:
+        Any: Payload with ``{{MODEL_ID}}`` replaced where applicable.
+    """
+    if isinstance(payload, dict):
+        return {key: _replace_template_values(value, model_id) for key, value in payload.items()}
+    if isinstance(payload, list):
+        return [_replace_template_values(item, model_id) for item in payload]
+    if isinstance(payload, str):
+        if payload == '{{MODEL_ID}}':
+            return model_id
+        return payload.replace('{{MODEL_ID}}', str(model_id))
+    return payload
+
+
+def load_scenario_input(file_name: str, model_id: int | None = None) -> dict[str, Any]:
+    """
+    Load a scenario input JSON from ``e2e/scenario_inputs``.
+
+    Args:
+        file_name: JSON file name inside ``e2e/scenario_inputs``.
+        model_id: Optional model id used to render ``{{MODEL_ID}}`` placeholders.
+
+    Return:
+        dict[str, Any]: Input payload ready to be passed to workflow/activity calls.
+    """
+    file_path = SCENARIO_INPUTS_DIR / file_name
+    with file_path.open('r', encoding='utf-8') as f:
+        payload = json.load(f)
+
+    if model_id is not None:
+        return _replace_template_values(payload, model_id)
+    return payload
+
+
+async def start_and_await_workflow(client, workflow_run, input_data: dict, workflow_id: str, timeout: float = 60.0):
+    """
+    Start a workflow and wait for its result.
+
+    Args:
+        client: Temporal client from WorkflowEnvironment.
+        workflow_run: Workflow run method (e.g. PredictionsBatch.run).
+        input_data: Workflow input payload.
+        workflow_id: Unique workflow id.
+        timeout: Max seconds to wait for completion.
+
+    Return:
+        Workflow result value.
+    """
+    handle = await client.start_workflow(
+        workflow_run,
+        input_data,
+        id=workflow_id,
+        task_queue='test-queue',
+    )
+    return await asyncio.wait_for(handle.result(), timeout=timeout)
+
+
+DEFAULT_BATCH_TIMESTAMP = '2024-01-01 12:00:00+00:00'
+DEFAULT_PREDICTION_HISTORY_TIMESTAMP = '2024-01-01 12:00:00+00:00'
+
+
+def insert_sample_data(
+    postgres_engine: Engine,
+    model_id: int,
+    values: list[Any],
+    *,
+    data_timestamp: str = DEFAULT_BATCH_TIMESTAMP,
+) -> None:
+    """
+    Replace laborious_data rows for a model_id with one row per value (sensor_1..n).
+
+    Args:
+        postgres_engine: SQLAlchemy engine.
+        model_id: Model id column value.
+        values: Per-sensor values; use string 'NULL' for SQL NULL.
+        data_timestamp: Timestamp and created_at for every inserted row; drives
+            ``last_timestamp`` on the MinIO/query payload (max row time).
+    """
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
+        values_sql = []
+        for i, value in enumerate(values):
+            values_sql.append(f"""
+                ({model_id}, 'sensor_{i + 1}', {value}, '{data_timestamp}', '{data_timestamp}')
+            """)
+        insert_sql = f"""
+            INSERT INTO sientia_data.laborious_data (model_id, variable, value, timestamp, created_at)
+            VALUES
+                {', '.join(values_sql)}
+        """
+        conn.execute(text(insert_sql))
+
+
+def insert_sample_prediction(
+    postgres_engine: Engine,
+    model_id: int,
+    *,
+    prediction_timestamp: str = DEFAULT_PREDICTION_HISTORY_TIMESTAMP,
+) -> tuple[int, Decimal, Decimal, str]:
+    """
+    Insert a single historical prediction row for REPEAT scenarios.
+
+    Args:
+        postgres_engine: SQLAlchemy engine.
+        model_id: Model id.
+        prediction_timestamp: Row ``timestamp`` (unique with model_id in tests).
+
+    Return:
+        tuple: (model_id, prediction, prediction_confidence, prediction_status) for assertions.
+    """
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.predictions WHERE model_id = {model_id}'))
+        insert_sql = f"""
+            INSERT INTO sientia_data.predictions (
+                model_id, timestamp, prediction, prediction_confidence, prediction_status, comments, response_time
+            )
+            VALUES (
+                {model_id}, '{prediction_timestamp}', 10, 0, 'Good', '', 0.1
+            )
+        """
+        conn.execute(text(insert_sql))
+    return (model_id, Decimal(10), Decimal(0), 'Good')
+
+
+def workflow_failure_message_chain(exc: BaseException) -> list[str]:
+    """
+    Collect ``str()`` / ``message`` from an exception and its ``__cause__`` chain.
+
+    Args:
+        exc: Root exception (e.g. from ``pytest.raises``).
+
+    Return:
+        list[str]: Messages from root to innermost cause.
+    """
+    messages: list[str] = []
+    current: BaseException | None = exc
+    while current is not None:
+        messages.append(getattr(current, 'message', None) or str(current) or repr(current))
+        current = current.__cause__
+    return messages
+
+
+def assert_postgres_unique_violation_in_chain(exc: BaseException) -> None:
+    """
+    Assert the exception chain mentions Postgres unique-constraint violation.
+
+    Args:
+        exc: Workflow or activity error from Temporal.
+
+    Raises:
+        AssertionError: If no link in the chain looks like UniqueViolation.
+    """
+    chain = ' | '.join(workflow_failure_message_chain(exc))
+    assert 'UniqueViolation' in chain or 'unique_model_id_timestamp' in chain, (
+        f'Expected unique constraint violation in error chain, got: {chain}'
+    )
+
+
+def assert_prediction_row_count(postgres_engine: Engine, model_id: int, expected: int) -> None:
+    """
+    Assert how many prediction rows exist for a model_id.
+
+    Args:
+        postgres_engine: SQLAlchemy engine.
+        model_id: Model id filter.
+        expected: Expected row count.
+    """
+    with postgres_engine.connect() as conn:
+        n = conn.execute(
+            text('SELECT COUNT(*) FROM sientia_data.predictions WHERE model_id = :m'),
+            {'m': model_id},
+        ).scalar()
+    assert n == expected, f'Expected {expected} prediction rows, got {n}'
+
+
+def assert_prediction(
+    postgres_engine: Engine,
+    model_id: int,
+    prediction: float = 0.5,
+    prediction_confidence: int | Decimal = 0,
+    prediction_status: str = 'Good',
+    comments: str = '',
+    comments_contains: str | None = None,
+) -> None:
+    """
+    Assert exactly one prediction row exists for model_id with expected columns.
+
+    Args:
+        postgres_engine: SQLAlchemy engine.
+        model_id: Expected model_id.
+        prediction: Expected prediction value.
+        prediction_confidence: Expected confidence (int or Decimal for numeric column).
+        prediction_status: Expected status string.
+        comments: Expected exact comments string (ignored when ``comments_contains`` is set).
+        comments_contains: When set, assert this substring appears in comments.
+    """
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, prediction, prediction_confidence, prediction_status, comments '
+                f'FROM sientia_data.predictions WHERE model_id = {model_id} '
+                f'ORDER BY created_at ASC'
+            )
+        )
+        prediction_rows = result_query.fetchall()
+        assert len(prediction_rows) == 1, f'Expected one prediction record, got {len(prediction_rows)}'
+        row = prediction_rows[0]
+        assert row[0] == model_id, f'Expected model_id={model_id}, got {row[0]}'
+        assert row[1] == prediction or Decimal(str(row[1])) == Decimal(str(prediction)), (
+            f'Expected prediction={prediction}, got {row[1]}'
+        )
+        assert row[2] == prediction_confidence or Decimal(str(row[2])) == Decimal(
+            str(prediction_confidence)
+        ), f'Expected prediction_confidence={prediction_confidence}, got {row[2]}'
+        assert row[3] == prediction_status, f"Expected prediction_status='{prediction_status}', got {row[3]}"
+        actual_comments = row[4] or ''
+        if comments_contains is not None:
+            assert comments_contains in actual_comments, (
+                f"Expected comments to contain '{comments_contains}', got '{actual_comments}'"
+            )
+        else:
+            assert actual_comments == comments, f"Expected comments='{comments}', got '{actual_comments}'"
+
+
+def assert_continue(
+    postgres_engine: Engine,
+    model_id: int,
+    prediction_confidence: Decimal = Decimal(2),
+    comments: str = 'Input data with bad quality',
+) -> None:
+    """Assert one default-style prediction row after CONTINUE gate path."""
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, prediction, prediction_confidence, prediction_status, comments '
+                f'FROM sientia_data.predictions WHERE model_id = {model_id}'
+            )
+        )
+        prediction_rows = result_query.fetchall()
+        assert len(prediction_rows) == 1, 'Expected one prediction record despite warnings'
+        row = prediction_rows[0]
+        assert row[1] == 0, f'Expected prediction=0, got {row[1]}'
+        assert row[2] == prediction_confidence, (
+            f'Expected prediction_confidence={prediction_confidence}, got {row[2]}'
+        )
+        assert row[3] == 'Bad', f"Expected prediction_status='Bad', got {row[3]}"
+        assert row[4] == comments, f"Expected comments='{comments}', got {row[4]}"
+
+
+def assert_stop(postgres_engine: Engine, model_id: int) -> None:
+    """Assert no prediction rows for model_id."""
+    import pytest
+
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(f'SELECT COUNT(*) FROM sientia_data.predictions WHERE model_id = {model_id}')
+        )
+        count = result_query.scalar()
+        assert count == 0, f'Expected no predictions, but found {count} records'
+
+
+def assert_repeat(postgres_engine: Engine, model_id: int, last_prediction: tuple) -> None:
+    """
+    Assert two prediction rows for model_id both match last_prediction.
+
+    Rows are compared in created_at order for stability.
+
+    Args:
+        postgres_engine: SQLAlchemy engine.
+        model_id: Model id.
+        last_prediction: Tuple (model_id, prediction, confidence, status) to match both rows.
+    """
+    import pytest
+
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, prediction, prediction_confidence, prediction_status '
+                f'FROM sientia_data.predictions WHERE model_id = {model_id} '
+                f'ORDER BY created_at ASC'
+            )
+        )
+        prediction_rows = result_query.fetchall()
+        assert len(prediction_rows) == 2, 'Expected two prediction records'
+        assert prediction_rows[0] == last_prediction, (
+            f'Expected first row {last_prediction}, got {prediction_rows[0]}'
+        )
+        assert prediction_rows[1] == last_prediction, (
+            f'Expected second row {last_prediction}, got {prediction_rows[1]}'
+        )
+
+
+def make_workflow_id(prefix: str) -> str:
+    """Build a unique workflow id using a prefix and current timestamp."""
+    return f'{prefix}-{datetime.now().timestamp()}'
+
+
+def insert_target_data_for_drift(
+    postgres_engine: Engine,
+    model_id: int,
+    timestamps: list[str],
+    variables_values: dict[str, list[float]],
+) -> None:
+    """
+    Insert one row per (timestamp, variable) pair into ``laborious_data``.
+
+    Used by drift scenarios that need wide-format input where the pivot keeps a
+    full row for every timestamp.
+
+    Args:
+     - postgres_engine: SQLAlchemy engine bound to the test container.
+     - model_id: Model id stamped on every row.
+     - timestamps: ISO-8601 strings used both as ``timestamp`` and ``created_at``.
+     - variables_values: Mapping of variable name to a list of values; each list
+       must be the same length as ``timestamps``.
+    """
+    for var_name, values in variables_values.items():
+        if len(values) != len(timestamps):
+            raise ValueError(
+                f"Variable '{var_name}' has {len(values)} values but {len(timestamps)} timestamps"
+            )
+
+    rows_sql = []
+    for index, ts in enumerate(timestamps):
+        for var_name, values in variables_values.items():
+            rows_sql.append(
+                f"({model_id}, '{var_name}', {values[index]}, '{ts}', '{ts}')"
+            )
+
+    with postgres_engine.begin() as conn:
+        conn.execute(
+            text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}')
+        )
+        if rows_sql:
+            conn.execute(
+                text(
+                    'INSERT INTO sientia_data.laborious_data '
+                    '(model_id, variable, value, "timestamp", created_at) VALUES '
+                    + ', '.join(rows_sql)
+                )
+            )
+
+

e2e/opc_test_server.py

@@ -0,0 +1,189 @@
+"""
+In-process OPC UA server for E2E tests (asyncua).
+
+Provides writable prediction/confidence nodes and optional write faults
+(Tier-1 BadSessionIdInvalid via PreWrite callback).
+"""
+
+from __future__ import annotations
+
+import socket
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from asyncua import Server, ua
+from asyncua.common.callback import CallbackType
+from asyncua.common.utils import ServiceError
+
+if TYPE_CHECKING:
+    from asyncua.common.node import Node
+
+
+UNKNOWN_NODE_ID = 'ns=99;i=9999'
+
+
+@dataclass(frozen=True)
+class OpcE2ENodeIds:
+    """NodeId strings used in opc_output_config for E2E workflows."""
+
+    prediction: str
+    confidence: str
+    unknown: str = UNKNOWN_NODE_ID
+
+
+class OpcE2ETestServer:
+    """
+    Ephemeral asyncua server with Laborious E2E variables and controllable faults.
+
+    Args:
+        host: Bind address (default 127.0.0.1).
+    """
+
+    def __init__(self, host: str = '127.0.0.1') -> None:
+        self._host = host
+        self._server: Server | None = None
+        self._prediction_node: Node | None = None
+        self._confidence_node: Node | None = None
+        self._session_bad_on_write = False
+        self._url: str | None = None
+        self._node_ids: OpcE2ENodeIds | None = None
+
+    @property
+    def url(self) -> str:
+        if self._url is None:
+            raise RuntimeError('OPC E2E server is not started')
+        return self._url
+
+    @property
+    def node_ids(self) -> OpcE2ENodeIds:
+        if self._node_ids is None:
+            raise RuntimeError('OPC E2E server is not started')
+        return self._node_ids
+
+    def set_session_bad_on_write(self, enabled: bool) -> None:
+        """
+        When enabled, every client Write is rejected with BadSessionIdInvalid.
+
+        Args:
+            enabled (bool): Turn Tier-1 session fault injection on or off.
+        """
+        self._session_bad_on_write = enabled
+
+    async def start(self) -> OpcE2ENodeIds:
+        """
+        Start the OPC UA server on a free TCP port.
+
+        Return:
+            OpcE2ENodeIds: NodeId strings for prediction and confidence tags.
+        """
+        port = _free_port(self._host)
+        self._url = f'opc.tcp://{self._host}:{port}/freeopcua/server/'
+
+        server = Server()
+        server.set_endpoint(self._url)
+        await server.init()
+        server.iserver.callback_service.addListener(
+            CallbackType.PreWrite,
+            self._pre_write_callback,
+        )
+
+        idx = await server.register_namespace('http://sientia.test/laborious-e2e')
+        e2e_object = await server.nodes.objects.add_object(idx, 'LaboriousE2E')
+        prediction = await e2e_object.add_variable(
+            idx,
+            'Prediction',
+            ua.Variant(0.0, ua.VariantType.Float),
+        )
+        confidence = await e2e_object.add_variable(
+            idx,
+            'Confidence',
+            ua.Variant(0.0, ua.VariantType.Float),
+        )
+        await prediction.set_writable()
+        await confidence.set_writable()
+
+        await server.start()
+        self._server = server
+        self._prediction_node = prediction
+        self._confidence_node = confidence
+        self._node_ids = OpcE2ENodeIds(
+            prediction=prediction.nodeid.to_string(),
+            confidence=confidence.nodeid.to_string(),
+        )
+        return self._node_ids
+
+    async def stop(self) -> None:
+        """Stop the OPC UA server and release the listening port."""
+        if self._server is not None:
+            await self._server.stop()
+            self._server = None
+        self._prediction_node = None
+        self._confidence_node = None
+        self._url = None
+        self._node_ids = None
+        self._session_bad_on_write = False
+
+    async def read_prediction(self) -> float:
+        """
+        Read the current prediction variable value from the address space.
+
+        Return:
+            float: Stored prediction value.
+        """
+        if self._prediction_node is None:
+            raise RuntimeError('OPC E2E server is not started')
+        value = await self._prediction_node.read_value()
+        return float(value)
+
+    async def read_confidence(self) -> float:
+        """
+        Read the current confidence variable value from the address space.
+
+        Return:
+            float: Stored confidence value.
+        """
+        if self._confidence_node is None:
+            raise RuntimeError('OPC E2E server is not started')
+        value = await self._confidence_node.read_value()
+        return float(value)
+
+    async def _pre_write_callback(self, _event, _service) -> None:
+        if self._session_bad_on_write:
+            raise ServiceError(ua.StatusCodes.BadSessionIdInvalid)
+
+
+def _free_port(host: str) -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind((host, 0))
+        return int(sock.getsockname()[1])
+
+
+def build_opc_output_config(
+    node_ids: OpcE2ENodeIds,
+    *,
+    prediction_tag: str | None = None,
+    confidence_tag: str | None = None,
+    prediction_only: bool = False,
+    server_key: str = '1',
+) -> dict[str, dict]:
+    """
+    Build opc_output_config for PredictionsBatch using real server NodeIds.
+
+    Args:
+        node_ids (OpcE2ENodeIds): Node ids from OpcE2ETestServer.
+        prediction_tag (str | None): Override prediction NodeId (default: node_ids.prediction).
+        confidence_tag (str | None): Override confidence NodeId (default: node_ids.confidence).
+        prediction_only (bool): When True, omit confidence_tags (single write per activity).
+        server_key (str): OPC server id key in opc_output_config.
+
+    Return:
+        dict: opc_output_config payload for workflow input.
+    """
+    pred = prediction_tag if prediction_tag is not None else node_ids.prediction
+    conf = confidence_tag if confidence_tag is not None else node_ids.confidence
+    server_config: dict = {
+        'prediction_tags': {pred: {'data_type': 'float'}},
+    }
+    if not prediction_only:
+        server_config['confidence_tags'] = {conf: {'data_type': 'float'}}
+    return {server_key: server_config}

e2e/scenario_inputs/drift_base.json

@@ -0,0 +1,14 @@
+{
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "schema": "sientia_data",
+  "source_table_name": "laborious_data",
+  "target_table_name": "drift_metrics",
+  "interval": 60,
+  "drift_metrics": ["kolmogorov_smirnov", "jensen_shannon", "wasserstein"],
+  "chunk_period": "min",
+  "model_config": {
+    "target": "sensor_1"
+  }
+}

e2e/scenario_inputs/format_export_base.json

@@ -0,0 +1,37 @@
+{
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "query": "SELECT timestamp, variable, value, created_at FROM sientia_data.laborious_data WHERE model_id = {{MODEL_ID}}",
+  "schema": "sientia_data",
+  "table_name": "predictions",
+  "transform_table_name": "transformed_data",
+  "input_filters": {
+    "EMPTY_DATA": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_transform_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_predict_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "path_priority": ["STOP", "CONTINUE", "REPEAT"],
+  "opc_output_config": {},
+  "pi_web_api_output_config": {},
+  "save_transform": true,
+  "prediction_store_policy": "lts:1",
+  "model_config": {
+    "retention_minutes": 0,
+    "target": "sensor_1"
+  },
+  "datetime_columns": ["timestamp", "created_at"]
+}

e2e/scenario_inputs/main_happy_path.json

@@ -0,0 +1,45 @@
+{
+  "metadata": {
+    "metadata": {
+      "model_id": "{{MODEL_ID}}",
+      "model_name": "test_model",
+      "schedule_name": "test-schedule",
+      "workflow_name": "predictions_batch"
+    }
+  },
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "query": "SELECT timestamp, variable, value, created_at FROM sientia_data.laborious_data WHERE model_id = {{MODEL_ID}}",
+  "schema": "sientia_data",
+  "table_name": "predictions",
+  "transform_table_name": "transformed_data",
+  "input_filters": {
+    "EMPTY_DATA": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_transform_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_predict_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "path_priority": ["STOP", "CONTINUE", "REPEAT"],
+  "opc_output_config": {},
+  "pi_web_api_output_config": {},
+  "save_transform": true,
+  "prediction_store_policy": "lts:1",
+  "model_config": {
+    "target": "sensor_1",
+    "retention_minutes": 0
+  },
+  "datetime_columns": ["timestamp", "created_at"]
+}

e2e/scenario_inputs/main_invalid_datetime.json

@@ -0,0 +1,45 @@
+{
+  "metadata": {
+    "metadata": {
+      "model_id": "{{MODEL_ID}}",
+      "model_name": "test_model",
+      "schedule_name": "test-schedule",
+      "workflow_name": "predictions_batch"
+    }
+  },
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "query": "SELECT timestamp, variable, value FROM sientia_data.laborious_data WHERE model_id = {{MODEL_ID}}",
+  "schema": "sientia_data",
+  "table_name": "predictions",
+  "transform_table_name": "transformed_data",
+  "input_filters": {
+    "EMPTY_DATA": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_transform_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_predict_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "path_priority": ["STOP", "CONTINUE", "REPEAT"],
+  "opc_output_config": {},
+  "pi_web_api_output_config": {},
+  "save_transform": true,
+  "prediction_store_policy": "lts:1",
+  "model_config": {
+    "target": "sensor_1",
+    "retention_minutes": 0
+  },
+  "datetime_columns": ["nonexistent_column"]
+}

e2e/scenario_inputs/main_missing_required.json

@@ -0,0 +1,16 @@
+{
+  "metadata": {
+    "metadata": {
+      "model_id": "{{MODEL_ID}}",
+      "model_name": "test_model",
+      "schedule_name": "test-schedule",
+      "workflow_name": "predictions_batch"
+    }
+  },
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "schema": "sientia_data",
+  "table_name": "predictions",
+  "transform_table_name": "transformed_data"
+}

e2e/scenario_inputs/main_sql_error.json

@@ -0,0 +1,44 @@
+{
+  "metadata": {
+    "metadata": {
+      "model_id": "{{MODEL_ID}}",
+      "model_name": "test_model",
+      "schedule_name": "test-schedule",
+      "workflow_name": "predictions_batch"
+    }
+  },
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "query": "SELECT * FROM nonexistent_table WHERE invalid_syntax =",
+  "schema": "sientia_data",
+  "table_name": "predictions",
+  "transform_table_name": "transformed_data",
+  "input_filters": {
+    "EMPTY_DATA": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_transform_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_predict_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "path_priority": ["STOP", "CONTINUE", "REPEAT"],
+  "opc_output_config": {},
+  "pi_web_api_output_config": {},
+  "save_transform": true,
+  "prediction_store_policy": "lts:1",
+  "model_config": {
+    "target": "sensor_1",
+    "retention_minutes": 0
+  }
+}

e2e/scenario_inputs/minimal_retrain_base.json

@@ -0,0 +1,12 @@
+{
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "query": "SELECT timestamp, variable, value, created_at FROM sientia_data.laborious_data WHERE model_id = {{MODEL_ID}}",
+  "schema": "sientia_data",
+  "table_name": "log_retrain",
+  "datetime_columns": ["timestamp", "created_at"],
+  "model_config": {
+    "target": "sensor_1"
+  }
+}

e2e/scenario_inputs/minio_offload_load_query.json

@@ -0,0 +1,11 @@
+{
+  "metadata": {
+    "schedule_name": "test-schedule",
+    "model_name": "test_model",
+    "model_id": "{{MODEL_ID}}",
+    "workflow_name": "predictions_batch"
+  },
+  "query": "SELECT timestamp, variable, value, created_at FROM sientia_data.laborious_data WHERE model_id = {{MODEL_ID}}",
+  "model_name": "test_model",
+  "datetime_columns": ["timestamp", "created_at"]
+}

e2e/scenario_inputs/minio_offload_workflow.json

@@ -0,0 +1,37 @@
+{
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "query": "SELECT timestamp, variable, value, created_at FROM sientia_data.laborious_data WHERE model_id = {{MODEL_ID}}",
+  "schema": "sientia_data",
+  "table_name": "predictions",
+  "transform_table_name": "transformed_data",
+  "input_filters": {
+    "EMPTY_DATA": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_transform_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_predict_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "path_priority": ["STOP", "CONTINUE", "REPEAT"],
+  "opc_output_config": {},
+  "pi_web_api_output_config": {},
+  "save_transform": false,
+  "prediction_store_policy": "lts:1",
+  "model_config": {
+    "retention_minutes": 0,
+    "target": "sensor_1"
+  },
+  "datetime_columns": ["timestamp", "created_at"]
+}

e2e/scenario_inputs/prediction_process_base.json

@@ -0,0 +1,39 @@
+{
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "query": "SELECT timestamp, variable, value, created_at FROM sientia_data.laborious_data WHERE model_id = {{MODEL_ID}}",
+  "schema": "sientia_data",
+  "table_name": "predictions",
+  "transform_table_name": "transformed_data",
+  "input_filters": {
+    "SPECIFIC_VARIABLES_NULL_VALUES": {
+      "POLICY": "CONTINUE",
+      "CONFIG": {
+        "variables": ["sensor_1"]
+      }
+    }
+  },
+  "mlflow_transform_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "mlflow_predict_filters": {
+    "API_ERROR": {
+      "POLICY": "STOP",
+      "CONFIG": {}
+    }
+  },
+  "path_priority": ["STOP", "CONTINUE", "REPEAT"],
+  "opc_output_config": {},
+  "pi_web_api_output_config": {},
+  "save_transform": true,
+  "prediction_store_policy": "lts:1",
+  "model_config": {
+    "retention_minutes": 0,
+    "target": "sensor_1"
+  },
+  "datetime_columns": ["timestamp", "created_at"]
+}

e2e/scenario_inputs/simple_metrics_base.json

@@ -0,0 +1,14 @@
+{
+  "schedule_name": "test-schedule",
+  "model_name": "test_model",
+  "model_id": "{{MODEL_ID}}",
+  "schema": "sientia_data",
+  "predictions_table_name": "predictions",
+  "data_table_name": "laborious_data",
+  "target_table_name": "simple_metrics",
+  "interval_minutes": 60,
+  "metrics": ["rmse", "mse", "mae", "r2"],
+  "model_config": {
+    "target": "sensor_target"
+  }
+}

e2e/scenarios.md

@@ -0,0 +1,455 @@
+# E2E Scenario Documentation - Predictions Batch
+
+This document describes the end-to-end scenarios for `predictions_batch` and its child workflows:
+`prediction_process` and `format_and_export_prediction`.
+
+It is a functional reference of scenario behavior, inputs, and expected outcomes.
+
+## Execution Context
+
+- Tests run under `e2e/` and are marked with `@pytest.mark.integration`.
+- PostgreSQL and MinIO are provisioned with testcontainers.
+- `test_minio_offload.py` uses real MinIO I/O; other scenario suites may use stubs/mocks for optional outputs.
+- Real OPC UA scenarios use `@pytest.mark.opc` and an in-process asyncua server (`e2e/test_opc_real_server.py`).
+
+### Local validation
+
+Use the existing project virtualenv and the shared `validate` script for unit/quality gates; run E2E separately (Docker required).
+
+```bash
+source ./venv/bin/activate
+
+# Auto-fix + static checks (no pytest)
+validate --fix --project-name=laborious
+
+# Full unit + quality gate
+validate --project-name=laborious
+
+# E2E (integration)
+pytest e2e/ --override-ini testpaths=e2e -m integration
+
+# E2E (real OPC server only)
+pytest e2e/test_opc_real_server.py --override-ini testpaths=e2e -m opc
+```
+
+---
+
+## 1. Main Workflow Scenarios
+Source: `e2e/test_predictions_batch_main_workflow.py`
+
+### 1.1.1 Happy Path - Complete Success
+**Summary**: Full workflow succeeds with valid query and default gate behavior.
+
+**Description**:
+- Query returns rows for a model.
+- `prediction_process` runs transform and predict paths.
+- Final prediction and transformed data are persisted.
+
+**Expected Outcome**:
+- Exactly one prediction row is created.
+- Transform rows are created.
+- Confidence/status/comments are success values.
+
+### 1.2.1 SQL Query Execution Error
+**Summary**: Invalid SQL leads to no persisted prediction.
+
+**Description**:
+- Input query is invalid.
+- Load step fails and workflow follows error/short-circuit path.
+
+**Expected Outcome**:
+- No prediction rows for the model.
+- Workflow does not require retry-loop assumptions in assertions.
+
+### 1.2.2 Missing Required Parameters
+**Summary**: Missing required fields prevent workflow completion path.
+
+**Description**:
+- Required input key (e.g. `query`) is omitted.
+- Workflow fails to produce actionable input for child flow.
+
+**Expected Outcome**:
+- No prediction rows are persisted.
+- Workflow handle may require explicit terminate in E2E harness.
+
+### 1.2.3 Invalid Datetime Column Specification (de-prioritized)
+**Summary**: Legacy invalid datetime-column case is retained only as low-priority legacy coverage.
+
+**Description**:
+- `datetime_columns` references non-existing columns.
+- Behavior may vary by query shape and parser fallback.
+
+**Expected Outcome**:
+- No predictions persisted in the covered legacy assertion path.
+- Scenario is not considered primary behavior coverage.
+
+---
+
+## 2. Prediction Process Scenarios
+Source: `e2e/test_predictions_batch_prediction_process.py`
+
+### 2.1 Input Gate Path Decisions
+
+#### 2.1.1 CONTINUE
+**Summary**: Input filter flags quality issue but allows continuation via default path.
+
+**Description**:
+- Input gate returns `CONTINUE`.
+- MLFlow transform/predict are skipped.
+- Export path persists default-style prediction with warning context.
+
+#### 2.1.2 STOP
+**Summary**: Input filter blocks processing.
+
+**Description**:
+- Input gate returns `STOP`.
+- Workflow exits without export.
+
+#### 2.1.3 REPEAT with history
+**Summary**: Prior prediction is reused.
+
+**Description**:
+- Input gate returns `REPEAT`.
+- `repeat_last_prediction` path is executed using existing historical row.
+
+#### 2.1.4 REPEAT without history
+**Summary**: Repeat requested but no previous prediction exists.
+
+**Description**:
+- Input gate returns `REPEAT`.
+- No prior row is available to duplicate.
+
+**Expected Outcome**:
+- No new prediction rows are created for the model.
+
+### 2.2 Transform Gate Decisions
+
+#### 2.2.1 CONTINUE on transform response error
+**Summary**: Transform response is degraded, but workflow continues.
+
+#### 2.2.2 STOP on transform response error
+**Summary**: Transform response error blocks downstream processing.
+
+#### 2.2.3 REPEAT on transform response error
+**Summary**: Transform response error triggers repeat-last-prediction path.
+
+#### 2.2.4 STOP on transform content NaN
+**Summary**: Content gate (`NAN_VALUES`) blocks on all-NaN transform payload.
+
+### 2.3 Predict Gate Decisions
+
+#### 2.3.1 CONTINUE on predict response error
+**Summary**: Predict response degraded; workflow exports with degraded metadata.
+
+#### 2.3.2 STOP on predict response error
+**Summary**: Predict response error blocks export.
+
+#### 2.3.3 REPEAT on predict response error
+**Summary**: Predict response error routes to repeat-last-prediction.
+
+### 2.4.1 Priority Conflict Resolution
+**Summary**: Deterministic selection when multiple filters produce different flags.
+
+**Description**:
+- Multiple filters may produce `STOP`, `CONTINUE`, and/or `REPEAT`.
+- `path_priority` defines precedence.
+
+**Expected Outcome**:
+- Highest-priority flag is applied consistently.
+- Executed branch matches configured priority ordering.
+
+---
+
+## 3. Format and Export Scenarios
+Source: `e2e/test_predictions_batch_format_export.py`
+
+### 3.1 Output Combination Scenarios
+
+#### 3.1.1 Default prediction export
+**Summary**: Non-`None` path flag uses `format_default_prediction`.
+
+**Description**:
+- Default prediction is generated.
+- Transform export is skipped.
+- Optional outputs (PI/OPC) still execute when configured.
+
+#### 3.1.2 OPC only
+**Summary**: Postgres + OPC writes, PI Web API disabled.
+
+#### 3.1.3 PI Web API only
+**Summary**: Postgres + PI writes, OPC disabled.
+
+#### 3.1.4 Postgres only
+**Summary**: Both optional outputs disabled; only Postgres persistence and metrics.
+
+#### 3.1.5 No transformed data export
+**Summary**: Prediction is persisted; transformed table is not written.
+
+### 3.2 Degraded-but-successful Completion
+
+#### 3.2.1 PI Web API write error
+**Summary**: PI write failure does not fail workflow.
+
+**Expected Outcome**:
+- Workflow completes.
+- Prediction persisted with degraded confidence/comments (PI error semantics).
+
+#### 3.2.2 OPC write error
+**Summary**: OPC write failure does not fail workflow.
+
+**Expected Outcome**:
+- Workflow completes.
+- Prediction persisted with OPC degraded confidence/comments.
+
+#### 3.2.3 PI Web API partial write error
+**Summary**: Partial PI acknowledgement is treated as degraded success.
+
+**Expected Outcome**:
+- Workflow completes.
+- Prediction persisted with PI error confidence and descriptive comment.
+
+#### 3.2.4 OPC session / channel error (confidence 14)
+**Summary**: Tier-1 `BadSessionIdInvalid` (or equivalent session error) degrades the prediction without failing the workflow.
+
+**Sources**:
+- Mock: `e2e/test_predictions_batch_format_export.py::test_scenario_3_2_4_opc_session_bad_mock`
+- Real server: `e2e/test_opc_real_server.py::test_scenario_3_2_4_opc_session_bad_real_server` (`@pytest.mark.opc`)
+
+**Expected Outcome**:
+- Workflow completes.
+- `prediction_confidence` is 14.
+- Comments contain `OPC UA session/channel error: BadSessionIdInvalid`.
+
+#### 3.2.5 OPC write blocked during reconnect (confidence 14)
+**Summary**: While reconnect holds the repository connection lock, writes fail fast with `reconnect_in_progress`.
+
+**Sources**:
+- Mock: `e2e/test_predictions_batch_format_export.py::test_scenario_3_2_5_opc_reconnect_in_progress_mock`
+- Real server: `e2e/test_opc_real_server.py::test_scenario_3_2_5_opc_write_blocked_during_reconnect_real_server` (`@pytest.mark.opc`)
+
+**Expected Outcome**:
+- Workflow completes.
+- `prediction_confidence` is 14.
+- Comments contain `OPC UA reconnect in progress`.
+
+### 3.3.1 Combined Optional Outputs (PI + OPC)
+**Summary**: Both external output channels are enabled together.
+
+**Description**:
+- PI Web API and OPC configs are both present.
+- Output mutation order matters for final persisted payload.
+
+**Expected Outcome**:
+- PI write executes before OPC write in workflow sequence.
+- Final Postgres payload reflects any confidence/comment updates.
+- OPC metrics are emitted when tag writes return response times.
+
+---
+
+## 4. MinIO Offload Scenarios
+Source: `e2e/test_minio_offload.py`
+
+### 4.1.1 Forced offload to MinIO
+**Summary**: Very low threshold forces parquet upload.
+
+**Description**:
+- Payload is offloaded (`object_key` present, inline data absent/empty).
+- Object is present in MinIO under `prediction_datasets/...`.
+- Retrieval reconstructs the dataframe.
+
+### 4.1.2 Full workflow with offloaded load payload
+**Summary**: Offload path works during full `predictions_batch` execution.
+
+**Expected Outcome**:
+- Workflow completes.
+- Prediction row is persisted.
+
+### 4.2.1 Inline payload below threshold
+**Summary**: Data remains inline when threshold is not exceeded.
+
+**Expected Outcome**:
+- Payload stores inline `data`.
+- `object_key` is `None`.
+- Downstream persistence behavior matches offload scenario semantics.
+
+---
+
+## 5. Drift Workflow Scenarios
+Source: `e2e/test_drift.py`
+
+The drift suite drives the **real** `sientia_model.analytics.drift_analysis.DriftAnalysis`
+analyzer (no stubs / mocks). Each scenario exercises the full pipeline:
+
+```
+laborious_data (Postgres) -> load_custom_query
+                          -> calculate_drift (DriftAnalysis univariate + multivariate)
+                          -> export_data_to_postgres (sientia_data.drift_metrics)
+```
+
+The `mlflow_repository_stub` provides the reference-data CSV via
+`download_artifacts`, and tests assert postgres rows in
+`sientia_data.drift_metrics` against this canonical schema:
+
+`id, model_id, feature, method, value, alert, chunk_index, chunk_start_date, chunk_end_date, accurate, timestamp, created_at`.
+
+Tests assert behavioral / structural properties (column presence, NOT NULL
+constraints, business-key invariants like uniform `timestamp` and stamped
+`model_id`) rather than exact numeric drift scores, since those depend on
+the real analyzer implementation and the synthetic data fed in.
+
+### 5.1 Happy paths
+
+#### D.1.1 Full pipeline persists all columns with reference data
+**Summary**: 10 minutes of target data are inserted; a 10-row reference CSV
+is configured via the MLflow stub. The `DriftAnalysis` runs end-to-end.
+
+**Expected Outcome**:
+- One row per `(chunk_index, feature, method)` plus a `multivariate` block
+  per chunk is persisted.
+- Every column in the DDL is populated; `feature` is the only nullable column
+  per the new schema.
+- `accurate=True` for every row (reference path).
+- All three default univariate methods reach the analyzer.
+- `model_id` is stamped as `text` and uniform across rows.
+- `timestamp` equals `max(target_data.timestamp)` and is uniform across rows.
+- `chunk_start_date` / `chunk_end_date` are persisted as ISO text and ordered.
+- `p_value` is dropped before persistence.
+
+#### D.1.2 30% fallback when reference data is unavailable
+**Summary**: MLflow alias resolution is forced to fail so
+`get_reference_data` returns `None`; `calculate_drift` falls back to the
+first 30% of target rows as reference.
+
+**Expected Outcome**:
+- All persisted rows carry `accurate=False`.
+- A `MODEL_METRICS_REFERENCE_DATA_WARNING` notification is emitted to MongoDB.
+
+### 5.2 Failure paths
+
+#### D.3.1 Empty target data short-circuits the workflow
+**Summary**: `load_custom_query` returns no rows.
+
+**Expected Outcome**:
+- The workflow returns early and writes nothing to `sientia_data.drift_metrics`.
+
+### 5.3 Configuration paths
+
+#### D.4.2 Invalid `chunk_period` raises ValueError
+**Summary**: Anything other than `min` / `s` is rejected by `calculate_drift`.
+
+**Expected Outcome**:
+- The workflow surfaces the `ValueError` ("Invalid chunk period: ...").
+- No rows are persisted.
+
+#### D.4.3 `chunk_period='s'` preserves seconds in `chunk_start_date`
+**Summary**: Target data spans two minutes with samples at second-30
+boundaries; the activity is configured with `chunk_period='s'`.
+
+**Expected Outcome**:
+- At least one persisted `chunk_start_date` carries `seconds=30`, proving
+  that the analyzer chunked at sub-minute granularity and the ISO-text
+  serialization preserved the boundary.
+
+---
+
+## 6. Simple Metrics Workflow Scenarios
+Source: `e2e/test_simple_metrics.py`
+
+Validates `sientia_data.simple_metrics` columns:
+`id, model_id, metric, value, timestamp, data_size, interval_minutes, created_at`.
+Note: ``timestamp`` is now nullable per the new DDL and ``model_id`` is ``text``.
+
+### 6.1 Happy paths
+
+#### S.1.1 rmse/mse/mae/r2 happy path
+**Summary**: Prediction/target pairs are inserted; the activity computes all
+four metrics with closed-form expected values.
+
+**Expected Outcome**:
+- One row per metric is persisted; all columns populated.
+- `data_size` matches the joined row count and `interval_minutes=60`.
+
+#### S.1.2 Subset metrics
+**Summary**: Requesting `metrics=['rmse']` writes only the rmse row.
+
+### 6.2 Edge cases
+
+#### S.2.1 Zero-variance target returns r2=0
+**Summary**: When all targets are equal, `ss_tot=0`; the activity must guard
+against division by zero and return `r2=0`.
+
+### 6.3 Failure paths
+
+#### S.3.1 No overlapping data short-circuits persistence
+**Summary**: With no `laborious_data` rows for the configured target variable
+the workflow exits before `calculate_simple_metrics` and writes nothing.
+
+---
+
+## 7. Minimal Retrain Workflow Scenarios
+Source: `e2e/test_minimal_retrain.py`
+
+The MLflow registry is fully mocked (no real artifacts in test container).
+Validates `sientia_data.log_retrain` columns:
+`mlflow_experiment_id, mlflow_run_id, model_id, model_name, status, timestamp, version`.
+Note: the new DDL drops the legacy ``id`` and ``created_at`` columns,
+``mlflow_experiment_id`` is now ``int8`` and ``model_id`` is ``text``.
+
+### 7.1 Happy path
+
+#### MR.1.1 Successful retrain + promotion
+**Summary**: Training data loads via MinIO offload, `wrapper.retrain` succeeds,
+the new version is promoted to the `production` alias.
+
+**Expected Outcome**:
+- Report row has success status, `version='7'`, `mlflow_run_id='retrain-run-id'`,
+  `mlflow_experiment_id=4242` (`int8`).
+- `mlflow.log_artifact` is called with the input CSV.
+- `promote_to_alias` is called once with the resolved version and alias.
+
+### 7.2 Failure paths
+
+#### MR.2.1 Wrapper retrain raises
+**Summary**: `wrapper.retrain` raises `RuntimeError`. The activity returns
+`success=False`, `update_production_model` is NOT invoked.
+
+**Expected Outcome**:
+- Report row carries the error message and `version`/`mlflow_*` columns are NULL.
+
+#### MR.2.2 Missing `model_config.target`
+**Summary**: Empty model config short-circuits before any MLflow call.
+
+**Expected Outcome**:
+- Report row carries the explicit guard message.
+- `get_cached_model` is never invoked.
+
+#### MR.3.1 No training data
+**Summary**: The training query returns no rows; the workflow does not
+persist any report row. The current code raises plain `ValueError` from the
+workflow function, which Temporal treats as a workflow-task failure (see
+`CODE_ISSUES.md` issue MR-1).
+
+---
+
+## Input Contract Reference
+
+Common scenario input fields:
+- `schedule_name`
+- `model_name`
+- `model_id`
+- `query`
+- `schema`
+- `table_name`
+- `transform_table_name`
+- `input_filters`
+- `mlflow_transform_filters`
+- `mlflow_predict_filters`
+- `path_priority` (default order: `STOP`, `CONTINUE`, `REPEAT`)
+- `save_transform`
+- `prediction_store_policy`
+- `model_config.target`
+- `datetime_columns` (when query returns temporal fields)
+
+Optional outputs:
+- `opc_output_config`
+- `pi_web_api_output_config`

e2e/test_child_workflows_e2e.py

@@ -0,0 +1,74 @@
+"""
+Direct E2E execution of child workflows (smaller surface than PredictionsBatch).
+"""
+
+from decimal import Decimal
+
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import make_workflow_id, start_and_await_workflow
+from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_format_and_export_prediction_default_path_e2e(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    postgres_engine,
+):
+    """
+    Run FormatAndExportPrediction with path_flag set (format_default_prediction path).
+    """
+    client = temporal_test_env.client
+    model_id = 401
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.predictions WHERE model_id = {model_id}'))
+
+    metadata = {
+        'metadata': {
+            'model_id': model_id,
+            'model_name': 'test_model',
+            'schedule_name': 'test-schedule',
+            'workflow_name': 'subworkflow.format_and_export_prediction',
+        }
+    }
+    input_data = {
+        'metadata': metadata,
+        'path_flag': 'CONTINUE',
+        'data': {'last_timestamp': '2024-01-01 12:00:00+00:00'},
+        'prediction_confidence': 2,
+        'timestamp': '2024-01-01 12:00:00+00:00',
+        'model_id': model_id,
+        'model_name': 'test_model',
+        'schema': 'sientia_data',
+        'table_name': 'predictions',
+        'transform_table_name': 'transformed_data',
+        'comment': 'e2e child workflow default path',
+        'opc_output_config': {},
+        'pi_web_api_output_config': {},
+        'prediction_store_policy': 'lts:1',
+    }
+
+    await start_and_await_workflow(
+        client,
+        FormatAndExportPrediction.run,
+        input_data,
+        make_workflow_id('e2e-format-export-child'),
+    )
+
+    with postgres_engine.connect() as conn:
+        row = conn.execute(
+            text(
+                f'SELECT prediction, prediction_confidence, prediction_status, comments '
+                f'FROM sientia_data.predictions WHERE model_id = {model_id}'
+            )
+        ).fetchone()
+    assert row is not None
+    assert row[0] == 0
+    assert row[1] == Decimal(2)
+    assert row[2] == 'Bad'
+    assert row[3] == 'e2e child workflow default path'

e2e/test_drift.py

@@ -0,0 +1,600 @@
+"""
+End-to-end tests for the Drift workflow.
+
+The drift suite drives the **real** ``sientia_model.analytics.drift_analysis.DriftAnalysis``
+analyzer (no mocking). Each scenario exercises the full pipeline:
+
+    laborious_data (Postgres)
+        -> load_custom_query
+        -> calculate_drift (DriftAnalysis univariate + multivariate)
+        -> export_data_to_postgres (sientia_data.drift_metrics)
+
+Coverage focus:
+
+- Happy path persists every column required by ``sientia_data.drift_metrics``
+  with a valid reference dataset downloaded from MLflow.
+- 30% fallback path activates when the MLflow reference is unavailable and
+  emits the ``MODEL_METRICS_REFERENCE_DATA_WARNING`` notification.
+- Empty target data short-circuits the workflow without persisting anything.
+- Invalid ``chunk_period`` is rejected by ``calculate_drift``.
+- ``chunk_period='s'`` preserves second-level precision in
+  ``chunk_start_date``.
+
+Tests assert behavioral / structural properties (column presence, NOT NULL
+constraints, business-key invariants) rather than exact numeric values, since
+those depend on the real analyzer implementation and synthetic data.
+"""
+
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pandas as pd
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import (
+    insert_target_data_for_drift,
+    load_scenario_input,
+    make_workflow_id,
+    start_and_await_workflow,
+)
+from laborious.activities.activities import Activities
+from laborious.activities.model_metrics import ModelMetrics
+from laborious.workflows.drift import Drift
+from sientia_model.analytics.drift_analysis import DriftAnalysis
+
+# Drift columns persisted on every row in ``sientia_data.drift_metrics`` —
+# mirrors the production DDL.
+EXPECTED_DRIFT_COLUMNS = [
+    'id',
+    'model_id',
+    'feature',
+    'method',
+    'value',
+    'alert',
+    'chunk_index',
+    'chunk_start_date',
+    'chunk_end_date',
+    'accurate',
+    'timestamp',
+    'created_at',
+]
+
+# Columns the DDL marks as NOT NULL. ``feature`` and ``timestamp`` are
+# nullable in the production schema (multivariate rows do not bind to a
+# single feature; ``timestamp`` is allowed to be empty when upstream data has
+# no usable instant).
+NON_NULL_DRIFT_COLUMNS = {
+    'id',
+    'model_id',
+    'method',
+    'value',
+    'alert',
+    'chunk_index',
+    'chunk_start_date',
+    'chunk_end_date',
+    'accurate',
+    'created_at',
+}
+
+DEFAULT_DRIFT_METHODS = ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']
+
+
+def _chunk_dataframe_skip_empty_groups(
+    self: DriftAnalysis,
+    df: pd.DataFrame,
+    timestamp_col: str,
+    chunk_period: str,
+) -> list[tuple[int, pd.DataFrame]]:
+    """
+    Same as ``DriftAnalysis._chunk_dataframe`` but omit empty time buckets.
+
+    ``pd.Grouper(freq='s')`` yields every second between min and max timestamp;
+    empty buckets still appear in the groupby iterator and produce invalid
+    drift rows (e.g. NaT timestamps) that ``calculate_drift`` later filters out
+    entirely. Production fix belongs in ``sientia_model``; this shim keeps the
+    e2e honest about second-level chunk boundaries with sparse samples.
+    """
+    grouped = df.groupby(pd.Grouper(key=timestamp_col, freq=chunk_period), dropna=True)
+    chunks: list[tuple[int, pd.DataFrame]] = []
+    idx = 0
+    for _, chunk in grouped:
+        if chunk.empty:
+            continue
+        chunks.append((idx, chunk.copy()))
+        idx += 1
+    return chunks
+
+
+def _drift_input(model_id: int, **overrides) -> dict:
+    """Load the base drift scenario JSON and apply ad-hoc overrides."""
+    input_data = load_scenario_input('drift_base.json', model_id=model_id)
+    input_data.update(overrides)
+    return input_data
+
+
+def _recent_minute_timestamps(count: int, offset_minutes: int = 6) -> list[str]:
+    """
+    Build ``count`` consecutive UTC minute timestamps placed in the recent past.
+
+    The Drift workflow filters target rows with ``timestamp > NOW() - INTERVAL``,
+    so timestamps must be recent for tests to retrieve any data. Snapping to
+    minute precision keeps the helper deterministic regardless of clock skew.
+
+    Args:
+     - count (int): How many consecutive minute timestamps to generate.
+     - offset_minutes (int): Minutes ago for the EARLIEST generated timestamp.
+
+    Return:
+     list[str]: ISO strings with ``+0000`` offset, one per minute.
+    """
+    base = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(
+        minutes=offset_minutes
+    )
+    return [
+        (base + timedelta(minutes=i)).strftime('%Y-%m-%d %H:%M:%S%z') for i in range(count)
+    ]
+
+
+def _configure_reference_csv(mlflow_repository_stub, reference_rows: pd.DataFrame) -> None:
+    """
+    Wire ``mlflow_repository_stub`` so ``get_reference_data`` returns
+    ``reference_rows`` by writing them to ``dst_path/retrain_input.csv``.
+
+    Args:
+     - mlflow_repository_stub: External MLflow repository fixture.
+     - reference_rows (pd.DataFrame): Rows to expose as the production reference.
+    """
+
+    def _download(run_id: str, artifact_path: str, dst_path: str, metadata=None):
+        target = Path(dst_path) / artifact_path
+        target.parent.mkdir(parents=True, exist_ok=True)
+        reference_rows.to_csv(target, index=False)
+
+    mlflow_repository_stub._client.get_model_version_by_alias.return_value = MagicMock(
+        run_id='fake-reference-run'
+    )
+    file_info = MagicMock()
+    file_info.path = 'retrain_input.csv'
+    mlflow_repository_stub._client.list_artifacts.return_value = [file_info]
+    mlflow_repository_stub.download_artifacts.side_effect = _download
+
+
+def _force_reference_unavailable(mlflow_repository_stub) -> None:
+    """Make ``get_reference_data`` return ``None`` by failing alias resolution."""
+    mlflow_repository_stub._client.get_model_version_by_alias.side_effect = Exception(
+        'no production alias registered'
+    )
+
+
+def _select_drift_rows(postgres_engine, model_id: int) -> list[dict]:
+    """Read every persisted drift row for ``model_id`` ordered by chunk/feature/method."""
+    with postgres_engine.connect() as conn:
+        rows = (
+            conn.execute(
+                text(
+                    'SELECT * FROM sientia_data.drift_metrics '
+                    'WHERE model_id = :m '
+                    'ORDER BY chunk_index, feature, method'
+                ),
+                {'m': str(model_id)},
+            )
+            .mappings()
+            .all()
+        )
+    return [dict(row) for row in rows]
+
+
+def _assert_required_columns_populated(rows: list[dict]) -> None:
+    """Validate column presence and NOT NULL constraints on every row."""
+    assert rows, 'expected at least one drift row to be persisted'
+    seen_columns = set(rows[0].keys())
+    for column in EXPECTED_DRIFT_COLUMNS:
+        assert column in seen_columns, f'Missing drift column in postgres: {column}'
+    for row in rows:
+        for column in NON_NULL_DRIFT_COLUMNS:
+            assert row[column] is not None, f"Column '{column}' is NULL in {row}"
+        assert 'p_value' not in row, 'p_value must not be persisted to drift_metrics'
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_drift_happy_path_persists_all_columns_with_reference_data(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_drift: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    Scenario D.1.1: Happy path with reference data downloaded from MLflow.
+
+    Drives the full pipeline against the real ``DriftAnalysis``. Asserts:
+
+    - One row is persisted per ``(chunk_index, feature, method)`` combination
+      plus the multivariate row block, with every column required by
+      ``sientia_data.drift_metrics`` populated.
+    - The three default univariate methods are forwarded to the analyzer.
+    - ``model_id`` and ``timestamp`` are stamped by the activity (not by the
+      analyzer); ``timestamp`` equals ``max(target_data.timestamp)`` and is
+      identical on every persisted row.
+    - ``chunk_start_date`` / ``chunk_end_date`` are persisted as ISO text so
+      the analyzer's nanosecond-precision boundaries survive the ``text``
+      column type.
+    - ``accurate=True`` because the reference dataset was available.
+    """
+    client = temporal_test_env.client
+    model_id = 411
+
+    target_timestamps = _recent_minute_timestamps(count=10)
+    insert_target_data_for_drift(
+        postgres_engine,
+        model_id=model_id,
+        timestamps=target_timestamps,
+        variables_values={
+            'sensor_1': [10.0 + i * 0.1 for i in range(10)],
+            'sensor_2': [20.0 + i * 0.5 for i in range(10)],
+        },
+    )
+
+    reference_df = pd.DataFrame(
+        {
+            'timestamp': [
+                f'2023-12-31 11:{minute:02d}:00+00:00' for minute in range(10)
+            ],
+            'sensor_1': [9.0 + i * 0.05 for i in range(10)],
+            'sensor_2': [18.0 + i * 0.25 for i in range(10)],
+        }
+    )
+    _configure_reference_csv(mlflow_repository_stub, reference_df)
+
+    input_data = _drift_input(model_id)
+    await start_and_await_workflow(
+        client, Drift.run, input_data, make_workflow_id('test-drift-happy-path')
+    )
+
+    rows = _select_drift_rows(postgres_engine, model_id)
+
+    exported_csv_path = '/tmp/test_drift_happy_path_exported.csv'
+    pd.DataFrame(rows).to_csv(exported_csv_path, index=False)
+    print(
+        f'\n[test_drift_happy_path] Exported drift dataframe '
+        f'({len(rows)} rows) -> {exported_csv_path}'
+    )
+
+    _assert_required_columns_populated(rows)
+
+    # The activity drops the target column from the feature list, so only
+    # ``sensor_2`` participates in univariate analysis (``sensor_1`` is the
+    # configured target). Multivariate produces one row per chunk regardless.
+    univariate_rows = [r for r in rows if r['feature'] != 'multivariate']
+    multivariate_rows = [r for r in rows if r['feature'] == 'multivariate']
+    assert univariate_rows, 'expected univariate drift rows for non-target features'
+    assert multivariate_rows, 'expected one multivariate drift row per chunk'
+
+    # All three default methods must reach the analyzer.
+    assert {r['method'] for r in univariate_rows} == set(DEFAULT_DRIFT_METHODS)
+    assert all(r['method'] == 'multivariate' for r in multivariate_rows)
+    assert {r['feature'] for r in univariate_rows} == {'sensor_2'}
+
+    # ``timestamp`` is stamped uniformly with ``max(target_data.timestamp)``.
+    expected_timestamp = pd.to_datetime(max(target_timestamps), utc=True)
+    persisted_timestamps = {pd.to_datetime(r['timestamp'], utc=True) for r in rows}
+    assert len(persisted_timestamps) == 1, (
+        'timestamp must be uniform across all drift rows '
+        f'(got {len(persisted_timestamps)} distinct values)'
+    )
+    assert pd.Timestamp(persisted_timestamps.pop()) == expected_timestamp, (
+        'timestamp must equal max(target_data.timestamp)'
+    )
+
+    # ``model_id`` is stamped by ``calculate_drift`` (not produced by the analyzer).
+    assert all(r['model_id'] == str(model_id) for r in rows), (
+        'model_id must be stamped on every drift row'
+    )
+
+    # Reference path → accurate=True.
+    assert all(r['accurate'] is True for r in rows), (
+        'reference path should mark all rows as accurate'
+    )
+
+    # ISO text serialization preserves ordering between start/end of each chunk.
+    for row in rows:
+        assert 'T' in row['chunk_start_date'], (
+            f"chunk_start_date should be ISO text, got {row['chunk_start_date']!r}"
+        )
+        assert 'T' in row['chunk_end_date'], (
+            f"chunk_end_date should be ISO text, got {row['chunk_end_date']!r}"
+        )
+        assert row['chunk_start_date'] <= row['chunk_end_date'], (
+            f'chunk_start_date must precede chunk_end_date '
+            f"(start={row['chunk_start_date']}, end={row['chunk_end_date']})"
+        )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_drift_uses_30pct_fallback_when_reference_unavailable(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_drift: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+    notification_inserts,
+):
+    """
+    Scenario D.1.2: ``get_reference_data`` returns ``None`` (production alias
+    missing), so ``calculate_drift`` falls back to using the first 30% of
+    target rows as reference. Persisted rows must report ``accurate=False``
+    and a ``MODEL_METRICS_REFERENCE_DATA_WARNING`` notification must be
+    emitted to mongo.
+    """
+    client = temporal_test_env.client
+    model_id = 412
+
+    target_timestamps = _recent_minute_timestamps(count=10)
+    insert_target_data_for_drift(
+        postgres_engine,
+        model_id=model_id,
+        timestamps=target_timestamps,
+        variables_values={
+            'sensor_1': [10.0 + i * 0.1 for i in range(10)],
+            'sensor_2': [20.0 + i * 0.5 for i in range(10)],
+        },
+    )
+
+    _force_reference_unavailable(mlflow_repository_stub)
+
+    input_data = _drift_input(model_id)
+    await start_and_await_workflow(
+        client, Drift.run, input_data, make_workflow_id('test-drift-fallback')
+    )
+
+    rows = _select_drift_rows(postgres_engine, model_id)
+    _assert_required_columns_populated(rows)
+
+    assert all(r['accurate'] is False for r in rows), (
+        'fallback path must mark all rows as inaccurate'
+    )
+
+    fallback_warnings = [
+        call
+        for call in notification_inserts.call_args_list
+        if call.args
+        and isinstance(call.args[0], dict)
+        and call.args[0].get('notification_id') == 'MODEL_METRICS_REFERENCE_DATA_WARNING'
+    ]
+    assert len(fallback_warnings) >= 1, 'expected reference fallback warning notification'
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_drift_empty_target_data_short_circuits_workflow(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_drift: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    Scenario D.3.1: When ``load_custom_query`` returns no rows the workflow
+    must return early without invoking the analyzer or writing any drift rows.
+    """
+    client = temporal_test_env.client
+    model_id = 431
+
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
+
+    _force_reference_unavailable(mlflow_repository_stub)
+
+    input_data = _drift_input(model_id)
+    await start_and_await_workflow(
+        client, Drift.run, input_data, make_workflow_id('test-drift-empty-target')
+    )
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM sientia_data.drift_metrics WHERE model_id = :m'),
+            {'m': str(model_id)},
+        ).scalar()
+    assert count == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_drift_invalid_chunk_period_raises_value_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_drift: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    Scenario D.4.2: ``calculate_drift`` validates ``chunk_period`` and rejects
+    anything other than ``min`` / ``s``. The workflow must surface the
+    ``ValueError`` and persist nothing.
+    """
+    client = temporal_test_env.client
+    model_id = 442
+
+    target_timestamps = _recent_minute_timestamps(count=5)
+    insert_target_data_for_drift(
+        postgres_engine,
+        model_id=model_id,
+        timestamps=target_timestamps,
+        variables_values={
+            'sensor_1': [1.0, 2.0, 3.0, 4.0, 5.0],
+            'sensor_2': [10.0, 20.0, 30.0, 40.0, 50.0],
+        },
+    )
+    _force_reference_unavailable(mlflow_repository_stub)
+
+    input_data = _drift_input(model_id, chunk_period='hour')
+
+    with pytest.raises(Exception) as excinfo:
+        await start_and_await_workflow(
+            client,
+            Drift.run,
+            input_data,
+            make_workflow_id('test-drift-bad-chunk-period'),
+        )
+
+    # Temporal wraps the activity ValueError in WorkflowFailureError; the
+    # message may live on ``.message`` or ``str(exc)`` depending on the SDK
+    # error class, so walk the cause chain looking for the guard text.
+    cause_descriptions = []
+    current: BaseException | None = excinfo.value
+    while current is not None:
+        cause_descriptions.append(
+            getattr(current, 'message', None) or str(current) or repr(current)
+        )
+        current = current.__cause__
+    assert any('Invalid chunk period' in msg for msg in cause_descriptions), (
+        f'Expected ValueError about chunk period in chain, got: {cause_descriptions}'
+    )
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM sientia_data.drift_metrics WHERE model_id = :m'),
+            {'m': str(model_id)},
+        ).scalar()
+    assert count == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_drift_empty_merge_skips_export_without_insufficient_notification(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_drift: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    Scenario D.4.3a: When the analyzer returns an empty merged frame (no metric rows),
+    ``calculate_drift`` yields ``[]``; the workflow skips export. Real insufficient-data
+    cases are signaled by ``DriftInsufficientDataError`` inside ``sientia_model``, not by
+    empty output alone.
+    """
+    client = temporal_test_env.client
+    model_id = 444
+
+    target_timestamps = _recent_minute_timestamps(count=5)
+    insert_target_data_for_drift(
+        postgres_engine,
+        model_id=model_id,
+        timestamps=target_timestamps,
+        variables_values={
+            'sensor_1': [1.0 + i * 0.1 for i in range(5)],
+            'sensor_2': [10.0 + i * 0.5 for i in range(5)],
+        },
+    )
+    _force_reference_unavailable(mlflow_repository_stub)
+
+    input_data = _drift_input(model_id, chunk_period='min')
+    empty_merge = pd.DataFrame(
+        columns=[
+            'timestamp',
+            'feature',
+            'method',
+            'value',
+            'alert',
+            'chunk_index',
+            'chunk_start_date',
+            'chunk_end_date',
+            'threshold',
+            'drift_type',
+        ]
+    )
+    with patch.object(ModelMetrics, 'get_drift_metrics', return_value=empty_merge):
+        await start_and_await_workflow(
+            client,
+            Drift.run,
+            input_data,
+            make_workflow_id('test-drift-empty-merge'),
+        )
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM sientia_data.drift_metrics WHERE model_id = :m'),
+            {'m': str(model_id)},
+        ).scalar()
+    assert count == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_drift_chunk_period_seconds_sufficient_data_preserves_seconds_in_chunk_start_date(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_drift: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    Scenario D.4.3b: With enough sub-minute samples and ``chunk_period='s'``, drift rows
+    persist and ``chunk_start_date`` keeps second-level precision (incl. second=30).
+
+    ``DriftAnalysis._chunk_dataframe`` is patched to skip empty ``pd.Grouper(freq='s')``
+    buckets so sparse seconds between samples do not flood the pipeline with NaT rows;
+    the durable fix belongs in ``sientia_model``.
+    """
+    client = temporal_test_env.client
+    model_id = 443
+
+    base = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(minutes=6)
+    target_timestamps = []
+    sensor_1_vals = []
+    sensor_2_vals = []
+    for minute_offset in range(6):
+        t0 = base + timedelta(minutes=minute_offset)
+        t1 = t0 + timedelta(seconds=30)
+        target_timestamps.append(t0.strftime('%Y-%m-%d %H:%M:%S%z'))
+        target_timestamps.append(t1.strftime('%Y-%m-%d %H:%M:%S%z'))
+        v0 = 1.0 + minute_offset * 0.1
+        v1 = v0 + 0.05
+        sensor_1_vals.extend([v0, v1])
+        sensor_2_vals.extend([10.0 + v0, 10.0 + v1])
+
+    insert_target_data_for_drift(
+        postgres_engine,
+        model_id=model_id,
+        timestamps=target_timestamps,
+        variables_values={
+            'sensor_1': sensor_1_vals,
+            'sensor_2': sensor_2_vals,
+        },
+    )
+    _force_reference_unavailable(mlflow_repository_stub)
+
+    input_data = _drift_input(model_id, chunk_period='s')
+    with patch.object(DriftAnalysis, '_chunk_dataframe', _chunk_dataframe_skip_empty_groups):
+        await start_and_await_workflow(
+            client,
+            Drift.run,
+            input_data,
+            make_workflow_id('test-drift-chunk-seconds-sufficient'),
+        )
+
+    with postgres_engine.connect() as conn:
+        rows = (
+            conn.execute(
+                text(
+                    'SELECT chunk_start_date FROM sientia_data.drift_metrics '
+                    'WHERE model_id = :m ORDER BY chunk_index'
+                ),
+                {'m': str(model_id)},
+            )
+            .mappings()
+            .all()
+        )
+
+    assert rows, 'expected at least one drift row to be persisted'
+    seconds_present = {pd.Timestamp(r['chunk_start_date']).second for r in rows}
+    assert 30 in seconds_present, (
+        f'expected at least one chunk_start_date with seconds=30, got {seconds_present}'
+    )

e2e/test_minimal_retrain.py

@@ -0,0 +1,430 @@
+"""
+End-to-end tests for the MinimalRetrain workflow.
+
+The MLflow registry is fully stubbed because no real artifacts exist in a
+test container; we only validate that the workflow:
+
+- Loads training data via ``load_query_with_minio_offload``.
+- Calls ``retrain_model`` with a payload pointing at MinIO.
+- Calls ``update_production_model`` only when retrain succeeds.
+- Persists ``sientia_data.log_retrain`` rows with all required columns;
+  success rows carry the new ``version`` / ``mlflow_run_id`` /
+  ``mlflow_experiment_id`` while failure rows leave them ``NULL``.
+
+The production DDL drops the legacy ``id`` / ``created_at`` columns and
+moves ``mlflow_experiment_id`` to ``int8`` and ``model_id`` to ``text``.
+The stubs used here therefore emit ``experiment_id`` as an integer to fit
+the new column type.
+"""
+
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from unittest.mock import MagicMock, patch
+
+import pytest
+import pandas as pd
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import (
+    insert_target_data_for_drift,
+    load_scenario_input,
+    make_workflow_id,
+    start_and_await_workflow,
+)
+from laborious.activities.activities import Activities
+from laborious.workflows.minimal_retrain import MinimalRetrain
+from sientia_model.wrappers.sientia_model import SientiaModel
+
+# Columns defined by the production DDL for ``sientia_data.log_retrain``.
+# The legacy ``retrain_reports`` table had ``id`` and ``created_at``; the new
+# DDL drops both. ``mlflow_experiment_id`` is ``int8`` and ``model_id`` is
+# ``text``.
+EXPECTED_RETRAIN_REPORT_COLUMNS = [
+    'mlflow_experiment_id',
+    'mlflow_run_id',
+    'model_id',
+    'model_name',
+    'status',
+    'timestamp',
+    'version',
+]
+
+# Matches ``retrain_model`` return ``message`` when ``success`` is True (also written to ``log_retrain.status``).
+RETRAIN_ACTIVITY_SUCCESS_MESSAGE = 'Model retrained successfully.'
+
+
+class _FakeSientiaModelForMinimalRetrain(SientiaModel):
+    """
+    Fake SientiaModel that uses the real SientiaModel lifecycle to surface
+    index-alignment issues during ``retrain()``.
+
+    It intentionally performs strict alignment inside ``_retrain_model``:
+    ``y.loc[x.index]``.
+    """
+
+    def __init__(self, *, target: str = 'sensor_1'):
+        super().__init__(
+            model_type='FakeMinimalRetrain',
+            model_version='0.0.0',
+            model=object(),
+            transformer=object(),
+        )
+        self.target = target
+        self.model_is_fitted = True
+        self.force_retrain_error = False
+
+    def store_model(  # type: ignore[override]
+        self,
+        name: str,
+        signature=None,
+        pip_requirements=None,
+        code_path=None,
+    ) -> None:
+        # No-op: E2E tests validate workflow persistence, not real MLflow artifacts.
+        return None
+
+    def _predict(self, data: pd.DataFrame):
+        pred = pd.DataFrame({'prediction': [0.5] * len(data)}, index=data.index)
+        return pred, {}
+
+    def _transform(self, data: pd.DataFrame):
+        out = data.drop(columns=[self.target], errors='ignore').copy()
+        out.index = data.index
+        return out, {}
+
+    def _train_transformer(self, train_data: pd.DataFrame, val_data: pd.DataFrame) -> None:
+        return None
+
+    def _train_model(
+        self,
+        x: pd.DataFrame,
+        y: pd.DataFrame,
+        x_val: pd.DataFrame | None = None,
+        y_val: pd.DataFrame | None = None,
+    ) -> None:
+        return None
+
+    def _retrain_transformer(self, data: pd.DataFrame) -> None:
+        return None
+
+    def _retrain_model(self, x: pd.DataFrame, y: pd.DataFrame | None) -> None:
+        if self.force_retrain_error:
+            raise RuntimeError('training did not converge')
+        if y is None:
+            return
+        # Strict alignment on purpose to reproduce the production failure mode.
+        _ = y.loc[x.index]
+
+
+@pytest.fixture
+def mlflow_repository_stub():
+    """
+    Override the shared E2E fixture: return a real fake ``SientiaModel`` wrapper
+    instead of a MagicMock wrapper.
+    """
+    repo = MagicMock()
+    repo._client = MagicMock()
+
+    wrapper = _FakeSientiaModelForMinimalRetrain(target='sensor_1')
+    repo.get_cached_model = MagicMock(return_value=wrapper)
+    return repo
+
+
+def _retrain_input(model_id: int, **overrides) -> dict:
+    """Load and override the minimal-retrain base scenario."""
+    payload = load_scenario_input('minimal_retrain_base.json', model_id=model_id)
+    payload.update(overrides)
+    return payload
+
+
+def _seed_retrain_training_rows(postgres_engine, model_id: int) -> None:
+    """
+    Insert training rows in long format that pivot cleanly into
+    ``index=timestamp`` / ``columns={sensor_1, sensor_2}`` for ``retrain_model``.
+    """
+    target_timestamps = [
+        (datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(minutes=10 - i))
+        .strftime('%Y-%m-%d %H:%M:%S%z')
+        for i in range(5)
+    ]
+    insert_target_data_for_drift(
+        postgres_engine,
+        model_id=model_id,
+        timestamps=target_timestamps,
+        variables_values={
+            'sensor_1': [10.0, 11.0, 12.0, 13.0, 14.0],
+            'sensor_2': [20.0, 21.0, 22.0, 23.0, 24.0],
+        },
+    )
+
+
+def _configure_retrain_happy_path(mlflow_repository_stub) -> None:
+    """
+    Wire ``mlflow_repository_stub`` so retrain + update_production succeed.
+
+    Mocks (in order of consumption):
+
+    - ``_client.get_model_version_by_alias``: returns ``mv`` with a stable
+      ``run_id`` (used as ``source_run_id``).
+    - ``start_run``: returns a context manager yielding a ``run_info`` with
+      run/experiment ids.
+    - ``log_params``: inert.
+    - ``_client.search_model_versions``: returns one registry entry whose
+      ``version`` is promoted by ``update_production_model``.
+    - ``promote_to_alias``: inert success.
+    """
+    mv_src = MagicMock()
+    mv_src.run_id = 'source-run-id'
+
+    new_version = MagicMock()
+    new_version.version = '7'
+    new_version.run_id = 'retrain-run-id'
+
+    mlflow_repository_stub._client.get_model_version_by_alias.return_value = mv_src
+
+    @contextmanager
+    def fake_start_run(**kwargs):
+        run_info = MagicMock()
+        run_info.run_id = 'retrain-run-id'
+        # ``mlflow_experiment_id`` is ``int8`` in the new DDL, so we feed an
+        # integer-compatible id from the stubbed run info.
+        run_info.experiment_id = 4242
+        yield run_info
+
+    mlflow_repository_stub.start_run.side_effect = fake_start_run
+    mlflow_repository_stub.log_params = MagicMock(return_value=None)
+    mlflow_repository_stub._client.search_model_versions.return_value = [new_version]
+    mlflow_repository_stub.promote_to_alias = MagicMock(return_value=None)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_minimal_retrain_happy_path_writes_success_report(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_minimal_retrain: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    Scenario MR.1.1: Retrain succeeds. ``sientia_data.log_retrain`` must
+    contain a success row with version/mlflow_run_id/mlflow_experiment_id
+    populated and the registry must have been told to promote the new version
+    to the configured alias.
+    """
+    client = temporal_test_env.client
+    model_id = 711
+
+    _seed_retrain_training_rows(postgres_engine, model_id)
+    _configure_retrain_happy_path(mlflow_repository_stub)
+
+    input_data = _retrain_input(model_id)
+
+    with patch('laborious.activities.mlflow.mlflow.log_artifact') as log_artifact_mock:
+        await start_and_await_workflow(
+            client,
+            MinimalRetrain.run,
+            input_data,
+            make_workflow_id('test-retrain-happy'),
+        )
+
+    with postgres_engine.connect() as conn:
+        rows = (
+            conn.execute(
+                text(
+                    'SELECT * FROM sientia_data.log_retrain '
+                    'WHERE model_id = :m'
+                ),
+                {'m': str(model_id)},
+            )
+            .mappings()
+            .all()
+        )
+
+    assert len(rows) == 1
+    row = rows[0]
+    for column in EXPECTED_RETRAIN_REPORT_COLUMNS:
+        assert column in row, f'Missing log_retrain column: {column}'
+
+    assert row['status'] == RETRAIN_ACTIVITY_SUCCESS_MESSAGE, (
+        "Expected retrain_model to return success (experiment_response['success'] is True). "
+        'Persisted log_retrain.status is the activity message; when success is False the run '
+        'never reaches mlflow.log_artifact — diagnose the retrain failure from status below, '
+        'not from a skipped artifact upload. '
+        f"Got status={row['status']!r}, version={row.get('version')!r}, "
+        f"mlflow_run_id={row.get('mlflow_run_id')!r}."
+    )
+
+    assert log_artifact_mock.called, (
+        'After a successful retrain, retrain_model must call mlflow.log_artifact for the '
+        'input CSV inside start_run.'
+    )
+
+    # ``model_id`` is now ``text``; compare against the stringified id.
+    assert row['model_id'] == str(model_id)
+    assert row['model_name'] == 'test_model'
+    assert row['version'] == '7'
+    assert row['mlflow_run_id'] == 'retrain-run-id'
+    # ``mlflow_experiment_id`` is now ``int8``; assert the integer value
+    # provided by the stubbed run info.
+    assert row['mlflow_experiment_id'] == 4242
+    assert row['timestamp'] is not None
+
+    mlflow_repository_stub.promote_to_alias.assert_called_once()
+    promote_kwargs = mlflow_repository_stub.promote_to_alias.call_args.kwargs
+    assert promote_kwargs['model_name'] == 'test_model'
+    assert promote_kwargs['version'] == '7'
+    assert promote_kwargs['alias'] == 'production'
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_minimal_retrain_failure_writes_report_without_version_columns(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_minimal_retrain: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    Scenario MR.2.1: ``wrapper.retrain`` raises. The activity must catch the
+    error, return ``success=False`` so ``update_production_model`` is skipped,
+    and ``format_retrain_report`` must produce a row with the error message
+    and NULL version columns.
+    """
+    client = temporal_test_env.client
+    model_id = 721
+
+    _seed_retrain_training_rows(postgres_engine, model_id)
+    _configure_retrain_happy_path(mlflow_repository_stub)
+    mlflow_repository_stub.get_cached_model.return_value.force_retrain_error = True
+
+    input_data = _retrain_input(model_id)
+
+    with patch('laborious.activities.mlflow.mlflow.log_artifact'):
+        await start_and_await_workflow(
+            client,
+            MinimalRetrain.run,
+            input_data,
+            make_workflow_id('test-retrain-failure'),
+        )
+
+    with postgres_engine.connect() as conn:
+        rows = (
+            conn.execute(
+                text(
+                    'SELECT * FROM sientia_data.log_retrain '
+                    'WHERE model_id = :m'
+                ),
+                {'m': str(model_id)},
+            )
+            .mappings()
+            .all()
+        )
+
+    assert len(rows) == 1
+    row = rows[0]
+    assert row['model_id'] == str(model_id)
+    assert row['model_name'] == 'test_model'
+    assert 'training did not converge' in row['status']
+    assert row['version'] is None
+    assert row['mlflow_run_id'] is None
+    assert row['mlflow_experiment_id'] is None
+
+    mlflow_repository_stub.promote_to_alias.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_minimal_retrain_missing_target_writes_failure_report(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_minimal_retrain: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    Scenario MR.2.2: ``model_config`` does not declare ``target``. The retrain
+    activity must short-circuit before any MLflow call and the report row must
+    carry the explicit guard message.
+    """
+    client = temporal_test_env.client
+    model_id = 722
+
+    _seed_retrain_training_rows(postgres_engine, model_id)
+    _configure_retrain_happy_path(mlflow_repository_stub)
+
+    input_data = _retrain_input(model_id, model_config={})
+
+    with patch('laborious.activities.mlflow.mlflow.log_artifact'):
+        await start_and_await_workflow(
+            client,
+            MinimalRetrain.run,
+            input_data,
+            make_workflow_id('test-retrain-missing-target'),
+        )
+
+    with postgres_engine.connect() as conn:
+        row = (
+            conn.execute(
+                text(
+                    'SELECT * FROM sientia_data.log_retrain '
+                    'WHERE model_id = :m'
+                ),
+                {'m': str(model_id)},
+            )
+            .mappings()
+            .first()
+        )
+
+    assert row is not None
+    assert 'target' in row['status'].lower(), (
+        f"expected target-missing message, got status={row['status']!r}"
+    )
+    assert row['version'] is None
+    mlflow_repository_stub.get_cached_model.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_minimal_retrain_no_training_data_does_not_persist_report(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_minimal_retrain: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    Scenario MR.3.1: When the training query returns no rows the workflow must
+    not persist any report row. The workflow currently raises plain
+    ``ValueError`` which Temporal treats as a workflow-task failure (causing
+    indefinite retries until the test environment times out), so the assertion
+    here is constrained to the persistence side-effect. See ``e2e/CODE_ISSUES.md``
+    issue MR-1 for the recommended ``ApplicationError`` fix.
+    """
+    client = temporal_test_env.client
+    model_id = 731
+
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
+
+    _configure_retrain_happy_path(mlflow_repository_stub)
+
+    input_data = _retrain_input(model_id)
+
+    with pytest.raises(Exception), patch('laborious.activities.mlflow.mlflow.log_artifact'):
+        await start_and_await_workflow(
+            client,
+            MinimalRetrain.run,
+            input_data,
+            make_workflow_id('test-retrain-no-data'),
+        )
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM sientia_data.log_retrain WHERE model_id = :m'),
+            {'m': str(model_id)},
+        ).scalar()
+    assert count == 0

e2e/test_minio_offload.py

@@ -0,0 +1,108 @@
+"""
+E2E-style tests for MinIO offload using a real MinIO testcontainer.
+"""
+
+from unittest.mock import patch
+
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import (
+    insert_sample_data,
+    load_scenario_input,
+    make_workflow_id,
+    start_and_await_workflow,
+)
+from laborious.activities.activities import Activities
+from laborious.utils.models import minio_dataframe_payload as mdp
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_load_query_with_minio_offload_writes_object_to_bucket(
+    postgres_engine,
+    minio_container,
+    test_activities_real_minio: Activities,
+):
+    """
+    With a tiny offload threshold, query results are uploaded as Parquet to MinIO.
+
+    Uses real MinioRepository against testcontainers MinIO (no MinIO mock).
+    """
+    model_id = 501
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
+    insert_sample_data(postgres_engine, model_id, [1.0, 2.0])
+
+    scenario_input = load_scenario_input('minio_offload_load_query.json', model_id=model_id)
+    metadata = {'metadata': scenario_input['metadata']}
+    with patch.object(mdp, 'OFFLOAD_THRESHOLD_BYTES', 1):
+        payload = test_activities_real_minio.load_query_with_minio_offload(scenario_input)
+    assert payload.object_key, 'offloaded payload must reference a MinIO object'
+    assert payload.data is None or payload.data == {}, 'large payloads should not inline tabular dict'
+
+    df = payload.retrieve(test_activities_real_minio.minio_repository, metadata['metadata'])
+    assert len(df) >= 1
+
+    client = minio_container.get_client()
+    listed = list(client.list_objects('test-bucket', recursive=True))
+    names = [getattr(o, 'object_name', None) or getattr(o, '_object_name', '') for o in listed]
+    assert any(n and 'prediction_datasets' in n for n in names), f'unexpected object listing: {names!r}'
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_predictions_batch_with_minio_offload_path(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_minio: Worker,
+    postgres_engine,
+    test_activities_real_minio: Activities,
+):
+    """
+    Full PredictionsBatch run with offload: load step stores Parquet in MinIO; pipeline completes.
+    """
+    model_id = 502
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
+        conn.execute(text(f'DELETE FROM sientia_data.predictions WHERE model_id = {model_id}'))
+        conn.execute(text(f'DELETE FROM sientia_data.transformed_data WHERE model_id = {model_id}'))
+    insert_sample_data(postgres_engine, model_id, [10.0, 20.0, 30.0])
+
+    input_data = load_scenario_input('minio_offload_workflow.json', model_id=model_id)
+
+    with patch.object(mdp, 'OFFLOAD_THRESHOLD_BYTES', 1):
+        await start_and_await_workflow(
+            temporal_test_env.client,
+            PredictionsBatch.run,
+            input_data,
+            make_workflow_id('test-batch-minio-offload'),
+        )
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text(f'SELECT COUNT(*) FROM sientia_data.predictions WHERE model_id = {model_id}')
+        ).scalar()
+    assert count == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_load_query_with_inline_payload_when_below_threshold(
+    postgres_engine,
+    test_activities_real_minio: Activities,
+):
+    """Scenario 4.2.1: payload stays inline when threshold is high enough."""
+    model_id = 503
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
+    insert_sample_data(postgres_engine, model_id, [1.0, 2.0])
+
+    scenario_input = load_scenario_input('minio_offload_load_query.json', model_id=model_id)
+    with patch.object(mdp, 'OFFLOAD_THRESHOLD_BYTES', 10**9):
+        payload = test_activities_real_minio.load_query_with_minio_offload(scenario_input)
+
+    assert payload.object_key is None
+    assert payload.data is not None

e2e/test_opc_real_server.py

@@ -0,0 +1,201 @@
+"""
+E2E tests for OPC export using an in-process asyncua server and real OpcRepository.
+
+Covers scenarios 3.1.2, 3.2.2, 3.2.4, and 3.2.5 from e2e/scenarios.md.
+Mock-based OPC tests remain in test_predictions_batch_format_export.py.
+"""
+
+import asyncio
+import threading
+import time
+
+import pytest
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import assert_prediction, insert_sample_data, make_workflow_id, start_and_await_workflow
+from e2e.opc_test_server import UNKNOWN_NODE_ID, OpcE2ETestServer, build_opc_output_config
+from e2e.test_predictions_batch_format_export import get_base_input_data
+from laborious.activities.activities import Activities
+from laborious.activities.opc import OPC_RECONNECT_IN_PROGRESS_COMMENT
+from laborious.utils.repository.opc_repository import OpcRepository
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+
+def _slow_reconnect_under_lock(repo: OpcRepository, hold_seconds: float = 0.75) -> None:
+    """
+    Hold the connection lock briefly so concurrent writes see reconnect_in_progress.
+
+    Args:
+        repo (OpcRepository): Connected repository.
+        hold_seconds (float): Time to keep the lock before reconnecting.
+    """
+    with repo._connection_lock:
+        time.sleep(hold_seconds)
+        repo._reconnect_locked()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.opc
+async def test_scenario_3_1_2_export_with_opc_only_real_server(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_opc: Worker,
+    test_activities_real_opc: Activities,
+    opc_e2e_server: OpcE2ETestServer,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.2 (real OPC): connect, write prediction and confidence, verify server values.
+    """
+    client = temporal_test_env.client
+    model_id = 412
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = build_opc_output_config(opc_e2e_server.node_ids)
+    input_data['pi_web_api_output_config'] = None
+
+    await start_and_await_workflow(
+        client,
+        PredictionsBatch.run,
+        input_data,
+        make_workflow_id('test-opc-real-happy'),
+    )
+
+    test_activities_real_opc.pi_web_api_client.write_value.assert_not_called()
+    assert await opc_e2e_server.read_prediction() == pytest.approx(0.5)
+    assert await opc_e2e_server.read_confidence() == pytest.approx(0.0)
+    assert_prediction(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.opc
+async def test_scenario_3_2_2_opc_write_error_real_server(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_opc: Worker,
+    test_activities_real_opc: Activities,
+    opc_e2e_server: OpcE2ETestServer,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.2 (real OPC): unknown NodeId yields generic write failure (confidence 12).
+    """
+    client = temporal_test_env.client
+    model_id = 422
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    node_ids = opc_e2e_server.node_ids
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = build_opc_output_config(
+        node_ids,
+        prediction_tag=UNKNOWN_NODE_ID,
+        confidence_tag=UNKNOWN_NODE_ID,
+    )
+    input_data['pi_web_api_output_config'] = None
+
+    await start_and_await_workflow(
+        client,
+        PredictionsBatch.run,
+        input_data,
+        make_workflow_id('test-opc-real-bad-node'),
+    )
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=12,
+        comments='Some data could not be written to OPC servers',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.opc
+async def test_scenario_3_2_4_opc_session_bad_real_server(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_opc: Worker,
+    test_activities_real_opc: Activities,
+    opc_e2e_server: OpcE2ETestServer,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.4 (real OPC): server PreWrite fault injects BadSessionIdInvalid (confidence 14).
+    """
+    client = temporal_test_env.client
+    model_id = 424
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    opc_e2e_server.set_session_bad_on_write(True)
+    try:
+        input_data = get_base_input_data(model_id)
+        input_data['opc_output_config'] = build_opc_output_config(
+            opc_e2e_server.node_ids,
+            prediction_only=True,
+        )
+        input_data['pi_web_api_output_config'] = None
+
+        await start_and_await_workflow(
+            client,
+            PredictionsBatch.run,
+            input_data,
+            make_workflow_id('test-opc-real-session-bad'),
+        )
+    finally:
+        opc_e2e_server.set_session_bad_on_write(False)
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=14,
+        comments_contains='OPC UA session/channel error: BadSessionIdInvalid',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+@pytest.mark.opc
+async def test_scenario_3_2_5_opc_write_blocked_during_reconnect_real_server(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_real_opc: Worker,
+    test_activities_real_opc: Activities,
+    opc_e2e_server: OpcE2ETestServer,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.5 (real OPC): writes rejected while reconnect holds the connection lock.
+    """
+    client = temporal_test_env.client
+    model_id = 425
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    repo = test_activities_real_opc.opc_repository['1']
+    repo._session_ready.clear()
+    reconnect_thread = threading.Thread(
+        target=_slow_reconnect_under_lock,
+        args=(repo,),
+        daemon=True,
+    )
+    reconnect_thread.start()
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = build_opc_output_config(opc_e2e_server.node_ids)
+    input_data['pi_web_api_output_config'] = None
+
+    try:
+        await start_and_await_workflow(
+            client,
+            PredictionsBatch.run,
+            input_data,
+            make_workflow_id('test-opc-real-reconnect-block'),
+        )
+    finally:
+        reconnect_thread.join(timeout=5.0)
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=14,
+        comments_contains=OPC_RECONNECT_IN_PROGRESS_COMMENT,
+    )

e2e/test_predictions_batch_format_export.py

@@ -0,0 +1,836 @@
+"""
+End-to-end tests for PredictionsBatch workflow - Format and Export scenarios.
+"""
+
+from decimal import Decimal
+from typing import Any, cast
+from unittest.mock import call
+
+import pytest
+from sientia_do.notifications.models import NotificationLevel
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import (
+    assert_prediction,
+    insert_sample_data,
+    load_scenario_input,
+    make_workflow_id,
+    start_and_await_workflow,
+)
+from laborious.activities.activities import Activities
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+def get_base_input_data(model_id):
+    return load_scenario_input('format_export_base.json', model_id=model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_1_default_prediction_export(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.1: Default prediction export (non-None path_flag).
+
+    Triggers input_gate CONTINUE via SPECIFIC_VARIABLES_NULL_VALUES so
+    PredictionProcess calls FormatAndExportPrediction with path_flag set.
+    That workflow uses format_default_prediction (not format_prediction) and
+    skips format_transformed_data / transform Postgres export.
+
+    Optional PI Web API and OPC outputs still run when configured.
+    """
+    client = temporal_test_env.client
+
+    model_id = 311
+
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f"DELETE FROM sientia_data.predictions WHERE model_id = {model_id}"))
+        conn.execute(text(f"DELETE FROM sientia_data.transformed_data WHERE model_id = {model_id}"))
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters'] = {
+        'SPECIFIC_VARIABLES_NULL_VALUES': {
+            'POLICY': 'CONTINUE',
+            'CONFIG': {'variables': ['sensor_1']},
+        },
+    }
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+
+    wid = make_workflow_id('test-default-prediction')
+
+    await start_and_await_workflow(client, PredictionsBatch.run, input_data, wid)
+
+    test_activities.pi_web_api_client.write_value.assert_has_calls(
+        [
+            call(
+                web_ids=['web_id_1'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0,
+                },
+                metadata={
+                    'model_id': 311,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                web_ids=['web_id_2'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 2,
+                },
+                metadata={
+                    'model_id': 311,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ],
+        any_order=True,
+    )
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_has_calls(
+        [
+            call(
+                'addr_1',
+                0,
+                'float',
+                {
+                    'model_id': 311,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                'addr_2',
+                2,
+                'float',
+                {
+                    'model_id': 311,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ]
+    )
+
+    with postgres_engine.connect() as conn:
+        tf_count = conn.execute(
+            text(f"SELECT COUNT(*) FROM sientia_data.transformed_data WHERE model_id = {model_id}")
+        ).scalar()
+        assert tf_count == 0, 'transform export must be skipped when path_flag is set'
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction=0,
+        prediction_confidence=Decimal(2),
+        prediction_status='Bad',
+        comments='Input data with bad quality',
+    )
+
+
+
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_2_export_with_opc_only(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.2: Export with OPC only
+    
+    Description:
+    Export to PostgreSQL and OPC server only (no PI Web API).
+    
+    Expected Behavior:
+    - Normal formatting
+    - PostgreSQL export executed
+    - OPC export executed
+    - PI Web API activity skipped
+    - Metrics written with OPC metrics
+    
+    Assertions:
+    - PI Web API activity NOT called
+    - OPC activity called
+    - PostgreSQL export called
+    - Metrics written with opc_metrics populated
+    """
+    client = temporal_test_env.client
+
+    model_id = 312
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+    input_data['pi_web_api_output_config'] = None  # No PI Web API config
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-opc-only')
+    )
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_has_calls(
+        [
+            call(
+                'addr_1',
+                0.5,
+                'float',
+                {
+                    'model_id': 312,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                'addr_2',
+                0,
+                'float',
+                {
+                    'model_id': 312,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ]
+    )
+
+    test_activities.pi_web_api_client.write_value.assert_not_called()
+
+    assert_prediction(postgres_engine, model_id)
+
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_3_export_with_pi_web_api_only(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.3: Export with PI Web API only
+    
+    Description:
+    Export to PostgreSQL and PI Web API only (no OPC).
+    
+    Expected Behavior:
+    - Normal formatting
+    - PostgreSQL export executed
+    - PI Web API export executed
+    - OPC activity skipped
+    - Metrics written without OPC metrics
+    
+    Assertions:
+    - OPC activity NOT called
+    - PI Web API activity called
+    - PostgreSQL export called
+    - Metrics written with empty opc_metrics
+    """
+    client = temporal_test_env.client
+
+    model_id = 313
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = None  # No OPC config
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-pi-api-only')
+    )
+
+    test_activities.pi_web_api_client.write_value.assert_has_calls(
+        [
+            call(
+                web_ids=['web_id_1'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0.5,
+                },
+                metadata={
+                    'model_id': 313,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                web_ids=['web_id_2'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0,
+                },
+                metadata={
+                    'model_id': 313,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ],
+        any_order=True,
+    )
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_not_called()
+
+    assert_prediction(postgres_engine, model_id)
+
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_4_export_without_optional_outputs(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.4: Export Without Optional Outputs
+    
+    Description:
+    Export only to PostgreSQL (no OPC or PI Web API).
+    
+    Expected Behavior:
+    - Normal formatting
+    - Only PostgreSQL export executed
+    - OPC and PI Web API activities skipped
+    - Metrics written without OPC metrics
+    
+    Assertions:
+    - PI Web API activity NOT called
+    - OPC activity NOT called
+    - PostgreSQL export called
+    - Metrics written with empty opc_metrics
+    """
+    client = temporal_test_env.client
+
+    model_id = 314
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = None  # No OPC config
+    input_data['pi_web_api_output_config'] = None  # No PI Web API config
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-no-optional-outputs')
+    )
+
+    test_activities.pi_web_api_client.write_value.assert_not_called()
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_not_called()
+
+    assert_prediction(postgres_engine, model_id)
+
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_1_5_export_without_transformed_data(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.1.5: Export Without Transformed Data
+    
+    Description:
+    Only prediction exported, no transform table.
+    
+    Expected Behavior:
+    - Only prediction formatted and exported
+    - Transform export skipped
+    - Single PostgreSQL write
+    
+    Assertions:
+    - format_transformed_data NOT called
+    - One PostgreSQL export
+    - Transform table remains empty
+    """
+    client = temporal_test_env.client
+
+    model_id = 315
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f"DELETE FROM sientia_data.transformed_data WHERE model_id = {model_id}"))
+
+    input_data = get_base_input_data(model_id)
+    input_data['save_transform'] = False  # Don't save transformed data
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-no-transform-export')
+    )
+
+    test_activities.pi_web_api_client.write_value.assert_has_calls(
+        [
+            call(
+                web_ids=['web_id_1'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0.5,
+                },
+                metadata={
+                    'model_id': 315,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                web_ids=['web_id_2'],
+                value={
+                    'Timestamp': '2024-01-01 12:00:00+0000',
+                    'Value': 0,
+                },
+                metadata={
+                    'model_id': 315,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ],
+        any_order=True,
+    )
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.assert_has_calls(
+        [
+            call(
+                'addr_1',
+                0.5,
+                'float',
+                {
+                    'model_id': 315,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+            call(
+                'addr_2',
+                0,
+                'float',
+                {
+                    'model_id': 315,
+                    'model_name': 'test_model',
+                    'schedule_name': 'test-schedule',
+                    'workflow_name': 'predictions_batch',
+                },
+            ),
+        ]
+    )
+
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(f"SELECT COUNT(*) FROM sientia_data.transformed_data WHERE model_id = {model_id}")
+        )
+        count = result_query.scalar()
+        assert count == 0, f"Expected transform table to be empty, but found {count} records"
+
+    assert_prediction(postgres_engine, model_id)
+
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_2_1_pi_web_api_write_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    notification_inserts,
+):
+    """
+    Scenario 3.2.1: PI Web API Write Error
+
+    Export failure is handled inside the activity; there is no retry loop. The
+    workflow completes and PostgreSQL stores prediction_confidence 13 and the
+    error message in comments.
+    """
+    client = temporal_test_env.client
+
+    model_id = 321
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    test_activities.pi_web_api_client.write_value.side_effect = Exception(
+        "PI Web API service unavailable")
+
+    input_data = get_base_input_data(model_id)
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-pi-api-error')
+    )
+
+    assert_prediction(
+        postgres_engine, model_id,
+        prediction_confidence=13,
+        comments='PI Web API service unavailable',
+    )
+    assert notification_inserts.call_count >= 1
+
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_2_2_opc_write_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.2: OPC Write Error
+
+    OPC failure is reported without failing the workflow; there is no retry
+    loop. PostgreSQL stores prediction_confidence 12 and OPC error comments.
+    """
+    client = temporal_test_env.client
+
+    model_id = 322
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.return_value = (False, {
+        'notification_id': 'OPC_WRITE_DATA_ERROR_1',
+        'message': 'OPC server unavailable',
+        'block': 'opc_repository',
+        'level': NotificationLevel.ERROR,
+        'attachment_content': 'OPC server unavailable',
+    })
+    
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-opc-error')
+    )
+
+    assert_prediction(
+        postgres_engine, model_id,
+        prediction_confidence=12,
+        comments='Some data could not be written to OPC servers',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_2_4_opc_session_bad_mock(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.4 (mock): Tier-1 session error maps to confidence 14.
+    """
+    client = temporal_test_env.client
+    model_id = 324
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.return_value = (
+        False,
+        {
+            'opc_error_kind': 'session_bad',
+            'opc_status': 'BadSessionIdInvalid',
+            'notification_id': 'OPC_WRITE_DATA_ERROR_1',
+            'message': 'OPC session invalid',
+            'block': 'opc_repository',
+            'level': NotificationLevel.ERROR,
+            'attachment_content': 'BadSessionIdInvalid',
+        },
+    )
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {'addr_1': {'data_type': 'float'}},
+            'confidence_tags': {'addr_2': {'data_type': 'float'}},
+        }
+    }
+    input_data['pi_web_api_output_config'] = None
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-opc-session-bad-mock')
+    )
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=14,
+        comments_contains='OPC UA session/channel error: BadSessionIdInvalid',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_2_5_opc_reconnect_in_progress_mock(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.5 (mock): reconnect_in_progress maps to confidence 14.
+    """
+    client = temporal_test_env.client
+    model_id = 325
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    opc_write_data.return_value = (
+        False,
+        {
+            'opc_error_kind': 'reconnect_in_progress',
+            'notification_id': 'OPC_WRITE_DATA_ERROR_1',
+            'message': 'OPC reconnect in progress',
+            'block': 'opc_repository',
+            'level': NotificationLevel.ERROR,
+            'attachment_content': 'reconnect',
+        },
+    )
+
+    input_data = get_base_input_data(model_id)
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {'addr_1': {'data_type': 'float'}},
+            'confidence_tags': {'addr_2': {'data_type': 'float'}},
+        }
+    }
+    input_data['pi_web_api_output_config'] = None
+
+    await start_and_await_workflow(
+        client,
+        PredictionsBatch.run,
+        input_data,
+        make_workflow_id('test-opc-reconnect-mock'),
+    )
+
+    assert_prediction(
+        postgres_engine,
+        model_id,
+        prediction_confidence=14,
+        comments_contains='OPC UA reconnect in progress',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_2_3_pi_web_api_partial_write_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.2.3: PI Web API Partial Write Error
+
+    Partial PI write: confidence 13, descriptive comments, workflow completes
+    without an activity retry loop.
+    """
+    client = temporal_test_env.client
+
+    model_id = 323
+
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    test_activities.pi_web_api_client.set_side_effect(
+        [
+            # Prediction batch: two web_ids requested, only one acknowledged.
+            [{'WebId': 'web_id_1', 'Errors': []}],
+            # Confidence write succeeds.
+            [{'WebId': 'web_id_2', 'Errors': []}],
+        ]
+    )
+
+    input_data = get_base_input_data(model_id)
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1', 'tag_3': 'web_id_3'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {
+                'addr_1': {
+                    'data_type': 'float',
+                }
+            },
+            'confidence_tags': {
+                'addr_2': {
+                    'data_type': 'float',
+                }
+            },
+        }
+    }
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-pi-api-partial-error')
+    )
+
+    assert_prediction(
+        postgres_engine, model_id,
+        prediction_confidence=13,
+        comments="The number of written tags does not match the number of tag names: Expected ['tag_1', 'tag_3'] tags, but ['tag_1'] tags were written.",
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_3_3_1_combined_pi_and_opc_outputs(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario 3.3.1: PI and OPC enabled together.
+    """
+    client = temporal_test_env.client
+    model_id = 333
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+
+    input_data = get_base_input_data(model_id)
+    input_data['pi_web_api_output_config'] = {
+        'endpoint': 'test_endpoint',
+        'prediction_tags': {'tag_1': 'web_id_1'},
+        'confidence_tags': {'tag_2': 'web_id_2'},
+    }
+    input_data['opc_output_config'] = {
+        '1': {
+            'prediction_tags': {'addr_1': {'data_type': 'float'}},
+            'confidence_tags': {'addr_2': {'data_type': 'float'}},
+        }
+    }
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-pi-opc-combined')
+    )
+
+    assert test_activities.pi_web_api_client.write_value.call_count == 2
+    opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
+    assert opc_write_data.call_count == 2
+    assert_prediction(postgres_engine, model_id)
+    

e2e/test_predictions_batch_main_workflow.py

@@ -0,0 +1,181 @@
+"""
+End-to-end tests for PredictionsBatch workflow - Main workflow scenarios.
+"""
+
+import asyncio
+
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+import pytest
+
+from e2e.helpers import load_scenario_input, make_workflow_id, start_and_await_workflow
+from laborious.activities.activities import Activities
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_1_1_1_happy_path_complete_success(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """Scenario 1.1.1: Happy path with SQL load, MLflow mocks, Postgres predictions and transforms."""
+    client = temporal_test_env.client
+
+    with postgres_engine.begin() as conn:
+        conn.execute(text('DELETE FROM sientia_data.laborious_data WHERE model_id = 123'))
+        insert_sql = """
+            INSERT INTO sientia_data.laborious_data (model_id, variable, value, timestamp, created_at)
+            VALUES
+                (123, 'sensor_1', 23.5, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
+                (123, 'sensor_2', 78.2, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
+                (123, 'sensor_3', 120.8, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
+        """
+        conn.execute(text(insert_sql))
+
+    input_data = load_scenario_input('main_happy_path.json', model_id=123)
+
+    await start_and_await_workflow(
+        client,
+        PredictionsBatch.run,
+        input_data,
+        make_workflow_id('test-predictions-batch'),
+    )
+
+    schema_name = 'sientia_data'
+    with postgres_engine.connect() as conn:
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, prediction, prediction_confidence, response_time, prediction_status, comments '
+                f'FROM {schema_name}.predictions WHERE model_id = 123'
+            )
+        )
+        prediction_rows = result_query.fetchall()
+        assert len(prediction_rows) == 1
+        row = prediction_rows[0]
+        assert row[0] == 123
+        assert row[1] == 0.5
+        assert row[2] == 0, f'Expected prediction_confidence=0, got {row[2]}'
+        assert row[3] is not None
+        assert row[4] == 'Good'
+        assert row[5] == ''
+
+        result_query = conn.execute(
+            text(
+                f'SELECT model_id, variable, value FROM {schema_name}.transformed_data WHERE model_id = 123'
+            )
+        )
+        transformed_rows = result_query.fetchall()
+        assert len(transformed_rows) == 2
+        assert transformed_rows[0][0] == 123
+        assert transformed_rows[0][1] == 'feature_1'
+        assert float(transformed_rows[0][2]) == 0.234
+        assert transformed_rows[1][0] == 123
+        assert transformed_rows[1][1] == 'feature_2'
+        assert float(transformed_rows[1][2]) == 0.783
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_1_2_1_sql_query_execution_error(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """Invalid SQL: workflow may complete with early exit; no prediction rows."""
+    client = temporal_test_env.client
+
+    input_data = load_scenario_input('main_sql_error.json', model_id=128)
+
+    await start_and_await_workflow(
+        client,
+        PredictionsBatch.run,
+        input_data,
+        make_workflow_id('test-sql-error'),
+    )
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM sientia_data.predictions WHERE model_id = 128')
+        ).scalar()
+    assert count == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_1_2_2_missing_required_parameters(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """Missing query: workflow does not produce predictions and is terminated explicitly."""
+    client = temporal_test_env.client
+
+    input_data = load_scenario_input('main_missing_required.json', model_id=129)
+
+    handle = await client.start_workflow(
+        PredictionsBatch.run,
+        input_data,
+        id=make_workflow_id('test-missing-param'),
+        task_queue='test-queue',
+    )
+
+    # Let Temporal process a few workflow tasks; for this case, result() can hang.
+    await asyncio.sleep(2.0)
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM sientia_data.predictions WHERE model_id = 129')
+        ).scalar()
+    assert count == 0
+
+    await handle.terminate('expected failure path in e2e test (missing required parameters)')
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_1_2_3_invalid_datetime_column_specification(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """Invalid datetime column: no predictions persisted; workflow terminated after validation."""
+    client = temporal_test_env.client
+
+    with postgres_engine.begin() as conn:
+        conn.execute(text('DELETE FROM sientia_data.laborious_data WHERE model_id = 130'))
+        conn.execute(
+            text(
+                """
+            INSERT INTO sientia_data.laborious_data (model_id, variable, value, timestamp, created_at)
+            VALUES (130, 'sensor_1', 23.5, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
+        """
+            )
+        )
+
+    input_data = load_scenario_input('main_invalid_datetime.json', model_id=130)
+
+    handle = await client.start_workflow(
+        PredictionsBatch.run,
+        input_data,
+        id=make_workflow_id('test-invalid-datetime-col'),
+        task_queue='test-queue',
+    )
+
+    # Let Temporal process and surface the failure path internally.
+    await asyncio.sleep(2.0)
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text('SELECT COUNT(*) FROM sientia_data.predictions WHERE model_id = 130')
+        ).scalar()
+    assert count == 0
+
+    await handle.terminate('expected failure path in e2e test (invalid datetime column)')

e2e/test_predictions_batch_prediction_process.py

@@ -0,0 +1,488 @@
+"""
+End-to-end tests for PredictionsBatch workflow - Prediction Process scenarios.
+"""
+
+from decimal import Decimal
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pandas as pd
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import (
+    assert_continue,
+    assert_postgres_unique_violation_in_chain,
+    assert_prediction,
+    assert_prediction_row_count,
+    assert_repeat,
+    assert_stop,
+    insert_sample_data,
+    insert_sample_prediction,
+    load_scenario_input,
+    make_workflow_id,
+    start_and_await_workflow,
+)
+from laborious.activities.activities import Activities
+from laborious.utils.models import minio_dataframe_payload as minio_payload_module
+from laborious.workflows.predictions_batch import PredictionsBatch
+
+DISTINCT_BATCH_TIMESTAMP = '2024-01-01 13:00:00+00:00'
+HISTORY_TIMESTAMP = '2024-01-01 12:00:00+00:00'
+
+
+def get_base_input_data(model_id):
+    return load_scenario_input('prediction_process_base.json', model_id=model_id)
+
+
+@pytest.fixture
+def bad_data_model(mlflow_repository_stub):
+    mlflow_repository_stub.stub_wrapper.transform = MagicMock(
+        side_effect=Exception('Bad data model')
+    )
+    return mlflow_repository_stub.stub_wrapper
+
+
+@pytest.fixture
+def bad_predict_model(mlflow_repository_stub):
+    wrapper = mlflow_repository_stub.stub_wrapper
+
+    def _good_transform(data):
+        result = pd.DataFrame(
+            {
+                'feature_1': [0.234] * len(data),
+                'feature_2': [0.783] * len(data),
+            }
+        )
+        result.index = data.index
+        return result, {}
+
+    wrapper.transform.side_effect = _good_transform
+    wrapper.predict = MagicMock(side_effect=Exception('Bad predict model'))
+    return wrapper
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_1_1_input_gate_triggers_continue(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """Input gate CONTINUE: export default prediction; MLflow transform/predict not used."""
+    client = temporal_test_env.client
+    model_id = 211
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+    input_data = get_base_input_data(model_id)
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-continue-policy')
+    )
+    assert_continue(postgres_engine, model_id)
+    mlflow_repository_stub.stub_wrapper.transform.assert_not_called()
+    mlflow_repository_stub.stub_wrapper.predict.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_1_2_input_gate_triggers_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """Input gate STOP: no export, no MLflow."""
+    client = temporal_test_env.client
+    model_id = 212
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'STOP'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-input-stop')
+    )
+    assert_stop(postgres_engine, model_id)
+    mlflow_repository_stub.stub_wrapper.transform.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_1_3_input_gate_repeat_batch_timestamp_equals_history_fails(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    REPEAT uses ``last_timestamp`` from the batch payload as the new row's ``timestamp``.
+    When it equals the only historical prediction row, Postgres rejects the duplicate key.
+    """
+    client = temporal_test_env.client
+    model_id = 213
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2], data_timestamp=HISTORY_TIMESTAMP)
+    insert_sample_prediction(postgres_engine, model_id, prediction_timestamp=HISTORY_TIMESTAMP)
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'REPEAT'
+    with pytest.raises(Exception) as excinfo:
+        await start_and_await_workflow(
+            client, PredictionsBatch.run, input_data, make_workflow_id('test-input-repeat-collision')
+        )
+    assert_postgres_unique_violation_in_chain(excinfo.value)
+    assert_prediction_row_count(postgres_engine, model_id, 1)
+    mlflow_repository_stub.stub_wrapper.transform.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_1_3_input_gate_repeat_distinct_batch_timestamp_inserts_second_row(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """REPEAT succeeds when batch ``last_timestamp`` differs from the historical prediction row."""
+    client = temporal_test_env.client
+    model_id = 2131
+    insert_sample_data(
+        postgres_engine, model_id, ['NULL', 78.2], data_timestamp=DISTINCT_BATCH_TIMESTAMP
+    )
+    data = insert_sample_prediction(postgres_engine, model_id, prediction_timestamp=HISTORY_TIMESTAMP)
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'REPEAT'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-input-repeat-ok')
+    )
+    assert_repeat(postgres_engine, model_id, data)
+    mlflow_repository_stub.stub_wrapper.transform.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_1_4_input_gate_repeat_without_prior_prediction(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """REPEAT when no prior row in predictions: repeat_last_prediction runs; still no new duplicate export path."""
+    client = temporal_test_env.client
+    model_id = 214
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.predictions WHERE model_id = {model_id}'))
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'REPEAT'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-input-repeat-no-history')
+    )
+    assert_stop(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_2_1_transform_gate_triggers_continue(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_data_model,
+):
+    client = temporal_test_env.client
+    model_id = 221
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'CONTINUE'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-continue')
+    )
+    assert_continue(
+        postgres_engine=postgres_engine,
+        model_id=model_id,
+        prediction_confidence=Decimal(10),
+        comments='Unknown MLFlow API error',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_2_2_transform_gate_triggers_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_data_model,
+    mlflow_repository_stub,
+):
+    client = temporal_test_env.client
+    model_id = 222
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'STOP'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-stop')
+    )
+    assert_stop(postgres_engine, model_id)
+    mlflow_repository_stub.stub_wrapper.predict.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_2_3_transform_gate_repeat_batch_timestamp_equals_history_fails(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_data_model,
+):
+    client = temporal_test_env.client
+    model_id = 223
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2], data_timestamp=HISTORY_TIMESTAMP)
+    insert_sample_prediction(postgres_engine, model_id, prediction_timestamp=HISTORY_TIMESTAMP)
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'REPEAT'
+    with pytest.raises(Exception) as excinfo:
+        await start_and_await_workflow(
+            client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-repeat-collision')
+        )
+    assert_postgres_unique_violation_in_chain(excinfo.value)
+    assert_prediction_row_count(postgres_engine, model_id, 1)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_2_3_transform_gate_repeat_distinct_batch_timestamp_inserts_second_row(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_data_model,
+):
+    client = temporal_test_env.client
+    model_id = 2231
+    insert_sample_data(
+        postgres_engine, model_id, [60.0, 78.2], data_timestamp=DISTINCT_BATCH_TIMESTAMP
+    )
+    data = insert_sample_prediction(postgres_engine, model_id, prediction_timestamp=HISTORY_TIMESTAMP)
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'REPEAT'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-repeat-ok')
+    )
+    assert_repeat(postgres_engine, model_id, data)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_2_4_transform_content_gate_nan_values_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """mlflow_content_gate triggers STOP when transform output is all NaN (NAN_VALUES filter)."""
+    client = temporal_test_env.client
+    model_id = 224
+
+    def all_nan_transform(data):
+        result = pd.DataFrame(
+            {'feature_1': [np.nan] * len(data), 'feature_2': [np.nan] * len(data)}
+        )
+        result.index = data.index
+        return result, {}
+
+    mlflow_repository_stub.stub_wrapper.transform = MagicMock(side_effect=all_nan_transform)
+
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters'] = {
+        'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
+        'NAN_VALUES': {'POLICY': 'STOP', 'CONFIG': {}},
+    }
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-content-stop')
+    )
+    assert_stop(postgres_engine, model_id)
+    mlflow_repository_stub.stub_wrapper.predict.assert_not_called()
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_3_1_predict_gate_triggers_continue(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_predict_model,
+):
+    client = temporal_test_env.client
+    model_id = 231
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'CONTINUE'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-predict-continue')
+    )
+    assert_continue(
+        postgres_engine=postgres_engine,
+        model_id=model_id,
+        prediction_confidence=Decimal(10),
+        comments='Unknown MLFlow API error',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_3_2_predict_gate_triggers_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_predict_model,
+):
+    client = temporal_test_env.client
+    model_id = 232
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'STOP'
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-predict-stop')
+    )
+    assert_stop(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_3_3_predict_gate_repeat_batch_timestamp_equals_history_fails(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_predict_model,
+):
+    client = temporal_test_env.client
+    model_id = 233
+    insert_sample_data(postgres_engine, model_id, [23.5, 78.2], data_timestamp=HISTORY_TIMESTAMP)
+    insert_sample_prediction(postgres_engine, model_id, prediction_timestamp=HISTORY_TIMESTAMP)
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'REPEAT'
+    input_data['path_priority'] = ['REPEAT', 'STOP', 'CONTINUE']
+    with pytest.raises(Exception) as excinfo:
+        await start_and_await_workflow(
+            client, PredictionsBatch.run, input_data, make_workflow_id('test-predict-repeat-collision')
+        )
+    assert_postgres_unique_violation_in_chain(excinfo.value)
+    assert_prediction_row_count(postgres_engine, model_id, 1)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_3_3_predict_gate_repeat_distinct_batch_timestamp_inserts_second_row(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_predict_model,
+):
+    client = temporal_test_env.client
+    model_id = 2331
+    insert_sample_data(
+        postgres_engine, model_id, [23.5, 78.2], data_timestamp=DISTINCT_BATCH_TIMESTAMP
+    )
+    data = insert_sample_prediction(postgres_engine, model_id, prediction_timestamp=HISTORY_TIMESTAMP)
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'REPEAT'
+    input_data['path_priority'] = ['REPEAT', 'STOP', 'CONTINUE']
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-predict-repeat-ok')
+    )
+    assert_repeat(postgres_engine, model_id, data)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_4_1_input_empty_data_stop(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """EMPTY_DATA filter with STOP when query returns no rows (offload payload empty)."""
+    client = temporal_test_env.client
+    model_id = 241
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
+    input_data = get_base_input_data(model_id)
+    input_data['input_filters'] = {'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}}}
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-empty-data-stop')
+    )
+    assert_stop(postgres_engine, model_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_scenario_2_4_1_priority_conflict_resolution(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    bad_data_model,
+):
+    """Conflicting filter outputs must honor configured path_priority order."""
+    client = temporal_test_env.client
+    model_id = 242
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    input_data['mlflow_transform_filters'] = {'API_ERROR': {'POLICY': 'CONTINUE', 'CONFIG': {}}}
+    input_data['mlflow_predict_filters'] = {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}}
+    input_data['path_priority'] = ['STOP', 'CONTINUE', 'REPEAT']
+
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-priority-conflict')
+    )
+    assert_continue(
+        postgres_engine=postgres_engine,
+        model_id=model_id,
+        prediction_confidence=Decimal(10),
+        comments='Unknown MLFlow API error',
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_e2e_request_predict_inline_minio_payload_with_datetimeindex(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mlflow_repository_stub,
+):
+    """
+    High offload threshold forces inline tabular dicts; ``DatetimeIndex`` must serialize as JSON
+    (string index keys via ``MinioDataFramePayload.from_dataframe``) so ``request_predict`` completes.
+    """
+    client = temporal_test_env.client
+    model_id = 252
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
+        conn.execute(text(f'DELETE FROM sientia_data.predictions WHERE model_id = {model_id}'))
+        conn.execute(text(f'DELETE FROM sientia_data.transformed_data WHERE model_id = {model_id}'))
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+
+    with patch.object(minio_payload_module, 'OFFLOAD_THRESHOLD_BYTES', 10**9):
+        await start_and_await_workflow(
+            client,
+            PredictionsBatch.run,
+            input_data,
+            make_workflow_id('test-predict-inline-json-datetimeindex'),
+        )
+
+    assert_prediction(postgres_engine, model_id, prediction=0.5, prediction_confidence=0)
+    mlflow_repository_stub.stub_wrapper.predict.assert_called()

e2e/test_simple_metrics.py

@@ -0,0 +1,333 @@
+"""
+End-to-end tests for the SimpleMetrics workflow.
+
+Coverage focus:
+
+- Happy path computes rmse/mse/mae/r2 from predictions joined against ``laborious_data``
+  and persists rows to ``sientia_data.simple_metrics`` with all required columns.
+- Subset metric selection (only rmse) writes exactly the requested rows.
+- Zero-variance target produces ``r2=0`` per division-by-zero guard.
+- Empty join (no overlapping data) short-circuits without persisting anything.
+"""
+
+from datetime import datetime, timedelta, timezone
+from decimal import Decimal
+
+import math
+import pytest
+from sqlalchemy import text
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import Worker
+
+from e2e.helpers import load_scenario_input, make_workflow_id, start_and_await_workflow
+from laborious.activities.activities import Activities
+from laborious.workflows.simple_metrics import SimpleMetrics
+
+# Columns defined by the production DDL for ``sientia_data.simple_metrics``.
+EXPECTED_SIMPLE_METRICS_COLUMNS = [
+    'id',
+    'model_id',
+    'metric',
+    'value',
+    'timestamp',
+    'data_size',
+    'interval_minutes',
+    'created_at',
+]
+
+# ``timestamp`` is now nullable per the new DDL (production code may write it
+# null when the upstream data has no usable instant); skip the non-null check
+# for it while still validating presence.
+NULLABLE_SIMPLE_METRICS_COLUMNS = {'timestamp'}
+
+
+def _simple_metrics_input(model_id: int, **overrides) -> dict:
+    """Load and override the simple-metrics base scenario."""
+    payload = load_scenario_input('simple_metrics_base.json', model_id=model_id)
+    payload.update(overrides)
+    return payload
+
+
+def _seed_predictions_and_targets(
+    postgres_engine,
+    model_id: int,
+    pairs: list[tuple[float, float]],
+    target_name: str = 'sensor_target',
+    offset_minutes: int = 6,
+) -> list[str]:
+    """
+    Insert matching prediction/target rows used by the SimpleMetrics SQL JOIN.
+
+    For each ``(prediction, target)`` pair we write a row in ``predictions`` and
+    a matching row in ``laborious_data`` with ``variable=target_name`` so the
+    inner join in the workflow query yields one row per pair.
+
+    Args:
+     - postgres_engine: SQLAlchemy engine bound to the test container.
+     - model_id: Model id stamped on every row.
+     - pairs: ``(prediction, target)`` pairs, one per minute.
+     - target_name: Variable name in ``laborious_data`` representing the target.
+     - offset_minutes: Earliest row sits this many minutes ago so timestamps fall
+       inside the workflow's recent-data window.
+
+    Return:
+     List of timestamp strings written for the inserted rows.
+    """
+    base = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(
+        minutes=offset_minutes
+    )
+    timestamps = [
+        (base + timedelta(minutes=i)).strftime('%Y-%m-%d %H:%M:%S%z')
+        for i in range(len(pairs))
+    ]
+
+    prediction_rows = []
+    target_rows = []
+    for index, (prediction, target_value) in enumerate(pairs):
+        ts = timestamps[index]
+        prediction_rows.append(
+            f"({model_id}, {prediction}, 0, 0, 'Good', '{ts}', '{ts}')"
+        )
+        target_rows.append(
+            f"({model_id}, '{target_name}', {target_value}, '{ts}', '{ts}')"
+        )
+
+    with postgres_engine.begin() as conn:
+        conn.execute(text(f'DELETE FROM sientia_data.predictions WHERE model_id = {model_id}'))
+        conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
+        # The SimpleMetrics SQL JOIN only filters ``predictions.model_id``; it does
+        # NOT filter ``laborious_data.model_id`` (see ``e2e/CODE_ISSUES.md`` issue
+        # SM-1). Without this cross-model cleanup, a previous test's target rows
+        # under the same variable name would join into this test's predictions
+        # whenever timestamps happened to overlap.
+        conn.execute(
+            text(
+                "DELETE FROM sientia_data.laborious_data "
+                "WHERE variable IN (:sensor_default, :target_name) "
+                "AND timestamp >= NOW() - INTERVAL '120 minutes'"
+            ),
+            {'sensor_default': 'sensor_target', 'target_name': target_name},
+        )
+        if prediction_rows:
+            conn.execute(
+                text(
+                    'INSERT INTO sientia_data.predictions '
+                    '(model_id, prediction, prediction_confidence, response_time, '
+                    'prediction_status, "timestamp", created_at) VALUES '
+                    + ', '.join(prediction_rows)
+                )
+            )
+            conn.execute(
+                text(
+                    'INSERT INTO sientia_data.laborious_data '
+                    '(model_id, variable, value, "timestamp", created_at) VALUES '
+                    + ', '.join(target_rows)
+                )
+            )
+    return timestamps
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_simple_metrics_happy_path_persists_all_metrics_and_columns(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_simple_metrics: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario S.1.1: rmse/mse/mae/r2 are calculated from a deterministic
+    prediction/target pair set and written one row per metric. Every column
+    expected by ``sientia_data.simple_metrics`` must be populated (except the
+    nullable ``timestamp`` column) and the numerical values must match
+    closed-form expectations.
+    """
+    client = temporal_test_env.client
+    model_id = 511
+
+    pairs = [
+        (1.0, 2.0),
+        (2.0, 4.0),
+        (3.0, 5.0),
+        (4.0, 9.0),
+        (5.0, 12.0),
+    ]
+    diffs = [target - prediction for prediction, target in pairs]
+    n = len(diffs)
+    expected_rmse = math.sqrt(sum(d * d for d in diffs) / n)
+    expected_mse = sum(d * d for d in diffs) / n
+    expected_mae = sum(abs(d) for d in diffs) / n
+    target_mean = sum(t for _, t in pairs) / n
+    ss_res = sum((target - prediction) ** 2 for prediction, target in pairs)
+    ss_tot = sum((t - target_mean) ** 2 for _, t in pairs)
+    expected_r2 = 1.0 - (ss_res / ss_tot)
+
+    _seed_predictions_and_targets(postgres_engine, model_id=model_id, pairs=pairs)
+
+    input_data = _simple_metrics_input(model_id)
+    await start_and_await_workflow(
+        client,
+        SimpleMetrics.run,
+        input_data,
+        make_workflow_id('test-simple-metrics-happy'),
+    )
+
+    with postgres_engine.connect() as conn:
+        rows = (
+            conn.execute(
+                text(
+                    'SELECT * FROM sientia_data.simple_metrics '
+                    'WHERE model_id = :m ORDER BY metric'
+                ),
+                {'m': str(model_id)},
+            )
+            .mappings()
+            .all()
+        )
+
+    assert len(rows) == 4, f'Expected 4 metric rows, got {len(rows)}'
+    for column in EXPECTED_SIMPLE_METRICS_COLUMNS:
+        assert column in rows[0], f'Missing simple_metrics column: {column}'
+    for row in rows:
+        for column in EXPECTED_SIMPLE_METRICS_COLUMNS:
+            if column in NULLABLE_SIMPLE_METRICS_COLUMNS:
+                continue
+            assert row[column] is not None, f"Column '{column}' is NULL in {dict(row)}"
+
+    by_metric = {row['metric']: row for row in rows}
+    assert set(by_metric) == {'rmse', 'mse', 'mae', 'r2'}
+
+    def _decimal_close(actual, expected, places: int = 6) -> bool:
+        return abs(float(actual) - expected) < 10 ** (-places)
+
+    assert _decimal_close(by_metric['rmse']['value'], expected_rmse)
+    assert _decimal_close(by_metric['mse']['value'], expected_mse)
+    assert _decimal_close(by_metric['mae']['value'], expected_mae)
+    assert _decimal_close(by_metric['r2']['value'], expected_r2)
+
+    assert all(row['data_size'] == n for row in rows), 'data_size must equal target row count'
+    assert all(row['interval_minutes'] == 60 for row in rows)
+    # ``model_id`` is now ``text`` in the new DDL, so we compare with the
+    # stringified test id rather than the numeric value.
+    assert all(row['model_id'] == str(model_id) for row in rows)
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_simple_metrics_subset_metrics_writes_only_requested_rows(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_simple_metrics: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario S.1.2: Requesting ``metrics=['rmse']`` must persist exactly one row
+    with metric ``rmse`` and skip mse/mae/r2.
+    """
+    client = temporal_test_env.client
+    model_id = 512
+
+    pairs = [(1.0, 2.0), (2.0, 4.0), (3.0, 6.0)]
+    _seed_predictions_and_targets(postgres_engine, model_id=model_id, pairs=pairs)
+
+    input_data = _simple_metrics_input(model_id, metrics=['rmse'])
+    await start_and_await_workflow(
+        client,
+        SimpleMetrics.run,
+        input_data,
+        make_workflow_id('test-simple-metrics-subset'),
+    )
+
+    with postgres_engine.connect() as conn:
+        metrics = [
+            r[0]
+            for r in conn.execute(
+                text(
+                    'SELECT metric FROM sientia_data.simple_metrics '
+                    'WHERE model_id = :m'
+                ),
+                {'m': str(model_id)},
+            ).all()
+        ]
+    assert metrics == ['rmse']
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_simple_metrics_zero_variance_target_returns_zero_r2(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_simple_metrics: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario S.2.1: When the target column has zero variance the activity must
+    return ``r2 = 0`` (division-by-zero guard) and still persist all four metrics.
+    """
+    client = temporal_test_env.client
+    model_id = 521
+
+    pairs = [(0.0, 5.0), (1.0, 5.0), (2.0, 5.0), (3.0, 5.0)]
+    _seed_predictions_and_targets(postgres_engine, model_id=model_id, pairs=pairs)
+
+    input_data = _simple_metrics_input(model_id)
+    await start_and_await_workflow(
+        client,
+        SimpleMetrics.run,
+        input_data,
+        make_workflow_id('test-simple-metrics-zero-variance'),
+    )
+
+    with postgres_engine.connect() as conn:
+        r2_value = conn.execute(
+            text(
+                "SELECT value FROM sientia_data.simple_metrics "
+                "WHERE model_id = :m AND metric = 'r2'"
+            ),
+            {'m': str(model_id)},
+        ).scalar()
+    assert r2_value is not None
+    assert Decimal(str(r2_value)) == Decimal('0'), f'expected r2=0, got {r2_value!r}'
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_simple_metrics_no_overlapping_data_short_circuits(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker_simple_metrics: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """
+    Scenario S.3.1: When the join produces no rows (no matching laborious_data
+    row for the configured ``target``), the workflow returns early without
+    invoking ``calculate_simple_metrics`` and writes nothing.
+    """
+    client = temporal_test_env.client
+    model_id = 531
+
+    # Insert predictions but no matching target rows for the configured variable.
+    _seed_predictions_and_targets(
+        postgres_engine,
+        model_id=model_id,
+        pairs=[(1.0, 1.0)],
+        target_name='wrong_variable_name',
+    )
+
+    input_data = _simple_metrics_input(model_id)
+    await start_and_await_workflow(
+        client,
+        SimpleMetrics.run,
+        input_data,
+        make_workflow_id('test-simple-metrics-empty-join'),
+    )
+
+    with postgres_engine.connect() as conn:
+        count = conn.execute(
+            text(
+                'SELECT COUNT(*) FROM sientia_data.simple_metrics '
+                'WHERE model_id = :m'
+            ),
+            {'m': str(model_id)},
+        ).scalar()
+    assert count == 0, 'Empty target data must short-circuit and skip persistence'