Merge release/SIENTIAPDE-1646 into feature/SIENTIAPDE-1986-simple-metrics-rewrite

Requirements follow the release; only sientia_model>=0.12.0 (RegressionMetrics),
model-library@0.13.1 and scikit-learn==1.8.0 (local) are kept from this branch.

laborious/activities/activities.py

@@ -7,6 +7,8 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.observability.logger import Logger
     from sientia_do.repository.minio_repository_sync import MinioRepository
+    from sientia_do.repository.mongodb_repository_sync import MongoDBRepository
+    from sientia_do.utils.connectors_config import build_mongodb_config
     from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
     from sientia_model.model_repository.plugin_store import PluginStore
 
@@ -15,18 +17,21 @@ with workflow.unsafe.imports_passed_through():
     from laborious.activities.mlflow import MLFlow
     from laborious.activities.model_metrics import ModelMetrics
     from laborious.activities.opc import OPC
+    from laborious.activities.prediction_metadata import PredictionMetadata
     from laborious.activities.storage import Storage
     from laborious.utils.connectors_config import build_mlflow_config
 
 
-class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
+class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API, PredictionMetadata):
     """
     Central orchestrator for all Temporal activities used by Laborious workflows.
 
     Composes Storage (Postgres + MinIO offload), MLFlow (wrapper-based inference and retrain
     via ``SientiaMLflowRepository``), Gates (data quality and ML response filters), OPC exports,
-    drift/simple metrics, and PI Web API writes. The worker constructs one ``Activities`` instance
-    per process and registers its callables on multiple workers bound to different task queues.
+    drift/simple metrics, PI Web API writes, and PredictionMetadata (the single write point for
+    gate metadata — see `laborious.activities.prediction_metadata`). The worker constructs one
+    ``Activities`` instance per process and registers its callables on multiple workers bound to
+    different task queues.
 
     MLflow connectivity: unless ``mlflow_repository`` is injected (tests only), this class builds
     ``SientiaMLflowRepository`` from ``build_mlflow_config()`` so tracking credentials and URL
@@ -48,6 +53,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
         notification_handler: NotificationHandler,
         metrics_controller: MetricsController | None = None,
         mlflow_repository: SientiaMLflowRepository | None = None,
+        mongo_repository: MongoDBRepository | None = None,
     ):
         """
         Wire Postgres, MinIO, MLflow, OPC, gates, metrics, and PI Web API into a single object.
@@ -67,6 +73,9 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
          - metrics_controller: Optional shared controller; if ``None``, a new one is created.
          - mlflow_repository: Optional ``SientiaMLflowRepository`` for unit/e2e tests; in production
            leave unset so the repository is built from environment via ``build_mlflow_config()``.
+         - mongo_repository: Optional ``MongoDBRepository`` (prediction metadata, best-effort) for
+           unit/e2e tests; in production leave unset so the repository is built from environment
+           via ``build_mongodb_config()``, same pattern as ``mlflow_repository`` above.
 
         Raises:
             Exception: If any parent ``__init__`` fails (e.g. invalid config keys).
@@ -89,6 +98,16 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
                 metrics_controller=mc,
             )
 
+        if mongo_repository is None:
+            mongo_cfg = build_mongodb_config()
+            mongo_repository = MongoDBRepository(
+                connection_string=mongo_cfg['connection_string'],
+                database_name=mongo_cfg['database_name'],
+                logger=logger,
+                notification_handler=notification_handler,
+                metrics_controller=mc,
+            )
+
         minio_repository = MinioRepository(
             endpoint=minio_config['endpoint_url'],
             access_key=minio_config['access_key'],
@@ -134,6 +153,14 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
             metrics_controller=mc,
         )
 
+        PredictionMetadata.__init__(
+            self,
+            mongo_repository=mongo_repository,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=mc,
+        )
+
         OPC.__init__(
             self,
             opc_servers=opc_config,
@@ -175,3 +202,4 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
         OPC.close(self)
         ModelMetrics.close(self)
         API.close(self)
+        PredictionMetadata.close(self)

laborious/activities/gates.py

@@ -27,6 +27,7 @@ with workflow.unsafe.imports_passed_through():
 InputFilterFunc = Callable[[DataFrame, dict[str, Any]], bool]
 ResponseFilterFunc = Callable[[dict[str, Any], dict[str, Any]], bool]
 ContentFilterFunc = Callable[[DataFrame, dict[str, Any]], bool]
+InputCounterFunc = Callable[[DataFrame, dict[str, Any]], int]
 
 # Input filter function mappings
 input_filter_functions: dict[str, InputFilterFunc] = {
@@ -34,6 +35,42 @@ input_filter_functions: dict[str, InputFilterFunc] = {
     'EMPTY_DATA': filter_empty_data,
 }
 
+
+def _count_specific_variables_null_values(data: DataFrame, config: dict[str, Any]) -> int:
+    """
+    Count rows with a null value in one of the configured variables.
+
+    Mirrors `filter_specific_variables_null_values`'s own trigger condition, so the count
+    reflects exactly what made that filter fire.
+
+    Return:
+        int: Number of rows with a null value in `config['variables']`.
+    """
+    if data.empty:
+        return 0
+    return int((data['variable'].isin(config['variables']) & data['value'].isna()).sum())
+
+
+def _count_empty_data(_data: DataFrame, _config: dict[str, Any]) -> int:
+    """
+    Count rows for the EMPTY_DATA filter.
+
+    `EMPTY_DATA` only triggers when there are no rows at all, so the count is always 0.
+
+    Return:
+        int: Always 0.
+    """
+    return 0
+
+
+# Per-filter row counts for the `counter` field in prediction metadata (gate-local: mirrors each
+# filter's own trigger condition instead of changing `conditional_filters.py`'s bool contract,
+# which `mlflow_content_gate` also depends on for EMPTY_DATA).
+input_filter_counters: dict[str, InputCounterFunc] = {
+    'SPECIFIC_VARIABLES_NULL_VALUES': _count_specific_variables_null_values,
+    'EMPTY_DATA': _count_empty_data,
+}
+
 # Confidence mappings kept separate from function maps to avoid Union types
 input_path_confidence: Mapping[str, int] = {
     'STOP': -1,
@@ -163,7 +200,9 @@ class Gates(SientiaMonitoring):
         return policy, filter_config
 
     @activity.defn(name='input_gate')
-    def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
+    def input_gate(
+        self, input_data: dict[str, Any]
+    ) -> tuple[str | None, int, str, dict[str, Any] | None]:
         """
         Apply input data quality filters and validation.
 
@@ -178,6 +217,9 @@ class Gates(SientiaMonitoring):
         3. Determines appropriate path decisions (STOP, CONTINUE, REPEAT)
         4. Provides confidence scores and detailed comments
         5. Handles errors gracefully with notification integration
+        6. Builds a metadata document describing which filter(s) triggered, with their policy
+           and a counter, whenever one does — this activity only computes it, it never writes it
+           (see `PredictionMetadata.write_prediction_metadata` for the actual write)
 
         Args:
             input_data: Configuration and data for input validation
@@ -188,10 +230,14 @@ class Gates(SientiaMonitoring):
                     - path_priority (list[str]): Priority order for path decisions
 
         Returns:
-            tuple: (path_flag, confidence, comment)
+            tuple: (path_flag, confidence, comment, document)
                 - path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
                 - confidence (int): Confidence score for the decision
                 - comment (str): Detailed explanation of the decision
+                - document (dict | None): Metadata describing the filter(s) that produced
+                    `path_flag`, shaped `{'filters': [{'name', 'policy', 'counter'}, ...]}`;
+                    `None` when nothing was filtered. Not written anywhere by this activity —
+                    the caller decides whether and how to persist it.
 
         Raises:
             Exception: If filter execution fails or configuration is invalid
@@ -205,7 +251,8 @@ class Gates(SientiaMonitoring):
         data = payload.retrieve(self.minio_repository, metadata)
         path_priority = input_data['path_priority']
 
-        filter_output = []
+        # (filter_name, policy, filter_config) per filter that triggered, in configured order
+        triggered_filters: list[tuple[str, str, dict[str, Any]]] = []
 
         self._debug_dataframe('Input data:', data, metadata)
         self.debug(f'Filters: {filters}', metadata)
@@ -219,7 +266,7 @@ class Gates(SientiaMonitoring):
             try:
                 if input_filter_functions[fil](data, filter_config):
                     self.debug(f'Data not passed the input filter {fil}:{config}', metadata)
-                    filter_output.append(policy)
+                    triggered_filters.append((fil, policy, filter_config))
             except Exception as e:
                 trace = traceback.format_exc()
                 self.send_notification(
@@ -231,16 +278,46 @@ class Gates(SientiaMonitoring):
                     attachment_content=trace,
                 )
 
+        triggered_policies = [policy for _, policy, _ in triggered_filters]
+
         for path_flag in path_priority:
-            if path_flag in filter_output:
+            if path_flag in triggered_policies:
                 self.info(f'Input gate result: {path_flag}', metadata)
-                return path_flag, input_path_confidence[path_flag], 'Input data with bad quality'
+
+                matched = [
+                    (fil, filter_config)
+                    for fil, policy, filter_config in triggered_filters
+                    if policy == path_flag
+                ]
+                document = {
+                    'filters': [
+                        {
+                            'name': fil,
+                            'policy': path_flag,
+                            # `.get(..., lambda *_: 0)`, not `[fil]`: a filter added to
+                            # `input_filter_functions` without a matching counter here degrades to
+                            # counter=0 instead of a raw KeyError crashing this prediction (review
+                            # finding, 2026-09-30 — see test enforcing key parity between the two).
+                            'counter': input_filter_counters.get(fil, lambda *_: 0)(
+                                data, filter_config
+                            ),
+                        }
+                        for fil, filter_config in matched
+                    ],
+                }
+
+                return (
+                    path_flag,
+                    input_path_confidence[path_flag],
+                    'Input data with bad quality',
+                    document,
+                )
 
         self.info('Nothing was filtered by the input gate', metadata)
 
         del data
 
-        return None, 0, ''
+        return None, 0, '', None
 
     @activity.defn(name='mlflow_response_gate')
     def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
@@ -636,6 +713,9 @@ class Gates(SientiaMonitoring):
                 - model_id (str): Unique identifier for the ML model
                 - prediction_confidence (float): Confidence score (typically low for errors)
                 - comment (str): Error description or operational comment
+                - metadata_id (str | None, optional): Id of the `prediction_metadata` Mongo
+                    document to reference for this row; `None`/absent when there's nothing to
+                    reference
 
         Returns:
             dict: Formatted default prediction data with error indicators
@@ -653,6 +733,7 @@ class Gates(SientiaMonitoring):
                 'prediction_confidence': [input_data['prediction_confidence']],
                 'prediction_status': ['Bad'],
                 'comments': [input_data['comment']],
+                'metadata_id': [input_data.get('metadata_id')],
             }
         )
 

laborious/activities/mlflow.py

@@ -29,6 +29,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_model.model_repository.plugin_store import PluginStore
 
     from laborious.utils.dataframe_debug import build_dataframe_debug_message
+    from laborious.utils.messages import MISSING_MINIO_REPOSITORY
     from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
 
 
@@ -445,7 +446,7 @@ class MLFlow(SientiaMonitoring):
         """
 
         if self.minio_repository is None:
-            raise ValueError('Minio repository not initialized')
+            raise ValueError(MISSING_MINIO_REPOSITORY)
 
         metadata = input_data['metadata']
 
@@ -528,22 +529,23 @@ class MLFlow(SientiaMonitoring):
             )
 
             # Keep heavy model fitting outside MLflow run timing.
-            prediction_data = wrapper.retrain(data)
-            prediction_data.rename(columns={target: 'prediction'}, inplace=True)
+            prediction_data = wrapper.retrain(data).rename(columns={target: 'prediction'})
 
-            # Merge prediction data with retrain data
-            evaluation_data = pd.merge(
-                data, prediction_data, left_index=True, right_index=True, how='left'
+            # Both frames are indexed by the pivoted timestamp: expose it as an explicit column
+            # and join on it. `validate` fails loudly instead of silently multiplying rows if the
+            # wrapper ever returns duplicated timestamps.
+            evaluation_data = (
+                data.reset_index(names='timestamp')
+                .merge(
+                    prediction_data.reset_index(names='timestamp'),
+                    on='timestamp',
+                    how='left',
+                    validate='one_to_one',
+                )
+                .rename(columns={target: 'target'})
+                .sort_values(by='timestamp', ascending=True, ignore_index=True)
             )
 
-            # Rename target column to "target"
-            evaluation_data.rename(columns={target: 'target'}, inplace=True)
-
-            # Reset index and put as column "timestamp"
-            evaluation_data['timestamp'] = evaluation_data.index
-            evaluation_data.reset_index(drop=True, inplace=True)
-            evaluation_data.sort_values(by='timestamp', inplace=True, ascending=True)
-
             run_name = f'{model_name}-retrain-{datetime.now().strftime("%Y%m%d%H%M%S")}'
 
             with self.mlflow_repository.start_run(

laborious/activities/prediction_metadata.py

@@ -0,0 +1,132 @@
+from temporalio import activity, workflow
+
+with workflow.unsafe.imports_passed_through():
+    from typing import Any
+
+    from bson import ObjectId
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+    from sientia_do.observability.logger import Logger
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
+    from sientia_do.repository.mongodb_repository_sync import MongoDBRepository
+
+
+class PredictionMetadata(SientiaMonitoring):
+    """
+    Centralized, best-effort writer for prediction gate metadata (triggered filter, policy,
+    counter) to MongoDB.
+
+    Gates only compute and return a metadata document; they never write it themselves. Keeping
+    the write here, instead of in each gate, puts the one real write point in the subworkflow
+    that's about to persist a `predictions` row — the only place that reliably knows whether one
+    is being written at all — and lets other metadata sources reuse the same activity instead of
+    duplicating write logic wherever they originate.
+    """
+
+    mongo_repository: MongoDBRepository | None = None
+    _PREDICTION_METADATA_COLLECTION = 'prediction_metadata'
+
+    def __init__(
+        self,
+        mongo_repository: MongoDBRepository | None = None,
+        logger: Logger | None = None,
+        notification_handler: NotificationHandler | None = None,
+        metrics_controller: MetricsController | None = None,
+    ):
+        """
+        Initialize the prediction metadata writer.
+
+        Args:
+            mongo_repository: MongoDB, where gate metadata (triggered filter, policy, counter) is
+                written best-effort. `None` is a valid state (see `write_prediction_metadata`).
+
+        Raises:
+            Exception: If BaseActivity initialization fails
+        """
+        self.mongo_repository = mongo_repository
+        SientiaMonitoring.__init__(
+            self,
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
+    def close(self) -> None:
+        """
+        Close the prediction metadata writer and clean up resources.
+        """
+        if self.mongo_repository is not None:
+            try:
+                self.mongo_repository.close()
+            finally:
+                self.mongo_repository = None
+        SientiaMonitoring.shutdown(self)
+
+    def __del__(self):
+        self.close()
+
+    @activity.defn(name='write_prediction_metadata')
+    def write_prediction_metadata(self, input_data: dict[str, Any]) -> str | None:
+        """
+        Best-effort write of one or more gate metadata documents to Mongo, combined into a single
+        `prediction_metadata` document.
+
+        Implements both business rules for this feature in one place:
+        - The prediction write must never be blocked or failed by a metadata write. A failed
+          `insert(...)` is swallowed, never re-raised, logged here with call-site context (which
+          `mongo_repository.insert(...)` itself can't know) — matching how `storage.py`'s
+          `cleanup_minio_objects_expired` also logs again at the call site even though the
+          repository already logs internally and emits its own Prometheus metric
+          (`MONGODB_WRITE_ERROR_COUNT`); the generic library log is for the metric/trace, this
+          one is for whoever is reading logs for this specific call.
+        - Nothing is written when there's nothing relevant to report. An empty/missing
+          `documents` list (or one full of falsy/empty entries) skips Mongo entirely, so callers
+          can call this unconditionally and get `None` back for "no metadata this time" instead
+          of deciding upfront whether there's something worth saving.
+
+        Takes a *list* of documents, not a single one: each caller contributes its own document
+        (shaped like `{'filters': [...]}`), and this flattens every `filters` entry across all of
+        them into one `prediction_metadata` document before the single insert. Today only one
+        gate ever contributes a document per prediction cycle (the gate chain short-circuits on
+        the first non-`None` `path_flag`), but the list shape means this activity won't need to
+        change again once more than one source contributes metadata for the same cycle.
+
+        `insert(...)` doesn't hand back the inserted document's `_id`, so it is generated here,
+        client-side, before the insert, as a plain string (not a BSON ObjectId) — matching this
+        codebase's convention of string `_id`s (e.g. the `models` collection) and the 24-hex-char
+        size of the `predictions.metadata_id` Postgres column that will reference it.
+
+        Args:
+            input_data: Dict with:
+                - documents (list[dict[str, Any]]): Metadata payloads to combine, each shaped
+                  like `{'filters': [...]}`. Missing/empty, or entries with no `filters`, skip
+                  the write entirely.
+                - metadata (dict): Workflow metadata for contextual logging.
+
+        Return:
+            The generated id, or `None` when there's nothing to write, Mongo isn't configured, or
+            the insert failed.
+        """
+        metadata = input_data['metadata']
+        documents = input_data.get('documents') or []
+
+        filters = [
+            entry for document in documents if document for entry in document.get('filters', [])
+        ]
+
+        if not filters or self.mongo_repository is None:
+            return None
+
+        generated_id = str(ObjectId())
+
+        try:
+            self.mongo_repository.insert(
+                self._PREDICTION_METADATA_COLLECTION,
+                {'filters': filters, '_id': generated_id},
+                metadata,
+            )
+        except Exception as e:
+            self.error(f'Failed to write prediction metadata: {e}', metadata)
+            return None
+
+        return generated_id

laborious/activities/storage.py

@@ -2,6 +2,7 @@ from temporalio import activity, workflow
 
 with workflow.unsafe.imports_passed_through():
     # Extend the Temporal Postgres activities for convenient query -> MinIO export
+    import time
     import traceback
     from datetime import timedelta
     from typing import Any
@@ -9,14 +10,18 @@ with workflow.unsafe.imports_passed_through():
     import pandas as pd
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.notifications.models import NotificationLevel
+    from sientia_do.observability import metrics
     from sientia_do.observability.logger import Logger
     from sientia_do.observability.metrics_controller import MetricsController
     from sientia_do.observability.sientia_monitoring import SientiaMonitoring
     from sientia_do.repository.minio_repository_sync import MinioRepository
     from sientia_do.temporal.activities.postgres_sync import Postgres
     from sientia_do.temporal.constants import now
+    from sqlalchemy import text
 
+    from laborious.utils.messages import MISSING_MINIO_REPOSITORY
     from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
+    from laborious.utils.time import to_naive
 
 _LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
 
@@ -24,7 +29,9 @@ _LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_thr
 class Storage(Postgres, SientiaMonitoring):
     """
     Extensions for Postgres activities with a helper to export query results
-    directly to MinIO as Parquet and return the object name.
+    directly to MinIO as Parquet and return the object name, and with
+    ``repeat_last_prediction``, a query specific to this repo's REPEAT path
+    (moved here from the shared library — nothing else consumes it).
     """
 
     minio_repository: MinioRepository | None = None
@@ -84,7 +91,7 @@ class Storage(Postgres, SientiaMonitoring):
             dict[str, Any]: Flat ``MinioDataFramePayload`` dict or ``success: False`` on failure.
         """
         if self.minio_repository is None:
-            raise ValueError('Minio repository not initialized')
+            raise ValueError(MISSING_MINIO_REPOSITORY)
 
         metadata: dict = input_data.get('metadata', {})
         model_name = input_data['model_name']
@@ -139,6 +146,110 @@ class Storage(Postgres, SientiaMonitoring):
             }
         )
 
+    def _build_repeat_last_prediction_query(self, schema: str, table_name: str):
+        table_identifier = f'"{schema}".{table_name}'
+        return text(  # noqa: S608
+            f"""
+        INSERT INTO {table_identifier} (
+            model_id,
+            prediction,
+            timestamp,
+            response_time,
+            prediction_status,
+            prediction_confidence,
+            metadata_id,
+            created_at
+        )
+        SELECT
+            model_id,
+            prediction,
+            :last_timestamp,
+            response_time,
+            prediction_status,
+            prediction_confidence,
+            :metadata_id,
+            NOW()
+        FROM {table_identifier}
+        WHERE model_id = :model
+        ORDER BY timestamp DESC
+        LIMIT 1;
+        """  # nosec B608
+        )
+
+    @activity.defn(name='repeat_last_prediction')
+    def repeat_last_prediction(self, query_items: dict[str, Any]):
+        """
+        Insert a new `predictions` row that copies the last one for a model forward in time.
+
+        REPEAT persists a new row (unlike STOP, which persists nothing), reusing the previous
+        row's `prediction`/`response_time`/`prediction_status`/`prediction_confidence` values with
+        a fresh `timestamp`/`created_at`. `metadata_id` references the `prediction_metadata`
+        document for this cycle, if any (see `PredictionMetadata.write_prediction_metadata`);
+        `None` is a valid value — the column is nullable.
+
+        Args (query_items):
+            metadata (dict): Workflow metadata for logging and metrics.
+            schema (str): Postgres schema of the `predictions` table.
+            table_name (str): `predictions` table name.
+            model (str): `model_id` to repeat the last prediction for.
+            last_timestamp: Timestamp to stamp the new row with.
+            metadata_id (str | None): `prediction_metadata` document id, or `None`.
+
+        Raises:
+            Exception: Re-raised after logging/metrics if the insert fails.
+        """
+        metadata = query_items['metadata']
+        schema = query_items['schema']
+        table_name = query_items['table_name']
+        model = query_items['model']
+        last_timestamp = query_items['last_timestamp']
+        metadata_id = query_items.get('metadata_id')
+
+        repeat_query = self._build_repeat_last_prediction_query(schema, table_name)
+        self.info(f'Repeating last prediction for model {model}', metadata)
+        self.debug(
+            f'Query parameters: schema={schema}, model={model}, last_timestamp={last_timestamp}',
+            metadata,
+        )
+        core_labels = {
+            **self.get_core_labels(metadata, 'execute_sql'),
+            'conflict_mode': 'error',
+        }
+
+        with self.session_factory.begin() as session:
+            try:
+                start_time = time.time()
+                session.execute(
+                    repeat_query,
+                    {
+                        'last_timestamp': last_timestamp,
+                        'model': model,
+                        'metadata_id': metadata_id,
+                    },
+                )
+            except Exception as e:
+                trace = traceback.format_exc()
+                self.send_notification(
+                    metadata=metadata,
+                    notification_id='ERROR_REPEATING_LAST_PREDICTION',
+                    message=f'Error repeating last prediction: {e}',
+                    block='repeat_last_prediction',
+                    level=NotificationLevel.ERROR,
+                    attachment_content=trace,
+                )
+                self.error(trace, metadata)
+                self.emit_metric_sync(
+                    metric_object=metrics.POSTGRES_WRITE_ERROR_COUNT,
+                    tags=core_labels,
+                )
+                raise
+            else:
+                self.observe_lag_sync(start_time, metrics.POSTGRES_WRITE_LAG, core_labels)
+                self.emit_metric_sync(
+                    metric_object=metrics.POSTGRES_WRITE_COUNT,
+                    tags=core_labels,
+                )
+
     @activity.defn(name='cleanup_minio_objects_expired')
     def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
@@ -152,15 +263,13 @@ class Storage(Postgres, SientiaMonitoring):
             dict[str, Any]: ``success``, ``deleted_count``, and optional ``message``.
         """
         if self.minio_repository is None:
-            raise ValueError('Minio repository not initialized')
+            raise ValueError(MISSING_MINIO_REPOSITORY)
 
         metadata = input_data.get('metadata', {})
         payload = MinioDataFramePayload.from_dict(input_data['data'])
         prefix = payload.cleanup_prefix()
         base = now()
-        cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta(
-            hours=self.retention_hours
-        )
+        cutoff = to_naive(base) - timedelta(hours=self.retention_hours)
 
         report: dict[str, Any] = {
             'failed': {},
@@ -223,7 +332,8 @@ class Storage(Postgres, SientiaMonitoring):
         sessions, and monitoring hooks), then closes the optional MinIO repository
         and clears the local reference to avoid accidental reuse after shutdown.
         """
-        Postgres.close(self)
+        if hasattr(self, 'engine'):
+            Postgres.close(self)
         if self.minio_repository is not None:
             try:
                 self.minio_repository.close()

laborious/utils/messages.py

@@ -0,0 +1 @@
+MISSING_MINIO_REPOSITORY = 'Minio repository not initialized'

laborious/utils/repository/opc_repository.py

@@ -87,7 +87,7 @@ def _opc_authentication_token_str(client: Client | None) -> str:
         if tok is None:
             return 'unknown'
         return str(tok)
-    except Exception:
+    except Exception:  # best-effort label for logs/metrics, must not fail loud
         return 'unknown'
 
 
@@ -278,12 +278,14 @@ class OpcRepository(SientiaMonitoring):
         Configure certificates and timeouts on the sync asyncua client.
 
         Raises:
-            ValueError: If cert paths or client are missing.
+            ValueError: If cert paths, server URI or client are missing.
         """
         if self.cert_path is None or self.private_key_path is None:
             raise ValueError(
                 'Certificate and private key paths must be provided for secure connection.'
             )
+        if self.server_uri is None:
+            raise ValueError('server_uri must be provided for secure connection.')
 
         cert = Path(self.cert_path)
         private_key = Path(self.private_key_path)
@@ -841,7 +843,10 @@ class OpcRepository(SientiaMonitoring):
         session_id = _opc_authentication_token_str(self.client)
 
         try:
-            assert self.client is not None
+            if self.client is None:
+                raise OpcClientNotInitializedError(
+                    f'OPC client is not initialized for server {self.id}; cannot write {node}'
+                )
             node_obj = self.client.get_node(node)
         except Exception as e:
             if is_reconnectable_opcua_bad(e):

laborious/utils/time.py

@@ -0,0 +1,6 @@
+from datetime import datetime
+
+
+def to_naive(moment: datetime) -> datetime:
+    """Strip tzinfo, if any. Postgres `TIMESTAMP WITHOUT TIME ZONE` columns need naive values."""
+    return moment.replace(tzinfo=None) if moment.tzinfo else moment

laborious/worker/worker.py

@@ -8,8 +8,8 @@ repository, single PluginStore handle) but polls a different task queue.
 Task queues (see ``sientia_do.temporal.worker.prepare_worker``):
 - ``predictions_batch-{runtime}-queue`` + sub-workflows on the same queue (ML-heavy path).
 - ``minimal_retrain-{runtime}-queue`` (retrain + promote + export).
-- ``drift-queue`` and ``simple_metrics-queue`` without a runtime suffix so existing schedulers
-  keep stable queue names.
+- ``drift-core-queue`` and ``simple_metrics-core-queue``: pinned to the ``core`` runtime because
+  drift and metrics do not load model code, so every runtime shares the same queues.
 
 Bootstrap order:
 1. Prometheus app metrics and Mongo-backed notification handler.
@@ -24,7 +24,7 @@ Environment Variables:
 - RUNTIME: Required non-empty string passed to ``install_runtime``.
 - STORE_* / PYPI_*: Plugin store and private index (see ``build_plugin_store_config``).
 - TEMPORAL_HOST, TEMPORAL_NAMESPACE: Cluster connection.
-- POD_ID, HTTP_METRICS_PORT, HTTP_SDK_METRICS_PORT: Observability.
+- HOSTNAME (pod id), HTTP_METRICS_PORT, HTTP_SDK_METRICS_PORT: Observability.
 - PROJECT_NAME, MONGODB_*: Notifications (via ``build_mongodb_config`` in handler).
 - POSTGRES_*, MINIO_*, OPC_*, PI_WEB_API_*, MLFLOW_*: Passed through ``Activities`` helpers.
 """
@@ -75,14 +75,14 @@ async def main():
 
     Exits the process with code 0 on normal completion of all worker tasks, or 1 after logging
     if any worker raises. ``finally`` always shuts down notifications and activities and sets
-    ``app_up`` to 0 before ``sys.exit``.
+    ``app_up`` to 0; ``sys.exit`` runs after it.
 
     Raises:
         Exception: Propagated from ``asyncio.gather`` only before ``finally`` handling; typically
             workers run until cancelled.
 
     Return:
-        None (process terminates via ``sys.exit`` from the ``finally`` block).
+        None (process terminates via ``sys.exit`` after the ``finally`` block).
     """
     host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
     logger = get_logger(__name__)
@@ -98,7 +98,7 @@ async def main():
     logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata)
 
     logger.custom_info('Starting prometheus client...', metadata)
-    start_prometheus_server()
+    start_prometheus_server(logger, metadata)
 
     logger.custom_info('Starting Notification Handler...', metadata)
 
@@ -157,7 +157,7 @@ async def main():
         metrics.APP_UP.labels(pod_id=POD_ID).set(0)
         sys.exit(1)
 
-    logger.custom_info('Starting Activities...', metadata)
+    logger.custom_info('Starting Activities...', metadata_runtime)
 
     activities = Activities(
         postgres_config=build_postgres_config(),
@@ -170,10 +170,13 @@ async def main():
         metrics_controller=metrics_controller,
     )
 
-    logger.custom_info('Initializing OPC...', metadata)
+    logger.custom_info('Initializing OPC...', metadata_runtime)
     activities.init_opc()
 
-    logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
+    logger.custom_info(
+        f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...',
+        metadata_runtime,
+    )
 
     new_runtime = Runtime(
         telemetry=TelemetryConfig(
@@ -181,7 +184,7 @@ async def main():
         )
     )
 
-    logger.custom_info(f'Starting Temporal Client at {host}...', metadata)
+    logger.custom_info(f'Starting Temporal Client at {host}...', metadata_runtime)
 
     temporal_client = await client.Client.connect(
         target_host=host,
@@ -189,7 +192,7 @@ async def main():
         runtime=new_runtime,
     )
 
-    logger.custom_info('Starting Workers...', metadata)
+    logger.custom_info(f'Starting Workers (runtime={runtime})...', metadata_runtime)
 
     workers = [
         prepare_worker(
@@ -246,6 +249,8 @@ async def main():
                 activities.format_transformed_data,
                 activities.format_prediction,
                 activities.format_default_prediction,
+                # Prediction metadata
+                activities.write_prediction_metadata,
                 # OPC
                 activities.write_opc_data,
                 # Postgres / MinIO offload
@@ -267,22 +272,29 @@ async def main():
     for w in workers:
         handlers.append(w.run())
 
-    logger.custom_info('Workers started successfully', metadata)
+    logger.custom_info('Workers started successfully', metadata_runtime)
 
     exit_code = 0
     try:
         await asyncio.gather(*handlers)
-    except BaseException as e:  # NOSONAR
-        logger.custom_error(f'An unhandled exception occurred: {e}', metadata)
+    except Exception as e:
+        logger.custom_error(f'An unhandled exception occurred: {e}', metadata_runtime)
         exit_code = 1
     finally:
+        # No `except (SystemExit, KeyboardInterrupt)` on purpose: `finally` runs on every path,
+        # caught or not, so cleanup still happens for a deliberate shutdown (Ctrl+C, a handler's
+        # own sys.exit) without us having to catch and re-raise it ourselves — which would only
+        # let us relabel its exit code before `sys.exit(exit_code)` below discards it anyway.
+        # Letting it propagate on its own keeps the original code (e.g. a handler's sys.exit(N))
+        # intact instead of collapsing it to 0.
         notification_handler.shutdown()
         activities.shutdown()
         metrics.APP_UP.labels(pod_id=POD_ID).set(0)  # Mark app as DOWN
-        sys.exit(exit_code)
+
+    sys.exit(exit_code)
 
 
-def start_prometheus_server():
+def start_prometheus_server(logger, metadata: dict):
     """
     Starts the Prometheus metrics server for monitoring and observability.
 
@@ -294,7 +306,7 @@ def start_prometheus_server():
 
     Environment Variables:
         HTTP_METRICS_PORT: Port for the metrics server (default: 9090)
-        POD_ID: Pod identifier for metrics labeling
+        HOSTNAME: Pod identifier for metrics labeling
 
     Raises:
         SystemExit: If the metrics server fails to start
@@ -302,10 +314,10 @@ def start_prometheus_server():
     try:
         port = int(os.getenv('HTTP_METRICS_PORT', 9090))
         start_http_server(port)
-        print(f'Prometheus server started on port {port}.')
+        logger.custom_info(f'Prometheus server started on port {port}.', metadata)
         metrics.APP_UP.labels(pod_id=POD_ID).set(1)  # Mark app as UP
     except Exception as e:
-        print(f'Failed to start Prometheus server: {e}')
+        logger.custom_critical(f'Failed to start Prometheus server: {e}', metadata)
         os._exit(1)
 
 

laborious/workflows/drift.py

@@ -37,6 +37,7 @@ class Drift:
         model_config = input_data['model_config']
         target_name = model_config['target']
 
+        # Built from internal Temporal workflow config, not user input (bandit B608).
         gathering_query = f"""
             SELECT *
             FROM "{input_data['schema']}"."{input_data['source_table_name']}"
@@ -44,7 +45,7 @@ class Drift:
                 model_id = '{input_data['model_id']}' AND
                 timestamp > NOW() - INTERVAL '{input_data['interval']} minutes'
             ORDER BY timestamp ASC
-        """  # nosec B608 - values come from internal Temporal workflow config, not user input
+        """  # nosec
 
         target_data_handler = workflow.start_activity_method(
             Activities.load_custom_query,

laborious/workflows/minimal_retrain.py

@@ -5,6 +5,7 @@ with workflow.unsafe.imports_passed_through():
     from typing import Any
 
     from sientia_do.temporal.policies import retry_policy
+    from temporalio.exceptions import ApplicationError
 
     from laborious.activities.activities import Activities
     from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
@@ -87,7 +88,11 @@ class MinimalRetrain:
 
         storage_payload = MinioDataFramePayload.from_dict(storage_result)
         if not storage_payload.has_data():
-            raise ValueError('No data returned from query')
+            raise ApplicationError(
+                'No data returned from query',
+                type='NoDataReturnedError',
+                non_retryable=True,
+            )
 
         experiment_response = await workflow.execute_activity_method(
             Activities.retrain_model,

laborious/workflows/simple_metrics.py

@@ -34,6 +34,7 @@ class SimpleMetrics:
         model_type = model_config.get('model_type')
         thresholds = model_config.get('simple_metrics_thresholds')
 
+        # Built from internal Temporal workflow config, not user input (bandit B608).
         query = f"""
             select p."timestamp", p.prediction, ld.value as "target"
             from "{input_data['schema']}"."{input_data['predictions_table_name']}" p
@@ -47,7 +48,7 @@ class SimpleMetrics:
                 p."timestamp" >= NOW() - INTERVAL '{interval_minutes} minutes'
             order by
                 p."timestamp" desc;
-        """  # nosec B608 - values come from internal Temporal workflow config, not user input
+        """  # nosec
 
         target_data = await workflow.execute_activity_method(
             Activities.load_custom_query,

laborious/workflows/sub_workflows/format_and_export_prediction.py

@@ -75,6 +75,11 @@ class FormatAndExportPrediction:
                         Required when path_flag is None
                     - comment (str): Operational comment or error description
                         Required when path_flag is not None
+                    - metadata_id (str | None): Id of the `prediction_metadata` Mongo document to
+                        reference for this row, already written by the caller; `None` when there's
+                        nothing to reference. Only meaningful when path_flag is not None — the
+                        normal (path_flag is None) path never has metadata to reference, so this
+                        key is irrelevant there.
 
         Returns:
             None: The workflow completes successfully when all export operations finish
@@ -154,6 +159,7 @@ class FormatAndExportPrediction:
                     'model_id': input_data['model_id'],
                     'prediction_confidence': prediction_confidence,
                     'comment': input_data['comment'],
+                    'metadata_id': input_data.get('metadata_id'),
                 },
                 retry_policy=retry_policy,
                 start_to_close_timeout=timedelta(seconds=60),

laborious/workflows/sub_workflows/prediction_process.py

@@ -132,7 +132,9 @@ class PredictionProcess:
             'path_priority': input_data['path_priority'],
         }
 
-        path_flag, confidence, comment = await workflow.execute_local_activity_method(
+        # `document` describes which filter(s) produced `path_flag` (`None` when nothing was
+        # filtered); `path_flag_handler` is the one that decides whether and how to persist it.
+        path_flag, confidence, comment, document = await workflow.execute_local_activity_method(
             Activities.input_gate,
             gate_input,
             retry_policy=retry_policy,
@@ -141,7 +143,7 @@ class PredictionProcess:
 
         # Handle path decision based on filter results
         if await self.path_flag_handler(
-            data, path_flag, input_data, confidence, last_timestamp, comment
+            data, path_flag, input_data, confidence, last_timestamp, comment, document
         ):
             return
 
@@ -247,6 +249,51 @@ class PredictionProcess:
             },
         )
 
+    async def _write_prediction_metadata(
+        self, metadata: dict[str, Any], document: dict[str, Any] | None
+    ) -> str | None:
+        """
+        Write a gate's metadata document via the centralized write activity.
+
+        Shared by the CONTINUE and REPEAT branches of `path_flag_handler` — both persist a new
+        `predictions` row (`REPEAT` reuses the previous row's values but still inserts a new
+        one), so both need a `metadata_id` to reference it.
+
+        Args:
+            metadata: Workflow metadata for contextual logging.
+            document: Metadata produced by the gate that resolved this `path_flag`; `None` when
+                nothing was filtered, or when the gate that produced this `path_flag` doesn't
+                build one yet.
+
+        Return:
+            The generated id, or `None` when there's nothing to write, including when the
+            activity call itself fails (see `PredictionMetadata.write_prediction_metadata`).
+        """
+        if not document:
+            # Skip the remote activity call entirely rather than dispatching it just to have
+            # `write_prediction_metadata` decide there's nothing to write — `mlflow_response_gate`
+            # and `mlflow_content_gate` don't build a `document` yet, so this is the common case
+            # today, not a rare edge case (review finding, 2026-10-01).
+            return None
+
+        try:
+            return await workflow.execute_activity_method(
+                Activities.write_prediction_metadata,
+                {**metadata, 'documents': [document]},
+                retry_policy=retry_policy,
+                start_to_close_timeout=timedelta(minutes=1),
+            )
+        except Exception as e:
+            # Business rule 1: writing metadata must never block or fail the prediction write.
+            # `write_prediction_metadata` already swallows Mongo-specific failures internally and
+            # returns `None`, but that protection doesn't cover the activity call itself failing
+            # to run at all (its `retry_policy` has `maximum_attempts=1` — no retry — so a
+            # timeout, a momentarily unavailable worker, or any other dispatch failure raises
+            # straight into the workflow). Catching it here, instead of just at the Mongo client
+            # level, is what actually makes the rule hold (review finding, 2026-10-01).
+            workflow.logger.warning(f'Failed to write prediction metadata, continuing anyway: {e}')
+            return None
+
     async def path_flag_handler(
         self,
         data: dict[str, Any],
@@ -255,6 +302,7 @@ class PredictionProcess:
         confidence: int,
         last_timestamp: str,
         comment: str,
+        document: dict[str, Any] | None = None,
     ) -> bool:
         """
         Handle path decisions based on filter results and confidence levels.
@@ -280,6 +328,13 @@ class PredictionProcess:
             confidence: Confidence level from filter validation
             last_timestamp: Last processed timestamp
             comment: Additional information about the filter result
+            document: Metadata describing the filter(s) that produced `path_flag`, as returned by
+                the gate (`None` when nothing was filtered, or when the gate that produced this
+                `path_flag` doesn't build one yet). Written via `write_prediction_metadata` for
+                `CONTINUE` and `REPEAT` alike — both persist a `predictions` row (`REPEAT` reuses
+                the previous row's values but still inserts a new one), so both can reference the
+                resulting `metadata_id`; `STOP` never reaches a write at all, so its `document` is
+                simply never used here.
 
         Returns:
             bool: True if processing should stop, False to continue
@@ -304,7 +359,10 @@ class PredictionProcess:
             # Stop processing and exit workflow
             return True
         elif path_flag == 'REPEAT':
-            # Repeat last prediction if available
+            metadata_id = await self._write_prediction_metadata(metadata, document)
+            # Repeat last prediction, copying the previous row's values forward with a new
+            # timestamp. `repeat_last_prediction` now lives in `Storage` (laborious), not the
+            # shared library — its INSERT...SELECT references `metadata_id` directly.
             await workflow.execute_activity_method(
                 Activities.repeat_last_prediction,
                 {
@@ -313,12 +371,14 @@ class PredictionProcess:
                     'table_name': table_name,
                     'model': model_id,
                     'last_timestamp': last_timestamp,
+                    'metadata_id': metadata_id,
                 },
                 retry_policy=retry_policy,
                 start_to_close_timeout=timedelta(minutes=1),
             )
             return True
         elif path_flag == 'CONTINUE':
+            metadata_id = await self._write_prediction_metadata(metadata, document)
             # call write workflow
             await workflow.execute_child_workflow(
                 'subworkflow.format_and_export_prediction',
@@ -335,6 +395,7 @@ class PredictionProcess:
                     'table_name': table_name,
                     'transform_table_name': transform_table_name,
                     'comment': comment,
+                    'metadata_id': metadata_id,
                     'opc_output_config': input_data['opc_output_config'],
                     'pi_web_api_output_config': input_data['pi_web_api_output_config'],
                     'prediction_store_policy': input_data['prediction_store_policy'],