Merge pull request #61 from Aignosi/rc/SIENTIAPDE-2167

Rc/sientiapde 2167 - Prediction Metadata — Passar metadata_id até o Postgres

e2e/conftest.py

@@ -48,8 +48,22 @@ def postgres_container():
 
 @pytest_asyncio.fixture(scope='session')
 def minio_container():
-    """MinIO testcontainer used by E2E offload and payload retrieval paths."""
+    """
-    minio = MinioContainer()
+    MinIO testcontainer used by E2E offload and payload retrieval paths.
+
+    `MinioContainer`'s default image (`minio/minio:RELEASE.2022-12-02T19-19-22Z`) no longer
+    pulls — Docker Hub removed the `minio/minio` repository entirely after MinIO's licensing
+    change. Production (`sientia-helm-chart`) already moved to Bitnami's post-Broadcom-migration
+    "legacy" namespace; mirroring that image here. Two more adjustments on top of just the image:
+    the image runs as non-root by default and can't write to the container's anonymous volume
+    without `user='root'`; and Bitnami deprecated `MINIO_ACCESS_KEY`/`MINIO_SECRET_KEY` (what
+    `MinioContainer` sets) in favor of `MINIO_ROOT_USER`/`MINIO_ROOT_PASSWORD` — reusing the same
+    values so `get_client()` still works unmodified.
+    """
+    minio = MinioContainer(image='docker.io/bitnamilegacy/minio:2025.7.23-debian-12-r3')
+    minio.with_env('MINIO_ROOT_USER', minio.access_key)
+    minio.with_env('MINIO_ROOT_PASSWORD', minio.secret_key)
+    minio.with_kwargs(user='root')
     minio.start()
     yield minio
     minio.stop()
@@ -368,6 +382,7 @@ def _worker_activity_list(test_activities: Activities):
         test_activities.mlflow_content_gate,
         test_activities.request_predict,
         test_activities.repeat_last_prediction,
+        test_activities.write_prediction_metadata,
         test_activities.format_prediction,
         test_activities.format_transformed_data,
         test_activities.format_default_prediction,
@@ -624,5 +639,3 @@ async def temporal_worker_real_opc(temporal_test_env, test_activities_real_opc):
             activity_executor=activity_executor,
         ) as worker:
             yield worker
-
-

e2e/test_drift.py

@@ -142,7 +142,7 @@ def _recent_minute_timestamps(count: int, offset_minutes: int = 6) -> list[str]:
 def _configure_reference_csv(mlflow_repository_stub, reference_rows: pd.DataFrame) -> None:
     """
     Wire ``mlflow_repository_stub`` so ``get_reference_data`` returns
-    ``reference_rows`` by writing them to ``dst_path/retrain_input.csv``.
+    ``reference_rows`` by writing them to ``dst_path/evaluation_data.csv``.
 
     Args:
      - mlflow_repository_stub: External MLflow repository fixture.
@@ -158,7 +158,8 @@ def _configure_reference_csv(mlflow_repository_stub, reference_rows: pd.DataFram
         run_id='fake-reference-run'
     )
     file_info = MagicMock()
-    file_info.path = 'retrain_input.csv'
+    # Matches `MLFlow._REFERENCE_ARTIFACT_CANDIDATES` (SIENTIAPDE-1646).
+    file_info.path = 'evaluation_data.csv'
     mlflow_repository_stub._client.list_artifacts.return_value = [file_info]
     mlflow_repository_stub.download_artifacts.side_effect = _download
 

e2e/test_predictions_batch_prediction_process.py

@@ -8,6 +8,7 @@ from unittest.mock import MagicMock, patch
 import numpy as np
 import pandas as pd
 import pytest
+from pymongo import MongoClient
 from sqlalchemy import text
 from temporalio.testing import WorkflowEnvironment
 from temporalio.worker import Worker
@@ -86,6 +87,76 @@ async def test_scenario_2_1_1_input_gate_triggers_continue(
     mlflow_repository_stub.stub_wrapper.predict.assert_not_called()
 
 
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_e2e_metadata_id_reaches_postgres_when_gate_triggers(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+    mongo_container,
+    mlflow_repository_stub,
+):
+    """
+    input_gate CONTINUE: the gate's document is written to Mongo, and the resulting
+    metadata_id lands on the predictions row — independently verified on both sides.
+    """
+    client = temporal_test_env.client
+    model_id = 261
+    # sensor_1 NULL, sensor_2 present — matches `input_filters.SPECIFIC_VARIABLES_NULL_VALUES`
+    # (`variables: ["sensor_1"]`, POLICY CONTINUE) in prediction_process_base.json.
+    insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
+    input_data = get_base_input_data(model_id)
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-metadata-id-continue')
+    )
+    assert_continue(postgres_engine, model_id)
+
+    with postgres_engine.connect() as conn:
+        metadata_id = conn.execute(
+            text(f'SELECT metadata_id FROM sientia_data.predictions WHERE model_id = {model_id}')
+        ).scalar()
+    assert metadata_id is not None, 'Expected a metadata_id on the predictions row'
+
+    # Independent readback: a fresh client against the same testcontainer, not the
+    # `mongo_repository`/`PredictionMetadata` instances the workflow itself wrote through.
+    mongo_port = mongo_container.get_exposed_port(27017)
+    verify_client = MongoClient(f'mongodb://localhost:{mongo_port}')
+    try:
+        document = verify_client['test_db']['prediction_metadata'].find_one({'_id': metadata_id})
+    finally:
+        verify_client.close()
+    assert document is not None, f'Expected a prediction_metadata document for id {metadata_id}'
+    assert document['filters'] == [
+        {'name': 'SPECIFIC_VARIABLES_NULL_VALUES', 'policy': 'CONTINUE', 'counter': 1}
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.integration
+async def test_e2e_metadata_id_is_null_when_nothing_triggers(
+    temporal_test_env: WorkflowEnvironment,
+    temporal_worker: Worker,
+    test_activities: Activities,
+    postgres_engine,
+):
+    """Clean prediction, no gate triggers: predictions.metadata_id stays NULL."""
+    client = temporal_test_env.client
+    model_id = 262
+    insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
+    input_data = get_base_input_data(model_id)
+    await start_and_await_workflow(
+        client, PredictionsBatch.run, input_data, make_workflow_id('test-metadata-id-none')
+    )
+    assert_prediction(postgres_engine, model_id, prediction=0.5, prediction_confidence=0)
+
+    with postgres_engine.connect() as conn:
+        metadata_id = conn.execute(
+            text(f'SELECT metadata_id FROM sientia_data.predictions WHERE model_id = {model_id}')
+        ).scalar()
+    assert metadata_id is None, f'Expected metadata_id to be NULL, got {metadata_id!r}'
+
+
 @pytest.mark.asyncio
 @pytest.mark.integration
 async def test_scenario_2_1_2_input_gate_triggers_stop(
@@ -204,7 +275,7 @@ async def test_scenario_2_2_1_transform_gate_triggers_continue(
         postgres_engine=postgres_engine,
         model_id=model_id,
         prediction_confidence=Decimal(10),
-        comments='Unknown MLFlow API error',
+        comments='Bad data model',
     )
 
 
@@ -332,7 +403,7 @@ async def test_scenario_2_3_1_predict_gate_triggers_continue(
         postgres_engine=postgres_engine,
         model_id=model_id,
         prediction_confidence=Decimal(10),
-        comments='Unknown MLFlow API error',
+        comments='Bad predict model',
     )
 
 
@@ -450,7 +521,7 @@ async def test_scenario_2_4_1_priority_conflict_resolution(
         postgres_engine=postgres_engine,
         model_id=model_id,
         prediction_confidence=Decimal(10),
-        comments='Unknown MLFlow API error',
+        comments='Bad data model',
     )
 
 

laborious/activities/gates.py

@@ -90,12 +90,34 @@ mlflow_response_path_confidence: Mapping[str, int] = {
     'REPEAT': -1,
 }
 
+
+def _count_nan_values(data: DataFrame, _config: dict[str, Any]) -> int:
+    """
+    Count rows for the NAN_VALUES filter.
+
+    `NAN_VALUES` only triggers when every value in the DataFrame is NaN, so every row in it
+    matched the condition — same convention as `_count_specific_variables_null_values`:
+    counting rows that satisfied the filter's own trigger condition.
+
+    Return:
+        int: Number of rows in the evaluated DataFrame.
+    """
+    return len(data)
+
+
 # MLFlow content filter function mappings
 mlflow_content_filter_functions: dict[str, ContentFilterFunc] = {
     'NAN_VALUES': nan_values_filter,
     'EMPTY_DATA': filter_empty_data,
 }
 
+# `EMPTY_DATA` reuses `_count_empty_data` — same function (`filter_empty_data`) and same
+# trigger condition (absence of rows) as `input_gate`'s own EMPTY_DATA.
+mlflow_content_filter_counters: dict[str, InputCounterFunc] = {
+    'NAN_VALUES': _count_nan_values,
+    'EMPTY_DATA': _count_empty_data,
+}
+
 mlflow_content_path_confidence: Mapping[str, int] = {
     'STOP': -1,
     'CONTINUE': 18,
@@ -457,7 +479,9 @@ class Gates(SientiaMonitoring):
         return None, 0, '', None
 
     @activity.defn(name='mlflow_content_gate')
-    def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
+    def mlflow_content_gate(
+        self, input_data: dict[str, Any]
+    ) -> tuple[str | None, int, str, dict[str, Any] | None]:
         """
         Validate MLFlow prediction content quality and integrity.
 
@@ -471,6 +495,9 @@ class Gates(SientiaMonitoring):
         3. Determines path decisions based on content validation results
         4. Provides confidence scores and detailed validation comments
         5. Handles content validation failures and quality issues
+        6. Builds a metadata document describing which filter(s) triggered, with their policy
+           and a counter, whenever one does — this activity only computes it, it never writes it
+           (see `PredictionMetadata.write_prediction_metadata` for the actual write)
 
         Args:
             input_data: Configuration and data for content validation
@@ -482,10 +509,14 @@ class Gates(SientiaMonitoring):
                     - path_priority (list[str]): Priority order for path decisions
 
         Returns:
-            tuple: (path_flag, confidence, comment)
+            tuple: (path_flag, confidence, comment, document)
                 - path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
                 - confidence (int): Confidence score for the decision
                 - comment (str): Detailed explanation of the decision
+                - document (dict | None): Metadata describing the filter(s) that produced
+                    `path_flag`, shaped `{'filters': [{'name', 'policy', 'counter'}, ...]}`;
+                    `None` when nothing was filtered. Not written anywhere by this activity —
+                    the caller decides whether and how to persist it.
 
         Raises:
             Exception: If content validation fails or configuration is invalid
@@ -501,7 +532,8 @@ class Gates(SientiaMonitoring):
         gate_type = input_data['type']
         path_priority = input_data['path_priority']
 
-        filter_output = []
+        # (filter_name, policy, filter_config) per filter that triggered, in configured order
+        triggered_filters: list[tuple[str, str, dict[str, Any]]] = []
 
         self._debug_dataframe('Input data:', data, metadata)
         self.debug(f'Filters: \n {filters}', metadata)
@@ -512,7 +544,7 @@ class Gates(SientiaMonitoring):
             policy, filter_config = self._read_filter_entry(config)
             try:
                 if mlflow_content_filter_functions[fil](data, filter_config):
-                    filter_output.append(policy)
+                    triggered_filters.append((fil, policy, filter_config))
                     self.send_notification(
                         metadata=metadata,
                         notification_id=f'{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}',
@@ -532,20 +564,46 @@ class Gates(SientiaMonitoring):
                     attachment_content=trace,
                 )
 
+        triggered_policies = [policy for _, policy, _ in triggered_filters]
+
         for path_flag in path_priority:
-            if path_flag in filter_output:
+            if path_flag in triggered_policies:
                 self.info(f'Mlflow content gate result: {path_flag}', metadata)
+
+                matched = [
+                    (fil, filter_config)
+                    for fil, policy, filter_config in triggered_filters
+                    if policy == path_flag
+                ]
+                document = {
+                    'filters': [
+                        {
+                            'name': fil,
+                            'policy': path_flag,
+                            # `.get(..., lambda *_: 0)`, not `[fil]`: same defensive fallback as
+                            # `input_gate` (review finding, 2026-09-30) — a filter added to
+                            # `mlflow_content_filter_functions` without a matching counter here
+                            # degrades to counter=0 instead of a raw KeyError.
+                            'counter': mlflow_content_filter_counters.get(fil, lambda *_: 0)(
+                                data, filter_config
+                            ),
+                        }
+                        for fil, filter_config in matched
+                    ],
+                }
+
                 return (
                     path_flag,
                     mlflow_content_path_confidence[path_flag],
                     'Transformed data not passed the content filter',
+                    document,
                 )
 
         self.info('Nothing was filtered by the mlflow content gate', metadata)
 
         del data
 
-        return None, 0, ''
+        return None, 0, '', None
 
     def get_prediction_store_policy(
         self, prediction_store_policy: str, metadata: dict[str, Any]

laborious/activities/mlflow.py

@@ -664,7 +664,7 @@ class MLFlow(SientiaMonitoring):
         """
         Pick the first available reference CSV artifact path from the MLflow run.
 
-        Candidates are checked in priority order: ``retrain_input.csv``, then ``train_data.csv``.
+        Candidates are checked in priority order: ``evaluation_data.csv``, then ``test_data.csv``.
         A path matches when it equals the candidate or ends with ``/<candidate>`` for nested layouts.
 
         Args:
@@ -703,9 +703,9 @@ class MLFlow(SientiaMonitoring):
         """
         Download reference training CSV from the MLflow run linked to the production alias.
 
-        Resolves ``retrain_input.csv`` or ``train_data.csv`` via artifact listing before download.
+        Resolves ``evaluation_data.csv`` or ``test_data.csv`` via artifact listing before download.
-        ``retrain_input.csv`` is preferred when both exist (most recent retrain snapshot). Used by
+        ``evaluation_data.csv`` is preferred when both exist. Used by drift workflows to compare
-        drift workflows to compare live data against the reference distribution logged with the model.
+        live data against the reference distribution logged with the model.
         Timestamps are normalized to ``DATETIME_FORMAT`` string columns before returning records.
 
         Args:

laborious/workflows/sub_workflows/prediction_process.py

@@ -175,7 +175,7 @@ class PredictionProcess:
         ):
             return
 
-        path_flag, confidence, comment = await workflow.execute_local_activity_method(
+        path_flag, confidence, comment, document = await workflow.execute_local_activity_method(
             Activities.mlflow_content_gate,
             {
                 **metadata,
@@ -189,7 +189,7 @@ class PredictionProcess:
         )
 
         if await self.path_flag_handler(
-            data, path_flag, input_data, confidence, last_timestamp, comment
+            data, path_flag, input_data, confidence, last_timestamp, comment, document
         ):
             return
 
@@ -262,8 +262,7 @@ class PredictionProcess:
         Args:
             metadata: Workflow metadata for contextual logging.
             document: Metadata produced by the gate that resolved this `path_flag`; `None` when
-                nothing was filtered, or when the gate that produced this `path_flag` doesn't
+                nothing was filtered.
-                build one yet.
 
         Return:
             The generated id, or `None` when there's nothing to write, including when the
@@ -271,9 +270,11 @@ class PredictionProcess:
         """
         if not document:
             # Skip the remote activity call entirely rather than dispatching it just to have
-            # `write_prediction_metadata` decide there's nothing to write — `mlflow_content_gate`
+            # `write_prediction_metadata` decide there's nothing to write (review finding,
-            # doesn't build a `document` yet, so this is the common case today, not a rare edge
+            # 2026-10-01). All 3 gates now build a `document` whenever they return a non-`None`
-            # case (review finding, 2026-10-01).
+            # `path_flag`, so this branch is a defensive fallback today, not the common case it
+            # was when this was first written — kept in case a future caller passes a `path_flag`
+            # without one.
             return None
 
         try:

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "laborious"
-version = "0.9.0-rc3"
+version = "0.9.0-rc4"
 description = "Sientia DataOps Laborious - ML Model Orchestration System"
 readme = "README.md"
 requires-python = ">=3.11"

tests/laborious/activities/test_gates.py

@@ -6,7 +6,13 @@ from sientia_do.notifications.models import NotificationLevel
 from sientia_do.observability.metrics_controller import MetricsController
 from sientia_do.observability.sientia_monitoring import SientiaMonitoring
 
-from laborious.activities.gates import Gates, input_filter_counters, input_filter_functions
+from laborious.activities.gates import (
+    Gates,
+    input_filter_counters,
+    input_filter_functions,
+    mlflow_content_filter_counters,
+    mlflow_content_filter_functions,
+)
 
 
 @fixture(autouse=True)
@@ -469,7 +475,7 @@ def test_mlflow_content_gate_invalid_filter(gates_activity):
     result = gates_activity.mlflow_content_gate(input_data)
 
     # Assert
-    assert result == (None, 0, '')
+    assert result == (None, 0, '', None)
 
 
 @patch('laborious.activities.gates.mlflow_content_filter_functions')
@@ -491,7 +497,7 @@ def test_mlflow_content_gate_filter_exception(mock_mlflow_content_filter_functio
     result = gates_activity.mlflow_content_gate(input_data)
 
     # Assert
-    assert result == (None, 0, '')
+    assert result == (None, 0, '', None)
     gates_activity.debug.assert_called()
     gates_activity.send_notification.assert_called_once_with(
         metadata=metadata['metadata'],
@@ -517,12 +523,12 @@ def test_mlflow_content_gate_no_filters(gates_activity):
     result = gates_activity.mlflow_content_gate(input_data)
 
     # Assert
-    assert result == (None, 0, '')
+    assert result == (None, 0, '', None)
     gates_activity.debug.assert_called()
 
 
 def test_mlflow_content_gate_with_filter(gates_activity):
-    # Arrange
+    # Arrange: 3 rows, all-NaN after the filter's own normalization — triggers NAN_VALUES.
     input_data = {
         **metadata,
         'filters': {'NAN_VALUES': {'POLICY': 'STOP', 'CONFIG': {}}},
@@ -532,10 +538,15 @@ def test_mlflow_content_gate_with_filter(gates_activity):
     }
 
     # Act
-    result = gates_activity.mlflow_content_gate(input_data)
+    path_flag, confidence, comment, document = gates_activity.mlflow_content_gate(input_data)
 
     # Assert
-    assert result == ('STOP', -1, 'Transformed data not passed the content filter')
+    assert (path_flag, confidence, comment) == (
+        'STOP',
+        -1,
+        'Transformed data not passed the content filter',
+    )
+    assert document == {'filters': [{'name': 'NAN_VALUES', 'policy': 'STOP', 'counter': 3}]}
     gates_activity.debug.assert_called()
     gates_activity.send_notification.assert_called()
 
@@ -554,7 +565,7 @@ def test_mlflow_content_gate_with_filter_not_caught(gates_activity):
     result = gates_activity.mlflow_content_gate(input_data)
 
     # Assert
-    assert result == (None, 0, '')
+    assert result == (None, 0, '', None)
     gates_activity.debug.assert_called()
 
 
@@ -569,10 +580,18 @@ def test_mlflow_content_gate_filter_returns_false(gates_activity):
 
     result = gates_activity.mlflow_content_gate(input_data)
 
-    assert result == (None, 0, '')
+    assert result == (None, 0, '', None)
     gates_activity.debug.assert_called()
 
 
+def test_mlflow_content_filter_counters_covers_every_mlflow_content_filter_function():
+    # Same parity guard as `test_input_filter_counters_covers_every_input_filter_function`
+    # (review finding, 2026-09-30): a filter added to `mlflow_content_filter_functions` without
+    # a matching counter here degrades to counter=0 instead of crashing, but this test still
+    # catches the gap before it ships.
+    assert set(mlflow_content_filter_counters) == set(mlflow_content_filter_functions)
+
+
 def test_get_prediction_store_policy_invalid_policy(gates_activity):
     # Arrange
     prediction_store_policy = 'INVALID_POLICY'

tests/laborious/workflows/subworkflows/test_prediction_process.py

@@ -61,7 +61,7 @@ async def test_run(workflow_mock, prediction_process):
         # mlflow_response_gate (transform)
         ('continue', 0.95, 'Error', None),
         # mlflow_content_gate (transform)
-        ('continue', 0.95, 'Transformed data not passed the content filter'),
+        ('continue', 0.95, 'Transformed data not passed the content filter', None),
         # mlflow_response_gate (predict)
         ('continue', 0.95, 'Error', None),
     ]
@@ -369,7 +369,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process
         # mlflow_response_gate (transform)
         ('continue', 0.95, 'Error', None),
         # mlflow_content_gate (transform)
-        ('continue', 0.95, 'Transformed data not passed the content filter'),
+        ('continue', 0.95, 'Transformed data not passed the content filter', None),
     ]
 
     # Act
@@ -480,7 +480,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p
         # mlflow_response_gate (transform)
         ('continue', 0.95, 'Error', None),
         # mlflow_content_gate (transform)
-        ('continue', 0.95, 'Transformed data not passed the content filter'),
+        ('continue', 0.95, 'Transformed data not passed the content filter', None),
         ('continue', 0.95, 'Error', None),  # mlflow_response_gate (predict)
     ]
 
@@ -695,10 +695,10 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process):
 async def test_path_flag_handler_repeat_no_document_skips_write_activity_call(
     workflow_mock, prediction_process
 ):
-    # Arrange: no document (e.g. REPEAT triggered by a gate that doesn't build one yet — today,
+    # Arrange: no document must not even dispatch the remote write_prediction_metadata
-    # `mlflow_content_gate`) must not even dispatch the remote write_prediction_metadata
+    # activity — that would be a wasted Temporal round-trip (review finding, 2026-10-01).
-    # activity — that would be a wasted Temporal round-trip on every REPEAT caused by that
+    # Defensive case: all 3 gates now always build a document alongside a non-`None`
-    # gate, not a rare edge case (review finding, 2026-10-01).
+    # `path_flag`, so this guards a caller that doesn't, not today's common case.
     data = {'test': 'data'}
 
     # Act
@@ -1076,7 +1076,7 @@ async def test_run_with_cleanup_prefixes(workflow_mock, prediction_process):
     workflow_mock.execute_local_activity_method.side_effect = [
         ('continue', 0.95, 'ok', None),  # input_gate
         ('continue', 0.95, '', None),  # mlflow_response_gate (transform)
-        ('continue', 0.95, ''),  # mlflow_content_gate (transform)
+        ('continue', 0.95, '', None),  # mlflow_content_gate (transform)
         ('continue', 0.95, '', None),  # mlflow_response_gate (predict)
     ]