11 Commits

Author SHA1 Message Date
github-actions[bot]
99392034e5 chore: bump version to 0.9.0-rc3 2026-10-08 13:51:09 +00:00
Carla Fernandes
f988a3ae20 Merge pull request #56 from Aignosi/rc/SIENTIAPDE-2165
Rc/sientiapde 2165
2026-10-08 15:50:39 +02:00
Carla Pinto
75b56251ef rc/SIENTIAPDE-2165 - fix: read MLflow error message/traceback from status['content'], not the top level 2026-10-08 14:10:40 +02:00
Carla Pinto
2649476571 rc/SIENTIAPDE-2165- refactor: inline counter=1 for API_ERROR, drop unnecessary dispatch (review finding) 2026-10-08 13:58:55 +02:00
Carla Pinto
bba4daaddd Merge branch 'release/SIENTIAPDE-1646' into rc/SIENTIAPDE-2165 2026-10-05 22:16:50 +02:00
Carla Pinto
0742078fad rc/SIENTIAPDE-2165- feat: rich metadata (filter/policy/counter) for mlflow_response_gate 2026-10-05 22:14:18 +02:00
github-actions[bot]
79605db260 chore: bump version to 0.9.0-rc2 2026-10-02 17:23:19 +00:00
4ae76cfd83 Merge pull request #55 from Aignosi/rc/SIENTIAPDE-2173
Rc/sientiapde 2173 - Mover repeat_last_prediction do sientia-dataops-library para o laborious, com suporte a metadata_id
2026-10-02 14:22:56 -03:00
Carla Pinto
14b27e4c30 rc/SIENTIAPDE-2173- fix: add missing metadata_id column to e2e/db_schema.sql 2026-10-02 18:57:56 +02:00
Carla Pinto
09349b6274 Merge branch 'release/SIENTIAPDE-1646' into rc/SIENTIAPDE-2173 2026-10-02 18:43:27 +02:00
Carla Pinto
2ace55c241 rc/SIENTIAPDE-2173- refactor: move repeat_last_prediction from sientia-dataops-library into Storage, add metadata_id 2026-10-02 18:41:18 +02:00
8 changed files with 297 additions and 46 deletions

View File

@@ -40,6 +40,7 @@ CREATE TABLE IF NOT EXISTS sientia_data.predictions (
"timestamp" timestamptz NOT NULL, "timestamp" timestamptz NOT NULL,
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL, created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
"comments" text NULL, "comments" text NULL,
metadata_id varchar(24) NULL,
CONSTRAINT unique_model_id_timestamp CONSTRAINT unique_model_id_timestamp
UNIQUE (model_id, "timestamp") UNIQUE (model_id, "timestamp")
); );

View File

@@ -78,6 +78,7 @@ input_path_confidence: Mapping[str, int] = {
'REPEAT': -1, 'REPEAT': -1,
} }
# MLFlow response filter function mappings # MLFlow response filter function mappings
mlflow_response_filter_functions: dict[str, ResponseFilterFunc] = { mlflow_response_filter_functions: dict[str, ResponseFilterFunc] = {
'API_ERROR': api_error_filter, 'API_ERROR': api_error_filter,
@@ -320,7 +321,9 @@ class Gates(SientiaMonitoring):
return None, 0, '', None return None, 0, '', None
@activity.defn(name='mlflow_response_gate') @activity.defn(name='mlflow_response_gate')
def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: def mlflow_response_gate(
self, input_data: dict[str, Any]
) -> tuple[str | None, int, str, dict[str, Any] | None]:
""" """
Validate MLFlow API response quality and integrity. Validate MLFlow API response quality and integrity.
@@ -334,6 +337,9 @@ class Gates(SientiaMonitoring):
3. Determines path decisions based on response validation results 3. Determines path decisions based on response validation results
4. Provides confidence scores and detailed validation comments 4. Provides confidence scores and detailed validation comments
5. Handles API errors and response validation failures 5. Handles API errors and response validation failures
6. Builds a metadata document describing which filter(s) triggered, with their policy
and a counter, whenever one does — this activity only computes it, it never writes it
(see `PredictionMetadata.write_prediction_metadata` for the actual write)
Args: Args:
input_data: Configuration and data for response validation input_data: Configuration and data for response validation
@@ -345,10 +351,14 @@ class Gates(SientiaMonitoring):
- path_priority (list[str]): Priority order for path decisions - path_priority (list[str]): Priority order for path decisions
Returns: Returns:
tuple: (path_flag, confidence, comment) tuple: (path_flag, confidence, comment, document)
- path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None) - path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
- confidence (int): Confidence score for the decision - confidence (int): Confidence score for the decision
- comment (str): Detailed explanation of the decision - comment (str): Detailed explanation of the decision
- document (dict | None): Metadata describing the filter(s) that produced
`path_flag`, shaped `{'filters': [{'name', 'policy', 'counter'}, ...]}`;
`None` when nothing was filtered. Not written anywhere by this activity —
the caller decides whether and how to persist it.
Raises: Raises:
Exception: If response validation fails or configuration is invalid Exception: If response validation fails or configuration is invalid
@@ -369,10 +379,11 @@ class Gates(SientiaMonitoring):
gate_type = input_data['type'] gate_type = input_data['type']
path_priority = input_data['path_priority'] path_priority = input_data['path_priority']
filter_output = []
comments = [] comments = []
# (filter_name, policy, filter_config) per filter that triggered, in configured order
triggered_filters: list[tuple[str, str, dict[str, Any]]] = []
status = payload.status or {} status = payload.status or {}
for fil, config in filters.items(): for fil, config in filters.items():
@@ -381,15 +392,21 @@ class Gates(SientiaMonitoring):
policy, filter_config = self._read_filter_entry(config) policy, filter_config = self._read_filter_entry(config)
try: try:
if mlflow_response_filter_functions[fil](status, filter_config): if mlflow_response_filter_functions[fil](status, filter_config):
filter_output.append(policy) triggered_filters.append((fil, policy, filter_config))
comments.append(status.get('message', 'Unknown MLFlow API error')) # The real error lives one level in, under `content` — `request_transform`/
# `request_predict` nest `message`/`traceback` there on failure (mlflow.py),
# reusing the same `content` key the success path uses for the DataFrame
# (review finding, 2026-10-08: reading `status['message']` directly always
# fell back to the generic default, in every real production occurrence).
error_content = status.get('content') or {}
comments.append(error_content.get('message', 'Unknown MLFlow API error'))
self.send_notification( self.send_notification(
metadata=metadata, metadata=metadata,
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}', notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
message=status.get('message', 'Unknown MLFlow API error'), message=error_content.get('message', 'Unknown MLFlow API error'),
block='mlflow_gate', block='mlflow_gate',
level=NotificationLevel.ERROR, level=NotificationLevel.ERROR,
attachment_content=status.get('traceback'), attachment_content=error_content.get('traceback'),
) )
except Exception as e: except Exception as e:
trace = traceback.format_exc() trace = traceback.format_exc()
@@ -402,16 +419,42 @@ class Gates(SientiaMonitoring):
attachment_content=trace, attachment_content=trace,
) )
triggered_policies = [policy for _, policy, _ in triggered_filters]
for path_flag in path_priority: for path_flag in path_priority:
if path_flag in filter_output: if path_flag in triggered_policies:
self.info(f'Mlflow response gate result: {path_flag}', metadata) self.info(f'Mlflow response gate result: {path_flag}', metadata)
return path_flag, mlflow_response_path_confidence[path_flag], ', '.join(comments)
matched = [fil for fil, policy, _ in triggered_filters if policy == path_flag]
document = {
'filters': [
{
'name': fil,
'policy': path_flag,
# No per-row/per-item unit to count here — `api_error_filter`
# evaluates a single API response as a whole, not tabular data —
# so, unlike `input_gate`'s per-filter counters, this is a fixed
# value: 1 (one API error event). Inlined rather than a counter
# function/dict, since there's only one response filter today and
# this isn't an override of anything (review finding).
'counter': 1,
}
for fil in matched
],
}
return (
path_flag,
mlflow_response_path_confidence[path_flag],
', '.join(comments),
document,
)
self.info('Nothing was filtered by the mlflow response gate', metadata) self.info('Nothing was filtered by the mlflow response gate', metadata)
del data del data
return None, 0, '' return None, 0, '', None
@activity.defn(name='mlflow_content_gate') @activity.defn(name='mlflow_content_gate')
def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:

View File

@@ -2,6 +2,7 @@ from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through(): with workflow.unsafe.imports_passed_through():
# Extend the Temporal Postgres activities for convenient query -> MinIO export # Extend the Temporal Postgres activities for convenient query -> MinIO export
import time
import traceback import traceback
from datetime import timedelta from datetime import timedelta
from typing import Any from typing import Any
@@ -9,12 +10,14 @@ with workflow.unsafe.imports_passed_through():
import pandas as pd import pandas as pd
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability import metrics
from sientia_do.observability.logger import Logger from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository_sync import MinioRepository from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_do.temporal.activities.postgres_sync import Postgres from sientia_do.temporal.activities.postgres_sync import Postgres
from sientia_do.temporal.constants import now from sientia_do.temporal.constants import now
from sqlalchemy import text
from laborious.utils.messages import MISSING_MINIO_REPOSITORY from laborious.utils.messages import MISSING_MINIO_REPOSITORY
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
@@ -26,7 +29,9 @@ _LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_thr
class Storage(Postgres, SientiaMonitoring): class Storage(Postgres, SientiaMonitoring):
""" """
Extensions for Postgres activities with a helper to export query results Extensions for Postgres activities with a helper to export query results
directly to MinIO as Parquet and return the object name. directly to MinIO as Parquet and return the object name, and with
``repeat_last_prediction``, a query specific to this repo's REPEAT path
(moved here from the shared library — nothing else consumes it).
""" """
minio_repository: MinioRepository | None = None minio_repository: MinioRepository | None = None
@@ -141,6 +146,110 @@ class Storage(Postgres, SientiaMonitoring):
} }
) )
def _build_repeat_last_prediction_query(self, schema: str, table_name: str):
table_identifier = f'"{schema}".{table_name}'
return text( # noqa: S608
f"""
INSERT INTO {table_identifier} (
model_id,
prediction,
timestamp,
response_time,
prediction_status,
prediction_confidence,
metadata_id,
created_at
)
SELECT
model_id,
prediction,
:last_timestamp,
response_time,
prediction_status,
prediction_confidence,
:metadata_id,
NOW()
FROM {table_identifier}
WHERE model_id = :model
ORDER BY timestamp DESC
LIMIT 1;
""" # nosec B608
)
@activity.defn(name='repeat_last_prediction')
def repeat_last_prediction(self, query_items: dict[str, Any]):
"""
Insert a new `predictions` row that copies the last one for a model forward in time.
REPEAT persists a new row (unlike STOP, which persists nothing), reusing the previous
row's `prediction`/`response_time`/`prediction_status`/`prediction_confidence` values with
a fresh `timestamp`/`created_at`. `metadata_id` references the `prediction_metadata`
document for this cycle, if any (see `PredictionMetadata.write_prediction_metadata`);
`None` is a valid value — the column is nullable.
Args (query_items):
metadata (dict): Workflow metadata for logging and metrics.
schema (str): Postgres schema of the `predictions` table.
table_name (str): `predictions` table name.
model (str): `model_id` to repeat the last prediction for.
last_timestamp: Timestamp to stamp the new row with.
metadata_id (str | None): `prediction_metadata` document id, or `None`.
Raises:
Exception: Re-raised after logging/metrics if the insert fails.
"""
metadata = query_items['metadata']
schema = query_items['schema']
table_name = query_items['table_name']
model = query_items['model']
last_timestamp = query_items['last_timestamp']
metadata_id = query_items.get('metadata_id')
repeat_query = self._build_repeat_last_prediction_query(schema, table_name)
self.info(f'Repeating last prediction for model {model}', metadata)
self.debug(
f'Query parameters: schema={schema}, model={model}, last_timestamp={last_timestamp}',
metadata,
)
core_labels = {
**self.get_core_labels(metadata, 'execute_sql'),
'conflict_mode': 'error',
}
with self.session_factory.begin() as session:
try:
start_time = time.time()
session.execute(
repeat_query,
{
'last_timestamp': last_timestamp,
'model': model,
'metadata_id': metadata_id,
},
)
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id='ERROR_REPEATING_LAST_PREDICTION',
message=f'Error repeating last prediction: {e}',
block='repeat_last_prediction',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
self.error(trace, metadata)
self.emit_metric_sync(
metric_object=metrics.POSTGRES_WRITE_ERROR_COUNT,
tags=core_labels,
)
raise
else:
self.observe_lag_sync(start_time, metrics.POSTGRES_WRITE_LAG, core_labels)
self.emit_metric_sync(
metric_object=metrics.POSTGRES_WRITE_COUNT,
tags=core_labels,
)
@activity.defn(name='cleanup_minio_objects_expired') @activity.defn(name='cleanup_minio_objects_expired')
def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]: def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
""" """

View File

@@ -156,7 +156,7 @@ class PredictionProcess:
) )
# Validate MLFlow transform response # Validate MLFlow transform response
path_flag, confidence, comment = await workflow.execute_local_activity_method( path_flag, confidence, comment, document = await workflow.execute_local_activity_method(
Activities.mlflow_response_gate, Activities.mlflow_response_gate,
{ {
**metadata, **metadata,
@@ -171,7 +171,7 @@ class PredictionProcess:
# Handle path decision based on transform validation # Handle path decision based on transform validation
if await self.path_flag_handler( if await self.path_flag_handler(
data, path_flag, input_data, confidence, last_timestamp, comment data, path_flag, input_data, confidence, last_timestamp, comment, document
): ):
return return
@@ -206,7 +206,7 @@ class PredictionProcess:
) )
# Validate MLFlow prediction response # Validate MLFlow prediction response
path_flag, confidence, comment = await workflow.execute_local_activity_method( path_flag, confidence, comment, document = await workflow.execute_local_activity_method(
Activities.mlflow_response_gate, Activities.mlflow_response_gate,
{ {
**metadata, **metadata,
@@ -221,7 +221,7 @@ class PredictionProcess:
# Handle path decision based on prediction validation # Handle path decision based on prediction validation
if await self.path_flag_handler( if await self.path_flag_handler(
data, path_flag, input_data, confidence, last_timestamp, comment data, path_flag, input_data, confidence, last_timestamp, comment, document
): ):
return return
@@ -271,9 +271,9 @@ class PredictionProcess:
""" """
if not document: if not document:
# Skip the remote activity call entirely rather than dispatching it just to have # Skip the remote activity call entirely rather than dispatching it just to have
# `write_prediction_metadata` decide there's nothing to write — `mlflow_response_gate` # `write_prediction_metadata` decide there's nothing to write — `mlflow_content_gate`
# and `mlflow_content_gate` don't build a `document` yet, so this is the common case # doesn't build a `document` yet, so this is the common case today, not a rare edge
# today, not a rare edge case (review finding, 2026-10-01). # case (review finding, 2026-10-01).
return None return None
try: try:
@@ -360,10 +360,9 @@ class PredictionProcess:
return True return True
elif path_flag == 'REPEAT': elif path_flag == 'REPEAT':
metadata_id = await self._write_prediction_metadata(metadata, document) metadata_id = await self._write_prediction_metadata(metadata, document)
# Repeat last prediction if available. `metadata_id` isn't consumed by # Repeat last prediction, copying the previous row's values forward with a new
# `repeat_last_prediction` yet (its INSERT...SELECT has no `metadata_id` column — # timestamp. `repeat_last_prediction` now lives in `Storage` (laborious), not the
# pending a change in `sientia-dataops-library`); passed through already so no further # shared library — its INSERT...SELECT references `metadata_id` directly.
# change is needed here once that lands.
await workflow.execute_activity_method( await workflow.execute_activity_method(
Activities.repeat_last_prediction, Activities.repeat_last_prediction,
{ {

View File

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project] [project]
name = "laborious" name = "laborious"
version = "0.9.0-rc1" version = "0.9.0-rc3"
description = "Sientia DataOps Laborious - ML Model Orchestration System" description = "Sientia DataOps Laborious - ML Model Orchestration System"
readme = "README.md" readme = "README.md"
requires-python = ">=3.11" requires-python = ">=3.11"

View File

@@ -314,7 +314,7 @@ def test_mlflow_response_gate_invalid_filter(gates_activity):
result = gates_activity.mlflow_response_gate(input_data) result = gates_activity.mlflow_response_gate(input_data)
# Assert # Assert
assert result == (None, 0, '') assert result == (None, 0, '', None)
@patch('laborious.activities.gates.mlflow_response_filter_functions') @patch('laborious.activities.gates.mlflow_response_filter_functions')
@@ -341,7 +341,7 @@ def test_mlflow_response_gate_filter_exception(
result = gates_activity.mlflow_response_gate(input_data) result = gates_activity.mlflow_response_gate(input_data)
# Assert # Assert
assert result == (None, 0, '') assert result == (None, 0, '', None)
gates_activity.send_notification.assert_called_once_with( gates_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'], metadata=metadata['metadata'],
notification_id='MLFLOW_GATE_RESPONSE_FILTER__INVALID_FILTER', notification_id='MLFLOW_GATE_RESPONSE_FILTER__INVALID_FILTER',
@@ -369,7 +369,7 @@ def test_mlflow_response_gate_no_filters(gates_activity):
result = gates_activity.mlflow_response_gate(input_data) result = gates_activity.mlflow_response_gate(input_data)
# Assert # Assert
assert result == (None, 0, '') assert result == (None, 0, '', None)
gates_activity.debug.assert_called() gates_activity.debug.assert_called()
@@ -380,19 +380,30 @@ def test_mlflow_response_gate_with_filter(gates_activity):
'filters': {'API_ERROR': {'POLICY': 'STOP'}}, 'filters': {'API_ERROR': {'POLICY': 'STOP'}},
'data': _minio_payload( 'data': _minio_payload(
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}}, {'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
status={'success': False, 'message': 'API error occurred'}, status={
'success': False,
'content': {'message': 'API error occurred', 'traceback': 'error trace'},
},
), ),
'type': 'test', 'type': 'test',
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
} }
# Act # Act
result = gates_activity.mlflow_response_gate(input_data) path_flag, confidence, comment, document = gates_activity.mlflow_response_gate(input_data)
# Assert # Assert
assert result == ('STOP', -1, 'API error occurred') assert (path_flag, confidence, comment) == ('STOP', -1, 'API error occurred')
assert document == {'filters': [{'name': 'API_ERROR', 'policy': 'STOP', 'counter': 1}]}
gates_activity.debug.assert_called() gates_activity.debug.assert_called()
gates_activity.send_notification.assert_called() gates_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='TEST_GATE_RESPONSE_FILTER__API_ERROR',
message='API error occurred',
block='mlflow_gate',
level=NotificationLevel.ERROR,
attachment_content='error trace',
)
def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity): def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
@@ -402,7 +413,10 @@ def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
'filters': {'API_ERROR': {'Policy': 'STOP'}}, 'filters': {'API_ERROR': {'Policy': 'STOP'}},
'data': _minio_payload( 'data': _minio_payload(
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}}, {'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
status={'success': False, 'message': 'API error occurred'}, status={
'success': False,
'content': {'message': 'API error occurred', 'traceback': 'error trace'},
},
), ),
'type': 'test', 'type': 'test',
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'], 'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
@@ -412,7 +426,12 @@ def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
result = gates_activity.mlflow_response_gate(input_data) result = gates_activity.mlflow_response_gate(input_data)
# Assert # Assert
assert result == ('STOP', -1, 'API error occurred') assert result == (
'STOP',
-1,
'API error occurred',
{'filters': [{'name': 'API_ERROR', 'policy': 'STOP', 'counter': 1}]},
)
def test_mlflow_response_gate_with_filter_not_caught(gates_activity): def test_mlflow_response_gate_with_filter_not_caught(gates_activity):
@@ -432,7 +451,7 @@ def test_mlflow_response_gate_with_filter_not_caught(gates_activity):
result = gates_activity.mlflow_response_gate(input_data) result = gates_activity.mlflow_response_gate(input_data)
# Assert # Assert
assert result == (None, 0, '') assert result == (None, 0, '', None)
gates_activity.debug.assert_called() gates_activity.debug.assert_called()

View File

@@ -257,6 +257,86 @@ def test_export_payload_to_postgres(storage):
assert result == {'success': True} assert result == {'success': True}
def test_build_repeat_last_prediction_query(storage):
query = storage._build_repeat_last_prediction_query('public', 'predictions')
query_str = str(query)
assert 'INSERT INTO "public".predictions' in query_str
assert 'metadata_id' in query_str
assert ':last_timestamp' in query_str
assert ':metadata_id' in query_str
assert ':model' in query_str
@patch('laborious.activities.storage.text')
def test_repeat_last_prediction_success(text_mock, storage):
storage.emit_metric_sync = MagicMock()
storage.observe_lag_sync = MagicMock()
query_items = {
**metadata,
'schema': 'public',
'table_name': 'predictions',
'model': 1,
'last_timestamp': '2022-01-01',
'metadata_id': 'abc123',
}
with patch('sqlalchemy.orm.session.Session.execute') as mock_execute:
storage.repeat_last_prediction(query_items)
text_mock.assert_called_once()
mock_execute.assert_called_once_with(
text_mock.return_value,
{'last_timestamp': '2022-01-01', 'model': 1, 'metadata_id': 'abc123'},
)
storage.observe_lag_sync.assert_called_once()
storage.emit_metric_sync.assert_called_once()
@patch('laborious.activities.storage.text')
def test_repeat_last_prediction_without_metadata_id(text_mock, storage):
storage.emit_metric_sync = MagicMock()
storage.observe_lag_sync = MagicMock()
query_items = {
**metadata,
'schema': 'public',
'table_name': 'predictions',
'model': 1,
'last_timestamp': '2022-01-01',
}
with patch('sqlalchemy.orm.session.Session.execute') as mock_execute:
storage.repeat_last_prediction(query_items)
mock_execute.assert_called_once_with(
text_mock.return_value,
{'last_timestamp': '2022-01-01', 'model': 1, 'metadata_id': None},
)
@patch('laborious.activities.storage.text')
def test_repeat_last_prediction_error(text_mock, storage):
storage.emit_metric_sync = MagicMock()
storage.send_notification = MagicMock()
query_items = {
**metadata,
'schema': 'public',
'table_name': 'predictions',
'model': 1,
'last_timestamp': '2022-01-01',
'metadata_id': 'abc123',
}
with (
patch('sqlalchemy.orm.session.Session.execute', side_effect=ValueError('db')),
raises(ValueError),
):
storage.repeat_last_prediction(query_items)
storage.send_notification.assert_called_once()
storage.emit_metric_sync.assert_called()
def test_cleanup_minio_objects_expired_minio_not_initialized(storage): def test_cleanup_minio_objects_expired_minio_not_initialized(storage):
storage.minio_repository = None storage.minio_repository = None

View File

@@ -59,11 +59,11 @@ async def test_run(workflow_mock, prediction_process):
workflow_mock.execute_local_activity_method.side_effect = [ workflow_mock.execute_local_activity_method.side_effect = [
('continue', 0.95, 'Input data with bad quality', None), # input_gate ('continue', 0.95, 'Input data with bad quality', None), # input_gate
# mlflow_response_gate (transform) # mlflow_response_gate (transform)
('continue', 0.95, 'Error'), ('continue', 0.95, 'Error', None),
# mlflow_content_gate (transform) # mlflow_content_gate (transform)
('continue', 0.95, 'Transformed data not passed the content filter'), ('continue', 0.95, 'Transformed data not passed the content filter'),
# mlflow_response_gate (predict) # mlflow_response_gate (predict)
('continue', 0.95, 'Error'), ('continue', 0.95, 'Error', None),
] ]
# Act # Act
@@ -276,7 +276,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_
] ]
workflow_mock.execute_local_activity_method.side_effect = [ workflow_mock.execute_local_activity_method.side_effect = [
('repeat', 0.95, 'Input data with bad quality', None), # input_gate ('repeat', 0.95, 'Input data with bad quality', None), # input_gate
('continue', 0.95, 'Error'), # mlflow_response_gate (transform) ('continue', 0.95, 'Error', None), # mlflow_response_gate (transform)
] ]
# Act # Act
@@ -367,7 +367,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process
workflow_mock.execute_local_activity_method.side_effect = [ workflow_mock.execute_local_activity_method.side_effect = [
('continue', 0.95, 'Input data with bad quality', None), # input_gate ('continue', 0.95, 'Input data with bad quality', None), # input_gate
# mlflow_response_gate (transform) # mlflow_response_gate (transform)
('continue', 0.95, 'Error'), ('continue', 0.95, 'Error', None),
# mlflow_content_gate (transform) # mlflow_content_gate (transform)
('continue', 0.95, 'Transformed data not passed the content filter'), ('continue', 0.95, 'Transformed data not passed the content filter'),
] ]
@@ -478,10 +478,10 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p
workflow_mock.execute_local_activity_method.side_effect = [ workflow_mock.execute_local_activity_method.side_effect = [
('continue', 0.95, 'Input data with bad quality', None), # input_gate ('continue', 0.95, 'Input data with bad quality', None), # input_gate
# mlflow_response_gate (transform) # mlflow_response_gate (transform)
('continue', 0.95, 'Error'), ('continue', 0.95, 'Error', None),
# mlflow_content_gate (transform) # mlflow_content_gate (transform)
('continue', 0.95, 'Transformed data not passed the content filter'), ('continue', 0.95, 'Transformed data not passed the content filter'),
('continue', 0.95, 'Error'), # mlflow_response_gate (predict) ('continue', 0.95, 'Error', None), # mlflow_response_gate (predict)
] ]
# Act # Act
@@ -696,9 +696,9 @@ async def test_path_flag_handler_repeat_no_document_skips_write_activity_call(
workflow_mock, prediction_process workflow_mock, prediction_process
): ):
# Arrange: no document (e.g. REPEAT triggered by a gate that doesn't build one yet — today, # Arrange: no document (e.g. REPEAT triggered by a gate that doesn't build one yet — today,
# `mlflow_response_gate`/`mlflow_content_gate`) must not even dispatch the remote # `mlflow_content_gate`) must not even dispatch the remote write_prediction_metadata
# write_prediction_metadata activity — that would be a wasted Temporal round-trip on every # activity — that would be a wasted Temporal round-trip on every REPEAT caused by that
# REPEAT caused by those two gates, not a rare edge case (review finding, 2026-10-01). # gate, not a rare edge case (review finding, 2026-10-01).
data = {'test': 'data'} data = {'test': 'data'}
# Act # Act
@@ -1075,9 +1075,9 @@ async def test_run_with_cleanup_prefixes(workflow_mock, prediction_process):
] ]
workflow_mock.execute_local_activity_method.side_effect = [ workflow_mock.execute_local_activity_method.side_effect = [
('continue', 0.95, 'ok', None), # input_gate ('continue', 0.95, 'ok', None), # input_gate
('continue', 0.95, ''), ('continue', 0.95, '', None), # mlflow_response_gate (transform)
('continue', 0.95, ''), ('continue', 0.95, ''), # mlflow_content_gate (transform)
('continue', 0.95, ''), ('continue', 0.95, '', None), # mlflow_response_gate (predict)
] ]
await prediction_process.run(input_data) await prediction_process.run(input_data)