Compare commits
13 Commits
c190267292
...
0.9.0-rc3
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
99392034e5 | ||
|
|
f988a3ae20 | ||
|
|
75b56251ef | ||
|
|
2649476571 | ||
|
|
bba4daaddd | ||
|
|
0742078fad | ||
|
|
79605db260 | ||
| 4ae76cfd83 | |||
|
|
14b27e4c30 | ||
|
|
09349b6274 | ||
|
|
2ace55c241 | ||
|
|
6981d09ac6 | ||
| 860bce1ba3 |
@@ -40,6 +40,7 @@ CREATE TABLE IF NOT EXISTS sientia_data.predictions (
|
||||
"timestamp" timestamptz NOT NULL,
|
||||
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
|
||||
"comments" text NULL,
|
||||
metadata_id varchar(24) NULL,
|
||||
CONSTRAINT unique_model_id_timestamp
|
||||
UNIQUE (model_id, "timestamp")
|
||||
);
|
||||
|
||||
@@ -78,6 +78,7 @@ input_path_confidence: Mapping[str, int] = {
|
||||
'REPEAT': -1,
|
||||
}
|
||||
|
||||
|
||||
# MLFlow response filter function mappings
|
||||
mlflow_response_filter_functions: dict[str, ResponseFilterFunc] = {
|
||||
'API_ERROR': api_error_filter,
|
||||
@@ -320,7 +321,9 @@ class Gates(SientiaMonitoring):
|
||||
return None, 0, '', None
|
||||
|
||||
@activity.defn(name='mlflow_response_gate')
|
||||
def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
def mlflow_response_gate(
|
||||
self, input_data: dict[str, Any]
|
||||
) -> tuple[str | None, int, str, dict[str, Any] | None]:
|
||||
"""
|
||||
Validate MLFlow API response quality and integrity.
|
||||
|
||||
@@ -334,6 +337,9 @@ class Gates(SientiaMonitoring):
|
||||
3. Determines path decisions based on response validation results
|
||||
4. Provides confidence scores and detailed validation comments
|
||||
5. Handles API errors and response validation failures
|
||||
6. Builds a metadata document describing which filter(s) triggered, with their policy
|
||||
and a counter, whenever one does — this activity only computes it, it never writes it
|
||||
(see `PredictionMetadata.write_prediction_metadata` for the actual write)
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for response validation
|
||||
@@ -345,10 +351,14 @@ class Gates(SientiaMonitoring):
|
||||
- path_priority (list[str]): Priority order for path decisions
|
||||
|
||||
Returns:
|
||||
tuple: (path_flag, confidence, comment)
|
||||
tuple: (path_flag, confidence, comment, document)
|
||||
- path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
|
||||
- confidence (int): Confidence score for the decision
|
||||
- comment (str): Detailed explanation of the decision
|
||||
- document (dict | None): Metadata describing the filter(s) that produced
|
||||
`path_flag`, shaped `{'filters': [{'name', 'policy', 'counter'}, ...]}`;
|
||||
`None` when nothing was filtered. Not written anywhere by this activity —
|
||||
the caller decides whether and how to persist it.
|
||||
|
||||
Raises:
|
||||
Exception: If response validation fails or configuration is invalid
|
||||
@@ -369,10 +379,11 @@ class Gates(SientiaMonitoring):
|
||||
gate_type = input_data['type']
|
||||
path_priority = input_data['path_priority']
|
||||
|
||||
filter_output = []
|
||||
|
||||
comments = []
|
||||
|
||||
# (filter_name, policy, filter_config) per filter that triggered, in configured order
|
||||
triggered_filters: list[tuple[str, str, dict[str, Any]]] = []
|
||||
|
||||
status = payload.status or {}
|
||||
|
||||
for fil, config in filters.items():
|
||||
@@ -381,15 +392,21 @@ class Gates(SientiaMonitoring):
|
||||
policy, filter_config = self._read_filter_entry(config)
|
||||
try:
|
||||
if mlflow_response_filter_functions[fil](status, filter_config):
|
||||
filter_output.append(policy)
|
||||
comments.append(status.get('message', 'Unknown MLFlow API error'))
|
||||
triggered_filters.append((fil, policy, filter_config))
|
||||
# The real error lives one level in, under `content` — `request_transform`/
|
||||
# `request_predict` nest `message`/`traceback` there on failure (mlflow.py),
|
||||
# reusing the same `content` key the success path uses for the DataFrame
|
||||
# (review finding, 2026-10-08: reading `status['message']` directly always
|
||||
# fell back to the generic default, in every real production occurrence).
|
||||
error_content = status.get('content') or {}
|
||||
comments.append(error_content.get('message', 'Unknown MLFlow API error'))
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
|
||||
message=status.get('message', 'Unknown MLFlow API error'),
|
||||
message=error_content.get('message', 'Unknown MLFlow API error'),
|
||||
block='mlflow_gate',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=status.get('traceback'),
|
||||
attachment_content=error_content.get('traceback'),
|
||||
)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
@@ -402,16 +419,42 @@ class Gates(SientiaMonitoring):
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
triggered_policies = [policy for _, policy, _ in triggered_filters]
|
||||
|
||||
for path_flag in path_priority:
|
||||
if path_flag in filter_output:
|
||||
if path_flag in triggered_policies:
|
||||
self.info(f'Mlflow response gate result: {path_flag}', metadata)
|
||||
return path_flag, mlflow_response_path_confidence[path_flag], ', '.join(comments)
|
||||
|
||||
matched = [fil for fil, policy, _ in triggered_filters if policy == path_flag]
|
||||
document = {
|
||||
'filters': [
|
||||
{
|
||||
'name': fil,
|
||||
'policy': path_flag,
|
||||
# No per-row/per-item unit to count here — `api_error_filter`
|
||||
# evaluates a single API response as a whole, not tabular data —
|
||||
# so, unlike `input_gate`'s per-filter counters, this is a fixed
|
||||
# value: 1 (one API error event). Inlined rather than a counter
|
||||
# function/dict, since there's only one response filter today and
|
||||
# this isn't an override of anything (review finding).
|
||||
'counter': 1,
|
||||
}
|
||||
for fil in matched
|
||||
],
|
||||
}
|
||||
|
||||
return (
|
||||
path_flag,
|
||||
mlflow_response_path_confidence[path_flag],
|
||||
', '.join(comments),
|
||||
document,
|
||||
)
|
||||
|
||||
self.info('Nothing was filtered by the mlflow response gate', metadata)
|
||||
|
||||
del data
|
||||
|
||||
return None, 0, ''
|
||||
return None, 0, '', None
|
||||
|
||||
@activity.defn(name='mlflow_content_gate')
|
||||
def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
|
||||
@@ -2,6 +2,7 @@ from temporalio import activity, workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
# Extend the Temporal Postgres activities for convenient query -> MinIO export
|
||||
import time
|
||||
import traceback
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
@@ -9,12 +10,14 @@ with workflow.unsafe.imports_passed_through():
|
||||
import pandas as pd
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability import metrics
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.repository.minio_repository_sync import MinioRepository
|
||||
from sientia_do.temporal.activities.postgres_sync import Postgres
|
||||
from sientia_do.temporal.constants import now
|
||||
from sqlalchemy import text
|
||||
|
||||
from laborious.utils.messages import MISSING_MINIO_REPOSITORY
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
@@ -26,7 +29,9 @@ _LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_thr
|
||||
class Storage(Postgres, SientiaMonitoring):
|
||||
"""
|
||||
Extensions for Postgres activities with a helper to export query results
|
||||
directly to MinIO as Parquet and return the object name.
|
||||
directly to MinIO as Parquet and return the object name, and with
|
||||
``repeat_last_prediction``, a query specific to this repo's REPEAT path
|
||||
(moved here from the shared library — nothing else consumes it).
|
||||
"""
|
||||
|
||||
minio_repository: MinioRepository | None = None
|
||||
@@ -141,6 +146,110 @@ class Storage(Postgres, SientiaMonitoring):
|
||||
}
|
||||
)
|
||||
|
||||
def _build_repeat_last_prediction_query(self, schema: str, table_name: str):
|
||||
table_identifier = f'"{schema}".{table_name}'
|
||||
return text( # noqa: S608
|
||||
f"""
|
||||
INSERT INTO {table_identifier} (
|
||||
model_id,
|
||||
prediction,
|
||||
timestamp,
|
||||
response_time,
|
||||
prediction_status,
|
||||
prediction_confidence,
|
||||
metadata_id,
|
||||
created_at
|
||||
)
|
||||
SELECT
|
||||
model_id,
|
||||
prediction,
|
||||
:last_timestamp,
|
||||
response_time,
|
||||
prediction_status,
|
||||
prediction_confidence,
|
||||
:metadata_id,
|
||||
NOW()
|
||||
FROM {table_identifier}
|
||||
WHERE model_id = :model
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 1;
|
||||
""" # nosec B608
|
||||
)
|
||||
|
||||
@activity.defn(name='repeat_last_prediction')
|
||||
def repeat_last_prediction(self, query_items: dict[str, Any]):
|
||||
"""
|
||||
Insert a new `predictions` row that copies the last one for a model forward in time.
|
||||
|
||||
REPEAT persists a new row (unlike STOP, which persists nothing), reusing the previous
|
||||
row's `prediction`/`response_time`/`prediction_status`/`prediction_confidence` values with
|
||||
a fresh `timestamp`/`created_at`. `metadata_id` references the `prediction_metadata`
|
||||
document for this cycle, if any (see `PredictionMetadata.write_prediction_metadata`);
|
||||
`None` is a valid value — the column is nullable.
|
||||
|
||||
Args (query_items):
|
||||
metadata (dict): Workflow metadata for logging and metrics.
|
||||
schema (str): Postgres schema of the `predictions` table.
|
||||
table_name (str): `predictions` table name.
|
||||
model (str): `model_id` to repeat the last prediction for.
|
||||
last_timestamp: Timestamp to stamp the new row with.
|
||||
metadata_id (str | None): `prediction_metadata` document id, or `None`.
|
||||
|
||||
Raises:
|
||||
Exception: Re-raised after logging/metrics if the insert fails.
|
||||
"""
|
||||
metadata = query_items['metadata']
|
||||
schema = query_items['schema']
|
||||
table_name = query_items['table_name']
|
||||
model = query_items['model']
|
||||
last_timestamp = query_items['last_timestamp']
|
||||
metadata_id = query_items.get('metadata_id')
|
||||
|
||||
repeat_query = self._build_repeat_last_prediction_query(schema, table_name)
|
||||
self.info(f'Repeating last prediction for model {model}', metadata)
|
||||
self.debug(
|
||||
f'Query parameters: schema={schema}, model={model}, last_timestamp={last_timestamp}',
|
||||
metadata,
|
||||
)
|
||||
core_labels = {
|
||||
**self.get_core_labels(metadata, 'execute_sql'),
|
||||
'conflict_mode': 'error',
|
||||
}
|
||||
|
||||
with self.session_factory.begin() as session:
|
||||
try:
|
||||
start_time = time.time()
|
||||
session.execute(
|
||||
repeat_query,
|
||||
{
|
||||
'last_timestamp': last_timestamp,
|
||||
'model': model,
|
||||
'metadata_id': metadata_id,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='ERROR_REPEATING_LAST_PREDICTION',
|
||||
message=f'Error repeating last prediction: {e}',
|
||||
block='repeat_last_prediction',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace,
|
||||
)
|
||||
self.error(trace, metadata)
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.POSTGRES_WRITE_ERROR_COUNT,
|
||||
tags=core_labels,
|
||||
)
|
||||
raise
|
||||
else:
|
||||
self.observe_lag_sync(start_time, metrics.POSTGRES_WRITE_LAG, core_labels)
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.POSTGRES_WRITE_COUNT,
|
||||
tags=core_labels,
|
||||
)
|
||||
|
||||
@activity.defn(name='cleanup_minio_objects_expired')
|
||||
def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
|
||||
@@ -156,7 +156,7 @@ class PredictionProcess:
|
||||
)
|
||||
|
||||
# Validate MLFlow transform response
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
path_flag, confidence, comment, document = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_response_gate,
|
||||
{
|
||||
**metadata,
|
||||
@@ -171,7 +171,7 @@ class PredictionProcess:
|
||||
|
||||
# Handle path decision based on transform validation
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment, document
|
||||
):
|
||||
return
|
||||
|
||||
@@ -206,7 +206,7 @@ class PredictionProcess:
|
||||
)
|
||||
|
||||
# Validate MLFlow prediction response
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
path_flag, confidence, comment, document = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_response_gate,
|
||||
{
|
||||
**metadata,
|
||||
@@ -221,7 +221,7 @@ class PredictionProcess:
|
||||
|
||||
# Handle path decision based on prediction validation
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment, document
|
||||
):
|
||||
return
|
||||
|
||||
@@ -271,9 +271,9 @@ class PredictionProcess:
|
||||
"""
|
||||
if not document:
|
||||
# Skip the remote activity call entirely rather than dispatching it just to have
|
||||
# `write_prediction_metadata` decide there's nothing to write — `mlflow_response_gate`
|
||||
# and `mlflow_content_gate` don't build a `document` yet, so this is the common case
|
||||
# today, not a rare edge case (review finding, 2026-10-01).
|
||||
# `write_prediction_metadata` decide there's nothing to write — `mlflow_content_gate`
|
||||
# doesn't build a `document` yet, so this is the common case today, not a rare edge
|
||||
# case (review finding, 2026-10-01).
|
||||
return None
|
||||
|
||||
try:
|
||||
@@ -360,10 +360,9 @@ class PredictionProcess:
|
||||
return True
|
||||
elif path_flag == 'REPEAT':
|
||||
metadata_id = await self._write_prediction_metadata(metadata, document)
|
||||
# Repeat last prediction if available. `metadata_id` isn't consumed by
|
||||
# `repeat_last_prediction` yet (its INSERT...SELECT has no `metadata_id` column —
|
||||
# pending a change in `sientia-dataops-library`); passed through already so no further
|
||||
# change is needed here once that lands.
|
||||
# Repeat last prediction, copying the previous row's values forward with a new
|
||||
# timestamp. `repeat_last_prediction` now lives in `Storage` (laborious), not the
|
||||
# shared library — its INSERT...SELECT references `metadata_id` directly.
|
||||
await workflow.execute_activity_method(
|
||||
Activities.repeat_last_prediction,
|
||||
{
|
||||
|
||||
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "laborious"
|
||||
version = "0.9.0"
|
||||
version = "0.9.0-rc3"
|
||||
description = "Sientia DataOps Laborious - ML Model Orchestration System"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
|
||||
@@ -314,7 +314,7 @@ def test_mlflow_response_gate_invalid_filter(gates_activity):
|
||||
result = gates_activity.mlflow_response_gate(input_data)
|
||||
|
||||
# Assert
|
||||
assert result == (None, 0, '')
|
||||
assert result == (None, 0, '', None)
|
||||
|
||||
|
||||
@patch('laborious.activities.gates.mlflow_response_filter_functions')
|
||||
@@ -341,7 +341,7 @@ def test_mlflow_response_gate_filter_exception(
|
||||
result = gates_activity.mlflow_response_gate(input_data)
|
||||
|
||||
# Assert
|
||||
assert result == (None, 0, '')
|
||||
assert result == (None, 0, '', None)
|
||||
gates_activity.send_notification.assert_called_once_with(
|
||||
metadata=metadata['metadata'],
|
||||
notification_id='MLFLOW_GATE_RESPONSE_FILTER__INVALID_FILTER',
|
||||
@@ -369,7 +369,7 @@ def test_mlflow_response_gate_no_filters(gates_activity):
|
||||
result = gates_activity.mlflow_response_gate(input_data)
|
||||
|
||||
# Assert
|
||||
assert result == (None, 0, '')
|
||||
assert result == (None, 0, '', None)
|
||||
gates_activity.debug.assert_called()
|
||||
|
||||
|
||||
@@ -380,19 +380,30 @@ def test_mlflow_response_gate_with_filter(gates_activity):
|
||||
'filters': {'API_ERROR': {'POLICY': 'STOP'}},
|
||||
'data': _minio_payload(
|
||||
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
|
||||
status={'success': False, 'message': 'API error occurred'},
|
||||
status={
|
||||
'success': False,
|
||||
'content': {'message': 'API error occurred', 'traceback': 'error trace'},
|
||||
},
|
||||
),
|
||||
'type': 'test',
|
||||
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
|
||||
}
|
||||
|
||||
# Act
|
||||
result = gates_activity.mlflow_response_gate(input_data)
|
||||
path_flag, confidence, comment, document = gates_activity.mlflow_response_gate(input_data)
|
||||
|
||||
# Assert
|
||||
assert result == ('STOP', -1, 'API error occurred')
|
||||
assert (path_flag, confidence, comment) == ('STOP', -1, 'API error occurred')
|
||||
assert document == {'filters': [{'name': 'API_ERROR', 'policy': 'STOP', 'counter': 1}]}
|
||||
gates_activity.debug.assert_called()
|
||||
gates_activity.send_notification.assert_called()
|
||||
gates_activity.send_notification.assert_called_once_with(
|
||||
metadata=metadata['metadata'],
|
||||
notification_id='TEST_GATE_RESPONSE_FILTER__API_ERROR',
|
||||
message='API error occurred',
|
||||
block='mlflow_gate',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content='error trace',
|
||||
)
|
||||
|
||||
|
||||
def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
|
||||
@@ -402,7 +413,10 @@ def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
|
||||
'filters': {'API_ERROR': {'Policy': 'STOP'}},
|
||||
'data': _minio_payload(
|
||||
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
|
||||
status={'success': False, 'message': 'API error occurred'},
|
||||
status={
|
||||
'success': False,
|
||||
'content': {'message': 'API error occurred', 'traceback': 'error trace'},
|
||||
},
|
||||
),
|
||||
'type': 'test',
|
||||
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
|
||||
@@ -412,7 +426,12 @@ def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
|
||||
result = gates_activity.mlflow_response_gate(input_data)
|
||||
|
||||
# Assert
|
||||
assert result == ('STOP', -1, 'API error occurred')
|
||||
assert result == (
|
||||
'STOP',
|
||||
-1,
|
||||
'API error occurred',
|
||||
{'filters': [{'name': 'API_ERROR', 'policy': 'STOP', 'counter': 1}]},
|
||||
)
|
||||
|
||||
|
||||
def test_mlflow_response_gate_with_filter_not_caught(gates_activity):
|
||||
@@ -432,7 +451,7 @@ def test_mlflow_response_gate_with_filter_not_caught(gates_activity):
|
||||
result = gates_activity.mlflow_response_gate(input_data)
|
||||
|
||||
# Assert
|
||||
assert result == (None, 0, '')
|
||||
assert result == (None, 0, '', None)
|
||||
gates_activity.debug.assert_called()
|
||||
|
||||
|
||||
|
||||
@@ -257,6 +257,86 @@ def test_export_payload_to_postgres(storage):
|
||||
assert result == {'success': True}
|
||||
|
||||
|
||||
def test_build_repeat_last_prediction_query(storage):
|
||||
query = storage._build_repeat_last_prediction_query('public', 'predictions')
|
||||
query_str = str(query)
|
||||
|
||||
assert 'INSERT INTO "public".predictions' in query_str
|
||||
assert 'metadata_id' in query_str
|
||||
assert ':last_timestamp' in query_str
|
||||
assert ':metadata_id' in query_str
|
||||
assert ':model' in query_str
|
||||
|
||||
|
||||
@patch('laborious.activities.storage.text')
|
||||
def test_repeat_last_prediction_success(text_mock, storage):
|
||||
storage.emit_metric_sync = MagicMock()
|
||||
storage.observe_lag_sync = MagicMock()
|
||||
query_items = {
|
||||
**metadata,
|
||||
'schema': 'public',
|
||||
'table_name': 'predictions',
|
||||
'model': 1,
|
||||
'last_timestamp': '2022-01-01',
|
||||
'metadata_id': 'abc123',
|
||||
}
|
||||
|
||||
with patch('sqlalchemy.orm.session.Session.execute') as mock_execute:
|
||||
storage.repeat_last_prediction(query_items)
|
||||
|
||||
text_mock.assert_called_once()
|
||||
mock_execute.assert_called_once_with(
|
||||
text_mock.return_value,
|
||||
{'last_timestamp': '2022-01-01', 'model': 1, 'metadata_id': 'abc123'},
|
||||
)
|
||||
storage.observe_lag_sync.assert_called_once()
|
||||
storage.emit_metric_sync.assert_called_once()
|
||||
|
||||
|
||||
@patch('laborious.activities.storage.text')
|
||||
def test_repeat_last_prediction_without_metadata_id(text_mock, storage):
|
||||
storage.emit_metric_sync = MagicMock()
|
||||
storage.observe_lag_sync = MagicMock()
|
||||
query_items = {
|
||||
**metadata,
|
||||
'schema': 'public',
|
||||
'table_name': 'predictions',
|
||||
'model': 1,
|
||||
'last_timestamp': '2022-01-01',
|
||||
}
|
||||
|
||||
with patch('sqlalchemy.orm.session.Session.execute') as mock_execute:
|
||||
storage.repeat_last_prediction(query_items)
|
||||
|
||||
mock_execute.assert_called_once_with(
|
||||
text_mock.return_value,
|
||||
{'last_timestamp': '2022-01-01', 'model': 1, 'metadata_id': None},
|
||||
)
|
||||
|
||||
|
||||
@patch('laborious.activities.storage.text')
|
||||
def test_repeat_last_prediction_error(text_mock, storage):
|
||||
storage.emit_metric_sync = MagicMock()
|
||||
storage.send_notification = MagicMock()
|
||||
query_items = {
|
||||
**metadata,
|
||||
'schema': 'public',
|
||||
'table_name': 'predictions',
|
||||
'model': 1,
|
||||
'last_timestamp': '2022-01-01',
|
||||
'metadata_id': 'abc123',
|
||||
}
|
||||
|
||||
with (
|
||||
patch('sqlalchemy.orm.session.Session.execute', side_effect=ValueError('db')),
|
||||
raises(ValueError),
|
||||
):
|
||||
storage.repeat_last_prediction(query_items)
|
||||
|
||||
storage.send_notification.assert_called_once()
|
||||
storage.emit_metric_sync.assert_called()
|
||||
|
||||
|
||||
def test_cleanup_minio_objects_expired_minio_not_initialized(storage):
|
||||
storage.minio_repository = None
|
||||
|
||||
|
||||
@@ -59,11 +59,11 @@ async def test_run(workflow_mock, prediction_process):
|
||||
workflow_mock.execute_local_activity_method.side_effect = [
|
||||
('continue', 0.95, 'Input data with bad quality', None), # input_gate
|
||||
# mlflow_response_gate (transform)
|
||||
('continue', 0.95, 'Error'),
|
||||
('continue', 0.95, 'Error', None),
|
||||
# mlflow_content_gate (transform)
|
||||
('continue', 0.95, 'Transformed data not passed the content filter'),
|
||||
# mlflow_response_gate (predict)
|
||||
('continue', 0.95, 'Error'),
|
||||
('continue', 0.95, 'Error', None),
|
||||
]
|
||||
|
||||
# Act
|
||||
@@ -276,7 +276,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_
|
||||
]
|
||||
workflow_mock.execute_local_activity_method.side_effect = [
|
||||
('repeat', 0.95, 'Input data with bad quality', None), # input_gate
|
||||
('continue', 0.95, 'Error'), # mlflow_response_gate (transform)
|
||||
('continue', 0.95, 'Error', None), # mlflow_response_gate (transform)
|
||||
]
|
||||
|
||||
# Act
|
||||
@@ -367,7 +367,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process
|
||||
workflow_mock.execute_local_activity_method.side_effect = [
|
||||
('continue', 0.95, 'Input data with bad quality', None), # input_gate
|
||||
# mlflow_response_gate (transform)
|
||||
('continue', 0.95, 'Error'),
|
||||
('continue', 0.95, 'Error', None),
|
||||
# mlflow_content_gate (transform)
|
||||
('continue', 0.95, 'Transformed data not passed the content filter'),
|
||||
]
|
||||
@@ -478,10 +478,10 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p
|
||||
workflow_mock.execute_local_activity_method.side_effect = [
|
||||
('continue', 0.95, 'Input data with bad quality', None), # input_gate
|
||||
# mlflow_response_gate (transform)
|
||||
('continue', 0.95, 'Error'),
|
||||
('continue', 0.95, 'Error', None),
|
||||
# mlflow_content_gate (transform)
|
||||
('continue', 0.95, 'Transformed data not passed the content filter'),
|
||||
('continue', 0.95, 'Error'), # mlflow_response_gate (predict)
|
||||
('continue', 0.95, 'Error', None), # mlflow_response_gate (predict)
|
||||
]
|
||||
|
||||
# Act
|
||||
@@ -696,9 +696,9 @@ async def test_path_flag_handler_repeat_no_document_skips_write_activity_call(
|
||||
workflow_mock, prediction_process
|
||||
):
|
||||
# Arrange: no document (e.g. REPEAT triggered by a gate that doesn't build one yet — today,
|
||||
# `mlflow_response_gate`/`mlflow_content_gate`) must not even dispatch the remote
|
||||
# write_prediction_metadata activity — that would be a wasted Temporal round-trip on every
|
||||
# REPEAT caused by those two gates, not a rare edge case (review finding, 2026-10-01).
|
||||
# `mlflow_content_gate`) must not even dispatch the remote write_prediction_metadata
|
||||
# activity — that would be a wasted Temporal round-trip on every REPEAT caused by that
|
||||
# gate, not a rare edge case (review finding, 2026-10-01).
|
||||
data = {'test': 'data'}
|
||||
|
||||
# Act
|
||||
@@ -1075,9 +1075,9 @@ async def test_run_with_cleanup_prefixes(workflow_mock, prediction_process):
|
||||
]
|
||||
workflow_mock.execute_local_activity_method.side_effect = [
|
||||
('continue', 0.95, 'ok', None), # input_gate
|
||||
('continue', 0.95, ''),
|
||||
('continue', 0.95, ''),
|
||||
('continue', 0.95, ''),
|
||||
('continue', 0.95, '', None), # mlflow_response_gate (transform)
|
||||
('continue', 0.95, ''), # mlflow_content_gate (transform)
|
||||
('continue', 0.95, '', None), # mlflow_response_gate (predict)
|
||||
]
|
||||
|
||||
await prediction_process.run(input_data)
|
||||
|
||||
Reference in New Issue
Block a user