gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rc/SIENTIAPDE-2165 - fix: read MLflow error message/traceback from status['content'], not the top level

Browse Source
This commit is contained in:
Carla Pinto
2026-10-08 14:10:40 +02:00
parent 2649476571
commit 75b56251ef
2 changed files with 25 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
laborious/activities/gates.py
View File

@@ -393,14 +393,20 @@ class Gates(SientiaMonitoring):
try:
if mlflow_response_filter_functions[fil](status, filter_config):
triggered_filters.append((fil, policy, filter_config))
comments.append(status.get('message', 'Unknown MLFlow API error'))
# The real error lives one level in, under `content` — `request_transform`/
# `request_predict` nest `message`/`traceback` there on failure (mlflow.py),
# reusing the same `content` key the success path uses for the DataFrame
# (review finding, 2026-10-08: reading `status['message']` directly always
# fell back to the generic default, in every real production occurrence).
error_content = status.get('content') or {}
comments.append(error_content.get('message', 'Unknown MLFlow API error'))
self.send_notification(
metadata=metadata,
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
message=status.get('message', 'Unknown MLFlow API error'),
message=error_content.get('message', 'Unknown MLFlow API error'),
block='mlflow_gate',
level=NotificationLevel.ERROR,
attachment_content=status.get('traceback'),
attachment_content=error_content.get('traceback'),
)
except Exception as e:
trace = traceback.format_exc()

19
tests/laborious/activities/test_gates.py
View File

@@ -380,7 +380,10 @@ def test_mlflow_response_gate_with_filter(gates_activity):
'filters': {'API_ERROR': {'POLICY': 'STOP'}},
'data': _minio_payload(
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
status={'success': False, 'message': 'API error occurred'},
status={
'success': False,
'content': {'message': 'API error occurred', 'traceback': 'error trace'},
},
),
'type': 'test',
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
@@ -393,7 +396,14 @@ def test_mlflow_response_gate_with_filter(gates_activity):
assert (path_flag, confidence, comment) == ('STOP', -1, 'API error occurred')
assert document == {'filters': [{'name': 'API_ERROR', 'policy': 'STOP', 'counter': 1}]}
gates_activity.debug.assert_called()
gates_activity.send_notification.assert_called()
gates_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='TEST_GATE_RESPONSE_FILTER__API_ERROR',
message='API error occurred',
block='mlflow_gate',
level=NotificationLevel.ERROR,
attachment_content='error trace',
)
def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
@@ -403,7 +413,10 @@ def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
'filters': {'API_ERROR': {'Policy': 'STOP'}},
'data': _minio_payload(
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
status={'success': False, 'message': 'API error occurred'},
status={
'success': False,
'content': {'message': 'API error occurred', 'traceback': 'error trace'},
},
),
'type': 'test',
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],