rc/SIENTIAPDE-2165 - fix: read MLflow error message/traceback from status['content'], not the top level
This commit is contained in:
@@ -393,14 +393,20 @@ class Gates(SientiaMonitoring):
|
|||||||
try:
|
try:
|
||||||
if mlflow_response_filter_functions[fil](status, filter_config):
|
if mlflow_response_filter_functions[fil](status, filter_config):
|
||||||
triggered_filters.append((fil, policy, filter_config))
|
triggered_filters.append((fil, policy, filter_config))
|
||||||
comments.append(status.get('message', 'Unknown MLFlow API error'))
|
# The real error lives one level in, under `content` — `request_transform`/
|
||||||
|
# `request_predict` nest `message`/`traceback` there on failure (mlflow.py),
|
||||||
|
# reusing the same `content` key the success path uses for the DataFrame
|
||||||
|
# (review finding, 2026-10-08: reading `status['message']` directly always
|
||||||
|
# fell back to the generic default, in every real production occurrence).
|
||||||
|
error_content = status.get('content') or {}
|
||||||
|
comments.append(error_content.get('message', 'Unknown MLFlow API error'))
|
||||||
self.send_notification(
|
self.send_notification(
|
||||||
metadata=metadata,
|
metadata=metadata,
|
||||||
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
|
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
|
||||||
message=status.get('message', 'Unknown MLFlow API error'),
|
message=error_content.get('message', 'Unknown MLFlow API error'),
|
||||||
block='mlflow_gate',
|
block='mlflow_gate',
|
||||||
level=NotificationLevel.ERROR,
|
level=NotificationLevel.ERROR,
|
||||||
attachment_content=status.get('traceback'),
|
attachment_content=error_content.get('traceback'),
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
trace = traceback.format_exc()
|
trace = traceback.format_exc()
|
||||||
|
|||||||
@@ -380,7 +380,10 @@ def test_mlflow_response_gate_with_filter(gates_activity):
|
|||||||
'filters': {'API_ERROR': {'POLICY': 'STOP'}},
|
'filters': {'API_ERROR': {'POLICY': 'STOP'}},
|
||||||
'data': _minio_payload(
|
'data': _minio_payload(
|
||||||
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
|
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
|
||||||
status={'success': False, 'message': 'API error occurred'},
|
status={
|
||||||
|
'success': False,
|
||||||
|
'content': {'message': 'API error occurred', 'traceback': 'error trace'},
|
||||||
|
},
|
||||||
),
|
),
|
||||||
'type': 'test',
|
'type': 'test',
|
||||||
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
|
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
|
||||||
@@ -393,7 +396,14 @@ def test_mlflow_response_gate_with_filter(gates_activity):
|
|||||||
assert (path_flag, confidence, comment) == ('STOP', -1, 'API error occurred')
|
assert (path_flag, confidence, comment) == ('STOP', -1, 'API error occurred')
|
||||||
assert document == {'filters': [{'name': 'API_ERROR', 'policy': 'STOP', 'counter': 1}]}
|
assert document == {'filters': [{'name': 'API_ERROR', 'policy': 'STOP', 'counter': 1}]}
|
||||||
gates_activity.debug.assert_called()
|
gates_activity.debug.assert_called()
|
||||||
gates_activity.send_notification.assert_called()
|
gates_activity.send_notification.assert_called_once_with(
|
||||||
|
metadata=metadata['metadata'],
|
||||||
|
notification_id='TEST_GATE_RESPONSE_FILTER__API_ERROR',
|
||||||
|
message='API error occurred',
|
||||||
|
block='mlflow_gate',
|
||||||
|
level=NotificationLevel.ERROR,
|
||||||
|
attachment_content='error trace',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
|
def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
|
||||||
@@ -403,7 +413,10 @@ def test_mlflow_response_gate_with_filter_capitalized_keys(gates_activity):
|
|||||||
'filters': {'API_ERROR': {'Policy': 'STOP'}},
|
'filters': {'API_ERROR': {'Policy': 'STOP'}},
|
||||||
'data': _minio_payload(
|
'data': _minio_payload(
|
||||||
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
|
{'content': {'message': 'API error occurred', 'traceback': 'error trace'}},
|
||||||
status={'success': False, 'message': 'API error occurred'},
|
status={
|
||||||
|
'success': False,
|
||||||
|
'content': {'message': 'API error occurred', 'traceback': 'error trace'},
|
||||||
|
},
|
||||||
),
|
),
|
||||||
'type': 'test',
|
'type': 'test',
|
||||||
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
|
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
|
||||||
|
|||||||
Reference in New Issue
Block a user