fix(simple_metrics): fail the run explicitly when the adaptive state update fails (SIENTIAPDE-2191)
Drop the try/except around update_simple_metrics_adaptive_state in the workflow and the broad except in the activity, so Mongo failures fail the run and show up in the Temporal metrics (PR #59 review). Invalid frequency_alert stays a warning + skip (the orchestrator ignores it too) and a malformed adaptive_state is read as not escalated.
This commit is contained in:
@@ -1424,16 +1424,33 @@ def test_adaptive_state_unchanged_does_not_write(model_metrics_activity, mongo):
|
||||
mongo.update_one.assert_not_called()
|
||||
|
||||
|
||||
def test_adaptive_state_invalid_frequency_alert_is_swallowed(model_metrics_activity, mongo):
|
||||
mongo.find_one.return_value = _pipeline(frequency_alert='5min')
|
||||
@mark.parametrize('frequency_alert', ['5min', '0m', 5])
|
||||
def test_adaptive_state_invalid_frequency_alert_is_skipped(
|
||||
model_metrics_activity, mongo, frequency_alert
|
||||
):
|
||||
mongo.find_one.return_value = _pipeline(frequency_alert=frequency_alert)
|
||||
|
||||
assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None
|
||||
mongo.update_one.assert_not_called()
|
||||
model_metrics_activity.error.assert_called_once()
|
||||
model_metrics_activity.warning.assert_called_once()
|
||||
|
||||
|
||||
def test_adaptive_state_mongo_error_is_swallowed(model_metrics_activity, mongo):
|
||||
mongo.find_one.side_effect = RuntimeError('mongo down')
|
||||
@mark.parametrize('adaptive_state', ['garbage', ['escalated']])
|
||||
def test_adaptive_state_malformed_state_is_not_escalated(
|
||||
model_metrics_activity, mongo, adaptive_state
|
||||
):
|
||||
mongo.find_one.return_value = _pipeline(frequency_alert='5m', adaptive_state=adaptive_state)
|
||||
|
||||
assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None
|
||||
model_metrics_activity.error.assert_called_once()
|
||||
result = model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT)
|
||||
|
||||
assert result == {'escalated': True, 'healthy_streak': 0}
|
||||
mongo.update_one.assert_called_once()
|
||||
|
||||
|
||||
@mark.parametrize('failing_call', ['find_one', 'update_one'])
|
||||
def test_adaptive_state_mongo_error_propagates(model_metrics_activity, mongo, failing_call):
|
||||
mongo.find_one.return_value = _pipeline(frequency_alert='5m')
|
||||
getattr(mongo, failing_call).side_effect = RuntimeError('mongo down')
|
||||
|
||||
with raises(RuntimeError, match='mongo down'):
|
||||
model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
|
||||
|
||||
from pytest import fixture, mark
|
||||
from pytest import fixture, mark, raises
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
@@ -218,7 +218,7 @@ async def test_run_default_metrics(workflow_mock: AsyncMock, simple_metrics: Sim
|
||||
# Act
|
||||
await simple_metrics.run(input_data)
|
||||
|
||||
# Assert - all three activities ran (a swallowed exhausted side_effect would warn)
|
||||
# Assert - all three activities ran
|
||||
assert workflow_mock.execute_activity_method.call_count == 3
|
||||
workflow_mock.logger.warning.assert_not_called()
|
||||
|
||||
@@ -402,13 +402,16 @@ async def test_run_below_min_samples_sends_no_rows_to_adaptive_state(workflow_mo
|
||||
|
||||
@mark.asyncio
|
||||
@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
|
||||
async def test_run_adaptive_state_failure_does_not_fail_workflow(workflow_mock, simple_metrics):
|
||||
async def test_run_adaptive_state_failure_fails_workflow(workflow_mock, simple_metrics):
|
||||
workflow_mock.logger = MagicMock()
|
||||
workflow_mock.execute_activity_method = AsyncMock(
|
||||
side_effect=[[_ROW] * 3, None, RuntimeError('dispatch failed')]
|
||||
)
|
||||
workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS)
|
||||
|
||||
await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS))
|
||||
with raises(RuntimeError, match='dispatch failed'):
|
||||
await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS))
|
||||
|
||||
workflow_mock.logger.warning.assert_called_once()
|
||||
# Metrics were exported before the failing call
|
||||
calls = workflow_mock.execute_activity_method.call_args_list
|
||||
assert calls[1].args[0] is Activities.export_data_to_postgres
|
||||
|
||||
Reference in New Issue
Block a user