fix(simple_metrics): fail the run explicitly when the adaptive state update fails (SIENTIAPDE-2191)

Drop the try/except around update_simple_metrics_adaptive_state in the workflow and the broad except in the activity, so Mongo failures fail the run and show up in the Temporal metrics (PR #59 review). Invalid frequency_alert stays a warning + skip (the orchestrator ignores it too) and a malformed adaptive_state is read as not escalated.
This commit is contained in:
PedroHMCosme
2026-10-08 17:45:44 -03:00
parent 877a0c7543
commit d265e3328c
4 changed files with 98 additions and 67 deletions

View File

@@ -1424,16 +1424,33 @@ def test_adaptive_state_unchanged_does_not_write(model_metrics_activity, mongo):
mongo.update_one.assert_not_called()
def test_adaptive_state_invalid_frequency_alert_is_swallowed(model_metrics_activity, mongo):
mongo.find_one.return_value = _pipeline(frequency_alert='5min')
@mark.parametrize('frequency_alert', ['5min', '0m', 5])
def test_adaptive_state_invalid_frequency_alert_is_skipped(
model_metrics_activity, mongo, frequency_alert
):
mongo.find_one.return_value = _pipeline(frequency_alert=frequency_alert)
assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None
mongo.update_one.assert_not_called()
model_metrics_activity.error.assert_called_once()
model_metrics_activity.warning.assert_called_once()
def test_adaptive_state_mongo_error_is_swallowed(model_metrics_activity, mongo):
mongo.find_one.side_effect = RuntimeError('mongo down')
@mark.parametrize('adaptive_state', ['garbage', ['escalated']])
def test_adaptive_state_malformed_state_is_not_escalated(
model_metrics_activity, mongo, adaptive_state
):
mongo.find_one.return_value = _pipeline(frequency_alert='5m', adaptive_state=adaptive_state)
assert model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT) is None
model_metrics_activity.error.assert_called_once()
result = model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT)
assert result == {'escalated': True, 'healthy_streak': 0}
mongo.update_one.assert_called_once()
@mark.parametrize('failing_call', ['find_one', 'update_one'])
def test_adaptive_state_mongo_error_propagates(model_metrics_activity, mongo, failing_call):
mongo.find_one.return_value = _pipeline(frequency_alert='5m')
getattr(mongo, failing_call).side_effect = RuntimeError('mongo down')
with raises(RuntimeError, match='mongo down'):
model_metrics_activity.update_simple_metrics_adaptive_state(_BREACH_INPUT)

View File

@@ -1,6 +1,6 @@
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
from pytest import fixture, mark
from pytest import fixture, mark, raises
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from laborious.activities.activities import Activities
@@ -218,7 +218,7 @@ async def test_run_default_metrics(workflow_mock: AsyncMock, simple_metrics: Sim
# Act
await simple_metrics.run(input_data)
# Assert - all three activities ran (a swallowed exhausted side_effect would warn)
# Assert - all three activities ran
assert workflow_mock.execute_activity_method.call_count == 3
workflow_mock.logger.warning.assert_not_called()
@@ -402,13 +402,16 @@ async def test_run_below_min_samples_sends_no_rows_to_adaptive_state(workflow_mo
@mark.asyncio
@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
async def test_run_adaptive_state_failure_does_not_fail_workflow(workflow_mock, simple_metrics):
async def test_run_adaptive_state_failure_fails_workflow(workflow_mock, simple_metrics):
workflow_mock.logger = MagicMock()
workflow_mock.execute_activity_method = AsyncMock(
side_effect=[[_ROW] * 3, None, RuntimeError('dispatch failed')]
)
workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS)
await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS))
with raises(RuntimeError, match='dispatch failed'):
await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS))
workflow_mock.logger.warning.assert_called_once()
# Metrics were exported before the failing call
calls = workflow_mock.execute_activity_method.call_args_list
assert calls[1].args[0] is Activities.export_data_to_postgres