feat(simple_metrics): update adaptive state after each run, empty windows included (SIENTIAPDE-2191)

tests/laborious/worker/test_worker.py

@@ -16,6 +16,7 @@ def _build_fake_activities():
     inst.export_data_to_postgres = MagicMock()
     inst.load_custom_query = MagicMock()
     inst.calculate_simple_metrics = MagicMock()
+    inst.update_simple_metrics_adaptive_state = MagicMock()
     inst.get_reference_data = MagicMock()
     inst.calculate_drift = MagicMock()
     inst.request_predict = MagicMock()
@@ -188,6 +189,11 @@ async def test_main_success_exit_zero(monkeypatch):
     # Drift and SimpleMetrics load no model code, so every runtime shares the `core` queues.
     assert prepare_calls[1].kwargs['runtime'] == 'core'
     assert prepare_calls[2].kwargs['runtime'] == 'core'
+    # prepare_calls[1] is the SimpleMetrics worker (order: MinimalRetrain, SimpleMetrics, Drift, PredictionsBatch).
+    assert (
+        fake_activities.update_simple_metrics_adaptive_state
+        in prepare_calls[1].kwargs['activities']
+    )
 
 
 @mark.asyncio

tests/laborious/workflows/test_simple_metrics.py

@@ -245,7 +245,7 @@ _METRICS = [{'metric': 'rmse', 'value': 9.0}]
 @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
 async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metrics, n_rows):
     workflow_mock.logger = MagicMock()
-    workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * n_rows, None])
+    workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * n_rows, None, None])
     workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS)
 
     await simple_metrics.run(
@@ -254,8 +254,8 @@ async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metr
 
     activity_input = workflow_mock.execute_local_activity_method.call_args.args[1]
     assert activity_input['thresholds'] is None
-    # Persist, don't alert.
-    assert workflow_mock.execute_activity_method.call_count == 2
+    # Persist, don't alert: load, export, adaptive state.
+    assert workflow_mock.execute_activity_method.call_count == 3
     export_call = workflow_mock.execute_activity_method.call_args_list[1]
     assert export_call.args[0] == Activities.export_data_to_postgres
     assert export_call.args[1]['data'] == _METRICS
@@ -265,7 +265,7 @@ async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metr
 @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
 async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics):
     workflow_mock.logger = MagicMock()
-    workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None])
+    workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None, None])
     workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS)
 
     await simple_metrics.run(
@@ -274,6 +274,43 @@ async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics
 
     activity_input = workflow_mock.execute_local_activity_method.call_args.args[1]
     assert activity_input['thresholds'] == {'rmse_max': 1.0}
+    assert workflow_mock.execute_activity_method.call_count == 3
+
+
+_BREACH_ROWS = [{'metric': 'rmse', 'value': 9.0}]
+_THRESHOLDS = {'rmse_max': 1.0}
+
+
+@mark.asyncio
+@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
+async def test_run_updates_adaptive_state_after_export(workflow_mock, simple_metrics):
+    workflow_mock.logger = MagicMock()
+    workflow_mock.execute_activity_method = AsyncMock(
+        side_effect=[[_ROW] * 3, None, {'escalated': True, 'healthy_streak': 0}]
+    )
+    workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS)
+
+    await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS))
+
+    calls = workflow_mock.execute_activity_method.call_args_list
+    assert calls[1].args[0] is Activities.export_data_to_postgres
+    assert calls[2].args[0] is Activities.update_simple_metrics_adaptive_state
+    assert calls[2].args[1] == {
+        **metadata,
+        'simple_metrics': _BREACH_ROWS,
+        'thresholds': _THRESHOLDS,
+        'interval_minutes': 60,
+    }
+
+
+@mark.asyncio
+@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
+async def test_run_without_thresholds_skips_adaptive_state(workflow_mock, simple_metrics):
+    workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 3, None])
+    workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS)
+
+    await simple_metrics.run(_input())
+
     assert workflow_mock.execute_activity_method.call_count == 2
 
 
@@ -284,7 +321,7 @@ async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics
 @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
 async def test_run_guard_off_keeps_thresholds(workflow_mock, simple_metrics, min_samples, warns):
     workflow_mock.logger = MagicMock()
-    workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None])
+    workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None, None])
     workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS)
 
     await simple_metrics.run(
@@ -294,3 +331,50 @@ async def test_run_guard_off_keeps_thresholds(workflow_mock, simple_metrics, min
     activity_input = workflow_mock.execute_local_activity_method.call_args.args[1]
     assert activity_input['thresholds'] == {'rmse_max': 1.0}
     assert workflow_mock.logger.warning.call_count == int(warns)
+
+
+@mark.asyncio
+@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
+async def test_run_empty_window_counts_as_healthy_run(workflow_mock, simple_metrics):
+    workflow_mock.logger = MagicMock()
+    workflow_mock.execute_activity_method = AsyncMock(side_effect=[[], None])
+    workflow_mock.execute_local_activity_method = AsyncMock()
+
+    await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS))
+
+    workflow_mock.execute_local_activity_method.assert_not_called()
+    calls = workflow_mock.execute_activity_method.call_args_list
+    assert len(calls) == 2  # load + adaptive state, no export
+    assert calls[1].args[0] is Activities.update_simple_metrics_adaptive_state
+    assert calls[1].args[1]['simple_metrics'] == []
+
+
+@mark.asyncio
+@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
+async def test_run_below_min_samples_sends_no_rows_to_adaptive_state(workflow_mock, simple_metrics):
+    workflow_mock.logger = MagicMock()
+    workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None, None])
+    workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS)
+
+    await simple_metrics.run(
+        _input(simple_metrics_thresholds=_THRESHOLDS, simple_metrics_min_samples=5)
+    )
+
+    calls = workflow_mock.execute_activity_method.call_args_list
+    assert calls[1].args[0] is Activities.export_data_to_postgres  # metrics still persisted
+    assert calls[2].args[0] is Activities.update_simple_metrics_adaptive_state
+    assert calls[2].args[1]['simple_metrics'] == []  # never a breach
+
+
+@mark.asyncio
+@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
+async def test_run_adaptive_state_failure_does_not_fail_workflow(workflow_mock, simple_metrics):
+    workflow_mock.logger = MagicMock()
+    workflow_mock.execute_activity_method = AsyncMock(
+        side_effect=[[_ROW] * 3, None, RuntimeError('dispatch failed')]
+    )
+    workflow_mock.execute_local_activity_method = AsyncMock(return_value=_BREACH_ROWS)
+
+    await simple_metrics.run(_input(simple_metrics_thresholds=_THRESHOLDS))
+
+    workflow_mock.logger.warning.assert_called_once()