diff --git a/README.md b/README.md index ede41bf..66f59be 100644 --- a/README.md +++ b/README.md @@ -487,7 +487,7 @@ The **SimpleMetrics** workflow calculates regression metrics (RMSE, MSE, MAE, R #### Execution Flow 1. **Data Loading**: Loads prediction vs target data via a JOIN query -2. **Metrics Calculation**: Calculates configured regression metrics +2. **Metrics Calculation**: Calculates configured regression metrics; alerts on `simple_metrics_thresholds` breaches (one alert per model, the message names the window) unless the window has fewer rows than `simple_metrics_min_samples` 3. **Data Export**: Exports metrics to PostgreSQL #### Architecture Diagram @@ -510,7 +510,12 @@ flowchart LR "data_table_name": "laborious_data", "target_table_name": "simple_metrics", "interval_minutes": 60, - "model_config": { "target": "temperature", "alias": "production" }, + "model_config": { + "target": "temperature", + "alias": "production", + "simple_metrics_thresholds": { "rmse_max": 1.5 }, + "simple_metrics_min_samples": 10 + }, "metrics": ["rmse", "mse", "mae", "r2"] } ``` diff --git a/laborious/workflows/simple_metrics.py b/laborious/workflows/simple_metrics.py index 475f871..fad9a89 100644 --- a/laborious/workflows/simple_metrics.py +++ b/laborious/workflows/simple_metrics.py @@ -34,6 +34,14 @@ class SimpleMetrics: model_type = model_config.get('model_type') thresholds = model_config.get('simple_metrics_thresholds') min_samples = model_config.get('simple_metrics_min_samples') + # Untyped Mongo config: anything but a positive int (bool excluded) turns the guard off + # instead of raising in workflow code; falsy values mean "unset" and stay silent. + if type(min_samples) is not int or min_samples <= 0: + if min_samples: + workflow.logger.warning( + f'Invalid simple_metrics_min_samples={min_samples!r}: min samples guard off' + ) + min_samples = None # Built from internal Temporal workflow config, not user input (bandit B608). query = f""" diff --git a/tests/laborious/workflows/test_simple_metrics.py b/tests/laborious/workflows/test_simple_metrics.py index c293685..bafd007 100644 --- a/tests/laborious/workflows/test_simple_metrics.py +++ b/tests/laborious/workflows/test_simple_metrics.py @@ -237,15 +237,16 @@ def _input(**model_config) -> dict: _ROW = {'timestamp': '2026-10-06 10:00:00+0000', 'prediction': 1.0, 'target': 2.0} +_METRICS = [{'metric': 'rmse', 'value': 9.0}] + + @mark.asyncio +@mark.parametrize('n_rows', [2, 4]) @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) -async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metrics): +async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metrics, n_rows): workflow_mock.logger = MagicMock() - # load, export, and the adaptive-state call S2 (Task 8) adds. - workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None, None]) - workflow_mock.execute_local_activity_method = AsyncMock( - return_value=[{'metric': 'rmse', 'value': 9.0}] - ) + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * n_rows, None]) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS) await simple_metrics.run( _input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=5) @@ -253,16 +254,19 @@ async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metr activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] assert activity_input['thresholds'] is None + # Persist, don't alert. + assert workflow_mock.execute_activity_method.call_count == 2 + export_call = workflow_mock.execute_activity_method.call_args_list[1] + assert export_call.args[0] == Activities.export_data_to_postgres + assert export_call.args[1]['data'] == _METRICS @mark.asyncio @patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics): workflow_mock.logger = MagicMock() - workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None, None]) - workflow_mock.execute_local_activity_method = AsyncMock( - return_value=[{'metric': 'rmse', 'value': 9.0}] - ) + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None]) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS) await simple_metrics.run( _input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=5) @@ -270,3 +274,23 @@ async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] assert activity_input['thresholds'] == {'rmse_max': 1.0} + assert workflow_mock.execute_activity_method.call_count == 2 + + +@mark.asyncio +@mark.parametrize( + ('min_samples', 'warns'), [(None, False), (0, False), ('5', True), (5.0, True), (True, True)] +) +@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock) +async def test_run_guard_off_keeps_thresholds(workflow_mock, simple_metrics, min_samples, warns): + workflow_mock.logger = MagicMock() + workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None]) + workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS) + + await simple_metrics.run( + _input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=min_samples) + ) + + activity_input = workflow_mock.execute_local_activity_method.call_args.args[1] + assert activity_input['thresholds'] == {'rmse_max': 1.0} + assert workflow_mock.logger.warning.call_count == int(warns)