fix(simple_metrics): ignore invalid simple_metrics_min_samples; tighten tests and docs (SIENTIAPDE-2189)

This commit is contained in:
PedroHMCosme
2026-10-08 08:43:01 -03:00
parent ccf4dc86e2
commit 9f8237a46e
3 changed files with 49 additions and 12 deletions

View File

@@ -487,7 +487,7 @@ The **SimpleMetrics** workflow calculates regression metrics (RMSE, MSE, MAE, R
#### Execution Flow
1. **Data Loading**: Loads prediction vs target data via a JOIN query
2. **Metrics Calculation**: Calculates configured regression metrics
2. **Metrics Calculation**: Calculates configured regression metrics; alerts on `simple_metrics_thresholds` breaches (one alert per model, the message names the window) unless the window has fewer rows than `simple_metrics_min_samples`
3. **Data Export**: Exports metrics to PostgreSQL
#### Architecture Diagram
@@ -510,7 +510,12 @@ flowchart LR
"data_table_name": "laborious_data",
"target_table_name": "simple_metrics",
"interval_minutes": 60,
"model_config": { "target": "temperature", "alias": "production" },
"model_config": {
"target": "temperature",
"alias": "production",
"simple_metrics_thresholds": { "rmse_max": 1.5 },
"simple_metrics_min_samples": 10
},
"metrics": ["rmse", "mse", "mae", "r2"]
}
```

View File

@@ -34,6 +34,14 @@ class SimpleMetrics:
model_type = model_config.get('model_type')
thresholds = model_config.get('simple_metrics_thresholds')
min_samples = model_config.get('simple_metrics_min_samples')
# Untyped Mongo config: anything but a positive int (bool excluded) turns the guard off
# instead of raising in workflow code; falsy values mean "unset" and stay silent.
if type(min_samples) is not int or min_samples <= 0:
if min_samples:
workflow.logger.warning(
f'Invalid simple_metrics_min_samples={min_samples!r}: min samples guard off'
)
min_samples = None
# Built from internal Temporal workflow config, not user input (bandit B608).
query = f"""

View File

@@ -237,15 +237,16 @@ def _input(**model_config) -> dict:
_ROW = {'timestamp': '2026-10-06 10:00:00+0000', 'prediction': 1.0, 'target': 2.0}
_METRICS = [{'metric': 'rmse', 'value': 9.0}]
@mark.asyncio
@mark.parametrize('n_rows', [2, 4])
@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metrics):
async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metrics, n_rows):
workflow_mock.logger = MagicMock()
# load, export, and the adaptive-state call S2 (Task 8) adds.
workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None, None])
workflow_mock.execute_local_activity_method = AsyncMock(
return_value=[{'metric': 'rmse', 'value': 9.0}]
)
workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * n_rows, None])
workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS)
await simple_metrics.run(
_input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=5)
@@ -253,16 +254,19 @@ async def test_run_below_min_samples_drops_thresholds(workflow_mock, simple_metr
activity_input = workflow_mock.execute_local_activity_method.call_args.args[1]
assert activity_input['thresholds'] is None
# Persist, don't alert.
assert workflow_mock.execute_activity_method.call_count == 2
export_call = workflow_mock.execute_activity_method.call_args_list[1]
assert export_call.args[0] == Activities.export_data_to_postgres
assert export_call.args[1]['data'] == _METRICS
@mark.asyncio
@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics):
workflow_mock.logger = MagicMock()
workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None, None])
workflow_mock.execute_local_activity_method = AsyncMock(
return_value=[{'metric': 'rmse', 'value': 9.0}]
)
workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 5, None])
workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS)
await simple_metrics.run(
_input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=5)
@@ -270,3 +274,23 @@ async def test_run_at_min_samples_keeps_thresholds(workflow_mock, simple_metrics
activity_input = workflow_mock.execute_local_activity_method.call_args.args[1]
assert activity_input['thresholds'] == {'rmse_max': 1.0}
assert workflow_mock.execute_activity_method.call_count == 2
@mark.asyncio
@mark.parametrize(
('min_samples', 'warns'), [(None, False), (0, False), ('5', True), (5.0, True), (True, True)]
)
@patch('laborious.workflows.simple_metrics.workflow', new_callable=AsyncMock)
async def test_run_guard_off_keeps_thresholds(workflow_mock, simple_metrics, min_samples, warns):
workflow_mock.logger = MagicMock()
workflow_mock.execute_activity_method = AsyncMock(side_effect=[[_ROW] * 2, None])
workflow_mock.execute_local_activity_method = AsyncMock(return_value=_METRICS)
await simple_metrics.run(
_input(simple_metrics_thresholds={'rmse_max': 1.0}, simple_metrics_min_samples=min_samples)
)
activity_input = workflow_mock.execute_local_activity_method.call_args.args[1]
assert activity_input['thresholds'] == {'rmse_max': 1.0}
assert workflow_mock.logger.warning.call_count == int(warns)