gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(simple_metrics): add threshold-based alerting via send_notification

Browse Source 
Compares each computed metric against optional per-model thresholds
from model_config.simple_metrics_thresholds. Convention:
- {metric}_max: breach when value > threshold (rmse, mse, mae)
- {metric}_min: breach when value < threshold (r2)

Fires WARNING notification on breach. Missing thresholds = no alerting.
Schema designed to be extensible for Card 2 (Drift) thresholds.

SIENTIAPDE-1986
This commit is contained in:
PedroHMCosme
2026-08-19 11:29:14 -03:00
parent 4e3b5756be
commit 5997210118
2 changed files with 162 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
tests/laborious/activities/test_model_metrics.py
View File

@@ -1037,3 +1037,135 @@ def test_calculate_simple_metrics_no_model_type_includes_r2(model_metrics_activi
assert result['metric'].values[0] == 'r2'
mock_class.is_r2_supported.assert_not_called()
def test_calculate_simple_metrics_threshold_breach_rmse(model_metrics_activity):
"""When rmse exceeds rmse_max, a WARNING notification fires."""
target_data = DataFrame(
{
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
'target': [1.0, 2.0],
'prediction': [1.1, 2.1],
}
)
input_data = {
**metadata,
'model_id': 'test_model_id',
'target_data': target_data.to_dict(),
'metrics': ['rmse'],
'interval_minutes': 5,
'thresholds': {'rmse_max': 0.05},
}
patcher, mock_class, mock_instance = _mock_regression_metrics_class(
[
{'metric': 'rmse', 'value': 0.1},
]
)
with patcher:
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
assert len(result) == 1
model_metrics_activity.send_notification.assert_called_once()
call_kwargs = model_metrics_activity.send_notification.call_args.kwargs
assert call_kwargs['notification_id'] == 'SIMPLE_METRICS_THRESHOLD_BREACH'
assert call_kwargs['level'] == NotificationLevel.WARNING
assert 'rmse' in call_kwargs['message']
def test_calculate_simple_metrics_threshold_no_breach(model_metrics_activity):
"""When rmse is below rmse_max, no notification fires."""
target_data = DataFrame(
{
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
'target': [1.0, 2.0],
'prediction': [1.1, 2.1],
}
)
input_data = {
**metadata,
'model_id': 'test_model_id',
'target_data': target_data.to_dict(),
'metrics': ['rmse'],
'interval_minutes': 5,
'thresholds': {'rmse_max': 1.0},
}
patcher, mock_class, mock_instance = _mock_regression_metrics_class(
[
{'metric': 'rmse', 'value': 0.1},
]
)
with patcher:
model_metrics_activity.calculate_simple_metrics(input_data)
model_metrics_activity.send_notification.assert_not_called()
def test_calculate_simple_metrics_threshold_r2_below_min(model_metrics_activity):
"""When r2 drops below r2_min, a WARNING notification fires."""
target_data = DataFrame(
{
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
'target': [1.0, 2.0],
'prediction': [1.1, 2.1],
}
)
input_data = {
**metadata,
'model_id': 'test_model_id',
'target_data': target_data.to_dict(),
'metrics': ['r2'],
'interval_minutes': 5,
'thresholds': {'r2_min': 0.95},
}
patcher, mock_class, mock_instance = _mock_regression_metrics_class(
[
{'metric': 'r2', 'value': 0.8},
]
)
with patcher:
model_metrics_activity.calculate_simple_metrics(input_data)
model_metrics_activity.send_notification.assert_called_once()
call_kwargs = model_metrics_activity.send_notification.call_args.kwargs
assert 'r2' in call_kwargs['message']
def test_calculate_simple_metrics_no_thresholds_no_alert(model_metrics_activity):
"""When thresholds is None (not configured), no alerting, no crash."""
target_data = DataFrame(
{
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
'target': [1.0, 2.0],
'prediction': [1.1, 2.1],
}
)
input_data = {
**metadata,
'model_id': 'test_model_id',
'target_data': target_data.to_dict(),
'metrics': ['rmse'],
'interval_minutes': 5,
# no thresholds key
}
patcher, mock_class, mock_instance = _mock_regression_metrics_class(
[
{'metric': 'rmse', 'value': 999.0},
]
)
with patcher:
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
assert len(result) == 1
model_metrics_activity.send_notification.assert_not_called()