gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(simple_metrics): model-level, window-aware threshold breach alert (SIENTIAPDE-2189)

Browse Source
This commit is contained in:
PedroHMCosme
2026-10-07 15:18:35 -03:00
parent 196b1aa708
commit 9c89cfbf8e
2 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
laborious/activities/model_metrics.py
View File

@@ -447,14 +447,18 @@ class ModelMetrics(SientiaMonitoring):
breach_msg = f'{metric_name}={value} below {min_key}={thresholds[min_key]}' breach_msg = f'{metric_name}={value} below {min_key}={thresholds[min_key]}'
if breach_msg: if breach_msg:
self.warning( message = (
f'Threshold breach for model {model_id}: {breach_msg}', f'Threshold breach for model {model_id} '
metadata, f'(window {interval_minutes} min): {breach_msg}'
) )
self.warning(message, metadata)
self.send_notification( self.send_notification(
metadata=metadata, # Model-level trigger: the orchestrator dedups alerts on
# `<trigger>:<notification_id>`, so one breach seen by several windows
# (one schedule each) becomes one alert (SIENTIAPDE-1995).
metadata={**metadata, 'schedule_name': f'{model_id}-simple_metrics'},
notification_id='SIMPLE_METRICS_THRESHOLD_BREACH', notification_id='SIMPLE_METRICS_THRESHOLD_BREACH',
message=f'Threshold breach for model {model_id}: {breach_msg}', message=message,
block='model_metrics', block='model_metrics',
level=NotificationLevel.WARNING, level=NotificationLevel.WARNING,
) )

4
tests/laborious/activities/test_model_metrics.py
View File

@@ -1074,6 +1074,10 @@ def test_calculate_simple_metrics_threshold_breach_rmse(model_metrics_activity):
assert call_kwargs['notification_id'] == 'SIMPLE_METRICS_THRESHOLD_BREACH' assert call_kwargs['notification_id'] == 'SIMPLE_METRICS_THRESHOLD_BREACH'
assert call_kwargs['level'] == NotificationLevel.WARNING assert call_kwargs['level'] == NotificationLevel.WARNING
assert 'rmse' in call_kwargs['message'] assert 'rmse' in call_kwargs['message']
assert '(window 5 min)' in call_kwargs['message']
# Model-level trigger so the orchestrator dedup merges windows (spec decision 9).
assert call_kwargs['metadata']['schedule_name'] == 'test_model_id-simple_metrics'
assert call_kwargs['metadata']['model_name'] == 'test_model'
def test_calculate_simple_metrics_threshold_no_breach(model_metrics_activity): def test_calculate_simple_metrics_threshold_no_breach(model_metrics_activity):