feat(simple_metrics): model-level, window-aware threshold breach alert (SIENTIAPDE-2189)
This commit is contained in:
@@ -447,14 +447,18 @@ class ModelMetrics(SientiaMonitoring):
|
||||
breach_msg = f'{metric_name}={value} below {min_key}={thresholds[min_key]}'
|
||||
|
||||
if breach_msg:
|
||||
self.warning(
|
||||
f'Threshold breach for model {model_id}: {breach_msg}',
|
||||
metadata,
|
||||
message = (
|
||||
f'Threshold breach for model {model_id} '
|
||||
f'(window {interval_minutes} min): {breach_msg}'
|
||||
)
|
||||
self.warning(message, metadata)
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
# Model-level trigger: the orchestrator dedups alerts on
|
||||
# `<trigger>:<notification_id>`, so one breach seen by several windows
|
||||
# (one schedule each) becomes one alert (SIENTIAPDE-1995).
|
||||
metadata={**metadata, 'schedule_name': f'{model_id}-simple_metrics'},
|
||||
notification_id='SIMPLE_METRICS_THRESHOLD_BREACH',
|
||||
message=f'Threshold breach for model {model_id}: {breach_msg}',
|
||||
message=message,
|
||||
block='model_metrics',
|
||||
level=NotificationLevel.WARNING,
|
||||
)
|
||||
|
||||
@@ -1074,6 +1074,10 @@ def test_calculate_simple_metrics_threshold_breach_rmse(model_metrics_activity):
|
||||
assert call_kwargs['notification_id'] == 'SIMPLE_METRICS_THRESHOLD_BREACH'
|
||||
assert call_kwargs['level'] == NotificationLevel.WARNING
|
||||
assert 'rmse' in call_kwargs['message']
|
||||
assert '(window 5 min)' in call_kwargs['message']
|
||||
# Model-level trigger so the orchestrator dedup merges windows (spec decision 9).
|
||||
assert call_kwargs['metadata']['schedule_name'] == 'test_model_id-simple_metrics'
|
||||
assert call_kwargs['metadata']['model_name'] == 'test_model'
|
||||
|
||||
|
||||
def test_calculate_simple_metrics_threshold_no_breach(model_metrics_activity):
|
||||
|
||||
Reference in New Issue
Block a user