feat(simple_metrics): add threshold-based alerting via send_notification
Compares each computed metric against optional per-model thresholds
from model_config.simple_metrics_thresholds. Convention:
- {metric}_max: breach when value > threshold (rmse, mse, mae)
- {metric}_min: breach when value < threshold (r2)
Fires WARNING notification on breach. Missing thresholds = no alerting.
Schema designed to be extensible for Card 2 (Drift) thresholds.
SIENTIAPDE-1986
This commit is contained in:
@@ -429,4 +429,34 @@ class ModelMetrics(SientiaMonitoring):
|
||||
|
||||
self._debug_dataframe(f'Simple metrics dataframe: Size {data.shape}', data, metadata)
|
||||
|
||||
# Threshold alerting (optional — no crash when absent)
|
||||
thresholds = input_data.get('thresholds')
|
||||
if thresholds:
|
||||
# Convention: _max thresholds breach when value > threshold,
|
||||
# _min thresholds breach when value < threshold.
|
||||
for row in output_data:
|
||||
metric_name = row['metric']
|
||||
value = row['value']
|
||||
max_key = f'{metric_name}_max'
|
||||
min_key = f'{metric_name}_min'
|
||||
|
||||
breach_msg = None
|
||||
if max_key in thresholds and value > thresholds[max_key]:
|
||||
breach_msg = f'{metric_name}={value} exceeds {max_key}={thresholds[max_key]}'
|
||||
elif min_key in thresholds and value < thresholds[min_key]:
|
||||
breach_msg = f'{metric_name}={value} below {min_key}={thresholds[min_key]}'
|
||||
|
||||
if breach_msg:
|
||||
self.warning(
|
||||
f'Threshold breach for model {model_id}: {breach_msg}',
|
||||
metadata,
|
||||
)
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='SIMPLE_METRICS_THRESHOLD_BREACH',
|
||||
message=f'Threshold breach for model {model_id}: {breach_msg}',
|
||||
block='model_metrics',
|
||||
level=NotificationLevel.WARNING,
|
||||
)
|
||||
|
||||
return data.to_dict(orient='records')
|
||||
|
||||
Reference in New Issue
Block a user