diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index b9b1841..1bc0a6d 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -429,4 +429,34 @@ class ModelMetrics(SientiaMonitoring): self._debug_dataframe(f'Simple metrics dataframe: Size {data.shape}', data, metadata) + # Threshold alerting (optional — no crash when absent) + thresholds = input_data.get('thresholds') + if thresholds: + # Convention: _max thresholds breach when value > threshold, + # _min thresholds breach when value < threshold. + for row in output_data: + metric_name = row['metric'] + value = row['value'] + max_key = f'{metric_name}_max' + min_key = f'{metric_name}_min' + + breach_msg = None + if max_key in thresholds and value > thresholds[max_key]: + breach_msg = f'{metric_name}={value} exceeds {max_key}={thresholds[max_key]}' + elif min_key in thresholds and value < thresholds[min_key]: + breach_msg = f'{metric_name}={value} below {min_key}={thresholds[min_key]}' + + if breach_msg: + self.warning( + f'Threshold breach for model {model_id}: {breach_msg}', + metadata, + ) + self.send_notification( + metadata=metadata, + notification_id='SIMPLE_METRICS_THRESHOLD_BREACH', + message=f'Threshold breach for model {model_id}: {breach_msg}', + block='model_metrics', + level=NotificationLevel.WARNING, + ) + return data.to_dict(orient='records') diff --git a/tests/laborious/activities/test_model_metrics.py b/tests/laborious/activities/test_model_metrics.py index ede6a27..a71e9ba 100644 --- a/tests/laborious/activities/test_model_metrics.py +++ b/tests/laborious/activities/test_model_metrics.py @@ -1037,3 +1037,135 @@ def test_calculate_simple_metrics_no_model_type_includes_r2(model_metrics_activi assert result['metric'].values[0] == 'r2' mock_class.is_r2_supported.assert_not_called() + + +def test_calculate_simple_metrics_threshold_breach_rmse(model_metrics_activity): + """When rmse exceeds rmse_max, a WARNING notification fires.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['rmse'], + 'interval_minutes': 5, + 'thresholds': {'rmse_max': 0.05}, + } + + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'rmse', 'value': 0.1}, + ] + ) + + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 1 + model_metrics_activity.send_notification.assert_called_once() + call_kwargs = model_metrics_activity.send_notification.call_args.kwargs + assert call_kwargs['notification_id'] == 'SIMPLE_METRICS_THRESHOLD_BREACH' + assert call_kwargs['level'] == NotificationLevel.WARNING + assert 'rmse' in call_kwargs['message'] + + +def test_calculate_simple_metrics_threshold_no_breach(model_metrics_activity): + """When rmse is below rmse_max, no notification fires.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['rmse'], + 'interval_minutes': 5, + 'thresholds': {'rmse_max': 1.0}, + } + + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'rmse', 'value': 0.1}, + ] + ) + + with patcher: + model_metrics_activity.calculate_simple_metrics(input_data) + + model_metrics_activity.send_notification.assert_not_called() + + +def test_calculate_simple_metrics_threshold_r2_below_min(model_metrics_activity): + """When r2 drops below r2_min, a WARNING notification fires.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['r2'], + 'interval_minutes': 5, + 'thresholds': {'r2_min': 0.95}, + } + + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'r2', 'value': 0.8}, + ] + ) + + with patcher: + model_metrics_activity.calculate_simple_metrics(input_data) + + model_metrics_activity.send_notification.assert_called_once() + call_kwargs = model_metrics_activity.send_notification.call_args.kwargs + assert 'r2' in call_kwargs['message'] + + +def test_calculate_simple_metrics_no_thresholds_no_alert(model_metrics_activity): + """When thresholds is None (not configured), no alerting, no crash.""" + target_data = DataFrame( + { + 'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'], + 'target': [1.0, 2.0], + 'prediction': [1.1, 2.1], + } + ) + + input_data = { + **metadata, + 'model_id': 'test_model_id', + 'target_data': target_data.to_dict(), + 'metrics': ['rmse'], + 'interval_minutes': 5, + # no thresholds key + } + + patcher, mock_class, mock_instance = _mock_regression_metrics_class( + [ + {'metric': 'rmse', 'value': 999.0}, + ] + ) + + with patcher: + result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data)) + + assert len(result) == 1 + model_metrics_activity.send_notification.assert_not_called()