feat(simple_metrics): rewrite calculate_simple_metrics to use RegressionMetrics

Replaces manual numpy if/elif chain with RegressionMetrics from
sientia_model. Fixes 3 known bugs in one pass:
- NaN now handled via _align_dropna (was silently propagated)
- r2 zero-variance uses sklearn r2_score (was divergent, pinned at 0.0)
- Unknown metric names raise ValueError (were silently dropped)

Also adds r2 lock: when model_type is known and non-linear,
r2 is excluded from calculation with a WARNING notification.

Drops the now-unused `numpy` import (the manual math it backed is gone,
and no other method in this file references it).

SIENTIAPDE-1986
This commit is contained in:
PedroHMCosme
2026-08-19 11:13:08 -03:00
parent 33b4ae8406
commit b7fdcebc15

View File

@@ -6,7 +6,6 @@ with workflow.unsafe.imports_passed_through():
import warnings
from typing import Any
import numpy as np
import pandas as pd
from pandas import DataFrame, Index, Series, to_datetime
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
@@ -16,6 +15,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_model.analytics.drift_analysis import DriftAnalysis, DriftInsufficientDataError
from sientia_model.metrics.regression import RegressionMetrics
from laborious import metrics
from laborious.utils.dataframe_debug import build_dataframe_debug_message
@@ -365,62 +365,62 @@ class ModelMetrics(SientiaMonitoring):
@activity.defn(name='calculate_simple_metrics')
def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]:
"""
Calculate simple metrics for a model. Metrics available are:
- rmse
- mse
- mae
- r2
- accuracy
- precision
- recall
- f1
Calculate simple regression metrics for a model using RegressionMetrics.
Args:
input_data (dict[str, Any]): Input data containing:
input_data: Input data containing:
- metadata (dict): Workflow execution metadata
- model_id (str): ID of the MLFlow model
- target_data (pd.DataFrame): Target data for calculating metrics, containing target and prediction columns
- metrics (list[str]): List of metrics to calculate
- target_data (list[dict]): Target data with target, prediction, timestamp columns
- metrics (list[str]): Metric names to calculate
- interval_minutes (int): Window interval in minutes
- model_type (str | None): Model algorithm type (for r2 lock)
Returns:
dict[Hashable, Any]: Dictionary containing the calculated metrics
list[dict]: Records with metric, value, model_id, timestamp, data_size, interval_minutes
"""
metadata = input_data['metadata']
model_id = input_data['model_id']
target_data = DataFrame(input_data['target_data'])
metric_names = input_data['metrics']
metric_names = list(input_data['metrics'])
interval_minutes = input_data['interval_minutes']
model_type = input_data.get('model_type')
data_size = target_data.shape[0]
output_data = []
# Filter r2 when model_type is known and unsupported
if (
model_type
and 'r2' in metric_names
and not RegressionMetrics.is_r2_supported(model_type)
):
metric_names = [m for m in metric_names if m != 'r2']
self.warning(
f'r2 excluded for model {model_id}: not supported for model_type={model_type}',
metadata,
)
self.send_notification(
metadata=metadata,
notification_id='SIMPLE_METRICS_R2_UNSUPPORTED',
message=f'r2 excluded: not a valid metric for model_type={model_type}',
block='model_metrics',
level=NotificationLevel.WARNING,
)
diff = target_data['target'] - target_data['prediction']
diff_squared = diff**2
if not metric_names:
self.info(f'No metrics to calculate for model {model_id} after filtering', metadata)
return []
self.info(f'Calculating simple metrics for model {model_id}: {metric_names}', metadata)
for metric in metric_names:
if metric == 'rmse':
output_data.append({'metric': 'rmse', 'value': np.sqrt(np.mean(diff_squared))})
elif metric == 'mse':
output_data.append({'metric': 'mse', 'value': np.mean(diff_squared)})
elif metric == 'mae':
output_data.append({'metric': 'mae', 'value': np.mean(np.abs(diff))})
elif metric == 'r2':
y_true = target_data['target']
y_mean = np.mean(y_true)
# Build Series with DatetimeIndex for RegressionMetrics
timestamps = pd.to_datetime(target_data['timestamp'])
real_data = Series(target_data['target'].values, index=timestamps, dtype=float)
predictions = Series(target_data['prediction'].values, index=timestamps, dtype=float)
ss_res = np.sum(diff_squared)
ss_tot = np.sum((y_true - y_mean) ** 2)
# Evita divisão por zero
if ss_tot == 0:
r2_score = 0.0
else:
r2_score = 1 - (ss_res / ss_tot)
output_data.append({'metric': 'r2', 'value': r2_score})
regression = RegressionMetrics(real_data, predictions)
output_data = regression.calculate(metric_names)
# Wrap with metadata columns matching the existing output schema
data = DataFrame(output_data)
data['model_id'] = model_id
data['timestamp'] = target_data['timestamp'].max()