diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py index cb44805..b9b1841 100644 --- a/laborious/activities/model_metrics.py +++ b/laborious/activities/model_metrics.py @@ -6,7 +6,6 @@ with workflow.unsafe.imports_passed_through(): import warnings from typing import Any - import numpy as np import pandas as pd from pandas import DataFrame, Index, Series, to_datetime from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler @@ -16,6 +15,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.sientia_monitoring import SientiaMonitoring from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from sientia_model.analytics.drift_analysis import DriftAnalysis, DriftInsufficientDataError + from sientia_model.metrics.regression import RegressionMetrics from laborious import metrics from laborious.utils.dataframe_debug import build_dataframe_debug_message @@ -365,62 +365,62 @@ class ModelMetrics(SientiaMonitoring): @activity.defn(name='calculate_simple_metrics') def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]: """ - Calculate simple metrics for a model. Metrics available are: - - rmse - - mse - - mae - - r2 - - accuracy - - precision - - recall - - f1 + Calculate simple regression metrics for a model using RegressionMetrics. + Args: - input_data (dict[str, Any]): Input data containing: + input_data: Input data containing: - metadata (dict): Workflow execution metadata - model_id (str): ID of the MLFlow model - - target_data (pd.DataFrame): Target data for calculating metrics, containing target and prediction columns - - metrics (list[str]): List of metrics to calculate + - target_data (list[dict]): Target data with target, prediction, timestamp columns + - metrics (list[str]): Metric names to calculate + - interval_minutes (int): Window interval in minutes + - model_type (str | None): Model algorithm type (for r2 lock) Returns: - dict[Hashable, Any]: Dictionary containing the calculated metrics + list[dict]: Records with metric, value, model_id, timestamp, data_size, interval_minutes """ - metadata = input_data['metadata'] model_id = input_data['model_id'] target_data = DataFrame(input_data['target_data']) - metric_names = input_data['metrics'] + metric_names = list(input_data['metrics']) interval_minutes = input_data['interval_minutes'] + model_type = input_data.get('model_type') data_size = target_data.shape[0] - output_data = [] + # Filter r2 when model_type is known and unsupported + if ( + model_type + and 'r2' in metric_names + and not RegressionMetrics.is_r2_supported(model_type) + ): + metric_names = [m for m in metric_names if m != 'r2'] + self.warning( + f'r2 excluded for model {model_id}: not supported for model_type={model_type}', + metadata, + ) + self.send_notification( + metadata=metadata, + notification_id='SIMPLE_METRICS_R2_UNSUPPORTED', + message=f'r2 excluded: not a valid metric for model_type={model_type}', + block='model_metrics', + level=NotificationLevel.WARNING, + ) - diff = target_data['target'] - target_data['prediction'] - diff_squared = diff**2 + if not metric_names: + self.info(f'No metrics to calculate for model {model_id} after filtering', metadata) + return [] self.info(f'Calculating simple metrics for model {model_id}: {metric_names}', metadata) - for metric in metric_names: - if metric == 'rmse': - output_data.append({'metric': 'rmse', 'value': np.sqrt(np.mean(diff_squared))}) - elif metric == 'mse': - output_data.append({'metric': 'mse', 'value': np.mean(diff_squared)}) - elif metric == 'mae': - output_data.append({'metric': 'mae', 'value': np.mean(np.abs(diff))}) - elif metric == 'r2': - y_true = target_data['target'] - y_mean = np.mean(y_true) + # Build Series with DatetimeIndex for RegressionMetrics + timestamps = pd.to_datetime(target_data['timestamp']) + real_data = Series(target_data['target'].values, index=timestamps, dtype=float) + predictions = Series(target_data['prediction'].values, index=timestamps, dtype=float) - ss_res = np.sum(diff_squared) - ss_tot = np.sum((y_true - y_mean) ** 2) - - # Evita divisão por zero - if ss_tot == 0: - r2_score = 0.0 - else: - r2_score = 1 - (ss_res / ss_tot) - - output_data.append({'metric': 'r2', 'value': r2_score}) + regression = RegressionMetrics(real_data, predictions) + output_data = regression.calculate(metric_names) + # Wrap with metadata columns matching the existing output schema data = DataFrame(output_data) data['model_id'] = model_id data['timestamp'] = target_data['timestamp'].max()