diff --git a/model_manager/activities/training.py b/model_manager/activities/training.py index 506a4b5..30c70a0 100644 --- a/model_manager/activities/training.py +++ b/model_manager/activities/training.py @@ -13,6 +13,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any import mlflow + import pandas as pd from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger @@ -231,6 +232,12 @@ class Training(SientiaMonitoring): train_data = train_result.train_data val_data = train_result.val_data + self.debug( + f'train_model prepared data (head 10):\ntrain:\n{train_data.head(10).to_string()}' + f'\nval:\n{val_data.head(10).to_string()}', + metadata, + ) + wrapper.train( train_data=train_data, val_data=val_data, @@ -245,9 +252,21 @@ class Training(SientiaMonitoring): transformed_train, _ = wrapper.transform(train_data) transformed_val, _ = wrapper.transform(val_data) + self.debug( + f'train_model transform (head 10):\ntrain:\n{transformed_train.head(10).to_string()}' + f'\nval:\n{transformed_val.head(10).to_string()}', + metadata, + ) + y_train_pred_df, _ = wrapper.predict({}, transformed_train) y_val_pred_df, _ = wrapper.predict({}, transformed_val) + self.debug( + f'train_model predict (head 10):\ntrain:\n{y_train_pred_df.head(10).to_string()}' + f'\nval:\n{y_val_pred_df.head(10).to_string()}', + metadata, + ) + y_train_pred_df.sort_index(inplace=True, ascending=False) y_val_pred_df.sort_index(inplace=True, ascending=False) @@ -258,6 +277,7 @@ class Training(SientiaMonitoring): train_result = self.data_manager_repository.compute_regression_metrics( train_result, wrapper, + metadata=metadata, ) self.info(f'Starting MLflow run for {train_params.model_type}', metadata) diff --git a/model_manager/utils/repository/data_manager_repository.py b/model_manager/utils/repository/data_manager_repository.py index 5e6b106..88e35b8 100644 --- a/model_manager/utils/repository/data_manager_repository.py +++ b/model_manager/utils/repository/data_manager_repository.py @@ -273,6 +273,7 @@ class DataManagerRepository(SientiaMonitoring): self, tmr: TrainModelResult, wrapper: SientiaModel, + metadata: dict[str, Any] | None = None, ) -> TrainModelResult: """ Compute regression metrics for training results. @@ -286,9 +287,11 @@ class DataManagerRepository(SientiaMonitoring): tmr: Training result containing: - train_data/val_data DataFrames with a target column - y_train_pred/y_pred populated (model predictions for train/val) + wrapper: Trained model wrapper (used for linear equation extraction). + metadata: Optional workflow metadata for debug logging. Return: - Updated TrainModelResult with mse_val, mae_val and r2_val fields populated. + TrainModelResult: Same object with mse_val, mae_val and r2_val set. """ if tmr.y_pred is None: raise ValueError('y_pred must be set before computing regression metrics') @@ -304,12 +307,28 @@ class DataManagerRepository(SientiaMonitoring): # Align by index to avoid metric calculation errors if ordering differs. common_index = y_true_val.index.intersection(y_pred_val.index) + head = min(5, len(y_true_val), len(y_pred_val)) + self.debug( + 'compute_regression_metrics index alignment: ' + f'val_n={len(y_true_val)} pred_n={len(y_pred_val)} common_n={len(common_index)}; ' + f'val_index_dtype={y_true_val.index.dtype} ' + f'pred_index_dtype={y_pred_val.index.dtype}; ' + f'val_index_sample={list(y_true_val.index[:head])} ' + f'pred_index_sample={list(y_pred_val.index[:head])}', + metadata, + ) + + if len(common_index) == 0: + raise ValueError( + 'No overlapping indices between val_data and y_pred. ' + f'val_n={len(y_true_val)} pred_n={len(y_pred_val)} ' + f'val_index_sample={list(y_true_val.index[:head])} ' + f'pred_index_sample={list(y_pred_val.index[:head])}' + ) + y_true_val = y_true_val.loc[common_index] y_pred_val = y_pred_val.loc[common_index] - if len(y_true_val) == 0: - raise ValueError('No overlapping indices between val_data and y_pred') - # Metrics helpers already round to 2 decimals. tmr.mse_val = mse(y_true_val, y_pred_val) tmr.mae_val = mae(y_true_val, y_pred_val)