gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1241: Fix: Correctly handle feature normalization and denormalization, and add scaler parameters to metadata

Browse Source
This commit is contained in:
Bruno Domingues
2025-10-22 21:26:11 -03:00
parent 42a7e82c82
commit 758ffb10b6
2 changed files with 25 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
model_manager/sientia/models.py
View File

@@ -535,8 +535,9 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
# Normalization # Normalization
if step == NORMALIZATION and self.scaler: if step == NORMALIZATION and self.scaler:
data_treat = data_treat[self.feature_names_order] # Only transform feature columns, preserve target and any other required columns
data_treat[existing_columns] = self.scaler.transform(data_treat[existing_columns]) feature_cols = self.feature_names_order
data_treat[feature_cols] = self.scaler.transform(data_treat[feature_cols])
# Feature Creation # Feature Creation
if step == 'Feature Creation': if step == 'Feature Creation':

29
model_manager/utils/repository/training_repository.py
View File

@@ -135,14 +135,28 @@ class TrainingRepository:
scaler = tmr.process_data.get_scaler() scaler = tmr.process_data.get_scaler()
self.logger.info('Denormalizing features') self.logger.info('Denormalizing features')
for col in params.variable_columns: # If using custom scaler with denormalize_* helpers
tmr.x_train[col] = scaler.denormalize_single_input(tmr.x_train[col], col) if hasattr(scaler, 'denormalize_single_input'):
tmr.x_test[col] = scaler.denormalize_single_input(tmr.x_test[col], col) for col in params.variable_columns:
tmr.x_train[col] = scaler.denormalize_single_input(tmr.x_train[col], col)
tmr.x_test[col] = scaler.denormalize_single_input(tmr.x_test[col], col)
self.logger.info('Denormalizing target variable') self.logger.info('Denormalizing target variable')
tmr.y_train = scaler.denormalize_single_input(tmr.y_train, params.target_variable) tmr.y_train = scaler.denormalize_single_input(tmr.y_train, params.target_variable)
tmr.y_test = scaler.denormalize_single_input(tmr.y_test, params.target_variable) tmr.y_test = scaler.denormalize_single_input(tmr.y_test, params.target_variable)
y_pred_array = scaler.denormalize_predictions(y_pred_array, params.target_variable) y_pred_array = scaler.denormalize_predictions(y_pred_array, params.target_variable)
else:
# Fallback for sklearn StandardScaler: only inverse-transform features
feature_cols = getattr(
tmr.process_data, 'feature_names_order', params.variable_columns
)
# Ensure columns are in the same order used during fit
x_train_features = tmr.x_train[feature_cols]
x_test_features = tmr.x_test[feature_cols]
tmr.x_train[feature_cols] = scaler.inverse_transform(x_train_features)
tmr.x_test[feature_cols] = scaler.inverse_transform(x_test_features)
# Target was not scaled with StandardScaler in preprocessing; leave y as-is
self.logger.info('Adding index to predictions') self.logger.info('Adding index to predictions')
tmr.y_pred = pd.Series(y_pred_array, index=tmr.y_test.index) tmr.y_pred = pd.Series(y_pred_array, index=tmr.y_test.index)
@@ -237,5 +251,6 @@ class TrainingRepository:
upp_lim=params.upp_lim, upp_lim=params.upp_lim,
window=params.window, window=params.window,
scaler_name='Standard Scaler' if params.use_scaler else 'None', scaler_name='Standard Scaler' if params.use_scaler else 'None',
scaler_params={} if params.use_scaler else None,
ar_var=params.target_variable if params.include_ar else None, ar_var=params.target_variable if params.include_ar else None,
) )