gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1241: Refactor: Remove redundant logging and fix duplicate logs

Browse Source 
This commit removes redundant logging statements from training and experiment tracking activities, preventing duplicate log entries. It also introduces a logger helper to disable log propagation, further addressing the duplicate logs issue. Additionally, the Makefile, run_coverage.sh, setup_port_forwards.sh, and simulator/Dockerfile files were removed as they are no longer needed.
This commit is contained in:
Bruno Domingues
2025-10-22 22:39:26 -03:00
parent ac97092ebf
commit f07bc8ff30
14 changed files with 86 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
model_manager/utils/repository/training_repository.py
View File

@@ -66,9 +66,7 @@ class TrainingRepository:
ValueError: If transformed data is empty
Exception: If data loading, preprocessing, or training fails
"""
self.logger.info('Loading data from BytesIO file')
data = load_data(uploaded_file, params.line_separator, params.decimal_separator)
self.logger.info('Initializing and fitting data preprocessor')
process_data = self._init_data_preprocessor(params)
process_data.fit(data)
data_view = process_data.transform(data)
@@ -76,7 +74,6 @@ class TrainingRepository:
if len(data_view) <= 0:
raise ValueError('Data view is empty after transformation')
self.logger.info('Splitting data into train/test sets')
x_train, x_test, y_train, y_test = split_train_test(
data_view[params.variable_columns],
data_view[params.target_variable],
@@ -85,17 +82,18 @@ class TrainingRepository:
random_state=42,
)
self.logger.info('Preparing training data')
data_train = pd.concat([x_train, y_train], axis=1)
scaler_dict = self._init_scaler_dict(process_data, params)
self.logger.info('Training linear regression model')
regr = LinearRegressionModel(
target_variable=params.target_variable,
variable_columns=params.variable_columns,
)
regr.fit(data_train)
self.logger.info(
f'Model trained successfully - experiment run id: {params.experiment_run_id}'
)
return TrainModelResult(
params=params,
@@ -128,12 +126,10 @@ class TrainingRepository:
TrainModelResult: Updated result with predictions, denormalized data,
and metrics (mse_val, mae_val, r2_val)
"""
self.logger.info('Making predictions on test set')
y_pred_array = tmr.regr.predict(tmr.x_test)
if params.use_scaler:
scaler = tmr.process_data.get_scaler()
self.logger.info('Denormalizing features')
# If using custom scaler with denormalize_* helpers
if hasattr(scaler, 'denormalize_single_input'):
@@ -141,7 +137,6 @@ class TrainingRepository:
tmr.x_train[col] = scaler.denormalize_single_input(tmr.x_train[col], col)
tmr.x_test[col] = scaler.denormalize_single_input(tmr.x_test[col], col)
self.logger.info('Denormalizing target variable')
tmr.y_train = scaler.denormalize_single_input(tmr.y_train, params.target_variable)
tmr.y_test = scaler.denormalize_single_input(tmr.y_test, params.target_variable)
y_pred_array = scaler.denormalize_predictions(y_pred_array, params.target_variable)
@@ -158,18 +153,15 @@ class TrainingRepository:
tmr.x_test[feature_cols] = scaler.inverse_transform(x_test_features)
# Target was not scaled with StandardScaler in preprocessing; leave y as-is
self.logger.info('Adding index to predictions')
tmr.y_pred = pd.Series(y_pred_array, index=tmr.y_test.index)
tmr.y_pred.name = f'{params.target_variable}_pred'
self.logger.info('Reordering all data by index')
tmr.x_train = tmr.x_train.sort_index()
tmr.x_test = tmr.x_test.sort_index()
tmr.y_train = tmr.y_train.sort_index()
tmr.y_test = tmr.y_test.sort_index()
tmr.y_pred = tmr.y_pred.sort_index()
self.logger.info('Calculating evaluation metrics')
assert tmr.y_pred is not None, 'y_pred should be set at this point'
tmr.mse_val = round(
@@ -183,6 +175,9 @@ class TrainingRepository:
)
tmr.r2_val = round(r2(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)), 2)
self.logger.info(
f'Model metrics calculated successfully - experiment run id: {params.experiment_run_id}'
)
return tmr
def _init_scaler_dict(self, process_data: DataPreprocessor, params: TrainModelParams) -> dict: