SIENTIAPDE-1241: refactor train_model workflow due to I/O errors.
This commit is contained in:
@@ -51,8 +51,7 @@ def build_mlflow_config() -> dict[str, Any]:
|
||||
dict: MLFlow configuration dictionary with all required parameters
|
||||
"""
|
||||
return {
|
||||
'host': getenv('MLFLOW_HOST', 'http://localhost'),
|
||||
'port': int(getenv('MLFLOW_PORT', '5080')),
|
||||
'url': getenv('MLFLOW_URL', 'http://localhost:5080'),
|
||||
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
||||
'password': getenv('MLFLOW_PASSWORD', 'aignosi'),
|
||||
}
|
||||
|
||||
12
model_manager/utils/exceptions.py
Normal file
12
model_manager/utils/exceptions.py
Normal file
@@ -0,0 +1,12 @@
|
||||
class ModelTrainingError(Exception):
|
||||
def __init__(self, model_trained: bool, model_saved: bool, message: str | None = None):
|
||||
self.model_trained = model_trained
|
||||
self.model_saved = model_saved
|
||||
|
||||
if message is None:
|
||||
message = (
|
||||
'Model training workflow failed '
|
||||
f'(model_trained={model_trained}, model_saved={model_saved})'
|
||||
)
|
||||
|
||||
super().__init__(message)
|
||||
@@ -184,22 +184,22 @@ class TrainModelParams:
|
||||
>>> params = TrainModelParams.from_dict(data)
|
||||
>>> params.validate_business_rules() # Raises ValueError if invalid
|
||||
"""
|
||||
# Validate train_size range (1-99%)
|
||||
if not 1 <= self.train_size <= 99:
|
||||
raise ValueError(f'train_size must be between 1 and 99, got {self.train_size}')
|
||||
# Validate train_size range (10-100%)
|
||||
if not 10 <= self.train_size <= 100:
|
||||
raise ValueError(f'train_size must be between 10 and 100, got {self.train_size}')
|
||||
|
||||
# Validate variable_columns is not empty
|
||||
if not self.variable_columns:
|
||||
raise ValueError('variable_columns cannot be empty')
|
||||
|
||||
# Validate positive integers
|
||||
if self.lag_train <= 0:
|
||||
if self.lag_train < 0:
|
||||
raise ValueError(f'lag_train must be positive, got {self.lag_train}')
|
||||
|
||||
if self.lag_val <= 0:
|
||||
if self.lag_val < 0:
|
||||
raise ValueError(f'lag_val must be positive, got {self.lag_val}')
|
||||
|
||||
if self.window <= 0:
|
||||
if self.window < 0:
|
||||
raise ValueError(f'window must be positive, got {self.window}')
|
||||
|
||||
# Validate low_lim and upp_lim consistency
|
||||
@@ -218,13 +218,6 @@ class TrainModelParams:
|
||||
f'Got low_lim={self.low_lim[var]}, upp_lim={self.upp_lim[var]}'
|
||||
)
|
||||
|
||||
# Validate target_variable is in variable_columns
|
||||
if self.target_variable not in self.variable_columns:
|
||||
raise ValueError(
|
||||
f'target_variable "{self.target_variable}" must be in variable_columns: '
|
||||
f'{self.variable_columns}'
|
||||
)
|
||||
|
||||
# Validate bucket_name and file_name are not empty
|
||||
if not self.bucket_name.strip():
|
||||
raise ValueError('bucket_name cannot be empty or whitespace')
|
||||
|
||||
@@ -9,6 +9,7 @@ and logging model runs to MLFlow.
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from os import makedirs, path
|
||||
@@ -22,14 +23,81 @@ from model_manager.sientia.reports import Reports # type: ignore[import-untyped
|
||||
from model_manager.utils.models.train_model_result import TrainModelResult
|
||||
|
||||
|
||||
class MLFlowRepository:
|
||||
def __init__(self, host, username, password, logger: Logger):
|
||||
self.model_serving = ModelServing(
|
||||
tracking_uri=host, username=username, password=password, logger=logger
|
||||
)
|
||||
class ModelRepository:
|
||||
def __init__(self, url, username, password, logger: Logger):
|
||||
self.model_serving = ModelServing(tracking_uri=url, username=username, password=password)
|
||||
|
||||
self.logger = logger
|
||||
|
||||
def get_next_run_name(self, experiment_name: str) -> str:
|
||||
def save_model(self, train_result: TrainModelResult) -> TrainModelResult:
|
||||
"""
|
||||
Save a trained ML model and its artifacts to MLflow.
|
||||
|
||||
This activity orchestrates the complete model saving pipeline:
|
||||
1. Generates the next run name for the experiment
|
||||
2. Creates and organizes artifacts (reports, data files)
|
||||
3. Logs model, parameters, metrics, and artifacts to MLflow
|
||||
|
||||
Args:
|
||||
input_data: Configuration for model saving operation
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- train_result (TrainModelResult): Training result with model and metrics
|
||||
|
||||
Returns:
|
||||
TrainModelResult: Updated training result with run_name and artifacts
|
||||
|
||||
Raises:
|
||||
Exception: If model saving fails (after sending notification)
|
||||
"""
|
||||
experiment_name = train_result.params.experiment_name
|
||||
self.logger.info(f'Starting model save for experiment: {experiment_name}')
|
||||
|
||||
# Step 1: Generate next run name
|
||||
self.logger.info('Generating run name')
|
||||
train_result.run_name = self._get_next_run_name(experiment_name)
|
||||
self.logger.info(f'Generated run name: {train_result.run_name}')
|
||||
|
||||
# Step 2: Generate artifacts (reports, CSV files)
|
||||
self.logger.info('Generating artifacts')
|
||||
train_result = self._generate_artifacts(train_result)
|
||||
self.logger.info('Artifacts generated successfully')
|
||||
|
||||
# Step 3: Save run to MLflow
|
||||
self.logger.info('Saving run to MLflow')
|
||||
self._save_run(train_result)
|
||||
|
||||
self.logger.info(
|
||||
f'Model saved successfully - Run: {train_result.run_name}, '
|
||||
f'Experiment: {experiment_name}'
|
||||
)
|
||||
|
||||
return train_result
|
||||
|
||||
def cleanup_run_directory(self, run_dir: str) -> None:
|
||||
"""
|
||||
Clean up temporary run directory after model training.
|
||||
|
||||
This activity deletes the temporary directory created during model training
|
||||
and artifact generation. It implements idempotent cleanup to handle cases
|
||||
where the directory may have already been deleted.
|
||||
|
||||
Args:
|
||||
run_dir (str): Path to the run directory to delete
|
||||
"""
|
||||
if not run_dir:
|
||||
self.logger.info('No run directory specified, skipping cleanup')
|
||||
return
|
||||
|
||||
self.logger.info(f'Cleaning up run directory: {run_dir}')
|
||||
|
||||
if os.path.exists(run_dir):
|
||||
shutil.rmtree(run_dir)
|
||||
self.logger.info(f'Run directory deleted successfully: {run_dir}')
|
||||
else:
|
||||
self.logger.info(f'Run directory already deleted: {run_dir}')
|
||||
|
||||
def _get_next_run_name(self, experiment_name: str) -> str:
|
||||
"""
|
||||
Generates the next run name for a given experiment.
|
||||
|
||||
@@ -46,7 +114,7 @@ class MLFlowRepository:
|
||||
next_run_number = len(runs) + 1
|
||||
return f'{experiment_name}-{next_run_number}'
|
||||
|
||||
def generate_artifacts(self, data: TrainModelResult) -> TrainModelResult:
|
||||
def _generate_artifacts(self, data: TrainModelResult) -> TrainModelResult:
|
||||
"""
|
||||
Generates and organizes artifacts related to the training process, such as reports and data files.
|
||||
|
||||
@@ -87,7 +155,7 @@ class MLFlowRepository:
|
||||
self._setup_run_directory(data.run_dir, header_file_path)
|
||||
return self._generate_report(reference_data, current_data, data)
|
||||
|
||||
def save_run(self, data: TrainModelResult):
|
||||
def _save_run(self, data: TrainModelResult):
|
||||
"""
|
||||
Logs the details of a machine learning run, including parameters, metrics, models, and artifacts,
|
||||
to the Sientia tracking system.
|
||||
@@ -122,60 +190,48 @@ class MLFlowRepository:
|
||||
self.logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
try:
|
||||
# Prepare parameters
|
||||
train_test_split = f'{data.params.train_size}-{100 - data.params.train_size}'
|
||||
interval_strs = [
|
||||
(str(interval[0]), str(interval[1]))
|
||||
for interval in (data.params.removed_intervals or [])
|
||||
]
|
||||
# Prepare parameters
|
||||
train_test_split = f'{data.params.train_size}-{100 - data.params.train_size}'
|
||||
|
||||
# Set experiment and create run
|
||||
self.model_serving.set_experiment(data.params.experiment_name)
|
||||
self.logger.info(
|
||||
f"Logging run '{data.run_name}' to experiment '{data.params.experiment_name}'"
|
||||
)
|
||||
interval_strs = [
|
||||
(str(interval[0]), str(interval[1]))
|
||||
for interval in (data.params.removed_intervals or [])
|
||||
]
|
||||
|
||||
with self.model_serving.save_experiment(
|
||||
run_name=data.run_name, description=data.params.experiment_name
|
||||
):
|
||||
# Log model parameters
|
||||
self.model_serving.log_param('model_type', 'Linear Regression')
|
||||
self.model_serving.log_param('target_variable', data.params.target_variable)
|
||||
self.model_serving.log_param('input_variables', data.params.variable_columns)
|
||||
self.model_serving.log_param('lag_train', data.params.lag_train)
|
||||
self.model_serving.log_param('lag_val', data.params.lag_val)
|
||||
self.model_serving.log_param('ma', data.params.window)
|
||||
self.model_serving.log_param('low_lim', data.params.low_lim)
|
||||
self.model_serving.log_param('upp_lim', data.params.upp_lim)
|
||||
self.model_serving.log_param('normalized', data.scaler_dict)
|
||||
self.model_serving.log_param('ar', data.params.include_ar)
|
||||
self.model_serving.log_param('Train_test_split', train_test_split)
|
||||
self.model_serving.log_param('Removed_intervals', interval_strs)
|
||||
self.model_serving.log_param('Retrain', False)
|
||||
# Set experiment and create run
|
||||
self.model_serving.set_experiment(data.params.experiment_name)
|
||||
|
||||
# Log evaluation metrics
|
||||
self.model_serving.log_metric('MSE', data.mse_val)
|
||||
self.model_serving.log_metric('R2', data.r2_val)
|
||||
self.model_serving.log_metric('MAE', data.mae_val)
|
||||
with self.model_serving.save_experiment(
|
||||
run_name=data.run_name, description=data.params.experiment_name
|
||||
):
|
||||
# Log model parameters
|
||||
self.model_serving.log_param('model_type', 'Linear Regression')
|
||||
self.model_serving.log_param('target_variable', data.params.target_variable)
|
||||
self.model_serving.log_param('input_variables', data.params.variable_columns)
|
||||
self.model_serving.log_param('lag_train', data.params.lag_train)
|
||||
self.model_serving.log_param('lag_val', data.params.lag_val)
|
||||
self.model_serving.log_param('ma', data.params.window)
|
||||
self.model_serving.log_param('low_lim', data.params.low_lim)
|
||||
self.model_serving.log_param('upp_lim', data.params.upp_lim)
|
||||
self.model_serving.log_param('normalized', data.scaler_dict)
|
||||
self.model_serving.log_param('ar', data.params.include_ar)
|
||||
self.model_serving.log_param('Train_test_split', train_test_split)
|
||||
self.model_serving.log_param('Removed_intervals', interval_strs)
|
||||
self.model_serving.log_param('Retrain', False)
|
||||
|
||||
# Log models
|
||||
self.model_serving.log_model(data.process_data, 'data_model')
|
||||
self.model_serving.log_model(data.regr, 'prediction_model')
|
||||
# Log evaluation metrics
|
||||
self.model_serving.log_metric('MSE', data.mse_val)
|
||||
self.model_serving.log_metric('R2', data.r2_val)
|
||||
self.model_serving.log_metric('MAE', data.mae_val)
|
||||
|
||||
# Log artifacts
|
||||
self.model_serving.log_artifact(data.report_path)
|
||||
self.model_serving.log_artifact(data.train_data_path)
|
||||
self.model_serving.log_artifact(data.test_data_path)
|
||||
# Log models
|
||||
self.model_serving.log_model(data.process_data, 'data_model')
|
||||
self.model_serving.log_model(data.regr, 'prediction_model')
|
||||
|
||||
self.logger.info(
|
||||
f"Successfully logged run '{data.run_name}' with metrics: MSE={data.mse_val:.4f}, R2={data.r2_val:.4f}, MAE={data.mae_val:.4f}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to save run '{data.run_name}' to MLflow: {str(e)}"
|
||||
self.logger.error(error_msg)
|
||||
raise RuntimeError(error_msg) from e
|
||||
# Log artifacts
|
||||
self.model_serving.log_artifact(data.report_path)
|
||||
self.model_serving.log_artifact(data.train_data_path)
|
||||
self.model_serving.log_artifact(data.test_data_path)
|
||||
|
||||
def _init_artifacts_data(self, data: TrainModelResult) -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
"""
|
||||
@@ -258,7 +314,6 @@ class MLFlowRepository:
|
||||
|
||||
try:
|
||||
makedirs(run_dir, exist_ok=True)
|
||||
self.logger.info(f'Created run directory: {run_dir}')
|
||||
return run_dir
|
||||
except PermissionError as e:
|
||||
error_msg = f'Permission denied when creating directory: {run_dir}'
|
||||
@@ -298,9 +353,6 @@ class MLFlowRepository:
|
||||
# Copy header file to run directory
|
||||
header_dest = path.join(run_dir, 'header.html')
|
||||
shutil.copy(header_file_path, header_dest)
|
||||
|
||||
self.logger.info(f'Run directory setup completed successfully in: {run_dir}')
|
||||
|
||||
except FileNotFoundError as e:
|
||||
error_msg = f'Header file not found: {header_file_path}'
|
||||
self.logger.error(error_msg)
|
||||
@@ -360,20 +412,16 @@ class MLFlowRepository:
|
||||
# Save HTML report
|
||||
data.report_path = path.join(data.run_dir, 'report.html')
|
||||
report.save_all_sections_html(data.report_path)
|
||||
self.logger.info(f'Generated HTML report: {data.report_path}')
|
||||
|
||||
# Save training data CSV
|
||||
data.train_data_path = path.join(data.run_dir, 'train_data.csv')
|
||||
reference_data.to_csv(data.train_data_path, index=False)
|
||||
self.logger.info(f'Saved training data: {data.train_data_path}')
|
||||
|
||||
# Save test data CSV
|
||||
data.test_data_path = path.join(data.run_dir, 'test_data.csv')
|
||||
current_data.to_csv(data.test_data_path, index=False)
|
||||
self.logger.info(f'Saved test data: {data.test_data_path}')
|
||||
|
||||
return data
|
||||
|
||||
except ValueError as e:
|
||||
error_msg = f'Failed to convert data to float64 for report generation: {str(e)}'
|
||||
self.logger.error(error_msg)
|
||||
|
||||
129
model_manager/utils/repository/storage_repository.py
Normal file
129
model_manager/utils/repository/storage_repository.py
Normal file
@@ -0,0 +1,129 @@
|
||||
from io import BytesIO
|
||||
|
||||
import boto3 # type: ignore[import-untyped]
|
||||
from botocore.config import Config # type: ignore[import-untyped]
|
||||
from sientia_do.observability.logger import Logger
|
||||
|
||||
|
||||
class StorageRepository:
|
||||
"""
|
||||
MinIO (S3-compatible) storage activities for file operations.
|
||||
|
||||
This class provides activities for interacting with MinIO object storage,
|
||||
including file download and deletion operations. It handles authentication,
|
||||
connection management, and comprehensive error handling.
|
||||
|
||||
The class implements best practices for S3/MinIO operations:
|
||||
- Connection reuse (boto3 client is thread-safe)
|
||||
- Automatic retry with exponential backoff
|
||||
- Comprehensive error handling and logging
|
||||
- Notification integration for critical errors
|
||||
|
||||
Attributes:
|
||||
endpoint_url (str): MinIO server endpoint URL
|
||||
access_key (str): MinIO access key ID
|
||||
secret_key (str): MinIO secret access key
|
||||
region (str): MinIO region name
|
||||
use_ssl (bool): Whether to use SSL/TLS for connections
|
||||
minio_client: Boto3 S3 client configured for MinIO
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
endpoint_url: str,
|
||||
access_key: str,
|
||||
secret_key: str,
|
||||
region: str,
|
||||
use_ssl: bool,
|
||||
max_retry_attempts: int,
|
||||
retry_mode: str,
|
||||
connect_timeout: int,
|
||||
read_timeout: int,
|
||||
logger: Logger,
|
||||
):
|
||||
"""
|
||||
Initialize a reusable MinIO client with retry configuration.
|
||||
|
||||
Args:
|
||||
endpoint_url: MinIO server endpoint URL (e.g., http://localhost:9000).
|
||||
access_key: MinIO access key ID for authentication.
|
||||
secret_key: MinIO secret access key for authentication.
|
||||
region: MinIO region name (e.g., us-east-1).
|
||||
use_ssl: Whether to use SSL/TLS for connections.
|
||||
max_retry_attempts: Maximum number of retry attempts (e.g., 3).
|
||||
retry_mode: Retry policy to apply (standard, legacy, adaptive).
|
||||
connect_timeout: Connection timeout in seconds.
|
||||
read_timeout: Read timeout in seconds.
|
||||
logger: Logger used for observability.
|
||||
"""
|
||||
self.endpoint_url = endpoint_url
|
||||
self.access_key = access_key
|
||||
self.secret_key = secret_key
|
||||
self.region = region
|
||||
self.use_ssl = use_ssl
|
||||
self.max_retry_attempts = max_retry_attempts
|
||||
self.retry_mode = retry_mode
|
||||
self.connect_timeout = connect_timeout
|
||||
self.read_timeout = read_timeout
|
||||
self.logger = logger
|
||||
|
||||
boto_config = Config(
|
||||
region_name=region,
|
||||
retries={
|
||||
'max_attempts': max_retry_attempts,
|
||||
'mode': retry_mode,
|
||||
},
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
)
|
||||
|
||||
self.minio_client = boto3.client(
|
||||
's3',
|
||||
endpoint_url=endpoint_url,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=boto_config,
|
||||
use_ssl=use_ssl,
|
||||
)
|
||||
|
||||
self.logger.info(f'MinIO client initialized successfully: {endpoint_url}')
|
||||
|
||||
def fetch_file(self, bucket_name: str, file_name: str) -> BytesIO:
|
||||
"""
|
||||
Fetch an object from MinIO and return its contents as `BytesIO`.
|
||||
|
||||
Args:
|
||||
bucket_name: MinIO bucket where the object resides.
|
||||
file_name: Object key to download inside the bucket.
|
||||
|
||||
Returns:
|
||||
BytesIO: File-like stream containing the downloaded bytes.
|
||||
|
||||
Raises:
|
||||
OSError: If the download fails (network, permissions, missing key, etc.).
|
||||
"""
|
||||
self.logger.info(f'Fetching file from MinIO: {bucket_name}/{file_name}')
|
||||
response = self.minio_client.get_object(Bucket=bucket_name, Key=file_name)
|
||||
|
||||
with response['Body'] as body:
|
||||
file_content = body.read()
|
||||
|
||||
file_size = len(file_content)
|
||||
|
||||
self.logger.info(
|
||||
f'File fetched successfully: {bucket_name}/{file_name} ({file_size} bytes)'
|
||||
)
|
||||
|
||||
return BytesIO(file_content)
|
||||
|
||||
def delete_file(self, bucket_name: str, file_name: str) -> None:
|
||||
"""
|
||||
Remove an object from MinIO storage.
|
||||
|
||||
Args:
|
||||
bucket_name: Bucket that contains the object.
|
||||
file_name: Object key to delete.
|
||||
"""
|
||||
self.logger.info(f'Deleting file from MinIO: {bucket_name}/{file_name}')
|
||||
self.minio_client.delete_object(Bucket=bucket_name, Key=file_name)
|
||||
self.logger.info(f'File deleted successfully: {bucket_name}/{file_name}')
|
||||
@@ -66,19 +66,17 @@ class TrainingRepository:
|
||||
ValueError: If transformed data is empty
|
||||
Exception: If data loading, preprocessing, or training fails
|
||||
"""
|
||||
# Load data from BytesIO
|
||||
self.logger.info('Loading data from BytesIO file')
|
||||
data = load_data(uploaded_file, params.line_separator, params.decimal_separator)
|
||||
|
||||
# Initialize and fit data preprocessor
|
||||
process_data = self.init_data_preprocessor(params)
|
||||
self.logger.info('Initializing and fitting data preprocessor')
|
||||
process_data = self._init_data_preprocessor(params)
|
||||
process_data.fit(data)
|
||||
data_view = process_data.transform(data)
|
||||
|
||||
# Validate transformed data
|
||||
if len(data_view) <= 0:
|
||||
raise ValueError('Data view is empty after transformation')
|
||||
|
||||
# Split data into train/test sets
|
||||
self.logger.info('Splitting data into train/test sets')
|
||||
x_train, x_test, y_train, y_test = split_train_test(
|
||||
data_view[params.variable_columns],
|
||||
data_view[params.target_variable],
|
||||
@@ -87,18 +85,18 @@ class TrainingRepository:
|
||||
random_state=42,
|
||||
)
|
||||
|
||||
# Prepare training data
|
||||
self.logger.info('Preparing training data')
|
||||
data_train = pd.concat([x_train, y_train], axis=1)
|
||||
scaler_dict = self.init_scaler_dict(process_data, params)
|
||||
scaler_dict = self._init_scaler_dict(process_data, params)
|
||||
|
||||
# Create and train linear regression model
|
||||
self.logger.info('Training linear regression model')
|
||||
regr = LinearRegressionModel(
|
||||
target_variable=params.target_variable,
|
||||
variable_columns=params.variable_columns,
|
||||
)
|
||||
|
||||
regr.fit(data_train)
|
||||
|
||||
# Return training result
|
||||
return TrainModelResult(
|
||||
params=params,
|
||||
process_data=process_data,
|
||||
@@ -110,7 +108,70 @@ class TrainingRepository:
|
||||
scaler_dict=scaler_dict,
|
||||
)
|
||||
|
||||
def init_scaler_dict(self, process_data: DataPreprocessor, params: TrainModelParams) -> dict:
|
||||
def after_train_calculation(
|
||||
self, params: TrainModelParams, tmr: TrainModelResult
|
||||
) -> TrainModelResult:
|
||||
"""
|
||||
Perform post-training calculations: predictions, denormalization, and metrics.
|
||||
|
||||
This method completes the training pipeline by:
|
||||
1. Making predictions on test set
|
||||
2. Denormalizing all data (if scaler was used)
|
||||
3. Reordering data by index
|
||||
4. Calculating evaluation metrics (MSE, MAE, R²)
|
||||
|
||||
Args:
|
||||
params: Training parameters used during model training
|
||||
tmr: Result object from training
|
||||
|
||||
Returns:
|
||||
TrainModelResult: Updated result with predictions, denormalized data,
|
||||
and metrics (mse_val, mae_val, r2_val)
|
||||
"""
|
||||
self.logger.info('Making predictions on test set')
|
||||
y_pred_array = tmr.regr.predict(tmr.x_test)
|
||||
|
||||
if params.use_scaler:
|
||||
scaler = tmr.process_data.get_scaler()
|
||||
self.logger.info('Denormalizing features')
|
||||
|
||||
for col in params.variable_columns:
|
||||
tmr.x_train[col] = scaler.denormalize_single_input(tmr.x_train[col], col)
|
||||
tmr.x_test[col] = scaler.denormalize_single_input(tmr.x_test[col], col)
|
||||
|
||||
self.logger.info('Denormalizing target variable')
|
||||
tmr.y_train = scaler.denormalize_single_input(tmr.y_train, params.target_variable)
|
||||
tmr.y_test = scaler.denormalize_single_input(tmr.y_test, params.target_variable)
|
||||
y_pred_array = scaler.denormalize_predictions(y_pred_array, params.target_variable)
|
||||
|
||||
self.logger.info('Adding index to predictions')
|
||||
tmr.y_pred = pd.Series(y_pred_array, index=tmr.y_test.index)
|
||||
tmr.y_pred.name = f'{params.target_variable}_pred'
|
||||
|
||||
self.logger.info('Reordering all data by index')
|
||||
tmr.x_train = tmr.x_train.sort_index()
|
||||
tmr.x_test = tmr.x_test.sort_index()
|
||||
tmr.y_train = tmr.y_train.sort_index()
|
||||
tmr.y_test = tmr.y_test.sort_index()
|
||||
tmr.y_pred = tmr.y_pred.sort_index()
|
||||
|
||||
self.logger.info('Calculating evaluation metrics')
|
||||
assert tmr.y_pred is not None, 'y_pred should be set at this point'
|
||||
|
||||
tmr.mse_val = round(
|
||||
mse(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)),
|
||||
2,
|
||||
)
|
||||
|
||||
tmr.mae_val = round(
|
||||
mae(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)),
|
||||
2,
|
||||
)
|
||||
|
||||
tmr.r2_val = round(r2(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)), 2)
|
||||
return tmr
|
||||
|
||||
def _init_scaler_dict(self, process_data: DataPreprocessor, params: TrainModelParams) -> dict:
|
||||
"""
|
||||
Initialize dictionary containing scaling parameters for features and target.
|
||||
|
||||
@@ -152,69 +213,7 @@ class TrainingRepository:
|
||||
|
||||
return scaler_dict
|
||||
|
||||
def after_train_calculation(
|
||||
self, params: TrainModelParams, tmr: TrainModelResult
|
||||
) -> TrainModelResult:
|
||||
"""
|
||||
Perform post-training calculations: predictions, denormalization, and metrics.
|
||||
|
||||
This method completes the training pipeline by:
|
||||
1. Making predictions on test set
|
||||
2. Denormalizing all data (if scaler was used)
|
||||
3. Reordering data by index
|
||||
4. Calculating evaluation metrics (MSE, MAE, R²)
|
||||
|
||||
Args:
|
||||
params: Training parameters used during model training
|
||||
tmr: Result object from training
|
||||
|
||||
Returns:
|
||||
TrainModelResult: Updated result with predictions, denormalized data,
|
||||
and metrics (mse_val, mae_val, r2_val)
|
||||
"""
|
||||
# Make predictions on test set
|
||||
y_pred_array = tmr.regr.predict(tmr.x_test)
|
||||
|
||||
# Denormalize data if scaler was used
|
||||
if params.use_scaler:
|
||||
scaler = tmr.process_data.get_scaler()
|
||||
|
||||
# Denormalize features
|
||||
for col in params.variable_columns:
|
||||
tmr.x_train[col] = scaler.denormalize_single_input(tmr.x_train[col], col)
|
||||
tmr.x_test[col] = scaler.denormalize_single_input(tmr.x_test[col], col)
|
||||
|
||||
# Denormalize target variable
|
||||
tmr.y_train = scaler.denormalize_single_input(tmr.y_train, params.target_variable)
|
||||
tmr.y_test = scaler.denormalize_single_input(tmr.y_test, params.target_variable)
|
||||
y_pred_array = scaler.denormalize_predictions(y_pred_array, params.target_variable)
|
||||
|
||||
# Add index to predictions
|
||||
tmr.y_pred = pd.Series(y_pred_array, index=tmr.y_test.index)
|
||||
tmr.y_pred.name = f'{params.target_variable}_pred'
|
||||
|
||||
# Reorder all data by index
|
||||
tmr.x_train = tmr.x_train.sort_index()
|
||||
tmr.x_test = tmr.x_test.sort_index()
|
||||
tmr.y_train = tmr.y_train.sort_index()
|
||||
tmr.y_test = tmr.y_test.sort_index()
|
||||
tmr.y_pred = tmr.y_pred.sort_index()
|
||||
|
||||
# Calculate evaluation metrics
|
||||
assert tmr.y_pred is not None, 'y_pred should be set at this point'
|
||||
tmr.mse_val = round(
|
||||
mse(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)),
|
||||
2,
|
||||
)
|
||||
tmr.mae_val = round(
|
||||
mae(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)),
|
||||
2,
|
||||
)
|
||||
tmr.r2_val = round(r2(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)), 2)
|
||||
|
||||
return tmr
|
||||
|
||||
def init_data_preprocessor(self, params: TrainModelParams) -> DataPreprocessor:
|
||||
def _init_data_preprocessor(self, params: TrainModelParams) -> DataPreprocessor:
|
||||
"""
|
||||
Initialize DataPreprocessor with training parameters.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user