SIENTIAPDE-1241: refactor train_model workflow due to I/O errors.

This commit is contained in:
Bruno Domingues
2025-10-22 15:37:56 -03:00
parent f2a1c88ff3
commit 5789a13023
31 changed files with 37878 additions and 6097 deletions

View File

@@ -51,8 +51,7 @@ def build_mlflow_config() -> dict[str, Any]:
dict: MLFlow configuration dictionary with all required parameters
"""
return {
'host': getenv('MLFLOW_HOST', 'http://localhost'),
'port': int(getenv('MLFLOW_PORT', '5080')),
'url': getenv('MLFLOW_URL', 'http://localhost:5080'),
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
'password': getenv('MLFLOW_PASSWORD', 'aignosi'),
}

View File

@@ -0,0 +1,12 @@
class ModelTrainingError(Exception):
def __init__(self, model_trained: bool, model_saved: bool, message: str | None = None):
self.model_trained = model_trained
self.model_saved = model_saved
if message is None:
message = (
'Model training workflow failed '
f'(model_trained={model_trained}, model_saved={model_saved})'
)
super().__init__(message)

View File

@@ -184,22 +184,22 @@ class TrainModelParams:
>>> params = TrainModelParams.from_dict(data)
>>> params.validate_business_rules() # Raises ValueError if invalid
"""
# Validate train_size range (1-99%)
if not 1 <= self.train_size <= 99:
raise ValueError(f'train_size must be between 1 and 99, got {self.train_size}')
# Validate train_size range (10-100%)
if not 10 <= self.train_size <= 100:
raise ValueError(f'train_size must be between 10 and 100, got {self.train_size}')
# Validate variable_columns is not empty
if not self.variable_columns:
raise ValueError('variable_columns cannot be empty')
# Validate positive integers
if self.lag_train <= 0:
if self.lag_train < 0:
raise ValueError(f'lag_train must be positive, got {self.lag_train}')
if self.lag_val <= 0:
if self.lag_val < 0:
raise ValueError(f'lag_val must be positive, got {self.lag_val}')
if self.window <= 0:
if self.window < 0:
raise ValueError(f'window must be positive, got {self.window}')
# Validate low_lim and upp_lim consistency
@@ -218,13 +218,6 @@ class TrainModelParams:
f'Got low_lim={self.low_lim[var]}, upp_lim={self.upp_lim[var]}'
)
# Validate target_variable is in variable_columns
if self.target_variable not in self.variable_columns:
raise ValueError(
f'target_variable "{self.target_variable}" must be in variable_columns: '
f'{self.variable_columns}'
)
# Validate bucket_name and file_name are not empty
if not self.bucket_name.strip():
raise ValueError('bucket_name cannot be empty or whitespace')

View File

@@ -9,6 +9,7 @@ and logging model runs to MLFlow.
"""
import os
import shutil
from datetime import datetime
from os import makedirs, path
@@ -22,14 +23,81 @@ from model_manager.sientia.reports import Reports # type: ignore[import-untyped
from model_manager.utils.models.train_model_result import TrainModelResult
class MLFlowRepository:
def __init__(self, host, username, password, logger: Logger):
self.model_serving = ModelServing(
tracking_uri=host, username=username, password=password, logger=logger
)
class ModelRepository:
def __init__(self, url, username, password, logger: Logger):
self.model_serving = ModelServing(tracking_uri=url, username=username, password=password)
self.logger = logger
def get_next_run_name(self, experiment_name: str) -> str:
def save_model(self, train_result: TrainModelResult) -> TrainModelResult:
"""
Save a trained ML model and its artifacts to MLflow.
This activity orchestrates the complete model saving pipeline:
1. Generates the next run name for the experiment
2. Creates and organizes artifacts (reports, data files)
3. Logs model, parameters, metrics, and artifacts to MLflow
Args:
input_data: Configuration for model saving operation
Required keys:
- metadata (dict): Workflow execution metadata
- train_result (TrainModelResult): Training result with model and metrics
Returns:
TrainModelResult: Updated training result with run_name and artifacts
Raises:
Exception: If model saving fails (after sending notification)
"""
experiment_name = train_result.params.experiment_name
self.logger.info(f'Starting model save for experiment: {experiment_name}')
# Step 1: Generate next run name
self.logger.info('Generating run name')
train_result.run_name = self._get_next_run_name(experiment_name)
self.logger.info(f'Generated run name: {train_result.run_name}')
# Step 2: Generate artifacts (reports, CSV files)
self.logger.info('Generating artifacts')
train_result = self._generate_artifacts(train_result)
self.logger.info('Artifacts generated successfully')
# Step 3: Save run to MLflow
self.logger.info('Saving run to MLflow')
self._save_run(train_result)
self.logger.info(
f'Model saved successfully - Run: {train_result.run_name}, '
f'Experiment: {experiment_name}'
)
return train_result
def cleanup_run_directory(self, run_dir: str) -> None:
"""
Clean up temporary run directory after model training.
This activity deletes the temporary directory created during model training
and artifact generation. It implements idempotent cleanup to handle cases
where the directory may have already been deleted.
Args:
run_dir (str): Path to the run directory to delete
"""
if not run_dir:
self.logger.info('No run directory specified, skipping cleanup')
return
self.logger.info(f'Cleaning up run directory: {run_dir}')
if os.path.exists(run_dir):
shutil.rmtree(run_dir)
self.logger.info(f'Run directory deleted successfully: {run_dir}')
else:
self.logger.info(f'Run directory already deleted: {run_dir}')
def _get_next_run_name(self, experiment_name: str) -> str:
"""
Generates the next run name for a given experiment.
@@ -46,7 +114,7 @@ class MLFlowRepository:
next_run_number = len(runs) + 1
return f'{experiment_name}-{next_run_number}'
def generate_artifacts(self, data: TrainModelResult) -> TrainModelResult:
def _generate_artifacts(self, data: TrainModelResult) -> TrainModelResult:
"""
Generates and organizes artifacts related to the training process, such as reports and data files.
@@ -87,7 +155,7 @@ class MLFlowRepository:
self._setup_run_directory(data.run_dir, header_file_path)
return self._generate_report(reference_data, current_data, data)
def save_run(self, data: TrainModelResult):
def _save_run(self, data: TrainModelResult):
"""
Logs the details of a machine learning run, including parameters, metrics, models, and artifacts,
to the Sientia tracking system.
@@ -122,60 +190,48 @@ class MLFlowRepository:
self.logger.error(error_msg)
raise ValueError(error_msg)
try:
# Prepare parameters
train_test_split = f'{data.params.train_size}-{100 - data.params.train_size}'
interval_strs = [
(str(interval[0]), str(interval[1]))
for interval in (data.params.removed_intervals or [])
]
# Prepare parameters
train_test_split = f'{data.params.train_size}-{100 - data.params.train_size}'
# Set experiment and create run
self.model_serving.set_experiment(data.params.experiment_name)
self.logger.info(
f"Logging run '{data.run_name}' to experiment '{data.params.experiment_name}'"
)
interval_strs = [
(str(interval[0]), str(interval[1]))
for interval in (data.params.removed_intervals or [])
]
with self.model_serving.save_experiment(
run_name=data.run_name, description=data.params.experiment_name
):
# Log model parameters
self.model_serving.log_param('model_type', 'Linear Regression')
self.model_serving.log_param('target_variable', data.params.target_variable)
self.model_serving.log_param('input_variables', data.params.variable_columns)
self.model_serving.log_param('lag_train', data.params.lag_train)
self.model_serving.log_param('lag_val', data.params.lag_val)
self.model_serving.log_param('ma', data.params.window)
self.model_serving.log_param('low_lim', data.params.low_lim)
self.model_serving.log_param('upp_lim', data.params.upp_lim)
self.model_serving.log_param('normalized', data.scaler_dict)
self.model_serving.log_param('ar', data.params.include_ar)
self.model_serving.log_param('Train_test_split', train_test_split)
self.model_serving.log_param('Removed_intervals', interval_strs)
self.model_serving.log_param('Retrain', False)
# Set experiment and create run
self.model_serving.set_experiment(data.params.experiment_name)
# Log evaluation metrics
self.model_serving.log_metric('MSE', data.mse_val)
self.model_serving.log_metric('R2', data.r2_val)
self.model_serving.log_metric('MAE', data.mae_val)
with self.model_serving.save_experiment(
run_name=data.run_name, description=data.params.experiment_name
):
# Log model parameters
self.model_serving.log_param('model_type', 'Linear Regression')
self.model_serving.log_param('target_variable', data.params.target_variable)
self.model_serving.log_param('input_variables', data.params.variable_columns)
self.model_serving.log_param('lag_train', data.params.lag_train)
self.model_serving.log_param('lag_val', data.params.lag_val)
self.model_serving.log_param('ma', data.params.window)
self.model_serving.log_param('low_lim', data.params.low_lim)
self.model_serving.log_param('upp_lim', data.params.upp_lim)
self.model_serving.log_param('normalized', data.scaler_dict)
self.model_serving.log_param('ar', data.params.include_ar)
self.model_serving.log_param('Train_test_split', train_test_split)
self.model_serving.log_param('Removed_intervals', interval_strs)
self.model_serving.log_param('Retrain', False)
# Log models
self.model_serving.log_model(data.process_data, 'data_model')
self.model_serving.log_model(data.regr, 'prediction_model')
# Log evaluation metrics
self.model_serving.log_metric('MSE', data.mse_val)
self.model_serving.log_metric('R2', data.r2_val)
self.model_serving.log_metric('MAE', data.mae_val)
# Log artifacts
self.model_serving.log_artifact(data.report_path)
self.model_serving.log_artifact(data.train_data_path)
self.model_serving.log_artifact(data.test_data_path)
# Log models
self.model_serving.log_model(data.process_data, 'data_model')
self.model_serving.log_model(data.regr, 'prediction_model')
self.logger.info(
f"Successfully logged run '{data.run_name}' with metrics: MSE={data.mse_val:.4f}, R2={data.r2_val:.4f}, MAE={data.mae_val:.4f}"
)
except Exception as e:
error_msg = f"Failed to save run '{data.run_name}' to MLflow: {str(e)}"
self.logger.error(error_msg)
raise RuntimeError(error_msg) from e
# Log artifacts
self.model_serving.log_artifact(data.report_path)
self.model_serving.log_artifact(data.train_data_path)
self.model_serving.log_artifact(data.test_data_path)
def _init_artifacts_data(self, data: TrainModelResult) -> tuple[pd.DataFrame, pd.DataFrame]:
"""
@@ -258,7 +314,6 @@ class MLFlowRepository:
try:
makedirs(run_dir, exist_ok=True)
self.logger.info(f'Created run directory: {run_dir}')
return run_dir
except PermissionError as e:
error_msg = f'Permission denied when creating directory: {run_dir}'
@@ -298,9 +353,6 @@ class MLFlowRepository:
# Copy header file to run directory
header_dest = path.join(run_dir, 'header.html')
shutil.copy(header_file_path, header_dest)
self.logger.info(f'Run directory setup completed successfully in: {run_dir}')
except FileNotFoundError as e:
error_msg = f'Header file not found: {header_file_path}'
self.logger.error(error_msg)
@@ -360,20 +412,16 @@ class MLFlowRepository:
# Save HTML report
data.report_path = path.join(data.run_dir, 'report.html')
report.save_all_sections_html(data.report_path)
self.logger.info(f'Generated HTML report: {data.report_path}')
# Save training data CSV
data.train_data_path = path.join(data.run_dir, 'train_data.csv')
reference_data.to_csv(data.train_data_path, index=False)
self.logger.info(f'Saved training data: {data.train_data_path}')
# Save test data CSV
data.test_data_path = path.join(data.run_dir, 'test_data.csv')
current_data.to_csv(data.test_data_path, index=False)
self.logger.info(f'Saved test data: {data.test_data_path}')
return data
except ValueError as e:
error_msg = f'Failed to convert data to float64 for report generation: {str(e)}'
self.logger.error(error_msg)

View File

@@ -0,0 +1,129 @@
from io import BytesIO
import boto3 # type: ignore[import-untyped]
from botocore.config import Config # type: ignore[import-untyped]
from sientia_do.observability.logger import Logger
class StorageRepository:
"""
MinIO (S3-compatible) storage activities for file operations.
This class provides activities for interacting with MinIO object storage,
including file download and deletion operations. It handles authentication,
connection management, and comprehensive error handling.
The class implements best practices for S3/MinIO operations:
- Connection reuse (boto3 client is thread-safe)
- Automatic retry with exponential backoff
- Comprehensive error handling and logging
- Notification integration for critical errors
Attributes:
endpoint_url (str): MinIO server endpoint URL
access_key (str): MinIO access key ID
secret_key (str): MinIO secret access key
region (str): MinIO region name
use_ssl (bool): Whether to use SSL/TLS for connections
minio_client: Boto3 S3 client configured for MinIO
"""
def __init__(
self,
endpoint_url: str,
access_key: str,
secret_key: str,
region: str,
use_ssl: bool,
max_retry_attempts: int,
retry_mode: str,
connect_timeout: int,
read_timeout: int,
logger: Logger,
):
"""
Initialize a reusable MinIO client with retry configuration.
Args:
endpoint_url: MinIO server endpoint URL (e.g., http://localhost:9000).
access_key: MinIO access key ID for authentication.
secret_key: MinIO secret access key for authentication.
region: MinIO region name (e.g., us-east-1).
use_ssl: Whether to use SSL/TLS for connections.
max_retry_attempts: Maximum number of retry attempts (e.g., 3).
retry_mode: Retry policy to apply (standard, legacy, adaptive).
connect_timeout: Connection timeout in seconds.
read_timeout: Read timeout in seconds.
logger: Logger used for observability.
"""
self.endpoint_url = endpoint_url
self.access_key = access_key
self.secret_key = secret_key
self.region = region
self.use_ssl = use_ssl
self.max_retry_attempts = max_retry_attempts
self.retry_mode = retry_mode
self.connect_timeout = connect_timeout
self.read_timeout = read_timeout
self.logger = logger
boto_config = Config(
region_name=region,
retries={
'max_attempts': max_retry_attempts,
'mode': retry_mode,
},
connect_timeout=connect_timeout,
read_timeout=read_timeout,
)
self.minio_client = boto3.client(
's3',
endpoint_url=endpoint_url,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=boto_config,
use_ssl=use_ssl,
)
self.logger.info(f'MinIO client initialized successfully: {endpoint_url}')
def fetch_file(self, bucket_name: str, file_name: str) -> BytesIO:
"""
Fetch an object from MinIO and return its contents as `BytesIO`.
Args:
bucket_name: MinIO bucket where the object resides.
file_name: Object key to download inside the bucket.
Returns:
BytesIO: File-like stream containing the downloaded bytes.
Raises:
OSError: If the download fails (network, permissions, missing key, etc.).
"""
self.logger.info(f'Fetching file from MinIO: {bucket_name}/{file_name}')
response = self.minio_client.get_object(Bucket=bucket_name, Key=file_name)
with response['Body'] as body:
file_content = body.read()
file_size = len(file_content)
self.logger.info(
f'File fetched successfully: {bucket_name}/{file_name} ({file_size} bytes)'
)
return BytesIO(file_content)
def delete_file(self, bucket_name: str, file_name: str) -> None:
"""
Remove an object from MinIO storage.
Args:
bucket_name: Bucket that contains the object.
file_name: Object key to delete.
"""
self.logger.info(f'Deleting file from MinIO: {bucket_name}/{file_name}')
self.minio_client.delete_object(Bucket=bucket_name, Key=file_name)
self.logger.info(f'File deleted successfully: {bucket_name}/{file_name}')

View File

@@ -66,19 +66,17 @@ class TrainingRepository:
ValueError: If transformed data is empty
Exception: If data loading, preprocessing, or training fails
"""
# Load data from BytesIO
self.logger.info('Loading data from BytesIO file')
data = load_data(uploaded_file, params.line_separator, params.decimal_separator)
# Initialize and fit data preprocessor
process_data = self.init_data_preprocessor(params)
self.logger.info('Initializing and fitting data preprocessor')
process_data = self._init_data_preprocessor(params)
process_data.fit(data)
data_view = process_data.transform(data)
# Validate transformed data
if len(data_view) <= 0:
raise ValueError('Data view is empty after transformation')
# Split data into train/test sets
self.logger.info('Splitting data into train/test sets')
x_train, x_test, y_train, y_test = split_train_test(
data_view[params.variable_columns],
data_view[params.target_variable],
@@ -87,18 +85,18 @@ class TrainingRepository:
random_state=42,
)
# Prepare training data
self.logger.info('Preparing training data')
data_train = pd.concat([x_train, y_train], axis=1)
scaler_dict = self.init_scaler_dict(process_data, params)
scaler_dict = self._init_scaler_dict(process_data, params)
# Create and train linear regression model
self.logger.info('Training linear regression model')
regr = LinearRegressionModel(
target_variable=params.target_variable,
variable_columns=params.variable_columns,
)
regr.fit(data_train)
# Return training result
return TrainModelResult(
params=params,
process_data=process_data,
@@ -110,7 +108,70 @@ class TrainingRepository:
scaler_dict=scaler_dict,
)
def init_scaler_dict(self, process_data: DataPreprocessor, params: TrainModelParams) -> dict:
def after_train_calculation(
self, params: TrainModelParams, tmr: TrainModelResult
) -> TrainModelResult:
"""
Perform post-training calculations: predictions, denormalization, and metrics.
This method completes the training pipeline by:
1. Making predictions on test set
2. Denormalizing all data (if scaler was used)
3. Reordering data by index
4. Calculating evaluation metrics (MSE, MAE, R²)
Args:
params: Training parameters used during model training
tmr: Result object from training
Returns:
TrainModelResult: Updated result with predictions, denormalized data,
and metrics (mse_val, mae_val, r2_val)
"""
self.logger.info('Making predictions on test set')
y_pred_array = tmr.regr.predict(tmr.x_test)
if params.use_scaler:
scaler = tmr.process_data.get_scaler()
self.logger.info('Denormalizing features')
for col in params.variable_columns:
tmr.x_train[col] = scaler.denormalize_single_input(tmr.x_train[col], col)
tmr.x_test[col] = scaler.denormalize_single_input(tmr.x_test[col], col)
self.logger.info('Denormalizing target variable')
tmr.y_train = scaler.denormalize_single_input(tmr.y_train, params.target_variable)
tmr.y_test = scaler.denormalize_single_input(tmr.y_test, params.target_variable)
y_pred_array = scaler.denormalize_predictions(y_pred_array, params.target_variable)
self.logger.info('Adding index to predictions')
tmr.y_pred = pd.Series(y_pred_array, index=tmr.y_test.index)
tmr.y_pred.name = f'{params.target_variable}_pred'
self.logger.info('Reordering all data by index')
tmr.x_train = tmr.x_train.sort_index()
tmr.x_test = tmr.x_test.sort_index()
tmr.y_train = tmr.y_train.sort_index()
tmr.y_test = tmr.y_test.sort_index()
tmr.y_pred = tmr.y_pred.sort_index()
self.logger.info('Calculating evaluation metrics')
assert tmr.y_pred is not None, 'y_pred should be set at this point'
tmr.mse_val = round(
mse(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)),
2,
)
tmr.mae_val = round(
mae(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)),
2,
)
tmr.r2_val = round(r2(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)), 2)
return tmr
def _init_scaler_dict(self, process_data: DataPreprocessor, params: TrainModelParams) -> dict:
"""
Initialize dictionary containing scaling parameters for features and target.
@@ -152,69 +213,7 @@ class TrainingRepository:
return scaler_dict
def after_train_calculation(
self, params: TrainModelParams, tmr: TrainModelResult
) -> TrainModelResult:
"""
Perform post-training calculations: predictions, denormalization, and metrics.
This method completes the training pipeline by:
1. Making predictions on test set
2. Denormalizing all data (if scaler was used)
3. Reordering data by index
4. Calculating evaluation metrics (MSE, MAE, R²)
Args:
params: Training parameters used during model training
tmr: Result object from training
Returns:
TrainModelResult: Updated result with predictions, denormalized data,
and metrics (mse_val, mae_val, r2_val)
"""
# Make predictions on test set
y_pred_array = tmr.regr.predict(tmr.x_test)
# Denormalize data if scaler was used
if params.use_scaler:
scaler = tmr.process_data.get_scaler()
# Denormalize features
for col in params.variable_columns:
tmr.x_train[col] = scaler.denormalize_single_input(tmr.x_train[col], col)
tmr.x_test[col] = scaler.denormalize_single_input(tmr.x_test[col], col)
# Denormalize target variable
tmr.y_train = scaler.denormalize_single_input(tmr.y_train, params.target_variable)
tmr.y_test = scaler.denormalize_single_input(tmr.y_test, params.target_variable)
y_pred_array = scaler.denormalize_predictions(y_pred_array, params.target_variable)
# Add index to predictions
tmr.y_pred = pd.Series(y_pred_array, index=tmr.y_test.index)
tmr.y_pred.name = f'{params.target_variable}_pred'
# Reorder all data by index
tmr.x_train = tmr.x_train.sort_index()
tmr.x_test = tmr.x_test.sort_index()
tmr.y_train = tmr.y_train.sort_index()
tmr.y_test = tmr.y_test.sort_index()
tmr.y_pred = tmr.y_pred.sort_index()
# Calculate evaluation metrics
assert tmr.y_pred is not None, 'y_pred should be set at this point'
tmr.mse_val = round(
mse(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)),
2,
)
tmr.mae_val = round(
mae(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)),
2,
)
tmr.r2_val = round(r2(tmr.y_test.astype(np.float64), tmr.y_pred.astype(np.float64)), 2)
return tmr
def init_data_preprocessor(self, params: TrainModelParams) -> DataPreprocessor:
def _init_data_preprocessor(self, params: TrainModelParams) -> DataPreprocessor:
"""
Initialize DataPreprocessor with training parameters.