feat: integrate PluginStore and MinIO repository into model manager activities

- Added PluginStore integration for model management.
- Replaced StorageRepository with MinIORepository in Activities, Cleanup, and Training classes.
- Updated training logic to handle validation files and improved data management.
- Enhanced configuration for MinIO and PluginStore in connectors.
- Removed deprecated model repository and storage repository files.
- Updated environment variable handling for new configurations.
This commit is contained in:
vitor-aignosi
2026-03-11 17:35:05 -03:00
parent 9d71c0cf80
commit cf5111e520
23 changed files with 1480 additions and 4588 deletions

View File

@@ -33,7 +33,8 @@ class TrainModelParams:
use_scaler (bool): Whether to use a scaler for data normalization.
include_ar (bool): Whether to include autoregressive variables.
bucket_name (str): Name of the MinIO bucket containing training data.
file_name (str): Name of the file in the MinIO bucket.
file_name (str): Name of the training file in the MinIO bucket.
validation_file_name (str | None): Optional name of the validation file in the same MinIO bucket as the training file.
line_separator (str): Line separator used in the CSV file.
decimal_separator (str): Decimal separator used in the CSV file.
date_column (str | None): Name of the date/time column. If set with date_format, the column is parsed as datetime.
@@ -66,6 +67,7 @@ class TrainModelParams:
include_ar: bool
bucket_name: str
file_name: str
validation_file_name: str | None
line_separator: str
decimal_separator: str
date_column: str | None
@@ -122,6 +124,9 @@ class TrainModelParams:
include_ar=cls._check_none(data.get('include_ar'), bool, 'include_ar'),
bucket_name=cls._check_none(data.get('bucket_name'), str, 'bucket_name'),
file_name=cls._check_none(data.get('file_name'), str, 'file_name'),
validation_file_name=cls._check_type(
data.get('validation_file_name'), str, 'validation_file_name'
),
line_separator=cls._check_none(data.get('line_separator'), str, 'line_separator'),
decimal_separator=cls._check_none(
data.get('decimal_separator'), str, 'decimal_separator'
@@ -220,6 +225,7 @@ class TrainModelParams:
self._validate_limits()
self._validate_required_strings()
self._validate_date_format()
self._validate_validation_file_name()
def _validate_numeric_ranges(self) -> None:
"""Validate numeric parameters are within acceptable ranges."""
@@ -335,3 +341,21 @@ class TrainModelParams:
"""Validate date_format is one of the allowed frontend formats when set."""
if self.date_format:
validate_frontend_date_format(self.date_format)
def _validate_validation_file_name(self) -> None:
"""
Validate that validation_file_name, when provided, is not empty or whitespace.
This field is optional; when present it must point to a valid object key in the
same MinIO bucket specified by bucket_name.
"""
if self.validation_file_name is None:
return
if not isinstance(self.validation_file_name, str):
raise TypeError(
f'validation_file_name must be a string, got {type(self.validation_file_name).__name__}'
)
if not self.validation_file_name.strip():
raise ValueError('validation_file_name cannot be empty or whitespace')

View File

@@ -2,7 +2,6 @@ from dataclasses import dataclass
import pandas as pd
from model_manager.sientia.models import DataPreprocessor, LinearRegressionModel
from model_manager.utils.models.train_model_params import TrainModelParams
@@ -12,18 +11,15 @@ class TrainModelResult:
A data container for storing the results of a machine learning training process.
This dataclass encapsulates all outputs from the training pipeline, including
the trained model, datasets, evaluation metrics, and paths to generated artifacts.
the prepared datasets, evaluation metrics, and paths to generated artifacts.
It is used to pass results between activities in the training workflow.
Attributes:
params (TrainModelParams): The parameters used to train the model.
process_data (DataPreprocessor): The data preprocessor object used to process the input data.
x_train (pd.DataFrame): The training dataset features.
x_test (pd.DataFrame): The testing dataset features.
y_train (pd.DataFrame): The training dataset target values.
y_test (pd.DataFrame): The testing dataset target values.
regr (LinearRegressionModel): The trained linear regression model.
scaler_dict (dict): A dictionary containing the scalers used to scale the features and target values.
y_pred (pd.Series | None): The predicted target values for the testing dataset. Default is None.
y_train_pred (pd.Series | None): The predicted target values for the training dataset. Default is None.
mse_val (float | None): The Mean Squared Error (MSE) of the predictions. Default is None.
@@ -35,17 +31,13 @@ class TrainModelResult:
report_path (str | None): The path to the generated HTML report file. Default is None.
train_data_path (str | None): The path to the training dataset CSV file. Default is None.
test_data_path (str | None): The path to the testing dataset CSV file. Default is None.
run_dir (str | None): The path to the run directory containing all artifacts. Default is None.
"""
params: TrainModelParams
process_data: DataPreprocessor
x_train: pd.DataFrame
x_test: pd.DataFrame
y_train: pd.Series
y_test: pd.Series
regr: LinearRegressionModel
scaler_dict: dict
y_pred: pd.Series | None = None
y_train_pred: pd.Series | None = None
mse_val: float | None = None
@@ -57,4 +49,3 @@ class TrainModelResult:
report_path: str | None = None
train_data_path: str | None = None
test_data_path: str | None = None
run_dir: str | None = None