feat: integrate PluginStore and MinIO repository into model manager activities
- Added PluginStore integration for model management. - Replaced StorageRepository with MinIORepository in Activities, Cleanup, and Training classes. - Updated training logic to handle validation files and improved data management. - Enhanced configuration for MinIO and PluginStore in connectors. - Removed deprecated model repository and storage repository files. - Updated environment variable handling for new configurations.
This commit is contained in:
@@ -33,7 +33,8 @@ class TrainModelParams:
|
||||
use_scaler (bool): Whether to use a scaler for data normalization.
|
||||
include_ar (bool): Whether to include autoregressive variables.
|
||||
bucket_name (str): Name of the MinIO bucket containing training data.
|
||||
file_name (str): Name of the file in the MinIO bucket.
|
||||
file_name (str): Name of the training file in the MinIO bucket.
|
||||
validation_file_name (str | None): Optional name of the validation file in the same MinIO bucket as the training file.
|
||||
line_separator (str): Line separator used in the CSV file.
|
||||
decimal_separator (str): Decimal separator used in the CSV file.
|
||||
date_column (str | None): Name of the date/time column. If set with date_format, the column is parsed as datetime.
|
||||
@@ -66,6 +67,7 @@ class TrainModelParams:
|
||||
include_ar: bool
|
||||
bucket_name: str
|
||||
file_name: str
|
||||
validation_file_name: str | None
|
||||
line_separator: str
|
||||
decimal_separator: str
|
||||
date_column: str | None
|
||||
@@ -122,6 +124,9 @@ class TrainModelParams:
|
||||
include_ar=cls._check_none(data.get('include_ar'), bool, 'include_ar'),
|
||||
bucket_name=cls._check_none(data.get('bucket_name'), str, 'bucket_name'),
|
||||
file_name=cls._check_none(data.get('file_name'), str, 'file_name'),
|
||||
validation_file_name=cls._check_type(
|
||||
data.get('validation_file_name'), str, 'validation_file_name'
|
||||
),
|
||||
line_separator=cls._check_none(data.get('line_separator'), str, 'line_separator'),
|
||||
decimal_separator=cls._check_none(
|
||||
data.get('decimal_separator'), str, 'decimal_separator'
|
||||
@@ -220,6 +225,7 @@ class TrainModelParams:
|
||||
self._validate_limits()
|
||||
self._validate_required_strings()
|
||||
self._validate_date_format()
|
||||
self._validate_validation_file_name()
|
||||
|
||||
def _validate_numeric_ranges(self) -> None:
|
||||
"""Validate numeric parameters are within acceptable ranges."""
|
||||
@@ -335,3 +341,21 @@ class TrainModelParams:
|
||||
"""Validate date_format is one of the allowed frontend formats when set."""
|
||||
if self.date_format:
|
||||
validate_frontend_date_format(self.date_format)
|
||||
|
||||
def _validate_validation_file_name(self) -> None:
|
||||
"""
|
||||
Validate that validation_file_name, when provided, is not empty or whitespace.
|
||||
|
||||
This field is optional; when present it must point to a valid object key in the
|
||||
same MinIO bucket specified by bucket_name.
|
||||
"""
|
||||
if self.validation_file_name is None:
|
||||
return
|
||||
|
||||
if not isinstance(self.validation_file_name, str):
|
||||
raise TypeError(
|
||||
f'validation_file_name must be a string, got {type(self.validation_file_name).__name__}'
|
||||
)
|
||||
|
||||
if not self.validation_file_name.strip():
|
||||
raise ValueError('validation_file_name cannot be empty or whitespace')
|
||||
|
||||
@@ -2,7 +2,6 @@ from dataclasses import dataclass
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from model_manager.sientia.models import DataPreprocessor, LinearRegressionModel
|
||||
from model_manager.utils.models.train_model_params import TrainModelParams
|
||||
|
||||
|
||||
@@ -12,18 +11,15 @@ class TrainModelResult:
|
||||
A data container for storing the results of a machine learning training process.
|
||||
|
||||
This dataclass encapsulates all outputs from the training pipeline, including
|
||||
the trained model, datasets, evaluation metrics, and paths to generated artifacts.
|
||||
the prepared datasets, evaluation metrics, and paths to generated artifacts.
|
||||
It is used to pass results between activities in the training workflow.
|
||||
|
||||
Attributes:
|
||||
params (TrainModelParams): The parameters used to train the model.
|
||||
process_data (DataPreprocessor): The data preprocessor object used to process the input data.
|
||||
x_train (pd.DataFrame): The training dataset features.
|
||||
x_test (pd.DataFrame): The testing dataset features.
|
||||
y_train (pd.DataFrame): The training dataset target values.
|
||||
y_test (pd.DataFrame): The testing dataset target values.
|
||||
regr (LinearRegressionModel): The trained linear regression model.
|
||||
scaler_dict (dict): A dictionary containing the scalers used to scale the features and target values.
|
||||
y_pred (pd.Series | None): The predicted target values for the testing dataset. Default is None.
|
||||
y_train_pred (pd.Series | None): The predicted target values for the training dataset. Default is None.
|
||||
mse_val (float | None): The Mean Squared Error (MSE) of the predictions. Default is None.
|
||||
@@ -35,17 +31,13 @@ class TrainModelResult:
|
||||
report_path (str | None): The path to the generated HTML report file. Default is None.
|
||||
train_data_path (str | None): The path to the training dataset CSV file. Default is None.
|
||||
test_data_path (str | None): The path to the testing dataset CSV file. Default is None.
|
||||
run_dir (str | None): The path to the run directory containing all artifacts. Default is None.
|
||||
"""
|
||||
|
||||
params: TrainModelParams
|
||||
process_data: DataPreprocessor
|
||||
x_train: pd.DataFrame
|
||||
x_test: pd.DataFrame
|
||||
y_train: pd.Series
|
||||
y_test: pd.Series
|
||||
regr: LinearRegressionModel
|
||||
scaler_dict: dict
|
||||
y_pred: pd.Series | None = None
|
||||
y_train_pred: pd.Series | None = None
|
||||
mse_val: float | None = None
|
||||
@@ -57,4 +49,3 @@ class TrainModelResult:
|
||||
report_path: str | None = None
|
||||
train_data_path: str | None = None
|
||||
test_data_path: str | None = None
|
||||
run_dir: str | None = None
|
||||
|
||||
Reference in New Issue
Block a user