gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1249: Implement data models for Model Manager and add unit tests. This commit introduces data transfer objects (DTOs) and model classes for experiment status, training parameters, and training results, along with corresponding unit tests to ensure their correct behavior.

Browse Source
This commit is contained in:
Bruno Domingues
2025-10-06 16:49:18 -03:00
parent 18ccfed78a
commit d8583cdae7
9 changed files with 940 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
model_manager/utils/models/train_model_params.py Normal file
View File

@@ -0,0 +1,166 @@
from typing import Any
class TrainModelParams:
"""
Parameters for machine learning model training.
This class encapsulates all configuration parameters required for the training
pipeline, including data processing settings, model configuration, and experiment
tracking information. All parameters are validated upon initialization to ensure
data integrity and prevent runtime errors.
Attributes:
variable_columns (List[str]): List of variable column names to use as features.
lag_train (int): Number of lags to apply during training phase.
lag_val (int): Number of lags to apply during validation phase.
target_variable (str): Name of the target variable to predict.
rem_static_win (bool): Whether to remove static windows from data.
low_lim (Dict[str, float]): Dictionary of lower limits for each variable.
upp_lim (Dict[str, float]): Dictionary of upper limits for each variable.
window (int): Window size for rolling operations.
use_scaler (bool): Whether to use a scaler for data normalization.
include_ar (bool): Whether to include autoregressive variables.
bucket_name (str): Name of the MinIO bucket containing training data.
file_name (str): Name of the file in the MinIO bucket.
line_separator (str): Line separator used in the CSV file.
decimal_separator (str): Decimal separator used in the CSV file.
train_size (int): Percentage of data to use for training (0-100).
shuffle (bool): Whether to shuffle the data during train/test split.
experiment_run_id (int): Unique identifier for the experiment run.
experiment_name (str): Name of the experiment for tracking.
experiment_description (str): Description of the experiment.
removed_intervals (list): List of time intervals to remove from the data.
"""
def __init__(
self,
variable_columns: Any | None,
lag_train: Any | None,
lag_val: Any | None,
target_variable: Any | None,
rem_static_win: Any | None,
low_lim: Any | None,
upp_lim: Any | None,
window: Any | None,
use_scaler: Any | None,
include_ar: Any | None,
bucket_name: Any | None,
file_name: Any | None,
line_separator: Any | None,
decimal_separator: Any | None,
train_size: Any | None,
shuffle: Any | None,
experiment_run_id: Any | None,
experiment_name: Any | None,
experiment_description: Any | None,
removed_intervals: Any | None,
):
"""
Initialize the TrainModelParams object and validate all input parameters.
This constructor validates that all required parameters are provided and
have the correct types. It raises descriptive errors if validation fails,
helping to catch configuration issues early in the pipeline.
Args:
variable_columns (list | None): List of variable column names.
lag_train (int | None): Number of lags to apply during training.
lag_val (int | None): Number of lags to apply during validation.
target_variable (str | None): Name of the target variable.
rem_static_win (bool | None): Whether to remove static windows.
low_lim (dict | None): Dictionary of lower limits for variables.
upp_lim (dict | None): Dictionary of upper limits for variables.
window (int | None): Window size for rolling operations.
use_scaler (bool | None): Whether to use a scaler for normalization.
include_ar (bool | None): Whether to include autoregressive variables.
bucket_name (str | None): Name of the MinIO bucket.
file_name (str | None): Name of the file in the MinIO bucket.
line_separator (str | None): Line separator used in the file.
decimal_separator (str | None): Decimal separator used in the file.
train_size (int | None): Percentage of data to use for training.
shuffle (bool | None): Whether to shuffle the data during splitting.
experiment_run_id (int | None): ID of the experiment run.
experiment_name (str | None): Name of the experiment.
experiment_description (str | None): Description of the experiment.
removed_intervals (list | None): List of intervals to remove from the data.
Raises:
ValueError: If any required parameter is None.
TypeError: If any parameter is not of the expected type.
"""
self.variable_columns: list[str] = self._check_none(
variable_columns, list, 'variable_columns'
)
self.lag_train: int = self._check_none(lag_train, int, 'lag_train')
self.lag_val: int = self._check_none(lag_val, int, 'lag_val')
self.target_variable: str = self._check_none(target_variable, str, 'target_variable')
self.rem_static_win: bool = self._check_none(rem_static_win, bool, 'rem_static_win')
self.low_lim: dict[str, float] = self._check_none(low_lim, dict, 'low_lim')
self.upp_lim: dict[str, float] = self._check_none(upp_lim, dict, 'upp_lim')
self.window: int = self._check_none(window, int, 'window')
self.use_scaler: bool = self._check_none(use_scaler, bool, 'use_scaler')
self.include_ar: bool = self._check_none(include_ar, bool, 'include_ar')
self.bucket_name: str = self._check_none(bucket_name, str, 'bucket_name')
self.file_name: str = self._check_none(file_name, str, 'file_name')
self.line_separator: str = self._check_none(line_separator, str, 'line_separator')
self.decimal_separator: str = self._check_none(decimal_separator, str, 'decimal_separator')
self.train_size: int = self._check_none(train_size, int, 'train_size')
self.shuffle: bool = self._check_none(shuffle, bool, 'shuffle')
self.experiment_run_id: int = self._check_none(experiment_run_id, int, 'experiment_run_id')
self.experiment_name: str = self._check_none(experiment_name, str, 'experiment_name')
self.experiment_description: str = self._check_none(
experiment_description, str, 'experiment_description'
)
self.removed_intervals: list = self._check_type(
removed_intervals, list, 'removed_intervals'
)
def _check_none(self, value: Any | None, expected_type: type, field_name: str):
"""
Validate that a value is not None and check its type.
This method ensures that required parameters are provided and have the
correct type, raising descriptive errors if validation fails.
Args:
value (Any | None): The value to validate.
expected_type (type): The expected type of the value.
field_name (str): The name of the field being validated (for error messages).
Returns:
Any: The validated value if it is not None and matches the expected type.
Raises:
ValueError: If the value is None.
TypeError: If the value is not of the expected type.
"""
if value is None:
error = f'{field_name} is required and cannot be None.'
raise ValueError(error)
return self._check_type(value, expected_type, field_name)
def _check_type(self, value: Any | None, expected_type: type, field_name: str):
"""
Validate that a value matches the expected type.
This method checks type compatibility and raises a descriptive error
if the value does not match the expected type.
Args:
value (Any | None): The value to validate.
expected_type (type): The expected type of the value.
field_name (str): The name of the field being validated (for error messages).
Returns:
Any: The validated value if it matches the expected type.
Raises:
TypeError: If the value is not of the expected type.
"""
if value is not None and not isinstance(value, expected_type):
error = f'{field_name} must be of type {expected_type.__name__}, but got {type(value).__name__}.'
raise TypeError(error)
return value