SIENTIAPDE-1430: Implement advanced model training capabilities and enhanced data preprocessing. This includes support for Polynomial Regression with configurable degree and interaction terms, flexible per-variable lag configurations, and new data filtering options by date range and removed intervals. Comprehensive business validations are now enforced for all parameters, and MLflow logging has been extended to capture these detailed configurations. Additionally, Reduced Coulomb Energy (RCE) metrics are added for drift detection, with a new changelog documenting all pipeline parameter updates.

model_manager/utils/models/train_model_params.py

@@ -17,8 +17,8 @@ class TrainModelParams:
 
     Attributes:
         variable_columns (list[str]): List of variable column names to use as features.
-        lag_train (int): Number of lags to apply during training phase.
-        lag_val (int): Number of lags to apply during validation phase.
+        lag_train (dict[str, int]): Dictionary of lags per variable for training phase.
+        lag_val (dict[str, int]): Dictionary of lags per variable for validation phase.
         target_variable (str): Name of the target variable to predict.
         rem_static_win (bool): Whether to remove static windows from data.
         low_lim (dict[str, float]): Dictionary of lower limits for each variable.
@@ -35,11 +35,19 @@ class TrainModelParams:
         experiment_run_id (int): Unique identifier for the experiment run.
         experiment_name (str): Name of the experiment for tracking.
         removed_intervals (list): List of time intervals to remove from the data.
+        model_name (str): Name of the model type ('Linear Regression' or 'Polynomial Regression').
+        degree (int): Degree of polynomial features (1 for linear, >1 for polynomial).
+        interaction_only (bool): If True, only interaction features are produced for polynomial.
+        nan_treatment (str): Treatment for NaN values ('drop' or 'linear interpolation').
+        start_date (str | None): Start date for filtering data.
+        end_date (str | None): End date for filtering data.
+        scaler_name (str): Name of the scaler to use ('Standard Scaler' or 'None').
+        support_filters (dict): Custom support filters per variable.
     """
 
     variable_columns: list[str]
-    lag_train: int
-    lag_val: int
+    lag_train: dict[str, int]
+    lag_val: dict[str, int]
     target_variable: str
     rem_static_win: bool
     low_lim: dict[str, float]
@@ -56,6 +64,14 @@ class TrainModelParams:
     experiment_run_id: int
     experiment_name: str
     removed_intervals: list
+    model_name: str
+    degree: int
+    interaction_only: bool
+    nan_treatment: str
+    start_date: str | None
+    end_date: str | None
+    scaler_name: str
+    support_filters: dict
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> 'TrainModelParams':
@@ -83,8 +99,8 @@ class TrainModelParams:
             variable_columns=cls._check_none(
                 data.get('variable_columns'), list, 'variable_columns'
             ),
-            lag_train=cls._check_none(data.get('lag_train'), int, 'lag_train'),
-            lag_val=cls._check_none(data.get('lag_val'), int, 'lag_val'),
+            lag_train=cls._check_none(data.get('lag_train'), dict, 'lag_train'),
+            lag_val=cls._check_none(data.get('lag_val'), dict, 'lag_val'),
             target_variable=cls._check_none(data.get('target_variable'), str, 'target_variable'),
             rem_static_win=cls._check_none(data.get('rem_static_win'), bool, 'rem_static_win'),
             low_lim=cls._check_none(data.get('low_lim'), dict, 'low_lim'),
@@ -107,6 +123,17 @@ class TrainModelParams:
             removed_intervals=cls._check_type(
                 data.get('removed_intervals'), list, 'removed_intervals'
             ),
+            model_name=cls._check_none(data.get('model_name'), str, 'model_name'),
+            degree=cls._check_none(data.get('degree'), int, 'degree'),
+            interaction_only=cls._check_none(
+                data.get('interaction_only'), bool, 'interaction_only'
+            ),
+            nan_treatment=cls._check_none(data.get('nan_treatment'), str, 'nan_treatment'),
+            start_date=cls._check_type(data.get('start_date'), str, 'start_date'),
+            end_date=cls._check_type(data.get('end_date'), str, 'end_date'),
+            scaler_name=cls._check_none(data.get('scaler_name'), str, 'scaler_name'),
+            support_filters=cls._check_type(data.get('support_filters'), dict, 'support_filters')
+            or {},
         )
 
     @staticmethod
@@ -172,25 +199,81 @@ class TrainModelParams:
         Raises:
             ValueError: If any business rule is violated
         """
-        # Validate train_size range (10-100%)
+        self._validate_numeric_ranges()
+        self._validate_model_params()
+        self._validate_intervals_and_dates()
+        self._validate_limits()
+        self._validate_required_strings()
+
+    def _validate_numeric_ranges(self) -> None:
+        """Validate numeric parameters are within acceptable ranges."""
         if not 10 <= self.train_size <= 100:
             raise ValueError(f'train_size must be between 10 and 100, got {self.train_size}')
 
-        # Validate variable_columns is not empty
         if not self.variable_columns:
             raise ValueError('variable_columns cannot be empty')
 
-        # Validate positive integers
-        if self.lag_train < 0:
-            raise ValueError(f'lag_train must be positive, got {self.lag_train}')
+        for var, lag in self.lag_train.items():
+            if lag < 0:
+                raise ValueError(f'lag_train for {var} must be non-negative, got {lag}')
 
-        if self.lag_val < 0:
-            raise ValueError(f'lag_val must be positive, got {self.lag_val}')
+        for var, lag in self.lag_val.items():
+            if lag < 0:
+                raise ValueError(f'lag_val for {var} must be non-negative, got {lag}')
 
         if self.window < 0:
-            raise ValueError(f'window must be positive, got {self.window}')
+            raise ValueError(f'window must be non-negative, got {self.window}')
 
-        # Validate low_lim and upp_lim consistency
+    def _validate_model_params(self) -> None:
+        """Validate model-related parameters."""
+        if self.degree < 1:
+            raise ValueError(f'degree must be at least 1, got {self.degree}')
+
+        valid_nan_treatments = ['drop', 'linear interpolation', 'fill linear']
+        if self.nan_treatment not in valid_nan_treatments:
+            raise ValueError(
+                f'nan_treatment must be one of {valid_nan_treatments}, got {self.nan_treatment}'
+            )
+
+        valid_scalers = ['Standard Scaler', 'None']
+        if self.scaler_name not in valid_scalers:
+            raise ValueError(f'scaler_name must be one of {valid_scalers}, got {self.scaler_name}')
+
+        valid_models = ['Linear Regression', 'Polynomial Regression']
+        if self.model_name not in valid_models:
+            raise ValueError(f'model_name must be one of {valid_models}, got {self.model_name}')
+
+        if self.model_name == 'Polynomial Regression' and self.degree < 2:
+            raise ValueError(
+                f'degree must be at least 2 for Polynomial Regression, got {self.degree}'
+            )
+
+        if self.model_name == 'Linear Regression' and self.degree != 1:
+            raise ValueError(f'degree must be 1 for Linear Regression, got {self.degree}')
+
+    def _validate_intervals_and_dates(self) -> None:
+        """Validate removed_intervals format and date parameters."""
+        if self.removed_intervals:
+            for i, interval in enumerate(self.removed_intervals):
+                if not isinstance(interval, (list, tuple)):
+                    raise ValueError(
+                        f'removed_intervals[{i}] must be a list or tuple, '
+                        f'got {type(interval).__name__}'
+                    )
+                if len(interval) < 2:
+                    raise ValueError(
+                        f'removed_intervals[{i}] must have at least 2 elements (start, end), '
+                        f'got {len(interval)}'
+                    )
+
+        if self.start_date is not None and not isinstance(self.start_date, str):
+            raise TypeError(f'start_date must be a string, got {type(self.start_date).__name__}')
+
+        if self.end_date is not None and not isinstance(self.end_date, str):
+            raise TypeError(f'end_date must be a string, got {type(self.end_date).__name__}')
+
+    def _validate_limits(self) -> None:
+        """Validate low_lim and upp_lim consistency."""
         if set(self.low_lim.keys()) != set(self.upp_lim.keys()):
             raise ValueError(
                 f'low_lim and upp_lim must have the same keys. '
@@ -198,7 +281,6 @@ class TrainModelParams:
                 f'upp_lim keys: {set(self.upp_lim.keys())}'
             )
 
-        # Validate that low_lim < upp_lim for each variable
         for var in self.low_lim:
             if self.low_lim[var] >= self.upp_lim[var]:
                 raise ValueError(
@@ -206,13 +288,16 @@ class TrainModelParams:
                     f'Got low_lim={self.low_lim[var]}, upp_lim={self.upp_lim[var]}'
                 )
 
-        # Validate bucket_name and file_name are not empty
+    def _validate_required_strings(self) -> None:
+        """Validate required string fields are not empty."""
+        if not self.target_variable.strip():
+            raise ValueError('target_variable cannot be empty or whitespace')
+
         if not self.bucket_name.strip():
             raise ValueError('bucket_name cannot be empty or whitespace')
 
         if not self.file_name.strip():
             raise ValueError('file_name cannot be empty or whitespace')
 
-        # Validate experiment_name is not empty
         if not self.experiment_name.strip():
             raise ValueError('experiment_name cannot be empty or whitespace')