diff --git a/model_manager/utils/models/train_model_params.py b/model_manager/utils/models/train_model_params.py index 8ab83f9..f79b536 100644 --- a/model_manager/utils/models/train_model_params.py +++ b/model_manager/utils/models/train_model_params.py @@ -34,6 +34,8 @@ class TrainModelParams: file_name (str): Name of the file in the MinIO bucket. line_separator (str): Line separator used in the CSV file. decimal_separator (str): Decimal separator used in the CSV file. + date_column (str | None): Name of the date/time column. If set with date_format, the column is parsed as datetime. + date_format (str | None): Format of the date column (e.g. dd/MM/yyyy HH:mm:ss). Used when date_column is set. train_size (int): Percentage of data to use for training (0-100). shuffle (bool): Whether to shuffle the data during train/test split. experiment_run_id (int): Unique identifier for the experiment run. @@ -64,6 +66,8 @@ class TrainModelParams: file_name: str line_separator: str decimal_separator: str + date_column: str | None + date_format: str | None train_size: int shuffle: bool experiment_run_id: int @@ -120,6 +124,8 @@ class TrainModelParams: decimal_separator=cls._check_none( data.get('decimal_separator'), str, 'decimal_separator' ), + date_column=data.get('date_column'), + date_format=data.get('date_format'), train_size=cls._check_none(data.get('train_size'), int, 'train_size'), shuffle=cls._check_none(data.get('shuffle'), bool, 'shuffle'), experiment_run_id=cls._check_none( diff --git a/model_manager/utils/repository/training_repository.py b/model_manager/utils/repository/training_repository.py index 7d5249c..89c0c02 100644 --- a/model_manager/utils/repository/training_repository.py +++ b/model_manager/utils/repository/training_repository.py @@ -21,6 +21,32 @@ from model_manager.utils.models.train_model_params import TrainModelParams from model_manager.utils.models.train_model_result import TrainModelResult +def _frontend_format_to_strftime(fmt: str) -> str: + """Convert front-end date format (dd/MM/yyyy HH:mm:ss) to Python strftime (%d/%m/%Y %H:%M:%S).""" + if not fmt: + return fmt + out = fmt.replace('yyyy', '%Y').replace('MM', '%m').replace('dd', '%d') + out = out.replace('HH', '%H').replace('mm', '%M').replace('ss', '%S') + return out + + +def _ensure_date_column_parsed(data: pd.DataFrame, params: TrainModelParams) -> pd.DataFrame: + """If date_column and date_format are set, parse the column as datetime to avoid comparison errors downstream.""" + if not params.date_column or not params.date_format or params.date_column not in data.columns: + return data + try: + python_fmt = _frontend_format_to_strftime(params.date_format) + data = data.copy() + data[params.date_column] = pd.to_datetime( + data[params.date_column], format=python_fmt, errors='coerce' + ) + except Exception as e: + raise ValueError( + f'Failed to parse date column "{params.date_column}" with format "{params.date_format}": {e}' + ) from e + return data + + class TrainingRepository: """ Repository for machine learning model training operations. @@ -67,10 +93,13 @@ class TrainingRepository: Exception: If data loading, preprocessing, or training fails """ data = load_data(uploaded_file, params.line_separator, params.decimal_separator) - - # Configure datetime index if timestamp column exists - # Required for TimeSeriesDiscontinuityAnalyzer (static window removal) - data = self._configure_datetime_index(data) + if data is None: + raise ValueError( + 'Failed to load CSV data: load_data returned None. ' + 'Check file encoding, line separator and decimal separator.' + ) + data = _ensure_date_column_parsed(data, params) + data = self._configure_datetime_index(data, params) process_data = self._init_data_preprocessor(params) process_data.fit(data) @@ -350,40 +379,41 @@ class TrainingRepository: 'original_features': params.variable_columns, } - def _configure_datetime_index(self, data: pd.DataFrame) -> pd.DataFrame: + def _configure_datetime_index( + self, data: pd.DataFrame | None, params: TrainModelParams + ) -> pd.DataFrame: """ Configure datetime index for the DataFrame. - This method attempts to identify a timestamp column and set it as the - DataFrame index with DatetimeIndex type. This is required for - TimeSeriesDiscontinuityAnalyzer (used in static window removal). - - The method looks for common timestamp column names and converts the - first matching column to datetime, then sets it as the index. - - Args: - data: Input DataFrame - - Returns: - pd.DataFrame: DataFrame with DatetimeIndex if timestamp column found, - otherwise returns original DataFrame unchanged + Guards against None to avoid 'NoneType' object has no attribute 'index' downstream. + Prefers params.date_column when set; otherwise looks for common timestamp column names. """ - # If index is already DatetimeIndex, just ensure it's sorted + if data is None: + raise ValueError( + 'Data is None after load_data. ' + 'Check file format, line separator and decimal separator.' + ) + if not isinstance(data, pd.DataFrame): + raise TypeError(f'Expected DataFrame, got {type(data).__name__}') + if isinstance(data.index, pd.DatetimeIndex): self.logger.info('DataFrame already has DatetimeIndex') return data.sort_index() - # Common timestamp column names - timestamp_columns = [ + common_timestamp_columns = [ 'timestamp', 'Timestamp', 'TIMESTAMP', 'date', 'Date', 'DATE', + 'DATA', 'datetime', 'DateTime', ] + timestamp_columns = ( + [params.date_column] if params.date_column else [] + ) + [c for c in common_timestamp_columns if c != params.date_column] for col in timestamp_columns: if col in data.columns: diff --git a/scripts/run_training_test.py b/scripts/run_training_test.py index 427a8bf..bb67558 100644 --- a/scripts/run_training_test.py +++ b/scripts/run_training_test.py @@ -194,6 +194,8 @@ def build_workflow_payload( 'file_name': file_name, 'line_separator': request_data['lineSeparator'], 'decimal_separator': request_data['decimalSeparator'], + 'date_column': request_data.get('dateColumn'), + 'date_format': request_data.get('dateFormat'), 'removed_intervals': request_data['removedIntervals'], # New parameters 'model_name': request_data.get('modelName', 'Linear Regression'),