diff --git a/docs/test-scenarios/11-linear-regression-static-threshold-custom.json b/docs/test-scenarios/11-linear-regression-static-threshold-custom.json index 8afa37e..c9330c3 100644 --- a/docs/test-scenarios/11-linear-regression-static-threshold-custom.json +++ b/docs/test-scenarios/11-linear-regression-static-threshold-custom.json @@ -18,6 +18,8 @@ "shuffle": true, "lineSeparator": ",", "decimalSeparator": ".", + "dateColumn": "timestamp", + "dateFormat": "yyyy-MM-dd HH:mm:ss", "removedIntervals": [], "degree": 1, "interactionOnly": false, diff --git a/docs/test-scenarios/12-angular-test-cv022-wit230.json b/docs/test-scenarios/12-angular-test-cv022-wit230.json index 2c74fda..096284e 100644 --- a/docs/test-scenarios/12-angular-test-cv022-wit230.json +++ b/docs/test-scenarios/12-angular-test-cv022-wit230.json @@ -17,6 +17,8 @@ "shuffle": true, "lineSeparator": ",", "decimalSeparator": ".", + "dateColumn": "DATA", + "dateFormat": "dd/MM/yyyy HH:mm:ss", "removedIntervals": [], "degree": 1, "interactionOnly": false, diff --git a/docs/test-scenarios/13-angular-test-double-date-column.json b/docs/test-scenarios/13-angular-test-double-date-column.json new file mode 100644 index 0000000..e94e224 --- /dev/null +++ b/docs/test-scenarios/13-angular-test-double-date-column.json @@ -0,0 +1,31 @@ +{ + "_description": "Cenário angular-test: CV022 WIT230 com ficheiro double date column e intervalo curto (00:00 a 00:05)", + "experimentName": "angular-test", + "username": "lucas.kou@aignosi.com.br", + "modelName": "Linear Regression", + "targetVariable": "03CV022/CORRENTE_N_M1_PV(Value)", + "variableColumns": ["303-WIT-230(Value)"], + "lagTrain": {"303-WIT-230(Value)": 0}, + "lagVal": {"303-WIT-230(Value)": 0}, + "remStaticWin": false, + "lowLim": {}, + "uppLim": {}, + "window": 0, + "useScaler": false, + "includeAr": false, + "trainSize": 80, + "shuffle": true, + "lineSeparator": ",", + "decimalSeparator": ".", + "dateColumn": "DATA", + "dateFormat": "dd/MM/yyyy HH:mm:ss", + "removedIntervals": [], + "degree": 1, + "interactionOnly": false, + "nanTreatment": "drop", + "startDate": "01/05/2022 00:00:00", + "endDate": "01/05/2022 00:05:10", + "scalerName": "None", + "supportFilters": {}, + "staticThreshold": null +} diff --git a/model_manager/sientia/models.py b/model_manager/sientia/models.py index b3abc3f..d7e8a3a 100644 --- a/model_manager/sientia/models.py +++ b/model_manager/sientia/models.py @@ -18,6 +18,15 @@ FEATURE_CREATION = 'Feature Creation' LAG_CREATION = 'Lag Creation' +def _frontend_date_format_to_strftime(fmt: str | None) -> str | None: + """Convert front-end date format (e.g. dd/MM/yyyy HH:mm:ss) to Python strftime.""" + if not fmt: + return None + out = fmt.replace('yyyy', '%Y').replace('MM', '%m').replace('dd', '%d') + out = out.replace('HH', '%H').replace('mm', '%M').replace('ss', '%S') + return out + + class LinearRegressionModel(BaseEstimator, TransformerMixin): """ Linear Regression Model for Time Series Analysis. @@ -244,6 +253,7 @@ class DataPreprocessor(BaseEstimator, TransformerMixin): lag_transform: dict[str, int] | None = None, start_date: str | None = None, end_date: str | None = None, + date_format: str | None = None, removed_intervals: list[tuple[str, str]] | None = None, static_threshold: int | None = None, low_lim: dict[str, float] | None = None, @@ -270,8 +280,9 @@ class DataPreprocessor(BaseEstimator, TransformerMixin): *Format: {'variable_name': lag}* lag_transform (dict): The lags for each variable to be applyed during transformation \\ *Format: {'variable_name': lag}* - start_date (str): The start date for filtering data (format: 'YYYY-MM-DD HH:MM:SS') - end_date (str): The end date for filtering data (format: 'YYYY-MM-DD HH:MM:SS') + start_date (str): The start date for filtering data + end_date (str): The end date for filtering data + date_format (str | None): Frontend date format for start/end (e.g. dd/MM/yyyy HH:mm:ss or MM/dd/yyyy HH:mm:ss). When set, parsing matches the CSV date column. removed_intervals (list): List of tuples with intervals to remove from data \\ *Format: [('start_date', 'end_date'), ...]* static_threshold (int): The number of repeated values to be considered as static @@ -314,6 +325,7 @@ class DataPreprocessor(BaseEstimator, TransformerMixin): self.lag_transform = lag_transform if lag_transform else {} self.start_date = start_date self.end_date = end_date + self.date_format = date_format self.removed_intervals = removed_intervals if removed_intervals else [] self.ar_var = ar_var self.self_operations = self_operations @@ -443,10 +455,17 @@ class DataPreprocessor(BaseEstimator, TransformerMixin): return input_data def _parse_datetime(self, date_str: str | None) -> pd.Timestamp | None: - """Parse a date string to Timestamp, returning None on failure.""" + """Parse a date string to Timestamp using date_format when set. + + When date_format is set (e.g. dd/MM/yyyy HH:mm:ss or MM/dd/yyyy HH:mm:ss), + parsing matches the CSV date column so start_date/end_date filter correctly. + """ if not date_str: return None try: + python_fmt = _frontend_date_format_to_strftime(self.date_format) if self.date_format else None + if python_fmt: + return pd.to_datetime(date_str, format=python_fmt) return pd.to_datetime(date_str) except (ValueError, TypeError): return None diff --git a/model_manager/utils/repository/training_repository.py b/model_manager/utils/repository/training_repository.py index 89c0c02..67abb17 100644 --- a/model_manager/utils/repository/training_repository.py +++ b/model_manager/utils/repository/training_repository.py @@ -311,6 +311,7 @@ class TrainingRepository: lag_transform=params.lag_val, start_date=params.start_date, end_date=params.end_date, + date_format=params.date_format, removed_intervals=removed_intervals, static_threshold=self._get_static_threshold(params), low_lim=params.low_lim, diff --git a/scripts/run_training_test.py b/scripts/run_training_test.py index bb67558..bfac41b 100644 --- a/scripts/run_training_test.py +++ b/scripts/run_training_test.py @@ -105,6 +105,19 @@ def load_scenario(scenario_name: str) -> dict: return json.load(f) +def _resolve_csv_path(csv_path: Path) -> Path: + """Resolve CSV path; if not found in project root, try docs/.""" + if csv_path.is_absolute(): + return csv_path + resolved = PROJECT_ROOT / csv_path + if resolved.exists(): + return resolved + docs_path = PROJECT_ROOT / 'docs' / csv_path.name + if docs_path.exists(): + return docs_path + return resolved + + def _ensure_source_file(path: Path) -> None: if not path.exists(): raise FileNotFoundError(f'Test dataset not found at {path.resolve()}') @@ -206,6 +219,7 @@ def build_workflow_payload( 'end_date': request_data.get('endDate'), 'scaler_name': request_data.get('scalerName', 'None'), 'support_filters': request_data.get('supportFilters', {}), + 'static_threshold': request_data.get('staticThreshold'), } @@ -292,8 +306,7 @@ def run_local_pipeline( result['error'] = str(exc) return result - if not csv_path.is_absolute(): - csv_path = PROJECT_ROOT / csv_path + csv_path = _resolve_csv_path(csv_path) _ensure_source_file(csv_path) payload = _build_local_payload(request_data, csv_path) try: @@ -439,6 +452,8 @@ def run_single_scenario(scenario_name: str, csv_path: Path) -> dict: result['error'] = str(exc) return result + csv_path = _resolve_csv_path(csv_path) + # Upload CSV to MinIO try: uploaded_file_name = upload_to_minio(csv_path)