SIENTIAPDE-1579: Fixed data preprocessor

docs/test-scenarios/11-linear-regression-static-threshold-custom.json

@@ -18,6 +18,8 @@
   "shuffle": true,
   "lineSeparator": ",",
   "decimalSeparator": ".",
+  "dateColumn": "timestamp",
+  "dateFormat": "yyyy-MM-dd HH:mm:ss",
   "removedIntervals": [],
   "degree": 1,
   "interactionOnly": false,

docs/test-scenarios/12-angular-test-cv022-wit230.json

@@ -17,6 +17,8 @@
   "shuffle": true,
   "lineSeparator": ",",
   "decimalSeparator": ".",
+  "dateColumn": "DATA",
+  "dateFormat": "dd/MM/yyyy HH:mm:ss",
   "removedIntervals": [],
   "degree": 1,
   "interactionOnly": false,

docs/test-scenarios/13-angular-test-double-date-column.json

@@ -0,0 +1,31 @@
+{
+  "_description": "Cenário angular-test: CV022 WIT230 com ficheiro double date column e intervalo curto (00:00 a 00:05)",
+  "experimentName": "angular-test",
+  "username": "lucas.kou@aignosi.com.br",
+  "modelName": "Linear Regression",
+  "targetVariable": "03CV022/CORRENTE_N_M1_PV(Value)",
+  "variableColumns": ["303-WIT-230(Value)"],
+  "lagTrain": {"303-WIT-230(Value)": 0},
+  "lagVal": {"303-WIT-230(Value)": 0},
+  "remStaticWin": false,
+  "lowLim": {},
+  "uppLim": {},
+  "window": 0,
+  "useScaler": false,
+  "includeAr": false,
+  "trainSize": 80,
+  "shuffle": true,
+  "lineSeparator": ",",
+  "decimalSeparator": ".",
+  "dateColumn": "DATA",
+  "dateFormat": "dd/MM/yyyy HH:mm:ss",
+  "removedIntervals": [],
+  "degree": 1,
+  "interactionOnly": false,
+  "nanTreatment": "drop",
+  "startDate": "01/05/2022 00:00:00",
+  "endDate": "01/05/2022 00:05:10",
+  "scalerName": "None",
+  "supportFilters": {},
+  "staticThreshold": null
+}

model_manager/sientia/models.py

@@ -18,6 +18,15 @@ FEATURE_CREATION = 'Feature Creation'
 LAG_CREATION = 'Lag Creation'
 
 
+def _frontend_date_format_to_strftime(fmt: str | None) -> str | None:
+    """Convert front-end date format (e.g. dd/MM/yyyy HH:mm:ss) to Python strftime."""
+    if not fmt:
+        return None
+    out = fmt.replace('yyyy', '%Y').replace('MM', '%m').replace('dd', '%d')
+    out = out.replace('HH', '%H').replace('mm', '%M').replace('ss', '%S')
+    return out
+
+
 class LinearRegressionModel(BaseEstimator, TransformerMixin):
     """
     Linear Regression Model for Time Series Analysis.
@@ -244,6 +253,7 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
         lag_transform: dict[str, int] | None = None,
         start_date: str | None = None,
         end_date: str | None = None,
+        date_format: str | None = None,
         removed_intervals: list[tuple[str, str]] | None = None,
         static_threshold: int | None = None,
         low_lim: dict[str, float] | None = None,
@@ -270,8 +280,9 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
                 *Format: {'variable_name': lag}*
             lag_transform (dict): The lags for each variable to be applyed during transformation \\
                 *Format: {'variable_name': lag}*
-            start_date (str): The start date for filtering data (format: 'YYYY-MM-DD HH:MM:SS')
+            start_date (str): The start date for filtering data
-            end_date (str): The end date for filtering data (format: 'YYYY-MM-DD HH:MM:SS')
+            end_date (str): The end date for filtering data
+            date_format (str | None): Frontend date format for start/end (e.g. dd/MM/yyyy HH:mm:ss or MM/dd/yyyy HH:mm:ss). When set, parsing matches the CSV date column.
             removed_intervals (list): List of tuples with intervals to remove from data \\
                 *Format: [('start_date', 'end_date'), ...]*
             static_threshold (int): The number of repeated values to be considered as static
@@ -314,6 +325,7 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
         self.lag_transform = lag_transform if lag_transform else {}
         self.start_date = start_date
         self.end_date = end_date
+        self.date_format = date_format
         self.removed_intervals = removed_intervals if removed_intervals else []
         self.ar_var = ar_var
         self.self_operations = self_operations
@@ -443,10 +455,17 @@ class DataPreprocessor(BaseEstimator, TransformerMixin):
         return input_data
 
     def _parse_datetime(self, date_str: str | None) -> pd.Timestamp | None:
-        """Parse a date string to Timestamp, returning None on failure."""
+        """Parse a date string to Timestamp using date_format when set.
+
+        When date_format is set (e.g. dd/MM/yyyy HH:mm:ss or MM/dd/yyyy HH:mm:ss),
+        parsing matches the CSV date column so start_date/end_date filter correctly.
+        """
         if not date_str:
             return None
         try:
+            python_fmt = _frontend_date_format_to_strftime(self.date_format) if self.date_format else None
+            if python_fmt:
+                return pd.to_datetime(date_str, format=python_fmt)
             return pd.to_datetime(date_str)
         except (ValueError, TypeError):
             return None

model_manager/utils/repository/training_repository.py

@@ -311,6 +311,7 @@ class TrainingRepository:
             lag_transform=params.lag_val,
             start_date=params.start_date,
             end_date=params.end_date,
+            date_format=params.date_format,
             removed_intervals=removed_intervals,
             static_threshold=self._get_static_threshold(params),
             low_lim=params.low_lim,

scripts/run_training_test.py

@@ -105,6 +105,19 @@ def load_scenario(scenario_name: str) -> dict:
         return json.load(f)
 
 
+def _resolve_csv_path(csv_path: Path) -> Path:
+    """Resolve CSV path; if not found in project root, try docs/."""
+    if csv_path.is_absolute():
+        return csv_path
+    resolved = PROJECT_ROOT / csv_path
+    if resolved.exists():
+        return resolved
+    docs_path = PROJECT_ROOT / 'docs' / csv_path.name
+    if docs_path.exists():
+        return docs_path
+    return resolved
+
+
 def _ensure_source_file(path: Path) -> None:
     if not path.exists():
         raise FileNotFoundError(f'Test dataset not found at {path.resolve()}')
@@ -206,6 +219,7 @@ def build_workflow_payload(
         'end_date': request_data.get('endDate'),
         'scaler_name': request_data.get('scalerName', 'None'),
         'support_filters': request_data.get('supportFilters', {}),
+        'static_threshold': request_data.get('staticThreshold'),
     }
 
 
@@ -292,8 +306,7 @@ def run_local_pipeline(
         result['error'] = str(exc)
         return result
 
-    if not csv_path.is_absolute():
+    csv_path = _resolve_csv_path(csv_path)
-        csv_path = PROJECT_ROOT / csv_path
     _ensure_source_file(csv_path)
     payload = _build_local_payload(request_data, csv_path)
     try:
@@ -439,6 +452,8 @@ def run_single_scenario(scenario_name: str, csv_path: Path) -> dict:
         result['error'] = str(exc)
         return result
 
+    csv_path = _resolve_csv_path(csv_path)
+
     # Upload CSV to MinIO
     try:
         uploaded_file_name = upload_to_minio(csv_path)