feat: require date_column in training parameters and update documentation

- Made `date_column` a required field in `TrainModelParams`, ensuring it must be present in the input data. - Updated related documentation in `input-sample.md`, `README.md`, and various test scenarios to reflect the change in requirement. - Adjusted the handling of `date_format` to default to `yyyy-MM-dd HH:mm:ss` if omitted, enhancing usability. - Refined test scenarios to include new examples and ensure compliance with the updated parameter structure. These changes improve the robustness of the model training workflow and clarify the expectations for input data.
2026-05-05 08:35:12 -03:00
parent 6bd30e3328
commit ba9eb3d7c7
38 changed files with 1027 additions and 261 deletions
--- a/docs/test-scenarios/10-linear-regression-with-ar.json
+++ b/docs/test-scenarios/10-linear-regression-with-ar.json
@@ -1,5 +1,5 @@
 {
-  "_description": "Regressão linear com variável autoregressiva (AR)",
+  "_description": "Linear regression placeholder for autoregressive features; include_ar is reserved for future wrapper support (see opt_params).",
  "experiment_run_id": 1010,
  "variable_columns": [
    "303-WIT-200(Value)"
@@ -36,5 +36,7 @@
    "interaction_only": false,
    "scaler_name": "None"
  },
-  "opt_params": {}
+  "opt_params": {
+    "include_ar": true
+  }
 }
--- a/docs/test-scenarios/12-angular-test-date-format.json
+++ b/docs/test-scenarios/12-angular-test-date-format.json
@@ -1,12 +1,12 @@
 {
-  "_description": "Cenário angular-test-01: CV022 WIT230 com lag e intervalo de datas",
+  "_description": "Alternate date column (DATA) and dd/MM/yyyy HH:mm:ss format; uses MinIO object training_data_dd_mm_yyyy.csv from E2E fixtures.",
  "experiment_run_id": 1012,
  "variable_columns": [
    "303-WIT-230(Value)"
  ],
  "target_variable": "03CV022/CORRENTE_N_M1_PV(Value)",
  "bucket_name": "model-training",
-  "file_name": "training_data.csv",
+  "file_name": "training_data_dd_mm_yyyy.csv",
  "line_separator": ",",
  "decimal_separator": ".",
  "date_column": "DATA",
--- a/docs/test-scenarios/13-angular-test-double-date-column.json
+++ b/docs/test-scenarios/13-angular-test-double-date-column.json
@@ -1,12 +1,12 @@
 {
-  "_description": "Cenário angular-test: CV022 WIT230 com ficheiro double date column e intervalo curto (00:00 a 00:05)",
+  "_description": "Same alternate CSV as scenario 12 (DATA + dd/MM/yyyy); narrow date window for regression coverage. Not a multi-date-column dataset.",
  "experiment_run_id": 1013,
  "variable_columns": [
    "303-WIT-230(Value)"
  ],
  "target_variable": "03CV022/CORRENTE_N_M1_PV(Value)",
  "bucket_name": "model-training",
-  "file_name": "training_data.csv",
+  "file_name": "training_data_dd_mm_yyyy.csv",
  "line_separator": ",",
  "decimal_separator": ".",
  "date_column": "DATA",
@@ -27,7 +27,7 @@
    "rem_static_win": false,
    "static_threshold": null,
    "start_date": "01/05/2022 00:00:00",
-    "end_date": "01/05/2022 00:05:10",
+    "end_date": "31/05/2022 23:59:59",
    "support_filters": {},
    "removed_intervals": []
  },
--- a/docs/test-scenarios/14-angular-test-polynomial-support-filters.json
+++ b/docs/test-scenarios/14-angular-test-polynomial-support-filters.json
@@ -26,8 +26,8 @@
    "nan_treatment": "drop",
    "rem_static_win": false,
    "static_threshold": null,
-    "start_date": "2025-06-02 00:00:05",
-    "end_date": "2025-06-06 15:02:01",
+    "start_date": "2025-06-02 00:00:00",
+    "end_date": "2025-06-08 23:59:59",
    "support_filters": {
      "303-WIT-200(Value)": {
        "upper_line": {
--- a/docs/test-scenarios/15-linear-regression-custom-target-column.json
+++ b/docs/test-scenarios/15-linear-regression-custom-target-column.json
@@ -0,0 +1,40 @@
+{
+  "_description": "Target column name is not ``target``; report/Evidently sections must use params.target_variable.",
+  "experiment_run_id": 1015,
+  "variable_columns": [
+    "303-WIT-200(Value)"
+  ],
+  "target_variable": "MY_CUSTOM_TARGET_COLUMN",
+  "bucket_name": "model-training",
+  "file_name": "training_data_custom_target.csv",
+  "line_separator": ",",
+  "decimal_separator": ".",
+  "date_column": "timestamp",
+  "date_format": "yyyy-MM-dd HH:mm:ss",
+  "train_size": 80,
+  "shuffle": true,
+  "random_state": 42,
+  "model_name": "Linear Regression",
+  "model_type": "linear_regression",
+  "data_model_kwargs": {
+    "lag_train": {
+      "303-WIT-200(Value)": 0
+    },
+    "lag_val": {
+      "303-WIT-200(Value)": 0
+    },
+    "nan_treatment": "drop",
+    "rem_static_win": false,
+    "static_threshold": null,
+    "start_date": null,
+    "end_date": null,
+    "support_filters": {},
+    "removed_intervals": []
+  },
+  "model_kwargs": {
+    "degree": 1,
+    "interaction_only": false,
+    "scaler_name": "None"
+  },
+  "opt_params": {}
+}
--- a/docs/test-scenarios/16-linear-regression-naive-timestamp-header.json
+++ b/docs/test-scenarios/16-linear-regression-naive-timestamp-header.json
@@ -0,0 +1,40 @@
+{
+  "_description": "Naive Timestamp column header; snake_case date_column/date_format and training_data_timestamp_naive.csv.",
+  "experiment_run_id": 1016,
+  "variable_columns": [
+    "303-WIT-200(Value)"
+  ],
+  "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
+  "bucket_name": "model-training",
+  "file_name": "training_data_timestamp_naive.csv",
+  "line_separator": ",",
+  "decimal_separator": ".",
+  "date_column": "Timestamp",
+  "date_format": "yyyy-MM-dd HH:mm:ss",
+  "train_size": 80,
+  "shuffle": true,
+  "random_state": 42,
+  "model_name": "Linear Regression",
+  "model_type": "linear_regression",
+  "data_model_kwargs": {
+    "lag_train": {
+      "303-WIT-200(Value)": 0
+    },
+    "lag_val": {
+      "303-WIT-200(Value)": 0
+    },
+    "nan_treatment": "drop",
+    "rem_static_win": false,
+    "static_threshold": null,
+    "start_date": null,
+    "end_date": null,
+    "support_filters": {},
+    "removed_intervals": []
+  },
+  "model_kwargs": {
+    "degree": 1,
+    "interaction_only": false,
+    "scaler_name": "None"
+  },
+  "opt_params": {}
+}
--- a/docs/test-scenarios/17-linear-regression-blank-timestamp-row.json
+++ b/docs/test-scenarios/17-linear-regression-blank-timestamp-row.json
@@ -0,0 +1,40 @@
+{
+  "_description": "One CSV row has an empty timestamp; pipeline should drop it and continue training.",
+  "experiment_run_id": 1017,
+  "variable_columns": [
+    "303-WIT-200(Value)"
+  ],
+  "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)",
+  "bucket_name": "model-training",
+  "file_name": "training_data_blank_timestamp_row.csv",
+  "line_separator": ",",
+  "decimal_separator": ".",
+  "date_column": "timestamp",
+  "date_format": "yyyy-MM-dd HH:mm:ss",
+  "train_size": 80,
+  "shuffle": true,
+  "random_state": 42,
+  "model_name": "Linear Regression",
+  "model_type": "linear_regression",
+  "data_model_kwargs": {
+    "lag_train": {
+      "303-WIT-200(Value)": 0
+    },
+    "lag_val": {
+      "303-WIT-200(Value)": 0
+    },
+    "nan_treatment": "drop",
+    "rem_static_win": false,
+    "static_threshold": null,
+    "start_date": null,
+    "end_date": null,
+    "support_filters": {},
+    "removed_intervals": []
+  },
+  "model_kwargs": {
+    "degree": 1,
+    "interaction_only": false,
+    "scaler_name": "None"
+  },
+  "opt_params": {}
+}
--- a/docs/train-model-workflow-io-diff-main-vs-current-branch.md
+++ b/docs/train-model-workflow-io-diff-main-vs-current-branch.md
@@ -272,8 +272,8 @@ Now:
 - `file_name` -> kept
 - `line_separator` -> kept
 - `decimal_separator` -> kept
- `date_column` -> kept optional
- `date_format` -> kept optional
+- `date_column` -> required (snake_case key; must exist in CSV)
+- `date_format` -> optional in payload; omitted/null/blank resolves to default `yyyy-MM-dd HH:mm:ss`
 - `train_size` -> kept
 - `shuffle` -> kept
 - `model_name` -> kept (now less coupled to legacy model enum)