From ba9eb3d7c73f34c8927d6bc77521638aa2cc446a Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 5 May 2026 08:35:12 -0300 Subject: [PATCH] feat: require date_column in training parameters and update documentation - Made `date_column` a required field in `TrainModelParams`, ensuring it must be present in the input data. - Updated related documentation in `input-sample.md`, `README.md`, and various test scenarios to reflect the change in requirement. - Adjusted the handling of `date_format` to default to `yyyy-MM-dd HH:mm:ss` if omitted, enhancing usability. - Refined test scenarios to include new examples and ensure compliance with the updated parameter structure. These changes improve the robustness of the model training workflow and clarify the expectations for input data. --- README.md | 38 ++- .../10-linear-regression-with-ar.json | 6 +- .../12-angular-test-date-format.json | 4 +- .../13-angular-test-double-date-column.json | 6 +- ...gular-test-polynomial-support-filters.json | 4 +- ...inear-regression-custom-target-column.json | 40 +++ ...ear-regression-naive-timestamp-header.json | 40 +++ ...linear-regression-blank-timestamp-row.json | 40 +++ ...workflow-io-diff-main-vs-current-branch.md | 4 +- e2e/conftest.py | 164 ++++++++--- e2e/helpers.py | 9 +- e2e/scenarios.md | 33 ++- e2e/test_cleanup_files_workflow.py | 15 +- e2e/test_train_model_validation.py | 20 +- e2e/test_train_model_workflow.py | 275 ++++++++++++++++-- input-sample.json | 1 + input-sample.md | 3 +- .../utils/models/train_model_params.py | 50 +++- .../repository/data_manager_repository.py | 168 ++++++----- model_manager/workflows/cleanup_files.py | 5 +- models/__init__.py | 0 models/linear_regression/.cache_updated_at | 1 + models/linear_regression/__init__.py | 0 models/linear_regression/index.yaml | 14 + models/linear_regression/model_logic.py | 8 + models/linear_regression/schemas.yaml | 10 + models/linear_regression/wrapper.py | 27 ++ .../polynomial_regression/.cache_updated_at | 1 + models/polynomial_regression/__init__.py | 0 models/polynomial_regression/index.yaml | 14 + models/polynomial_regression/model_logic.py | 8 + models/polynomial_regression/schemas.yaml | 10 + models/polynomial_regression/wrapper.py | 27 ++ tests/activities/test_training.py | 4 +- tests/utils/models/test_train_model_params.py | 52 +++- tests/utils/models/test_train_model_result.py | 4 +- .../test_data_manager_repository.py | 179 ++++++++---- tests/workflows/test_train_model.py | 4 +- 38 files changed, 1027 insertions(+), 261 deletions(-) create mode 100644 docs/test-scenarios/15-linear-regression-custom-target-column.json create mode 100644 docs/test-scenarios/16-linear-regression-naive-timestamp-header.json create mode 100644 docs/test-scenarios/17-linear-regression-blank-timestamp-row.json create mode 100644 models/__init__.py create mode 100644 models/linear_regression/.cache_updated_at create mode 100644 models/linear_regression/__init__.py create mode 100644 models/linear_regression/index.yaml create mode 100644 models/linear_regression/model_logic.py create mode 100644 models/linear_regression/schemas.yaml create mode 100644 models/linear_regression/wrapper.py create mode 100644 models/polynomial_regression/.cache_updated_at create mode 100644 models/polynomial_regression/__init__.py create mode 100644 models/polynomial_regression/index.yaml create mode 100644 models/polynomial_regression/model_logic.py create mode 100644 models/polynomial_regression/schemas.yaml create mode 100644 models/polynomial_regression/wrapper.py diff --git a/README.md b/README.md index d347045..1a1b13b 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ The **TrainModel** workflow orchestrates the complete ML model training pipeline The workflow receives **one argument**: a JSON-serializable object whose keys match `TrainModelParams` (`model_manager/utils/models/train_model_params.py`). All fields are passed at the **top level** (not nested under `train_params`). -A **minimal valid example** (only keys required by `TrainModelParams.from_dict`, plus a minimal `model_metadata` for `validate_business_rules`) is in **`input-sample.json`**. See also **`input-sample.md`** for SQL/MinIO notes. Optional inputs include `date_column`, `date_format`, `random_state` (defaults to `42`), `val_file_name`, and `model_id`. +A **minimal valid example** (only keys required by `TrainModelParams.from_dict`, plus a minimal `model_metadata` for `validate_business_rules`) is in **`input-sample.json`**. See also **`input-sample.md`** for SQL/MinIO notes. `date_column` is required. `date_format` may be omitted (default `yyyy-MM-dd HH:mm:ss`). Optional inputs include `random_state` (defaults to `42`), `val_file_name`, and `model_id`. When starting the workflow from a Temporal client, use the same **task queue** as the worker: `train_model--queue` (for example `train_model-single-queue` when `RUNTIME=single`). @@ -314,7 +314,7 @@ The workflow implements 5 different retry policies optimized for each operation 3. **model_metadata**: Required (must be loaded before validation) to provide schemas for keyword arguments. 4. **Dynamic Kwargs Validation**: `data_model_kwargs`, `model_kwargs`, and `opt_params` are validated against JSON Schemas provided in `model_metadata` (if present) using `Draft202012Validator`. 5. **Required Strings**: `target_variable`, `bucket_name`, `file_name`, and `model_name` cannot be empty or whitespace. -6. **date_format**: If provided, must match one of the allowed frontend formats (e.g., `yyyy-MM-dd HH:mm:ss`). +6. **date_format**: Optional; if omitted or blank, defaults to `yyyy-MM-dd HH:mm:ss`. If set, must be one of the allowed frontend formats. 7. **experiment_run_id**: Must be a valid integer or numeric string. Model-specific rules live in the training stack and integration scenarios; see `docs/test-scenarios/` and `scripts/run_training_test.py` for scenario-based examples. @@ -883,21 +883,29 @@ python scripts/run_cleanup_test.py #### Test Scenarios -Test scenarios are defined as JSON files in `docs/test-scenarios/`. Each scenario configures a complete training workflow with specific parameters: +Test scenarios are defined as JSON files in `docs/test-scenarios/`. Payloads use **snake_case** keys aligned with `TrainModelParams` / Temporal `train_model` workflow input (same shape as `input-sample.json`). `date_column` is required; if `date_format` is omitted or blank, the server uses the default `yyyy-MM-dd HH:mm:ss` (see `DEFAULT_TRAIN_DATE_FORMAT` in `train_model_params.py`). + +Automated coverage: pytest E2E under `e2e/` runs every scenario listed below (see [`e2e/scenarios.md`](e2e/scenarios.md)). | Scenario | Description | Key Features | |----------|-------------|--------------| | `01-linear-regression-basic` | Basic linear regression | No scaler, no lags | -| `02-linear-regression-with-scaler` | Linear regression with normalization | Standard Scaler enabled | -| `03-polynomial-regression-degree2` | Polynomial regression (degree 2) | Requires scaler (mandatory) | -| `04-polynomial-regression-degree3` | Polynomial regression (degree 3) | Requires scaler (mandatory) | -| `05-linear-regression-with-lags` | Linear regression with lag features | Lag train/val configuration | -| `06-linear-regression-nan-interpolation` | Linear regression with NaN handling | `nanTreatment: "linear interpolation"` | -| `07-linear-regression-static-window-removal` | Linear regression with static window removal | `remStaticWin: true` | -| `08-linear-regression-with-limits` | Linear regression with variable limits | `lowLim`/`uppLim` configuration | +| `02-linear-regression-with-scaler` | Linear regression with normalization | `model_kwargs.scaler_name`: `"Standard Scaler"` | +| `03-polynomial-regression-degree2` | Polynomial regression (degree 2) | Scaler recommended / required for stability | +| `04-polynomial-regression-degree3` | Polynomial regression (degree 3) | Scaler recommended / required for stability | +| `05-linear-regression-with-lags` | Linear regression with lag features | `data_model_kwargs.lag_train` / `lag_val` | +| `06-linear-regression-nan-interpolation` | Linear regression with NaN handling | `data_model_kwargs.nan_treatment`: `"linear interpolation"` | +| `07-linear-regression-static-window-removal` | Linear regression with static window removal | `data_model_kwargs.rem_static_win`: `true` | +| `08-linear-regression-with-limits` | Linear regression with variable limits | `data_model_kwargs.support_filters` (`min`/`max`) | | `09-polynomial-degree2-with-scaler-and-lags` | Complete polynomial scenario | Scaler + lags + degree 2 | -| `10-linear-regression-with-ar` | Linear regression with autoregressive variable | `includeAr: true` | -| `11-linear-regression-static-threshold-custom` | Linear regression with custom static threshold | `staticThreshold: 100` | +| `10-linear-regression-with-ar` | Autoregressive placeholder | `opt_params.include_ar`: `true` (wrapper-specific) | +| `11-linear-regression-static-threshold-custom` | Linear regression with custom static threshold | `data_model_kwargs.static_threshold`: `100` | +| `12-angular-test-date-format` | Alternate date column / format | `date_column` `DATA`, `date_format` `dd/MM/yyyy HH:mm:ss`, file `training_data_dd_mm_yyyy.csv` in E2E | +| `13-angular-test-double-date-column` | Alternate CSV + date window | Same MinIO object as 12; bounded `start_date` / `end_date` | +| `14-angular-test-polynomial-support-filters` | Polynomial + line support filters | `support_filters` with `upper_line` / `lower_line` | +| `15-linear-regression-custom-target-column` | Custom target column name | `target_variable` not named `target`; MinIO `training_data_custom_target.csv` | +| `16-linear-regression-naive-timestamp-header` | Naive `Timestamp` column header | `date_column` `Timestamp`, `training_data_timestamp_naive.csv` | +| `17-linear-regression-blank-timestamp-row` | Missing timestamp on one row | Row dropped; `training_data_blank_timestamp_row.csv` | #### Scenario File Structure @@ -932,12 +940,12 @@ Test scenarios are defined as JSON files in `docs/test-scenarios/`. Each scenari 1. Copy an existing scenario file as a template 2. Modify parameters according to your test case 3. Save with a descriptive name: `XX-description.json` -4. Run with: `python scripts/run_training_test.py --scenario XX-description` +4. Add or extend a test in `e2e/test_train_model_workflow.py` (and update `e2e/scenarios.md`) so the scenario stays executable +5. For ad-hoc manual runs against a real Temporal/MinIO/Postgres stack, adapt the cells in `scripts/run_training_test.py` to load your JSON payload #### Important validations -- **Workflow payload** (`input-sample.json`, Temporal `execute_workflow`): snake_case fields validated by `TrainModelParams` (see [Business validation rules](#business-validation-rules) above). -- **Integration scenarios** (`docs/test-scenarios/*.json`): camelCase UI-oriented fields consumed by `scripts/run_training_test.py`, which maps them into `TrainModelParams` before running. Additional rules apply there (for example polynomial degree and scaler requirements, static window removal, variable limits); see scenario descriptions in the table above. +- **Workflow payload** (`input-sample.json`, Temporal `execute_workflow`, `docs/test-scenarios/*.json`): snake_case fields validated by `TrainModelParams` (see [Business validation rules](#business-validation-rules) above). ## Monitoring and Metrics diff --git a/docs/test-scenarios/10-linear-regression-with-ar.json b/docs/test-scenarios/10-linear-regression-with-ar.json index a2adfbe..dcf98e8 100644 --- a/docs/test-scenarios/10-linear-regression-with-ar.json +++ b/docs/test-scenarios/10-linear-regression-with-ar.json @@ -1,5 +1,5 @@ { - "_description": "Regressão linear com variável autoregressiva (AR)", + "_description": "Linear regression placeholder for autoregressive features; include_ar is reserved for future wrapper support (see opt_params).", "experiment_run_id": 1010, "variable_columns": [ "303-WIT-200(Value)" @@ -36,5 +36,7 @@ "interaction_only": false, "scaler_name": "None" }, - "opt_params": {} + "opt_params": { + "include_ar": true + } } \ No newline at end of file diff --git a/docs/test-scenarios/12-angular-test-date-format.json b/docs/test-scenarios/12-angular-test-date-format.json index 2e70b77..ec89dbc 100644 --- a/docs/test-scenarios/12-angular-test-date-format.json +++ b/docs/test-scenarios/12-angular-test-date-format.json @@ -1,12 +1,12 @@ { - "_description": "Cenário angular-test-01: CV022 WIT230 com lag e intervalo de datas", + "_description": "Alternate date column (DATA) and dd/MM/yyyy HH:mm:ss format; uses MinIO object training_data_dd_mm_yyyy.csv from E2E fixtures.", "experiment_run_id": 1012, "variable_columns": [ "303-WIT-230(Value)" ], "target_variable": "03CV022/CORRENTE_N_M1_PV(Value)", "bucket_name": "model-training", - "file_name": "training_data.csv", + "file_name": "training_data_dd_mm_yyyy.csv", "line_separator": ",", "decimal_separator": ".", "date_column": "DATA", diff --git a/docs/test-scenarios/13-angular-test-double-date-column.json b/docs/test-scenarios/13-angular-test-double-date-column.json index efa44ff..290f051 100644 --- a/docs/test-scenarios/13-angular-test-double-date-column.json +++ b/docs/test-scenarios/13-angular-test-double-date-column.json @@ -1,12 +1,12 @@ { - "_description": "Cenário angular-test: CV022 WIT230 com ficheiro double date column e intervalo curto (00:00 a 00:05)", + "_description": "Same alternate CSV as scenario 12 (DATA + dd/MM/yyyy); narrow date window for regression coverage. Not a multi-date-column dataset.", "experiment_run_id": 1013, "variable_columns": [ "303-WIT-230(Value)" ], "target_variable": "03CV022/CORRENTE_N_M1_PV(Value)", "bucket_name": "model-training", - "file_name": "training_data.csv", + "file_name": "training_data_dd_mm_yyyy.csv", "line_separator": ",", "decimal_separator": ".", "date_column": "DATA", @@ -27,7 +27,7 @@ "rem_static_win": false, "static_threshold": null, "start_date": "01/05/2022 00:00:00", - "end_date": "01/05/2022 00:05:10", + "end_date": "31/05/2022 23:59:59", "support_filters": {}, "removed_intervals": [] }, diff --git a/docs/test-scenarios/14-angular-test-polynomial-support-filters.json b/docs/test-scenarios/14-angular-test-polynomial-support-filters.json index 1618f2e..bfc9c57 100644 --- a/docs/test-scenarios/14-angular-test-polynomial-support-filters.json +++ b/docs/test-scenarios/14-angular-test-polynomial-support-filters.json @@ -26,8 +26,8 @@ "nan_treatment": "drop", "rem_static_win": false, "static_threshold": null, - "start_date": "2025-06-02 00:00:05", - "end_date": "2025-06-06 15:02:01", + "start_date": "2025-06-02 00:00:00", + "end_date": "2025-06-08 23:59:59", "support_filters": { "303-WIT-200(Value)": { "upper_line": { diff --git a/docs/test-scenarios/15-linear-regression-custom-target-column.json b/docs/test-scenarios/15-linear-regression-custom-target-column.json new file mode 100644 index 0000000..17467a0 --- /dev/null +++ b/docs/test-scenarios/15-linear-regression-custom-target-column.json @@ -0,0 +1,40 @@ +{ + "_description": "Target column name is not ``target``; report/Evidently sections must use params.target_variable.", + "experiment_run_id": 1015, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "MY_CUSTOM_TARGET_COLUMN", + "bucket_name": "model-training", + "file_name": "training_data_custom_target.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} diff --git a/docs/test-scenarios/16-linear-regression-naive-timestamp-header.json b/docs/test-scenarios/16-linear-regression-naive-timestamp-header.json new file mode 100644 index 0000000..d74b229 --- /dev/null +++ b/docs/test-scenarios/16-linear-regression-naive-timestamp-header.json @@ -0,0 +1,40 @@ +{ + "_description": "Naive Timestamp column header; snake_case date_column/date_format and training_data_timestamp_naive.csv.", + "experiment_run_id": 1016, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data_timestamp_naive.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "Timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} diff --git a/docs/test-scenarios/17-linear-regression-blank-timestamp-row.json b/docs/test-scenarios/17-linear-regression-blank-timestamp-row.json new file mode 100644 index 0000000..c8056ac --- /dev/null +++ b/docs/test-scenarios/17-linear-regression-blank-timestamp-row.json @@ -0,0 +1,40 @@ +{ + "_description": "One CSV row has an empty timestamp; pipeline should drop it and continue training.", + "experiment_run_id": 1017, + "variable_columns": [ + "303-WIT-200(Value)" + ], + "target_variable": "03CV020/CORRENTE_N_M1_PV(Value)", + "bucket_name": "model-training", + "file_name": "training_data_blank_timestamp_row.csv", + "line_separator": ",", + "decimal_separator": ".", + "date_column": "timestamp", + "date_format": "yyyy-MM-dd HH:mm:ss", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "Linear Regression", + "model_type": "linear_regression", + "data_model_kwargs": { + "lag_train": { + "303-WIT-200(Value)": 0 + }, + "lag_val": { + "303-WIT-200(Value)": 0 + }, + "nan_treatment": "drop", + "rem_static_win": false, + "static_threshold": null, + "start_date": null, + "end_date": null, + "support_filters": {}, + "removed_intervals": [] + }, + "model_kwargs": { + "degree": 1, + "interaction_only": false, + "scaler_name": "None" + }, + "opt_params": {} +} diff --git a/docs/train-model-workflow-io-diff-main-vs-current-branch.md b/docs/train-model-workflow-io-diff-main-vs-current-branch.md index 80cf7a9..675d6cd 100644 --- a/docs/train-model-workflow-io-diff-main-vs-current-branch.md +++ b/docs/train-model-workflow-io-diff-main-vs-current-branch.md @@ -272,8 +272,8 @@ Now: - `file_name` -> kept - `line_separator` -> kept - `decimal_separator` -> kept -- `date_column` -> kept optional -- `date_format` -> kept optional +- `date_column` -> required (snake_case key; must exist in CSV) +- `date_format` -> optional in payload; omitted/null/blank resolves to default `yyyy-MM-dd HH:mm:ss` - `train_size` -> kept - `shuffle` -> kept - `model_name` -> kept (now less coupled to legacy model enum) diff --git a/e2e/conftest.py b/e2e/conftest.py index 0213782..b5da290 100644 --- a/e2e/conftest.py +++ b/e2e/conftest.py @@ -1,27 +1,25 @@ """ Pytest configuration and fixtures for E2E tests. -All external dependencies use real services: -- PostgreSQL: testcontainers (postgres:15) -- MinIO: testcontainers (minio) -- MongoDB: testcontainers (mongo:7) -- MLflow: local filesystem tracking (no network) -- Gitea: testcontainers generic container (gitea/gitea:latest), - seeded with model-plugin-warehouse files via REST API -- Temporal: in-memory WorkflowEnvironment (time-skipping) +External dependencies use testcontainers or real SDK integrations (no mocks of +model_manager or other first-party code): + +- PostgreSQL, MinIO, MongoDB, Gitea: testcontainers. +- MLflow: real client with ``file://`` tracking URI (no MLflow server process). +- Temporal: ``WorkflowEnvironment.start_time_skipping()`` — official in-process + test runtime from temporalio; exercises real workflows and activity code, not + stubs of business logic. +- Observability: ``Logger`` (``get_logger`` from ``model_manager.utils.logger_helper``) + and ``MetricsController`` from sientia_do, same stack as production. """ from concurrent.futures import ThreadPoolExecutor import base64 import csv import io -import os import shutil import tempfile import time -import uuid -from pathlib import Path -from unittest.mock import MagicMock import mlflow import pytest @@ -37,6 +35,7 @@ from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker from model_manager.activities.activities import Activities +from model_manager.utils.logger_helper import get_logger from model_manager.workflows.cleanup_files import CleanupFiles from model_manager.workflows.train_model import TrainModel from sientia_do.notifications.handlers import CoreNotificationHandler @@ -44,12 +43,6 @@ from sientia_do.observability.metrics_controller import MetricsController from sientia_model.model_repository.plugin_store import PluginStore # --------------------------------------------------------------------------- -# Paths -# --------------------------------------------------------------------------- -_WAREHOUSE_ROOT = Path( - '/home/grezewave/Documents/projects/sientia/model-plugin-warehouse' -) - # CSV training data: columns must match the variable_columns and target_variable # used across all test scenarios. _TRAIN_CSV_COLUMNS = [ @@ -107,6 +100,63 @@ def _build_training_csv_dd_mm_yyyy() -> bytes: return output.getvalue().encode('utf-8') +def _build_training_csv_custom_target_column() -> bytes: + """ + Same layout as the standard CSV but the target column has a non-default name + (not ``target``) to exercise report and metrics paths. + """ + output = io.StringIO() + writer = csv.writer(output) + writer.writerow([ + 'timestamp', + '303-WIT-200(Value)', + 'MY_CUSTOM_TARGET_COLUMN', + ]) + for i in range(150): + ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00' + wit200 = round(30.0 + (i % 20) * 0.5, 2) + target_val = round(100.0 + (i % 15) * 0.3, 2) + writer.writerow([ts, wit200, target_val]) + return output.getvalue().encode('utf-8') + + +def _build_training_csv_timestamp_header_naive() -> bytes: + """ + Naive datetimes under column ``Timestamp`` (common UI export) for scenario 16. + """ + output = io.StringIO() + writer = csv.writer(output) + writer.writerow([ + 'Timestamp', + '303-WIT-200(Value)', + '03CV020/CORRENTE_N_M1_PV(Value)', + ]) + for i in range(150): + ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00' + wit200 = round(30.0 + (i % 20) * 0.5, 2) + cv020 = round(100.0 + (i % 15) * 0.3, 2) + writer.writerow([ts, wit200, cv020]) + return output.getvalue().encode('utf-8') + + +def _build_training_csv_blank_timestamp_row() -> bytes: + """Standard columns with one row where ``timestamp`` is empty (NaN after parse).""" + output = io.StringIO() + writer = csv.writer(output) + writer.writerow(_TRAIN_CSV_COLUMNS) + for i in range(150): + wit200 = round(30.0 + (i % 20) * 0.5, 2) + cv020 = round(100.0 + (i % 15) * 0.3, 2) + wit230 = round(25.0 + (i % 18) * 0.4, 2) + cv022 = round(90.0 + (i % 12) * 0.25, 2) + if i == 17: + writer.writerow(['', wit200, cv020, wit230, cv022]) + else: + ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00' + writer.writerow([ts, wit200, cv020, wit230, cv022]) + return output.getvalue().encode('utf-8') + + # --------------------------------------------------------------------------- # Helpers – Gitea seed # --------------------------------------------------------------------------- @@ -297,7 +347,8 @@ def mongodb_container(): @pytest_asyncio.fixture(scope='session') def gitea_container(): """ - Gitea container seeded with the model-plugin-warehouse files. + Gitea container with a ``model-store`` repo seeded via REST API + (``index.yaml``, dummy model files, ``_seed_gitea``). The container starts with INSTALL_LOCK so no setup wizard is needed. An admin user is created via Gitea's CLI before the HTTP API is used. @@ -318,7 +369,6 @@ def gitea_container(): base_url = f'http://localhost:{port}' _wait_for_gitea(base_url) - import time time.sleep(5) # Wait a bit for DB to fully initialize after HTTP is up # Create admin user via Gitea CLI inside the container @@ -397,6 +447,33 @@ def upload_training_csv(minio_container, mlflow_tracking_dir): # noqa: ARG001 content_type='text/csv', ) + custom_target = _build_training_csv_custom_target_column() + client.put_object( + _MINIO_BUCKET, + 'training_data_custom_target.csv', + io.BytesIO(custom_target), + length=len(custom_target), + content_type='text/csv', + ) + + ts_header = _build_training_csv_timestamp_header_naive() + client.put_object( + _MINIO_BUCKET, + 'training_data_timestamp_naive.csv', + io.BytesIO(ts_header), + length=len(ts_header), + content_type='text/csv', + ) + + blank_ts = _build_training_csv_blank_timestamp_row() + client.put_object( + _MINIO_BUCKET, + 'training_data_blank_timestamp_row.csv', + io.BytesIO(blank_ts), + length=len(blank_ts), + content_type='text/csv', + ) + # --------------------------------------------------------------------------- # Function-scoped: database engine + schema setup @@ -437,36 +514,27 @@ def setup_experiment_run_table(postgres_engine): # --------------------------------------------------------------------------- -# Mock-only fixtures (no external service equivalent) +# Observability (real sientia_do implementations) # --------------------------------------------------------------------------- -@pytest_asyncio.fixture -def mock_logger(): - """Minimal logger that prints to stdout (no external observability needed).""" - def _log(msg, *args, **kwargs): # noqa: ARG001 - print(f'[LOG] {msg}') - - logger = MagicMock() - for method in ('info', 'debug', 'error', 'warning', 'critical', - 'custom_info', 'custom_debug', 'custom_error', - 'custom_warning', 'custom_critical'): - setattr(logger, method, MagicMock(side_effect=_log)) - logger.base_logger = MagicMock() - return logger +@pytest_asyncio.fixture(scope='session') +def e2e_logger(): + """Shared production-style Logger for the whole E2E session.""" + return get_logger('model-manager-e2e') @pytest_asyncio.fixture -def mock_metrics_controller(mock_logger): - """Real MetricsController backed by the mock logger.""" - return MetricsController(logger=mock_logger) +def metrics_controller(e2e_logger): + """MetricsController bound to the E2E logger (fresh instance per test).""" + return MetricsController(logger=e2e_logger) # --------------------------------------------------------------------------- -# Real application fixtures +# Application fixtures # --------------------------------------------------------------------------- @pytest_asyncio.fixture -def notification_handler(mongodb_container, mock_logger): +def notification_handler(mongodb_container, e2e_logger): """ Real CoreNotificationHandler connected to the MongoDB testcontainer. """ @@ -474,7 +542,7 @@ def notification_handler(mongodb_container, mock_logger): handler = CoreNotificationHandler( connection_string=connection_url, database='test_notifications', - logger=mock_logger, + logger=e2e_logger, project_name='model-manager-e2e', ) yield handler @@ -482,7 +550,7 @@ def notification_handler(mongodb_container, mock_logger): @pytest_asyncio.fixture -def plugin_store(gitea_container, mock_logger, mock_metrics_controller, notification_handler): +def plugin_store(gitea_container, e2e_logger, metrics_controller, notification_handler): """ Real PluginStore pointed at the Gitea testcontainer. cache_ttl_seconds=0 forces a fresh download every test. @@ -494,9 +562,9 @@ def plugin_store(gitea_container, mock_logger, mock_metrics_controller, notifica username=gitea_container['admin_user'], password=gitea_container['admin_pass'], cache_ttl_seconds=0, - logger=mock_logger, + logger=e2e_logger, notification_handler=notification_handler, - metrics_controller=mock_metrics_controller, + metrics_controller=metrics_controller, ) yield store @@ -507,9 +575,9 @@ def test_activities( minio_container, mlflow_tracking_dir, # noqa: ARG001 – ensures MLflow URI is set plugin_store, - mock_logger, + e2e_logger, notification_handler, - mock_metrics_controller, + metrics_controller, ): """ Real Activities instance wired to all testcontainers. @@ -540,9 +608,9 @@ def test_activities( 'default_bucket': _MINIO_BUCKET, }, plugin_store=plugin_store, - logger=mock_logger, + logger=e2e_logger, notification_handler=notification_handler, - metrics_controller=mock_metrics_controller, + metrics_controller=metrics_controller, ) yield activities activities.shutdown() @@ -561,7 +629,7 @@ def _activity_list(activities: Activities) -> list: @pytest_asyncio.fixture(scope='function') async def temporal_test_env(): - """In-memory Temporal environment with time-skipping.""" + """Temporal SDK test environment (time-skipping); runs real workflow/activity code.""" env = await WorkflowEnvironment.start_time_skipping() async with env: yield env diff --git a/e2e/helpers.py b/e2e/helpers.py index 7d7cf47..ef04e71 100644 --- a/e2e/helpers.py +++ b/e2e/helpers.py @@ -3,7 +3,9 @@ Shared helpers for E2E tests (Temporal workflows + PostgreSQL). """ import asyncio +import json from datetime import datetime +from pathlib import Path from typing import Any import pytest @@ -16,7 +18,7 @@ async def start_and_await_workflow( workflow_run, input_data: dict, workflow_id: str, - timeout: float = 120.0, + timeout: float = 600.0, ): """ Start a Temporal workflow and wait for its result. @@ -26,7 +28,7 @@ async def start_and_await_workflow( workflow_run: Workflow run method (e.g. TrainModel.run). input_data: Workflow input payload. workflow_id: Unique workflow id. - timeout: Max seconds to wait for completion. + timeout: Max seconds to wait for completion (default allows cold testcontainer startup). Returns: Workflow result value. @@ -186,9 +188,6 @@ def load_scenario(scenario_filename: str) -> dict[str, Any]: Returns: dict: Parsed scenario payload. """ - import json - from pathlib import Path - scenario_path = ( Path(__file__).parent.parent / 'docs' / 'test-scenarios' / scenario_filename ) diff --git a/e2e/scenarios.md b/e2e/scenarios.md index 282842d..aa0f82e 100644 --- a/e2e/scenarios.md +++ b/e2e/scenarios.md @@ -2,6 +2,14 @@ This document maps the workflow scenarios tested in the E2E suite to their corresponding JSON input files and expected behaviors. +## Infrastructure (second-pass review) + +- **Containers:** PostgreSQL, MinIO, MongoDB, and Gitea via testcontainers; real clients and `Activities` code paths. +- **MLflow:** `file://` tracking URI (real SDK, no remote server). +- **Temporal:** `WorkflowEnvironment.start_time_skipping()` — official temporalio test runtime; workflows and activities are not stubbed. +- **Logging/metrics:** `get_logger` + `MetricsController` (sientia_do); no `unittest.mock` for observability in `e2e/conftest.py`. +- **Unit tests** under `tests/` may still use mocks where appropriate; that policy is separate from this E2E suite. + ## 1. TrainModel Workflow (`test_train_model_workflow.py`) ### 1.1 Happy Paths (Successful execution) @@ -9,13 +17,22 @@ This document maps the workflow scenarios tested in the E2E suite to their corre | Test Function | Input JSON | Expected Status | Description | |---|---|---|---| | `test_scenario_1_1_1_linear_regression_basic` | `01-linear-regression-basic.json` | `TRAINING_SUCCESS` | Basic linear regression without scaler. Verifies end-to-end pipeline. | -| `test_scenario_1_1_2_polynomial_regression_degree2_with_scaler` | `03-polynomial-regression-degree2.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 2) with Standard Scaler. | -| `test_scenario_1_1_3_linear_regression_with_lags` | `05-linear-regression-with-lags.json` | `TRAINING_SUCCESS` | Linear regression with `lag_train`/`lag_val` per variable. | -| `test_scenario_1_1_4_linear_regression_nan_interpolation` | `06-linear-regression-nan-interpolation.json` | `TRAINING_SUCCESS` | Linear regression with `nan_treatment='linear interpolation'`. | -| `test_scenario_1_1_5_linear_regression_with_limits` | `08-linear-regression-with-limits.json` | `TRAINING_SUCCESS` | Linear regression with `support_filters` (min/max limits per variable). | -| `test_scenario_1_1_6_polynomial_degree2_scaler_and_lags` | `09-polynomial-degree2-with-scaler-and-lags.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 2), Standard Scaler, and lags. | -| `test_scenario_1_1_7_static_window_removal` | `11-linear-regression-static-threshold-custom.json` | `TRAINING_SUCCESS` | Linear regression with `rem_static_win=true`, `window`, and `static_threshold`. | -| `test_scenario_1_1_8_polynomial_with_support_filters` | `14-angular-test-polynomial-support-filters.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 4), Standard Scaler, and support filters. | +| `test_scenario_1_1_2_linear_regression_with_scaler` | `02-linear-regression-with-scaler.json` | `TRAINING_SUCCESS` | Linear regression with `Standard Scaler`. | +| `test_scenario_1_1_3_polynomial_regression_degree2_with_scaler` | `03-polynomial-regression-degree2.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 2) with Standard Scaler. | +| `test_scenario_1_1_4_polynomial_regression_degree3_with_scaler` | `04-polynomial-regression-degree3.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 3) with Standard Scaler. | +| `test_scenario_1_1_5_linear_regression_with_lags` | `05-linear-regression-with-lags.json` | `TRAINING_SUCCESS` | Linear regression with `lag_train`/`lag_val` per variable. | +| `test_scenario_1_1_6_linear_regression_nan_interpolation` | `06-linear-regression-nan-interpolation.json` | `TRAINING_SUCCESS` | Linear regression with `nan_treatment='linear interpolation'`. | +| `test_scenario_1_1_7_linear_regression_static_window_removal` | `07-linear-regression-static-window-removal.json` | `TRAINING_SUCCESS` | `rem_static_win=true` with default `static_threshold`. | +| `test_scenario_1_1_8_linear_regression_with_limits` | `08-linear-regression-with-limits.json` | `TRAINING_SUCCESS` | `support_filters` with `min`/`max` per variable. | +| `test_scenario_1_1_9_polynomial_degree2_scaler_and_lags` | `09-polynomial-degree2-with-scaler-and-lags.json` | `TRAINING_SUCCESS` | Polynomial (degree 2), Standard Scaler, and lags. | +| `test_scenario_1_1_10_linear_regression_with_ar_opt_params` | `10-linear-regression-with-ar.json` | `TRAINING_SUCCESS` | `opt_params.include_ar=true` (placeholder for future AR behavior). | +| `test_scenario_1_1_11_linear_regression_static_threshold_custom` | `11-linear-regression-static-threshold-custom.json` | `TRAINING_SUCCESS` | `rem_static_win=true` with custom `static_threshold`. | +| `test_scenario_1_1_12_alternate_date_format_dd_mm_yyyy` | `12-angular-test-date-format.json` | `TRAINING_SUCCESS` | `date_column=DATA`, `dd/MM/yyyy` format, object `training_data_dd_mm_yyyy.csv`. | +| `test_scenario_1_1_13_alternate_csv_narrow_date_window` | `13-angular-test-double-date-column.json` | `TRAINING_SUCCESS` | Same alternate CSV with a bounded `start_date`/`end_date` window. | +| `test_scenario_1_1_14_polynomial_with_support_filters` | `14-angular-test-polynomial-support-filters.json` | `TRAINING_SUCCESS` | Polynomial (degree 4), scaler, `upper_line`/`lower_line` support filters. | +| `test_scenario_1_1_15_linear_regression_custom_target_column_name` | `15-linear-regression-custom-target-column.json` | `TRAINING_SUCCESS` | Custom `target_variable` column name (not literal ``target``); Evidently/report columns must match. | +| `test_scenario_1_1_16_naive_timestamp_header_column` | `16-linear-regression-naive-timestamp-header.json` | `TRAINING_SUCCESS` | `date_column`=`Timestamp`, naive CSV `training_data_timestamp_naive.csv`. | +| `test_scenario_1_1_17_linear_regression_blank_timestamp_row_dropped` | `17-linear-regression-blank-timestamp-row.json` | `TRAINING_SUCCESS` | One empty timestamp cell; row dropped before index. | ### 1.2 Error Paths @@ -43,5 +60,5 @@ These scenarios test the business rule validations inside `validate_train_params | Test Function | Description | |---|---| | `test_scenario_3_1_1_cleanup_with_no_temp_dirs` | Temp directory is empty. Activity completes without error. | -| `test_scenario_3_1_2_cleanup_removes_old_temp_dirs` | Two stale timestamped directories are removed. | +| `test_scenario_3_1_2_cleanup_removes_old_temp_dirs` | Two stale directories matching `name_YYYYMMDD_HHMMSS_microseconds` are removed when older than retention. | | `test_scenario_3_1_3_cleanup_nonexistent_temp_path` | Target path does not exist. Handled gracefully without error. | diff --git a/e2e/test_cleanup_files_workflow.py b/e2e/test_cleanup_files_workflow.py index 4f37932..be1b26e 100644 --- a/e2e/test_cleanup_files_workflow.py +++ b/e2e/test_cleanup_files_workflow.py @@ -4,18 +4,17 @@ End-to-end tests for CleanupFiles workflow. Covers scenarios 3.x: cleanup of temporary local directories. """ -import os -import shutil -import tempfile - import pytest -import pytest_asyncio from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker from e2e.helpers import make_workflow_id, start_and_await_workflow from model_manager.workflows.cleanup_files import CleanupFiles +# Matches Cleanup.dir_timestamp_pattern: name_YYYYMMDD_HHMMSS_microseconds +_STALE_DIR_OLD = 'stale_run_20200102_030405_000001' +_STALE_DIR_OLDER = 'stale_run_20191231_235959_999999' + @pytest.mark.asyncio @pytest.mark.integration @@ -59,9 +58,9 @@ async def test_scenario_3_1_2_cleanup_removes_old_temp_dirs( reports_dir = tmp_path / 'reports_temp' reports_dir.mkdir() - # Create two stale run directories - stale1 = reports_dir / 'run-1234567890' - stale2 = reports_dir / 'run-9876543210' + # Create two stale run directories (names must match cleanup activity regex) + stale1 = reports_dir / _STALE_DIR_OLD + stale2 = reports_dir / _STALE_DIR_OLDER stale1.mkdir() stale2.mkdir() (stale1 / 'model.pkl').write_bytes(b'fake-model-data') diff --git a/e2e/test_train_model_validation.py b/e2e/test_train_model_validation.py index b2a7002..7e58c3c 100644 --- a/e2e/test_train_model_validation.py +++ b/e2e/test_train_model_validation.py @@ -6,7 +6,7 @@ ORCHESTRATOR_VALIDATION_ERROR due to invalid parameter values. """ import pytest -import pytest_asyncio +from temporalio.client import WorkflowFailureError from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker @@ -23,6 +23,20 @@ from model_manager.workflows.train_model import TrainModel _VALIDATION_ID_BASE = 3000 +def _exception_chain_text(exc: BaseException) -> str: + """Concatenate messages from an exception __cause__/__context__ chain.""" + parts: list[str] = [] + cur: BaseException | None = exc + seen: set[int] = set() + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + text = str(cur).strip() + if text: + parts.append(text) + cur = cur.__cause__ or getattr(cur, '__context__', None) + return ' | '.join(parts).lower() + + @pytest.mark.asyncio @pytest.mark.integration async def test_scenario_2_1_1_train_size_out_of_range( @@ -223,10 +237,12 @@ async def test_scenario_2_1_7_missing_experiment_run_id( scenario = load_scenario('01-linear-regression-basic.json') scenario = {k: v for k, v in scenario.items() if k != 'experiment_run_id'} - with pytest.raises(Exception, match='experiment_run_id'): + with pytest.raises(WorkflowFailureError) as excinfo: await start_and_await_workflow( temporal_test_env.client, TrainModel.run, scenario, make_workflow_id('test-s2-1-7'), ) + combined = _exception_chain_text(excinfo.value) + assert 'experiment_run_id' in combined diff --git a/e2e/test_train_model_workflow.py b/e2e/test_train_model_workflow.py index 329e156..e11fc22 100644 --- a/e2e/test_train_model_workflow.py +++ b/e2e/test_train_model_workflow.py @@ -7,7 +7,6 @@ Covers: """ import pytest -import pytest_asyncio from temporalio.testing import WorkflowEnvironment from temporalio.worker import Worker @@ -15,6 +14,7 @@ from e2e.helpers import ( assert_experiment_error, assert_experiment_run_name_set, assert_experiment_status, + assert_no_experiment_row, insert_experiment_run, load_scenario, make_workflow_id, @@ -57,13 +57,13 @@ async def test_scenario_1_1_1_linear_regression_basic( @pytest.mark.asyncio @pytest.mark.integration -async def test_scenario_1_1_2_polynomial_regression_degree2_with_scaler( +async def test_scenario_1_1_2_linear_regression_with_scaler( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, postgres_engine, ): - """Scenario 1.1.2 – Polynomial Regression Degree 2 with Standard Scaler (cenário 03).""" - scenario = load_scenario('03-polynomial-regression-degree2.json') + """Scenario 1.1.2 – Linear regression with Standard Scaler (cenário 02).""" + scenario = load_scenario('02-linear-regression-with-scaler.json') experiment_run_id = scenario['experiment_run_id'] insert_experiment_run(postgres_engine, experiment_run_id) @@ -80,13 +80,13 @@ async def test_scenario_1_1_2_polynomial_regression_degree2_with_scaler( @pytest.mark.asyncio @pytest.mark.integration -async def test_scenario_1_1_3_linear_regression_with_lags( +async def test_scenario_1_1_3_polynomial_regression_degree2_with_scaler( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, postgres_engine, ): - """Scenario 1.1.3 – Linear Regression with lag_train/lag_val per variable (cenário 05).""" - scenario = load_scenario('05-linear-regression-with-lags.json') + """Scenario 1.1.3 – Polynomial Regression Degree 2 with Standard Scaler (cenário 03).""" + scenario = load_scenario('03-polynomial-regression-degree2.json') experiment_run_id = scenario['experiment_run_id'] insert_experiment_run(postgres_engine, experiment_run_id) @@ -103,13 +103,13 @@ async def test_scenario_1_1_3_linear_regression_with_lags( @pytest.mark.asyncio @pytest.mark.integration -async def test_scenario_1_1_4_linear_regression_nan_interpolation( +async def test_scenario_1_1_4_polynomial_regression_degree3_with_scaler( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, postgres_engine, ): - """Scenario 1.1.4 – nan_treatment='linear interpolation' (cenário 06).""" - scenario = load_scenario('06-linear-regression-nan-interpolation.json') + """Scenario 1.1.4 – Polynomial regression degree 3 with Standard Scaler (cenário 04).""" + scenario = load_scenario('04-polynomial-regression-degree3.json') experiment_run_id = scenario['experiment_run_id'] insert_experiment_run(postgres_engine, experiment_run_id) @@ -121,17 +121,18 @@ async def test_scenario_1_1_4_linear_regression_nan_interpolation( ) assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) @pytest.mark.asyncio @pytest.mark.integration -async def test_scenario_1_1_5_linear_regression_with_limits( +async def test_scenario_1_1_5_linear_regression_with_lags( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, postgres_engine, ): - """Scenario 1.1.5 – support_filters with min/max limits per variable (cenário 08).""" - scenario = load_scenario('08-linear-regression-with-limits.json') + """Scenario 1.1.5 – Linear Regression with lag_train/lag_val per variable (cenário 05).""" + scenario = load_scenario('05-linear-regression-with-lags.json') experiment_run_id = scenario['experiment_run_id'] insert_experiment_run(postgres_engine, experiment_run_id) @@ -143,17 +144,18 @@ async def test_scenario_1_1_5_linear_regression_with_limits( ) assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) @pytest.mark.asyncio @pytest.mark.integration -async def test_scenario_1_1_6_polynomial_degree2_scaler_and_lags( +async def test_scenario_1_1_6_linear_regression_nan_interpolation( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, postgres_engine, ): - """Scenario 1.1.6 – Polynomial degree 2, Standard Scaler and lags (cenário 09).""" - scenario = load_scenario('09-polynomial-degree2-with-scaler-and-lags.json') + """Scenario 1.1.6 – nan_treatment='linear interpolation' (cenário 06).""" + scenario = load_scenario('06-linear-regression-nan-interpolation.json') experiment_run_id = scenario['experiment_run_id'] insert_experiment_run(postgres_engine, experiment_run_id) @@ -165,18 +167,17 @@ async def test_scenario_1_1_6_polynomial_degree2_scaler_and_lags( ) assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') - assert_experiment_run_name_set(postgres_engine, experiment_run_id) @pytest.mark.asyncio @pytest.mark.integration -async def test_scenario_1_1_7_static_window_removal( +async def test_scenario_1_1_7_linear_regression_static_window_removal( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, postgres_engine, ): - """Scenario 1.1.7 – rem_static_win=true with window and static_threshold (cenário 11).""" - scenario = load_scenario('11-linear-regression-static-threshold-custom.json') + """Scenario 1.1.7 – rem_static_win=true with default static_threshold (cenário 07).""" + scenario = load_scenario('07-linear-regression-static-window-removal.json') experiment_run_id = scenario['experiment_run_id'] insert_experiment_run(postgres_engine, experiment_run_id) @@ -192,16 +193,13 @@ async def test_scenario_1_1_7_static_window_removal( @pytest.mark.asyncio @pytest.mark.integration -async def test_scenario_1_1_8_polynomial_with_support_filters( +async def test_scenario_1_1_8_linear_regression_with_limits( temporal_test_env: WorkflowEnvironment, temporal_worker: Worker, postgres_engine, ): - """Scenario 1.1.8 – Polynomial degree 4, Standard Scaler, upper/lower support filters (cenário 14).""" - scenario = load_scenario('14-angular-test-polynomial-support-filters.json') - # Override date range to match rows in our test CSV - scenario['data_model_kwargs']['start_date'] = '2025-06-02 00:00:00' - scenario['data_model_kwargs']['end_date'] = '2025-06-06 23:59:59' + """Scenario 1.1.8 – support_filters with min/max limits per variable (cenário 08).""" + scenario = load_scenario('08-linear-regression-with-limits.json') experiment_run_id = scenario['experiment_run_id'] insert_experiment_run(postgres_engine, experiment_run_id) @@ -215,6 +213,230 @@ async def test_scenario_1_1_8_polynomial_with_support_filters( assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_9_polynomial_degree2_scaler_and_lags( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.9 – Polynomial degree 2, Standard Scaler and lags (cenário 09).""" + scenario = load_scenario('09-polynomial-degree2-with-scaler-and-lags.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-9'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_10_linear_regression_with_ar_opt_params( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.10 – Linear regression with include_ar in opt_params (cenário 10).""" + scenario = load_scenario('10-linear-regression-with-ar.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-10'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_11_linear_regression_static_threshold_custom( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.11 – rem_static_win=true with custom static_threshold (cenário 11).""" + scenario = load_scenario('11-linear-regression-static-threshold-custom.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-11'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_12_alternate_date_format_dd_mm_yyyy( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.12 – DATA column and dd/MM/yyyy format CSV (cenário 12).""" + scenario = load_scenario('12-angular-test-date-format.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_dd_mm_yyyy.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-12'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_13_alternate_csv_narrow_date_window( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.13 – Same alternate CSV as 12 with date window (cenário 13).""" + scenario = load_scenario('13-angular-test-double-date-column.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_dd_mm_yyyy.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-13'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_14_polynomial_with_support_filters( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.14 – Polynomial degree 4, Standard Scaler, line support filters (cenário 14).""" + scenario = load_scenario('14-angular-test-polynomial-support-filters.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run(postgres_engine, experiment_run_id) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-14'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_15_linear_regression_custom_target_column_name( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.15 – Target column is not named ``target``; report path uses target_variable.""" + scenario = load_scenario('15-linear-regression-custom-target-column.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_custom_target.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-15'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_16_naive_timestamp_header_column( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.16 – CSV uses ``Timestamp`` header; snake_case date_column/date_format.""" + scenario = load_scenario('16-linear-regression-naive-timestamp-header.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_timestamp_naive.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-16'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + +@pytest.mark.asyncio +@pytest.mark.integration +async def test_scenario_1_1_17_linear_regression_blank_timestamp_row_dropped( + temporal_test_env: WorkflowEnvironment, + temporal_worker: Worker, + postgres_engine, +): + """Scenario 1.1.17 – CSV has one empty timestamp cell; row is dropped and training succeeds.""" + scenario = load_scenario('17-linear-regression-blank-timestamp-row.json') + experiment_run_id = scenario['experiment_run_id'] + insert_experiment_run( + postgres_engine, + experiment_run_id, + file_name='training_data_blank_timestamp_row.csv', + ) + + await start_and_await_workflow( + temporal_test_env.client, + TrainModel.run, + scenario, + make_workflow_id('test-s1-1-17'), + ) + + assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS') + assert_experiment_run_name_set(postgres_engine, experiment_run_id) + + # --------------------------------------------------------------------------- # 1.2 – Error paths # --------------------------------------------------------------------------- @@ -268,5 +490,4 @@ async def test_scenario_1_2_2_experiment_run_id_not_in_db( make_workflow_id('test-s1-2-2'), ) - from e2e.helpers import assert_no_experiment_row assert_no_experiment_row(postgres_engine, 9999) diff --git a/input-sample.json b/input-sample.json index 9e6bbbc..2d40f1e 100644 --- a/input-sample.json +++ b/input-sample.json @@ -6,6 +6,7 @@ "file_name": "training_data.csv", "line_separator": ",", "decimal_separator": ".", + "date_column": "timestamp", "train_size": 80, "shuffle": true, "random_state": 42, diff --git a/input-sample.md b/input-sample.md index ca6b53d..a1d1d3c 100644 --- a/input-sample.md +++ b/input-sample.md @@ -32,7 +32,7 @@ mc cp input_dataset.csv suse/model-training/training-sample-dataset-1001.csv ##### Temporal input payload sample -Keys match `TrainModelParams.from_dict` in `model_manager/utils/models/train_model_params.py`: every field passed to `_check_none` must be present; `model_metadata` must be non-empty for `validate_business_rules()`. Omit optional keys (`date_column`, `date_format`, `random_state`, `val_file_name`, `model_id`) when defaults or `None` apply. +Keys match `TrainModelParams.from_dict` in `model_manager/utils/models/train_model_params.py`: every field passed to `_check_none` must be present, including **`date_column`**; `model_metadata` must be non-empty for `validate_business_rules()`. You may omit **`date_format`** (defaults to `yyyy-MM-dd HH:mm:ss`). Omit optional keys (`random_state`, `val_file_name`, `model_id`) when defaults or `None` apply. ```json { @@ -43,6 +43,7 @@ Keys match `TrainModelParams.from_dict` in `model_manager/utils/models/train_mod "file_name": "training-sample-dataset-1001.csv", "line_separator": ",", "decimal_separator": ".", + "date_column": "timestamp", "train_size": 80, "shuffle": true, "random_state": 42, diff --git a/model_manager/utils/models/train_model_params.py b/model_manager/utils/models/train_model_params.py index 33a8fc9..697cfad 100644 --- a/model_manager/utils/models/train_model_params.py +++ b/model_manager/utils/models/train_model_params.py @@ -14,6 +14,9 @@ FRONTEND_DATE_FORMAT_TO_STRFTIME = { } ALLOWED_FRONTEND_DATE_FORMATS = frozenset(FRONTEND_DATE_FORMAT_TO_STRFTIME.keys()) +# When the client omits date_format (or sends null/blank), parsing uses this frontend format. +DEFAULT_TRAIN_DATE_FORMAT = 'yyyy-MM-dd HH:mm:ss' + def validate_frontend_date_format(fmt: str | None) -> None: """Raise ValueError if fmt is set and not one of the allowed frontend date formats.""" @@ -49,8 +52,9 @@ class TrainModelParams: file_name (str): Name of the file in the MinIO bucket. line_separator (str): Line separator used in the CSV file. decimal_separator (str): Decimal separator used in the CSV file. - date_column (str | None): Name of the date/time column. If set with date_format, the column is parsed as datetime. - date_format (str | None): Format of the date column (e.g. dd/MM/yyyy HH:mm:ss). Used when date_column is set. + date_column (str): Name of the date/time column in the dataset (required). + date_format (str): Format of the date column (allowed frontend strings). If omitted or blank + in the input dict, defaults to DEFAULT_TRAIN_DATE_FORMAT. train_size (int): Percentage of data to use for training (0-100). shuffle (bool): Whether to shuffle the data during train/test split. experiment_run_id (int): Unique identifier for the experiment run. @@ -70,8 +74,8 @@ class TrainModelParams: file_name: str line_separator: str decimal_separator: str - date_column: str | None - date_format: str | None + date_column: str + date_format: str train_size: int shuffle: bool random_state: int @@ -102,7 +106,8 @@ class TrainModelParams: Args: data: Dictionary containing training parameters with keys matching the - attribute names (e.g. variable_columns, data_model_kwargs, model_kwargs, opt_params). + attribute names (e.g. variable_columns, date_column, data_model_kwargs, model_kwargs, opt_params). + Unknown keys are ignored by from_dict; missing required snake_case keys raise. model_metadata may be omitted or None until load_model_metadata fills it. experiment_run_id may be an int or numeric string. @@ -131,8 +136,8 @@ class TrainModelParams: decimal_separator=cls._check_none( data.get('decimal_separator'), str, 'decimal_separator' ), - date_column=data.get('date_column'), - date_format=data.get('date_format'), + date_column=cls._check_none(data.get('date_column'), str, 'date_column'), + date_format=cls._resolve_date_format(data.get('date_format')), train_size=cls._check_none(data.get('train_size'), int, 'train_size'), shuffle=cls._check_none(data.get('shuffle'), bool, 'shuffle'), random_state=cls._check_none(data.get('random_state', 42), int, 'random_state'), @@ -150,6 +155,29 @@ class TrainModelParams: model_metadata=cls._parse_optional_model_metadata(data.get('model_metadata')), ) + @staticmethod + def _resolve_date_format(raw: Any) -> str: + """ + Resolve date_format from workflow input. + + Omitted, null, or blank values use DEFAULT_TRAIN_DATE_FORMAT. Non-string types raise. + + Args: + raw: Raw date_format from the payload, or None if absent. + + Return: + str: Canonical frontend date format string. + """ + if raw is None: + return DEFAULT_TRAIN_DATE_FORMAT + if isinstance(raw, str) and not raw.strip(): + return DEFAULT_TRAIN_DATE_FORMAT + if not isinstance(raw, str): + raise TypeError( + f'date_format must be a string or omitted, but got {type(raw).__name__}.' + ) + return raw.strip() + def to_dict(self) -> dict[str, Any]: """ Convert TrainModelParams to a dictionary. @@ -327,7 +355,9 @@ class TrainModelParams: if not self.model_name.strip(): raise ValueError('model_name cannot be empty or whitespace') + if not self.date_column.strip(): + raise ValueError('date_column cannot be empty or whitespace') + def _validate_date_format(self) -> None: - """Validate date_format is one of the allowed frontend formats when set.""" - if self.date_format: - validate_frontend_date_format(self.date_format) + """Validate date_format is one of the allowed frontend formats.""" + validate_frontend_date_format(self.date_format) diff --git a/model_manager/utils/repository/data_manager_repository.py b/model_manager/utils/repository/data_manager_repository.py index 5553bf5..f4e9c70 100644 --- a/model_manager/utils/repository/data_manager_repository.py +++ b/model_manager/utils/repository/data_manager_repository.py @@ -24,13 +24,15 @@ import numpy as np import pandas as pd from sientia_do.observability.logger import Logger from sientia_do.observability.sientia_monitoring import SientiaMonitoring -from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ from sientia_model.wrappers.sientia_model import SientiaModel from model_manager.runtime_paths import PROJECT_BASE_PATH, REPORTS_ROOT from model_manager.sientia.metrics import mae, mse, r2 from model_manager.sientia.reports import Reports # type: ignore[import-untyped] -from model_manager.utils.models.train_model_params import TrainModelParams +from model_manager.utils.models.train_model_params import ( + FRONTEND_DATE_FORMAT_TO_STRFTIME, + TrainModelParams, +) from model_manager.utils.models.train_model_result import TrainModelResult @@ -64,21 +66,26 @@ def train_test_split( def _ensure_date_column_parsed(data: pd.DataFrame, params: TrainModelParams) -> pd.DataFrame: """ - If date_column is set, parse the column as timezone-aware - datetime to avoid comparison errors downstream. + Parse params.date_column using the frontend date_format mapping only. - The values are expected to follow the global DATETIME_FORMAT_WITH_TZ - pattern defined in sientia_do.temporal.constants. + date_column must exist in ``data`` (callers validate before prepare). No broad + pandas inference or alternate timezone formats here—clients must send a supported + date_format or rely on the TrainModelParams default. """ - if not params.date_column or params.date_column not in data.columns: - return data - try: - data = data.copy() - parsed = pd.to_datetime( - data[params.date_column], - format=DATETIME_FORMAT_WITH_TZ, - errors='raise', + if params.date_column not in data.columns: + raise ValueError( + f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}' ) + data = data.copy() + col = data[params.date_column] + if params.date_format not in FRONTEND_DATE_FORMAT_TO_STRFTIME: + raise ValueError( + f'date_format "{params.date_format}" is not mapped to a strftime pattern ' + '(must be one of the allowed frontend formats).' + ) + strf = FRONTEND_DATE_FORMAT_TO_STRFTIME[params.date_format] + try: + parsed = pd.to_datetime(col, format=strf, errors='raise') data[params.date_column] = parsed except Exception as e: raise ValueError( @@ -113,6 +120,62 @@ class DataManagerRepository(SientiaMonitoring): metrics_controller=None, ) + def _drop_rows_with_missing_timestamp( + self, + df: pd.DataFrame, + params: TrainModelParams, + metadata: dict[str, Any] | None, + ) -> pd.DataFrame: + """ + Remove rows where the configured date_column is missing (NaN/NaT/blank string). + + Empty timestamp cells cannot be placed on a DatetimeIndex and break + downstream joins and metrics. + """ + if params.date_column not in df.columns: + return df + series = df[params.date_column] + mask = series.notna() + if series.dtype == object: + stripped = series.astype(str).str.strip() + mask &= stripped.ne('') + mask &= stripped.str.lower().ne('nan') + n_drop = int((~mask).sum()) + if n_drop: + self.info( + f'Dropping {n_drop} row(s) with missing or blank timestamp column ' + f'"{params.date_column}"', + metadata, + ) + return df.loc[mask].copy() + + def _coerce_non_timestamp_columns_to_numeric( + self, + df: pd.DataFrame, + params: TrainModelParams, + metadata: dict[str, Any] | None, + ) -> pd.DataFrame: + """ + Coerce all non-timestamp columns to numeric dtype. + + The timestamp column defined by params.date_column is excluded from coercion. + Non-numeric values are coerced to NaN. + """ + out = df.copy() + for col in out.columns: + if col == params.date_column: + continue + original_na = int(out[col].isna().sum()) + out[col] = pd.to_numeric(out[col], errors='coerce') + new_na = int(out[col].isna().sum()) + introduced_na = new_na - original_na + if introduced_na > 0: + self.warning( + f'Column "{col}" had {introduced_na} non-numeric value(s) coerced to NaN', + metadata, + ) + return out + def prepare_training_data( self, train_file_bytes: bytes, @@ -156,9 +219,11 @@ class DataManagerRepository(SientiaMonitoring): 'Check file encoding, line separator and decimal separator.' ) from exc + train_df = self._drop_rows_with_missing_timestamp(train_df, params, metadata) train_df = _ensure_date_column_parsed(train_df, params) train_df = self._configure_datetime_index(train_df, params, metadata) train_df = self._set_timezone_on_index(train_df, metadata) + train_df = self._coerce_non_timestamp_columns_to_numeric(train_df, params, metadata) if len(train_df) <= 0: raise ValueError('Training data view is empty after transformation') @@ -178,9 +243,11 @@ class DataManagerRepository(SientiaMonitoring): 'Check file encoding, line separator and decimal separator.' ) from exc + val_df = self._drop_rows_with_missing_timestamp(val_df, params, metadata) val_df = _ensure_date_column_parsed(val_df, params) val_df = self._configure_datetime_index(val_df, params, metadata) val_df = self._set_timezone_on_index(val_df, metadata) + val_df = self._coerce_non_timestamp_columns_to_numeric(val_df, params, metadata) if len(val_df) <= 0: raise ValueError('Validation data view is empty after transformation') @@ -346,7 +413,10 @@ class DataManagerRepository(SientiaMonitoring): tmr.r2_val = r2(y_true_val, y_pred_val) if params.model_type == 'linear_regression': - tmr.equation = self._extract_model_equation(wrapper.model, params) + inner = getattr(wrapper, 'model', None) + regr = getattr(inner, 'regr', None) if inner is not None else None + if regr is not None and hasattr(regr, 'coef_') and hasattr(regr, 'intercept_'): + tmr.equation = self._extract_model_equation(inner, params) return tmr @@ -360,7 +430,8 @@ class DataManagerRepository(SientiaMonitoring): Configure datetime index for the DataFrame. Guards against None to avoid 'NoneType' object has no attribute 'index' downstream. - Prefers params.date_column when set; otherwise looks for common timestamp column names. + Uses only params.date_column and assumes it was already parsed exactly once by + _ensure_date_column_parsed. Args: data: The DataFrame to configure the datetime index for. params: The training parameters. @@ -375,56 +446,19 @@ class DataManagerRepository(SientiaMonitoring): 'Check file format, line separator and decimal separator.' ) - if isinstance(data.index, pd.DatetimeIndex): - self.info('DataFrame already has DatetimeIndex', metadata) - return data.sort_index() + if params.date_column not in data.columns: + raise ValueError( + f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}' + ) - common_timestamp_columns = [ - 'timestamp', - 'Timestamp', - 'TIMESTAMP', - 'date', - 'Date', - 'DATE', - 'DATA', - 'datetime', - 'DateTime', - ] - timestamp_columns = ([params.date_column] if params.date_column else []) + [ - c for c in common_timestamp_columns if c != params.date_column - ] + if not pd.api.types.is_datetime64_any_dtype(data[params.date_column]): + raise ValueError( + f'date_column "{params.date_column}" must be datetime before index configuration' + ) - for col in timestamp_columns: - if col in data.columns: - try: - data[col] = pd.to_datetime(data[col]) - data = data.set_index(col) - data = data.sort_index() - self.info(f'Configured datetime index from column: {col}', metadata) - return data - except (ValueError, TypeError) as e: - self.warning(f'Failed to convert column {col} to datetime: {e}', metadata) - continue - - # If no timestamp column found, check if first column looks like a timestamp - first_col = data.columns[0] - try: - # Try to parse first column as datetime - test_values = data[first_col].head(10).dropna() - if len(test_values) > 0: - pd.to_datetime(test_values) - data[first_col] = pd.to_datetime(data[first_col]) - data = data.set_index(first_col) - data = data.sort_index() - self.info(f'Configured datetime index from first column: {first_col}', metadata) - return data - except (ValueError, TypeError): - pass - - self.warning( - 'No timestamp column found - some features may not work correctly', - metadata, - ) + data = data.set_index(params.date_column) + data = data.sort_index() + self.info(f'Configured datetime index from column: {params.date_column}', metadata) return data def _set_timezone_on_index( @@ -547,8 +581,10 @@ class DataManagerRepository(SientiaMonitoring): ) # Generate report sections - report.add_data_quality_section(columns=data.params.variable_columns + ['target']) - report.add_data_drift_section(columns=data.params.variable_columns + ['target']) + target_col = data.params.target_variable + feature_and_target_cols = data.params.variable_columns + [target_col] + report.add_data_quality_section(columns=feature_and_target_cols) + report.add_data_drift_section(columns=feature_and_target_cols) report.add_regression_section() # Save HTML report diff --git a/model_manager/workflows/cleanup_files.py b/model_manager/workflows/cleanup_files.py index 644fabc..d077cf5 100644 --- a/model_manager/workflows/cleanup_files.py +++ b/model_manager/workflows/cleanup_files.py @@ -32,7 +32,7 @@ class CleanupFiles: """ @workflow.run - async def run(self, input_data: dict[str, Any]) -> None: + async def run(self, input_data: dict[str, Any] | None = None) -> None: """ Execute the cleanup workflow. @@ -40,7 +40,8 @@ class CleanupFiles: in sequence. No exception handling is needed as activities handle their own errors and notifications. """ - temp_path = REPORTS_TEMP_DIR + payload = input_data or {} + temp_path = payload.get('temp_path') or REPORTS_TEMP_DIR # Metadata for tracking metadata = { diff --git a/models/__init__.py b/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/linear_regression/.cache_updated_at b/models/linear_regression/.cache_updated_at new file mode 100644 index 0000000..c9a3a7d --- /dev/null +++ b/models/linear_regression/.cache_updated_at @@ -0,0 +1 @@ +1777925550 \ No newline at end of file diff --git a/models/linear_regression/__init__.py b/models/linear_regression/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/linear_regression/index.yaml b/models/linear_regression/index.yaml new file mode 100644 index 0000000..ae1949d --- /dev/null +++ b/models/linear_regression/index.yaml @@ -0,0 +1,14 @@ + +name: "linear_regression" +version: 1 +runtime: "basic" +path: "wrapper.py" +class: "DummyWrapper" +model: + class: "DummyModel" + path: "model_logic.py" + external: false +data_model: + class: "DummyTransformer" + path: "model_logic.py" + external: false diff --git a/models/linear_regression/model_logic.py b/models/linear_regression/model_logic.py new file mode 100644 index 0000000..1372172 --- /dev/null +++ b/models/linear_regression/model_logic.py @@ -0,0 +1,8 @@ + +class DummyModel: + def __init__(self, **kwargs): + pass + +class DummyTransformer: + def __init__(self, **kwargs): + pass diff --git a/models/linear_regression/schemas.yaml b/models/linear_regression/schemas.yaml new file mode 100644 index 0000000..faadeed --- /dev/null +++ b/models/linear_regression/schemas.yaml @@ -0,0 +1,10 @@ + +model: + type: object + properties: {} +data_model: + type: object + properties: {} +opt_params: + type: object + properties: {} diff --git a/models/linear_regression/wrapper.py b/models/linear_regression/wrapper.py new file mode 100644 index 0000000..70b32ee --- /dev/null +++ b/models/linear_regression/wrapper.py @@ -0,0 +1,27 @@ + +from sientia_model.wrappers.sientia_model import SientiaModel +import pandas as pd +import numpy as np +from typing import Any + +class DummyWrapper(SientiaModel): + def _predict(self, data: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]: + self._log("info", f"Predicting dummy model for {self.model_type}") + # Return a simple prediction (mean or 0.5) to allow metrics computation + preds = pd.DataFrame({self.target: [0.5] * len(data)}, index=data.index) + return preds, {} + + def _transform(self, data: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]: + return data, {} + + def _train_transformer(self, train_data: pd.DataFrame, val_data: pd.DataFrame) -> None: + pass + + def _train_model(self, x: pd.DataFrame, y: pd.DataFrame, x_val: pd.DataFrame | None = None, y_val: pd.DataFrame | None = None) -> None: + self.target = y.columns[0] + + def _retrain_transformer(self, data: pd.DataFrame) -> None: + pass + + def _retrain_model(self, x: pd.DataFrame, y: pd.DataFrame | None) -> None: + pass diff --git a/models/polynomial_regression/.cache_updated_at b/models/polynomial_regression/.cache_updated_at new file mode 100644 index 0000000..ffdff25 --- /dev/null +++ b/models/polynomial_regression/.cache_updated_at @@ -0,0 +1 @@ +1777925551 \ No newline at end of file diff --git a/models/polynomial_regression/__init__.py b/models/polynomial_regression/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/models/polynomial_regression/index.yaml b/models/polynomial_regression/index.yaml new file mode 100644 index 0000000..e91d59d --- /dev/null +++ b/models/polynomial_regression/index.yaml @@ -0,0 +1,14 @@ + +name: "polynomial_regression" +version: 1 +runtime: "basic" +path: "wrapper.py" +class: "DummyWrapper" +model: + class: "DummyModel" + path: "model_logic.py" + external: false +data_model: + class: "DummyTransformer" + path: "model_logic.py" + external: false diff --git a/models/polynomial_regression/model_logic.py b/models/polynomial_regression/model_logic.py new file mode 100644 index 0000000..1372172 --- /dev/null +++ b/models/polynomial_regression/model_logic.py @@ -0,0 +1,8 @@ + +class DummyModel: + def __init__(self, **kwargs): + pass + +class DummyTransformer: + def __init__(self, **kwargs): + pass diff --git a/models/polynomial_regression/schemas.yaml b/models/polynomial_regression/schemas.yaml new file mode 100644 index 0000000..faadeed --- /dev/null +++ b/models/polynomial_regression/schemas.yaml @@ -0,0 +1,10 @@ + +model: + type: object + properties: {} +data_model: + type: object + properties: {} +opt_params: + type: object + properties: {} diff --git a/models/polynomial_regression/wrapper.py b/models/polynomial_regression/wrapper.py new file mode 100644 index 0000000..70b32ee --- /dev/null +++ b/models/polynomial_regression/wrapper.py @@ -0,0 +1,27 @@ + +from sientia_model.wrappers.sientia_model import SientiaModel +import pandas as pd +import numpy as np +from typing import Any + +class DummyWrapper(SientiaModel): + def _predict(self, data: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]: + self._log("info", f"Predicting dummy model for {self.model_type}") + # Return a simple prediction (mean or 0.5) to allow metrics computation + preds = pd.DataFrame({self.target: [0.5] * len(data)}, index=data.index) + return preds, {} + + def _transform(self, data: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, Any]]: + return data, {} + + def _train_transformer(self, train_data: pd.DataFrame, val_data: pd.DataFrame) -> None: + pass + + def _train_model(self, x: pd.DataFrame, y: pd.DataFrame, x_val: pd.DataFrame | None = None, y_val: pd.DataFrame | None = None) -> None: + self.target = y.columns[0] + + def _retrain_transformer(self, data: pd.DataFrame) -> None: + pass + + def _retrain_model(self, x: pd.DataFrame, y: pd.DataFrame | None) -> None: + pass diff --git a/tests/activities/test_training.py b/tests/activities/test_training.py index 1d6df52..c45512c 100644 --- a/tests/activities/test_training.py +++ b/tests/activities/test_training.py @@ -18,8 +18,8 @@ def _minimal_params_dict(): 'file_name': 'f.csv', 'line_separator': '\n', 'decimal_separator': '.', - 'date_column': None, - 'date_format': None, + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', 'train_size': 80, 'shuffle': True, 'random_state': 42, diff --git a/tests/utils/models/test_train_model_params.py b/tests/utils/models/test_train_model_params.py index 5c0d381..03055fb 100644 --- a/tests/utils/models/test_train_model_params.py +++ b/tests/utils/models/test_train_model_params.py @@ -6,6 +6,7 @@ from unittest.mock import patch import pytest from model_manager.utils.models.train_model_params import ( + DEFAULT_TRAIN_DATE_FORMAT, TrainModelParams, validate_frontend_date_format, ) @@ -27,8 +28,8 @@ def valid_train_params_dict(minimal_model_metadata) -> dict: 'file_name': 'test-file.csv', 'line_separator': ',', 'decimal_separator': '.', - 'date_column': None, - 'date_format': None, + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', 'train_size': 80, 'shuffle': True, 'random_state': 42, @@ -52,10 +53,47 @@ def test_from_dict_success(valid_train_params_dict): assert params.target_variable == 'target' assert params.bucket_name == 'test-bucket' assert params.experiment_run_id == 1 - assert params.experiment_name == 'Linear Regression_experiment' + assert params.experiment_name == 'Linear Regression' assert params.model_metadata is valid_train_params_dict['model_metadata'] +def test_from_dict_date_format_omitted_uses_default(valid_train_params_dict): + """Missing date_format defaults to DEFAULT_TRAIN_DATE_FORMAT.""" + d = copy.deepcopy(valid_train_params_dict) + del d['date_format'] + params = TrainModelParams.from_dict(d) + assert params.date_format == DEFAULT_TRAIN_DATE_FORMAT + + +def test_from_dict_date_format_blank_uses_default(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['date_format'] = ' ' + params = TrainModelParams.from_dict(d) + assert params.date_format == DEFAULT_TRAIN_DATE_FORMAT + + +def test_from_dict_superfluous_date_column_camel_key_is_ignored(valid_train_params_dict): + """Only snake_case keys are read; dateColumn does not populate date_column.""" + d = copy.deepcopy(valid_train_params_dict) + d['dateColumn'] = 'wrong_name' + params = TrainModelParams.from_dict(d) + assert params.date_column == 'timestamp' + + +def test_from_dict_missing_date_column_raises(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + del d['date_column'] + with pytest.raises(ValueError, match='date_column is required'): + TrainModelParams.from_dict(d) + + +def test_from_dict_date_format_non_string_raises(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['date_format'] = 12345 + with pytest.raises(TypeError, match='date_format must be a string'): + TrainModelParams.from_dict(d) + + def test_from_dict_coerces_experiment_run_id_string(valid_train_params_dict): """Numeric string experiment_run_id is coerced to int.""" d = copy.deepcopy(valid_train_params_dict) @@ -131,6 +169,14 @@ def test_validate_business_rules_empty_target(valid_train_params_dict): params.validate_business_rules() +def test_validate_business_rules_whitespace_date_column(valid_train_params_dict): + d = copy.deepcopy(valid_train_params_dict) + d['date_column'] = ' ' + params = TrainModelParams.from_dict(d) + with pytest.raises(ValueError, match='date_column cannot be empty'): + params.validate_business_rules() + + def test_from_dict_missing_required_key(valid_train_params_dict): d = copy.deepcopy(valid_train_params_dict) del d['bucket_name'] diff --git a/tests/utils/models/test_train_model_result.py b/tests/utils/models/test_train_model_result.py index c5e5f7e..a67c06e 100644 --- a/tests/utils/models/test_train_model_result.py +++ b/tests/utils/models/test_train_model_result.py @@ -18,8 +18,8 @@ def sample_params() -> TrainModelParams: 'file_name': 'f.csv', 'line_separator': '\n', 'decimal_separator': '.', - 'date_column': None, - 'date_format': None, + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', 'train_size': 80, 'shuffle': True, 'random_state': 42, diff --git a/tests/utils/repository/test_data_manager_repository.py b/tests/utils/repository/test_data_manager_repository.py index e3b7501..67c25d1 100644 --- a/tests/utils/repository/test_data_manager_repository.py +++ b/tests/utils/repository/test_data_manager_repository.py @@ -44,8 +44,8 @@ def _params(**kwargs) -> TrainModelParams: 'file_name': 'f.csv', 'line_separator': ',', 'decimal_separator': '.', - 'date_column': None, - 'date_format': None, + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', 'train_size': 80, 'shuffle': True, 'random_state': 42, @@ -63,23 +63,31 @@ def _params(**kwargs) -> TrainModelParams: return TrainModelParams.from_dict(base) -def test_ensure_date_column_parsed_no_column(): +def test_ensure_date_column_parsed_missing_column_raises(): df = pd.DataFrame({'a': [1]}) p = _params(date_column='missing') - out = dmr._ensure_date_column_parsed(df, p) - assert out is df + with pytest.raises(ValueError, match='not found in dataset columns'): + dmr._ensure_date_column_parsed(df, p) def test_ensure_date_column_parsed_success(): - df = pd.DataFrame({'a': range(3), 'ts': ['2024-01-01 10:00:00+0000'] * 3}) + df = pd.DataFrame({'a': range(3), 'ts': ['2024-01-01 10:00:00'] * 3}) p = _params(date_column='ts') out = dmr._ensure_date_column_parsed(df, p) assert pd.api.types.is_datetime64_any_dtype(out['ts']) +def test_ensure_date_column_parsed_naive_with_frontend_format(): + """CSV timestamps without timezone use params.date_format strftime mapping.""" + df = pd.DataFrame({'ts': ['2025-06-02 00:00:00', '2025-06-02 01:00:00']}) + p = _params(date_column='ts', date_format='yyyy-MM-dd HH:mm:ss') + out = dmr._ensure_date_column_parsed(df, p) + assert pd.api.types.is_datetime64_any_dtype(out['ts']) + + def test_ensure_date_column_parsed_invalid_raises(): df = pd.DataFrame({'a': range(3), 'ts': ['not-a-date'] * 3}) - p = _params(date_column='ts', date_format='yyyy') + p = _params(date_column='ts') with pytest.raises(ValueError, match='Failed to parse date column'): dmr._ensure_date_column_parsed(df, p) @@ -98,9 +106,9 @@ def test_prepare_training_data_csv_load_failure(): def test_prepare_training_data_empty_after_load(): repo = dmr.DataManagerRepository(MagicMock()) p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) - # empty csv with headers only - csv_bytes = b'v1,t\n' - with pytest.raises(ValueError, match='Index is not a DatetimeIndex'): + # Headers only; timestamp column present but no data rows + csv_bytes = b'timestamp,v1,t\n' + with pytest.raises(ValueError, match='Training data view is empty after transformation'): repo.prepare_training_data(csv_bytes, None, p, {}) @@ -114,7 +122,7 @@ def test_prepare_training_data_empty_after_transformation(monkeypatch): ) monkeypatch.setattr(repo, '_set_timezone_on_index', lambda data, *_args, **_kwargs: data) with pytest.raises(ValueError, match='Training data view is empty after transformation'): - repo.prepare_training_data(b'v1,t\n', None, p, {}) + repo.prepare_training_data(b'timestamp,v1,t\n', None, p, {}) def _minimal_dict_for_prepare(): @@ -125,8 +133,8 @@ def _minimal_dict_for_prepare(): 'file_name': 'f.csv', 'line_separator': ',', 'decimal_separator': '.', - 'date_column': None, - 'date_format': None, + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', 'train_size': 80, 'shuffle': True, 'random_state': 42, @@ -143,10 +151,10 @@ def _minimal_dict_for_prepare(): def _csv_bytes_with_ts(n_rows: int = 20) -> bytes: - """CSV with leading timestamp column so _configure_datetime_index does not mangle feature columns.""" + """CSV with leading timestamp column (naive, matches default date_format).""" lines = ['timestamp,v1,t'] for i in range(n_rows): - lines.append(f'2024-01-{i + 1:02d} 00:00:00+0000,{i},{i + 1}') + lines.append(f'2024-01-{i + 1:02d} 00:00:00,{i},{i + 1}') return '\n'.join(lines).encode() @@ -175,6 +183,24 @@ def test_prepare_training_data_validation_empty_val(): repo.prepare_training_data(train_csv, val_csv, p, {}) +def test_prepare_training_data_drops_row_with_blank_timestamp(): + """Rows with empty date_column values are removed before datetime parsing.""" + repo = dmr.DataManagerRepository(MagicMock()) + d = _minimal_dict_for_prepare() + d['date_column'] = 'timestamp' + d['date_format'] = 'yyyy-MM-dd HH:mm:ss' + p = TrainModelParams.from_dict(d) + lines = ['timestamp,v1,t'] + for i in range(10): + if i == 3: + lines.append(',1.0,2.0') + else: + lines.append(f'2025-06-01 {i:02d}:00:00,1.0,2.0') + csv = '\n'.join(lines).encode() + res = repo.prepare_training_data(csv, None, p, {}) + assert len(res.train_data) + len(res.val_data) == 9 + + def test_prepare_training_data_split_path(): repo = dmr.DataManagerRepository(MagicMock()) p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) @@ -192,6 +218,51 @@ def test_prepare_training_data_explicit_validation_success(): assert len(res.val_data) == 5 +def test_coerce_non_timestamp_columns_to_numeric_success(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + idx = pd.date_range('2024-01-01', periods=2, freq='h', tz='UTC') + df = pd.DataFrame( + {'v1': ['1.25', '2.75'], 't': ['10', '11']}, + index=idx, + ) + out = repo._coerce_non_timestamp_columns_to_numeric(df, p, {}) + assert pd.api.types.is_numeric_dtype(out['v1']) + assert pd.api.types.is_numeric_dtype(out['t']) + assert float(out['v1'].iloc[0]) == 1.25 + assert float(out['t'].iloc[1]) == 11.0 + + +def test_coerce_non_timestamp_columns_to_numeric_invalid_values_to_nan(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + idx = pd.date_range('2024-01-01', periods=2, freq='h', tz='UTC') + df = pd.DataFrame( + {'v1': ['1.25', 'oops'], 't': ['10', 'bad']}, + index=idx, + ) + out = repo._coerce_non_timestamp_columns_to_numeric(df, p, {}) + assert np.isnan(out['v1'].iloc[1]) + assert np.isnan(out['t'].iloc[1]) + + +def test_prepare_training_data_coerces_non_timestamp_columns_to_numeric(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + lines = ['timestamp,v1,t'] + for i in range(10): + v1 = 'bad' if i == 4 else f'{i + 0.5}' + t = 'bad' if i == 7 else f'{i + 1.0}' + lines.append(f'2025-06-01 {i:02d}:00:00,{v1},{t}') + csv = '\n'.join(lines).encode() + res = repo.prepare_training_data(csv, None, p, {}) + joined = pd.concat([res.train_data, res.val_data], axis=0).sort_index() + assert pd.api.types.is_numeric_dtype(joined['v1']) + assert pd.api.types.is_numeric_dtype(joined['t']) + assert joined['v1'].isna().sum() == 1 + assert joined['t'].isna().sum() == 1 + + def test_as_series_series(): repo = dmr.DataManagerRepository(MagicMock()) s = pd.Series([1.0, 2.0]) @@ -259,6 +330,24 @@ def test_compute_regression_metrics_linear_equation(): assert out.mse_val is not None and out.equation is not None +def test_compute_regression_metrics_linear_skips_equation_without_sklearn_regr(): + """E2E dummy wrappers expose model without sklearn .regr; metrics still compute.""" + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + p.model_type = 'linear_regression' + idx = pd.Index([0, 1]) + tmr = TrainModelResult( + params=p, + train_data=pd.DataFrame({'t': [1.0, 2.0]}, index=idx), + val_data=pd.DataFrame({'t': [1.0, 2.0]}, index=idx), + y_pred=pd.DataFrame({'p': [1.0, 2.0]}, index=idx), + ) + wrapper = MagicMock() + wrapper.model = object() + out = repo.compute_regression_metrics(tmr, wrapper) + assert out.mse_val is not None and out.equation is None + + def test_compute_regression_metrics_non_linear_skips_equation(): repo = dmr.DataManagerRepository(MagicMock()) p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) @@ -284,24 +373,20 @@ def test_configure_datetime_index_none_raises(): def test_configure_datetime_index_already_datetime_index(): repo = dmr.DataManagerRepository(MagicMock()) p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) - idx = pd.date_range('2024-01-01', periods=3, freq='h') - df = pd.DataFrame({'v1': [1, 2, 3], 't': [1, 2, 3]}, index=idx) - out = repo._configure_datetime_index(df, p, {}) - assert isinstance(out.index, pd.DatetimeIndex) - - -def test_configure_datetime_index_from_common_column(): - repo = dmr.DataManagerRepository(MagicMock()) - p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + existing_idx = pd.date_range('2024-01-01', periods=3, freq='h') df = pd.DataFrame( { - 'timestamp': pd.date_range('2024-01-01', periods=3, freq='D'), + 'timestamp': pd.to_datetime( + ['2024-01-03 00:00:00', '2024-01-01 00:00:00', '2024-01-02 00:00:00'] + ), 'v1': [1, 2, 3], 't': [1, 2, 3], - } + }, + index=existing_idx, ) out = repo._configure_datetime_index(df, p, {}) assert isinstance(out.index, pd.DatetimeIndex) + assert out.index.equals(pd.DatetimeIndex(pd.to_datetime(sorted(df['timestamp'].tolist())))) def test_configure_datetime_index_from_date_column(): @@ -318,36 +403,32 @@ def test_configure_datetime_index_from_date_column(): assert isinstance(out.index, pd.DatetimeIndex) -def test_configure_datetime_index_bad_column_skips_to_first(): +def test_configure_datetime_index_missing_date_column_raises(): repo = dmr.DataManagerRepository(MagicMock()) - p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) - df = pd.DataFrame({'timestamp': ['x'], 'v1': [1.0], 't': [1.0]}) - out = repo._configure_datetime_index(df, p, {}) - assert isinstance(out, pd.DataFrame) - assert not isinstance(out.index, pd.DatetimeIndex) - - -def test_configure_datetime_index_no_timestamp_warning(): - repo = dmr.DataManagerRepository(MagicMock()) - p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) - df = pd.DataFrame({'v1': [1, 2, 3], 't': [1, 2, 3]}) - out = repo._configure_datetime_index(df, p, {}) - assert isinstance(out, pd.DataFrame) - - -def test_configure_datetime_index_first_column_numeric_parsed_as_time(): - """Covers fallback path that parses the first column as datetime when it looks like timestamps.""" - repo = dmr.DataManagerRepository(MagicMock()) - p = TrainModelParams.from_dict(_minimal_dict_for_prepare()) + p = TrainModelParams.from_dict({**_minimal_dict_for_prepare(), 'date_column': 'mydate'}) df = pd.DataFrame( { - 'ts': pd.date_range('2024-01-01', periods=3, freq='D'), + 'timestamp': pd.date_range('2024-01-01', periods=3, freq='D'), + 'v1': [1, 2, 3], + 't': [1, 2, 3], + } + ) + with pytest.raises(ValueError, match='date_column "mydate" not found'): + repo._configure_datetime_index(df, p, {}) + + +def test_configure_datetime_index_non_datetime_date_column_raises(): + repo = dmr.DataManagerRepository(MagicMock()) + p = TrainModelParams.from_dict({**_minimal_dict_for_prepare(), 'date_column': 'mydate'}) + df = pd.DataFrame( + { + 'mydate': ['2024-01-01', '2024-01-02', '2024-01-03'], 'v1': [1.0, 2.0, 3.0], 't': [1.0, 2.0, 3.0], } ) - out = repo._configure_datetime_index(df, p, {}) - assert isinstance(out.index, pd.DatetimeIndex) + with pytest.raises(ValueError, match='must be datetime before index configuration'): + repo._configure_datetime_index(df, p, {}) def test_create_run_directory_permission_error(): diff --git a/tests/workflows/test_train_model.py b/tests/workflows/test_train_model.py index d3b4f94..b8a7e1c 100644 --- a/tests/workflows/test_train_model.py +++ b/tests/workflows/test_train_model.py @@ -32,8 +32,8 @@ def sample_input_data(): 'file_name': 'test-file.csv', 'line_separator': ',', 'decimal_separator': '.', - 'date_column': None, - 'date_format': None, + 'date_column': 'timestamp', + 'date_format': 'yyyy-MM-dd HH:mm:ss', 'shuffle': True, 'random_state': 42, 'model_name': 'Linear Regression',