feat: require date_column in training parameters and update documentation
- Made `date_column` a required field in `TrainModelParams`, ensuring it must be present in the input data. - Updated related documentation in `input-sample.md`, `README.md`, and various test scenarios to reflect the change in requirement. - Adjusted the handling of `date_format` to default to `yyyy-MM-dd HH:mm:ss` if omitted, enhancing usability. - Refined test scenarios to include new examples and ensure compliance with the updated parameter structure. These changes improve the robustness of the model training workflow and clarify the expectations for input data.
This commit is contained in:
164
e2e/conftest.py
164
e2e/conftest.py
@@ -1,27 +1,25 @@
|
||||
"""
|
||||
Pytest configuration and fixtures for E2E tests.
|
||||
|
||||
All external dependencies use real services:
|
||||
- PostgreSQL: testcontainers (postgres:15)
|
||||
- MinIO: testcontainers (minio)
|
||||
- MongoDB: testcontainers (mongo:7)
|
||||
- MLflow: local filesystem tracking (no network)
|
||||
- Gitea: testcontainers generic container (gitea/gitea:latest),
|
||||
seeded with model-plugin-warehouse files via REST API
|
||||
- Temporal: in-memory WorkflowEnvironment (time-skipping)
|
||||
External dependencies use testcontainers or real SDK integrations (no mocks of
|
||||
model_manager or other first-party code):
|
||||
|
||||
- PostgreSQL, MinIO, MongoDB, Gitea: testcontainers.
|
||||
- MLflow: real client with ``file://`` tracking URI (no MLflow server process).
|
||||
- Temporal: ``WorkflowEnvironment.start_time_skipping()`` — official in-process
|
||||
test runtime from temporalio; exercises real workflows and activity code, not
|
||||
stubs of business logic.
|
||||
- Observability: ``Logger`` (``get_logger`` from ``model_manager.utils.logger_helper``)
|
||||
and ``MetricsController`` from sientia_do, same stack as production.
|
||||
"""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import base64
|
||||
import csv
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import mlflow
|
||||
import pytest
|
||||
@@ -37,6 +35,7 @@ from temporalio.testing import WorkflowEnvironment
|
||||
from temporalio.worker import Worker
|
||||
|
||||
from model_manager.activities.activities import Activities
|
||||
from model_manager.utils.logger_helper import get_logger
|
||||
from model_manager.workflows.cleanup_files import CleanupFiles
|
||||
from model_manager.workflows.train_model import TrainModel
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler
|
||||
@@ -44,12 +43,6 @@ from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_model.model_repository.plugin_store import PluginStore
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paths
|
||||
# ---------------------------------------------------------------------------
|
||||
_WAREHOUSE_ROOT = Path(
|
||||
'/home/grezewave/Documents/projects/sientia/model-plugin-warehouse'
|
||||
)
|
||||
|
||||
# CSV training data: columns must match the variable_columns and target_variable
|
||||
# used across all test scenarios.
|
||||
_TRAIN_CSV_COLUMNS = [
|
||||
@@ -107,6 +100,63 @@ def _build_training_csv_dd_mm_yyyy() -> bytes:
|
||||
return output.getvalue().encode('utf-8')
|
||||
|
||||
|
||||
def _build_training_csv_custom_target_column() -> bytes:
|
||||
"""
|
||||
Same layout as the standard CSV but the target column has a non-default name
|
||||
(not ``target``) to exercise report and metrics paths.
|
||||
"""
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow([
|
||||
'timestamp',
|
||||
'303-WIT-200(Value)',
|
||||
'MY_CUSTOM_TARGET_COLUMN',
|
||||
])
|
||||
for i in range(150):
|
||||
ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00'
|
||||
wit200 = round(30.0 + (i % 20) * 0.5, 2)
|
||||
target_val = round(100.0 + (i % 15) * 0.3, 2)
|
||||
writer.writerow([ts, wit200, target_val])
|
||||
return output.getvalue().encode('utf-8')
|
||||
|
||||
|
||||
def _build_training_csv_timestamp_header_naive() -> bytes:
|
||||
"""
|
||||
Naive datetimes under column ``Timestamp`` (common UI export) for scenario 16.
|
||||
"""
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow([
|
||||
'Timestamp',
|
||||
'303-WIT-200(Value)',
|
||||
'03CV020/CORRENTE_N_M1_PV(Value)',
|
||||
])
|
||||
for i in range(150):
|
||||
ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00'
|
||||
wit200 = round(30.0 + (i % 20) * 0.5, 2)
|
||||
cv020 = round(100.0 + (i % 15) * 0.3, 2)
|
||||
writer.writerow([ts, wit200, cv020])
|
||||
return output.getvalue().encode('utf-8')
|
||||
|
||||
|
||||
def _build_training_csv_blank_timestamp_row() -> bytes:
|
||||
"""Standard columns with one row where ``timestamp`` is empty (NaN after parse)."""
|
||||
output = io.StringIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(_TRAIN_CSV_COLUMNS)
|
||||
for i in range(150):
|
||||
wit200 = round(30.0 + (i % 20) * 0.5, 2)
|
||||
cv020 = round(100.0 + (i % 15) * 0.3, 2)
|
||||
wit230 = round(25.0 + (i % 18) * 0.4, 2)
|
||||
cv022 = round(90.0 + (i % 12) * 0.25, 2)
|
||||
if i == 17:
|
||||
writer.writerow(['', wit200, cv020, wit230, cv022])
|
||||
else:
|
||||
ts = f'2025-06-{(i // 24) + 2:02d} {i % 24:02d}:00:00'
|
||||
writer.writerow([ts, wit200, cv020, wit230, cv022])
|
||||
return output.getvalue().encode('utf-8')
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers – Gitea seed
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -297,7 +347,8 @@ def mongodb_container():
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
def gitea_container():
|
||||
"""
|
||||
Gitea container seeded with the model-plugin-warehouse files.
|
||||
Gitea container with a ``model-store`` repo seeded via REST API
|
||||
(``index.yaml``, dummy model files, ``_seed_gitea``).
|
||||
|
||||
The container starts with INSTALL_LOCK so no setup wizard is needed.
|
||||
An admin user is created via Gitea's CLI before the HTTP API is used.
|
||||
@@ -318,7 +369,6 @@ def gitea_container():
|
||||
base_url = f'http://localhost:{port}'
|
||||
|
||||
_wait_for_gitea(base_url)
|
||||
import time
|
||||
time.sleep(5) # Wait a bit for DB to fully initialize after HTTP is up
|
||||
|
||||
# Create admin user via Gitea CLI inside the container
|
||||
@@ -397,6 +447,33 @@ def upload_training_csv(minio_container, mlflow_tracking_dir): # noqa: ARG001
|
||||
content_type='text/csv',
|
||||
)
|
||||
|
||||
custom_target = _build_training_csv_custom_target_column()
|
||||
client.put_object(
|
||||
_MINIO_BUCKET,
|
||||
'training_data_custom_target.csv',
|
||||
io.BytesIO(custom_target),
|
||||
length=len(custom_target),
|
||||
content_type='text/csv',
|
||||
)
|
||||
|
||||
ts_header = _build_training_csv_timestamp_header_naive()
|
||||
client.put_object(
|
||||
_MINIO_BUCKET,
|
||||
'training_data_timestamp_naive.csv',
|
||||
io.BytesIO(ts_header),
|
||||
length=len(ts_header),
|
||||
content_type='text/csv',
|
||||
)
|
||||
|
||||
blank_ts = _build_training_csv_blank_timestamp_row()
|
||||
client.put_object(
|
||||
_MINIO_BUCKET,
|
||||
'training_data_blank_timestamp_row.csv',
|
||||
io.BytesIO(blank_ts),
|
||||
length=len(blank_ts),
|
||||
content_type='text/csv',
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Function-scoped: database engine + schema setup
|
||||
@@ -437,36 +514,27 @@ def setup_experiment_run_table(postgres_engine):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock-only fixtures (no external service equivalent)
|
||||
# Observability (real sientia_do implementations)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def mock_logger():
|
||||
"""Minimal logger that prints to stdout (no external observability needed)."""
|
||||
def _log(msg, *args, **kwargs): # noqa: ARG001
|
||||
print(f'[LOG] {msg}')
|
||||
|
||||
logger = MagicMock()
|
||||
for method in ('info', 'debug', 'error', 'warning', 'critical',
|
||||
'custom_info', 'custom_debug', 'custom_error',
|
||||
'custom_warning', 'custom_critical'):
|
||||
setattr(logger, method, MagicMock(side_effect=_log))
|
||||
logger.base_logger = MagicMock()
|
||||
return logger
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
def e2e_logger():
|
||||
"""Shared production-style Logger for the whole E2E session."""
|
||||
return get_logger('model-manager-e2e')
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def mock_metrics_controller(mock_logger):
|
||||
"""Real MetricsController backed by the mock logger."""
|
||||
return MetricsController(logger=mock_logger)
|
||||
def metrics_controller(e2e_logger):
|
||||
"""MetricsController bound to the E2E logger (fresh instance per test)."""
|
||||
return MetricsController(logger=e2e_logger)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Real application fixtures
|
||||
# Application fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def notification_handler(mongodb_container, mock_logger):
|
||||
def notification_handler(mongodb_container, e2e_logger):
|
||||
"""
|
||||
Real CoreNotificationHandler connected to the MongoDB testcontainer.
|
||||
"""
|
||||
@@ -474,7 +542,7 @@ def notification_handler(mongodb_container, mock_logger):
|
||||
handler = CoreNotificationHandler(
|
||||
connection_string=connection_url,
|
||||
database='test_notifications',
|
||||
logger=mock_logger,
|
||||
logger=e2e_logger,
|
||||
project_name='model-manager-e2e',
|
||||
)
|
||||
yield handler
|
||||
@@ -482,7 +550,7 @@ def notification_handler(mongodb_container, mock_logger):
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def plugin_store(gitea_container, mock_logger, mock_metrics_controller, notification_handler):
|
||||
def plugin_store(gitea_container, e2e_logger, metrics_controller, notification_handler):
|
||||
"""
|
||||
Real PluginStore pointed at the Gitea testcontainer.
|
||||
cache_ttl_seconds=0 forces a fresh download every test.
|
||||
@@ -494,9 +562,9 @@ def plugin_store(gitea_container, mock_logger, mock_metrics_controller, notifica
|
||||
username=gitea_container['admin_user'],
|
||||
password=gitea_container['admin_pass'],
|
||||
cache_ttl_seconds=0,
|
||||
logger=mock_logger,
|
||||
logger=e2e_logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mock_metrics_controller,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
yield store
|
||||
|
||||
@@ -507,9 +575,9 @@ def test_activities(
|
||||
minio_container,
|
||||
mlflow_tracking_dir, # noqa: ARG001 – ensures MLflow URI is set
|
||||
plugin_store,
|
||||
mock_logger,
|
||||
e2e_logger,
|
||||
notification_handler,
|
||||
mock_metrics_controller,
|
||||
metrics_controller,
|
||||
):
|
||||
"""
|
||||
Real Activities instance wired to all testcontainers.
|
||||
@@ -540,9 +608,9 @@ def test_activities(
|
||||
'default_bucket': _MINIO_BUCKET,
|
||||
},
|
||||
plugin_store=plugin_store,
|
||||
logger=mock_logger,
|
||||
logger=e2e_logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mock_metrics_controller,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
yield activities
|
||||
activities.shutdown()
|
||||
@@ -561,7 +629,7 @@ def _activity_list(activities: Activities) -> list:
|
||||
|
||||
@pytest_asyncio.fixture(scope='function')
|
||||
async def temporal_test_env():
|
||||
"""In-memory Temporal environment with time-skipping."""
|
||||
"""Temporal SDK test environment (time-skipping); runs real workflow/activity code."""
|
||||
env = await WorkflowEnvironment.start_time_skipping()
|
||||
async with env:
|
||||
yield env
|
||||
|
||||
@@ -3,7 +3,9 @@ Shared helpers for E2E tests (Temporal workflows + PostgreSQL).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -16,7 +18,7 @@ async def start_and_await_workflow(
|
||||
workflow_run,
|
||||
input_data: dict,
|
||||
workflow_id: str,
|
||||
timeout: float = 120.0,
|
||||
timeout: float = 600.0,
|
||||
):
|
||||
"""
|
||||
Start a Temporal workflow and wait for its result.
|
||||
@@ -26,7 +28,7 @@ async def start_and_await_workflow(
|
||||
workflow_run: Workflow run method (e.g. TrainModel.run).
|
||||
input_data: Workflow input payload.
|
||||
workflow_id: Unique workflow id.
|
||||
timeout: Max seconds to wait for completion.
|
||||
timeout: Max seconds to wait for completion (default allows cold testcontainer startup).
|
||||
|
||||
Returns:
|
||||
Workflow result value.
|
||||
@@ -186,9 +188,6 @@ def load_scenario(scenario_filename: str) -> dict[str, Any]:
|
||||
Returns:
|
||||
dict: Parsed scenario payload.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
scenario_path = (
|
||||
Path(__file__).parent.parent / 'docs' / 'test-scenarios' / scenario_filename
|
||||
)
|
||||
|
||||
@@ -2,6 +2,14 @@
|
||||
|
||||
This document maps the workflow scenarios tested in the E2E suite to their corresponding JSON input files and expected behaviors.
|
||||
|
||||
## Infrastructure (second-pass review)
|
||||
|
||||
- **Containers:** PostgreSQL, MinIO, MongoDB, and Gitea via testcontainers; real clients and `Activities` code paths.
|
||||
- **MLflow:** `file://` tracking URI (real SDK, no remote server).
|
||||
- **Temporal:** `WorkflowEnvironment.start_time_skipping()` — official temporalio test runtime; workflows and activities are not stubbed.
|
||||
- **Logging/metrics:** `get_logger` + `MetricsController` (sientia_do); no `unittest.mock` for observability in `e2e/conftest.py`.
|
||||
- **Unit tests** under `tests/` may still use mocks where appropriate; that policy is separate from this E2E suite.
|
||||
|
||||
## 1. TrainModel Workflow (`test_train_model_workflow.py`)
|
||||
|
||||
### 1.1 Happy Paths (Successful execution)
|
||||
@@ -9,13 +17,22 @@ This document maps the workflow scenarios tested in the E2E suite to their corre
|
||||
| Test Function | Input JSON | Expected Status | Description |
|
||||
|---|---|---|---|
|
||||
| `test_scenario_1_1_1_linear_regression_basic` | `01-linear-regression-basic.json` | `TRAINING_SUCCESS` | Basic linear regression without scaler. Verifies end-to-end pipeline. |
|
||||
| `test_scenario_1_1_2_polynomial_regression_degree2_with_scaler` | `03-polynomial-regression-degree2.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 2) with Standard Scaler. |
|
||||
| `test_scenario_1_1_3_linear_regression_with_lags` | `05-linear-regression-with-lags.json` | `TRAINING_SUCCESS` | Linear regression with `lag_train`/`lag_val` per variable. |
|
||||
| `test_scenario_1_1_4_linear_regression_nan_interpolation` | `06-linear-regression-nan-interpolation.json` | `TRAINING_SUCCESS` | Linear regression with `nan_treatment='linear interpolation'`. |
|
||||
| `test_scenario_1_1_5_linear_regression_with_limits` | `08-linear-regression-with-limits.json` | `TRAINING_SUCCESS` | Linear regression with `support_filters` (min/max limits per variable). |
|
||||
| `test_scenario_1_1_6_polynomial_degree2_scaler_and_lags` | `09-polynomial-degree2-with-scaler-and-lags.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 2), Standard Scaler, and lags. |
|
||||
| `test_scenario_1_1_7_static_window_removal` | `11-linear-regression-static-threshold-custom.json` | `TRAINING_SUCCESS` | Linear regression with `rem_static_win=true`, `window`, and `static_threshold`. |
|
||||
| `test_scenario_1_1_8_polynomial_with_support_filters` | `14-angular-test-polynomial-support-filters.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 4), Standard Scaler, and support filters. |
|
||||
| `test_scenario_1_1_2_linear_regression_with_scaler` | `02-linear-regression-with-scaler.json` | `TRAINING_SUCCESS` | Linear regression with `Standard Scaler`. |
|
||||
| `test_scenario_1_1_3_polynomial_regression_degree2_with_scaler` | `03-polynomial-regression-degree2.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 2) with Standard Scaler. |
|
||||
| `test_scenario_1_1_4_polynomial_regression_degree3_with_scaler` | `04-polynomial-regression-degree3.json` | `TRAINING_SUCCESS` | Polynomial regression (degree 3) with Standard Scaler. |
|
||||
| `test_scenario_1_1_5_linear_regression_with_lags` | `05-linear-regression-with-lags.json` | `TRAINING_SUCCESS` | Linear regression with `lag_train`/`lag_val` per variable. |
|
||||
| `test_scenario_1_1_6_linear_regression_nan_interpolation` | `06-linear-regression-nan-interpolation.json` | `TRAINING_SUCCESS` | Linear regression with `nan_treatment='linear interpolation'`. |
|
||||
| `test_scenario_1_1_7_linear_regression_static_window_removal` | `07-linear-regression-static-window-removal.json` | `TRAINING_SUCCESS` | `rem_static_win=true` with default `static_threshold`. |
|
||||
| `test_scenario_1_1_8_linear_regression_with_limits` | `08-linear-regression-with-limits.json` | `TRAINING_SUCCESS` | `support_filters` with `min`/`max` per variable. |
|
||||
| `test_scenario_1_1_9_polynomial_degree2_scaler_and_lags` | `09-polynomial-degree2-with-scaler-and-lags.json` | `TRAINING_SUCCESS` | Polynomial (degree 2), Standard Scaler, and lags. |
|
||||
| `test_scenario_1_1_10_linear_regression_with_ar_opt_params` | `10-linear-regression-with-ar.json` | `TRAINING_SUCCESS` | `opt_params.include_ar=true` (placeholder for future AR behavior). |
|
||||
| `test_scenario_1_1_11_linear_regression_static_threshold_custom` | `11-linear-regression-static-threshold-custom.json` | `TRAINING_SUCCESS` | `rem_static_win=true` with custom `static_threshold`. |
|
||||
| `test_scenario_1_1_12_alternate_date_format_dd_mm_yyyy` | `12-angular-test-date-format.json` | `TRAINING_SUCCESS` | `date_column=DATA`, `dd/MM/yyyy` format, object `training_data_dd_mm_yyyy.csv`. |
|
||||
| `test_scenario_1_1_13_alternate_csv_narrow_date_window` | `13-angular-test-double-date-column.json` | `TRAINING_SUCCESS` | Same alternate CSV with a bounded `start_date`/`end_date` window. |
|
||||
| `test_scenario_1_1_14_polynomial_with_support_filters` | `14-angular-test-polynomial-support-filters.json` | `TRAINING_SUCCESS` | Polynomial (degree 4), scaler, `upper_line`/`lower_line` support filters. |
|
||||
| `test_scenario_1_1_15_linear_regression_custom_target_column_name` | `15-linear-regression-custom-target-column.json` | `TRAINING_SUCCESS` | Custom `target_variable` column name (not literal ``target``); Evidently/report columns must match. |
|
||||
| `test_scenario_1_1_16_naive_timestamp_header_column` | `16-linear-regression-naive-timestamp-header.json` | `TRAINING_SUCCESS` | `date_column`=`Timestamp`, naive CSV `training_data_timestamp_naive.csv`. |
|
||||
| `test_scenario_1_1_17_linear_regression_blank_timestamp_row_dropped` | `17-linear-regression-blank-timestamp-row.json` | `TRAINING_SUCCESS` | One empty timestamp cell; row dropped before index. |
|
||||
|
||||
### 1.2 Error Paths
|
||||
|
||||
@@ -43,5 +60,5 @@ These scenarios test the business rule validations inside `validate_train_params
|
||||
| Test Function | Description |
|
||||
|---|---|
|
||||
| `test_scenario_3_1_1_cleanup_with_no_temp_dirs` | Temp directory is empty. Activity completes without error. |
|
||||
| `test_scenario_3_1_2_cleanup_removes_old_temp_dirs` | Two stale timestamped directories are removed. |
|
||||
| `test_scenario_3_1_2_cleanup_removes_old_temp_dirs` | Two stale directories matching `name_YYYYMMDD_HHMMSS_microseconds` are removed when older than retention. |
|
||||
| `test_scenario_3_1_3_cleanup_nonexistent_temp_path` | Target path does not exist. Handled gracefully without error. |
|
||||
|
||||
@@ -4,18 +4,17 @@ End-to-end tests for CleanupFiles workflow.
|
||||
Covers scenarios 3.x: cleanup of temporary local directories.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from temporalio.testing import WorkflowEnvironment
|
||||
from temporalio.worker import Worker
|
||||
|
||||
from e2e.helpers import make_workflow_id, start_and_await_workflow
|
||||
from model_manager.workflows.cleanup_files import CleanupFiles
|
||||
|
||||
# Matches Cleanup.dir_timestamp_pattern: name_YYYYMMDD_HHMMSS_microseconds
|
||||
_STALE_DIR_OLD = 'stale_run_20200102_030405_000001'
|
||||
_STALE_DIR_OLDER = 'stale_run_20191231_235959_999999'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
@@ -59,9 +58,9 @@ async def test_scenario_3_1_2_cleanup_removes_old_temp_dirs(
|
||||
reports_dir = tmp_path / 'reports_temp'
|
||||
reports_dir.mkdir()
|
||||
|
||||
# Create two stale run directories
|
||||
stale1 = reports_dir / 'run-1234567890'
|
||||
stale2 = reports_dir / 'run-9876543210'
|
||||
# Create two stale run directories (names must match cleanup activity regex)
|
||||
stale1 = reports_dir / _STALE_DIR_OLD
|
||||
stale2 = reports_dir / _STALE_DIR_OLDER
|
||||
stale1.mkdir()
|
||||
stale2.mkdir()
|
||||
(stale1 / 'model.pkl').write_bytes(b'fake-model-data')
|
||||
|
||||
@@ -6,7 +6,7 @@ ORCHESTRATOR_VALIDATION_ERROR due to invalid parameter values.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from temporalio.client import WorkflowFailureError
|
||||
from temporalio.testing import WorkflowEnvironment
|
||||
from temporalio.worker import Worker
|
||||
|
||||
@@ -23,6 +23,20 @@ from model_manager.workflows.train_model import TrainModel
|
||||
_VALIDATION_ID_BASE = 3000
|
||||
|
||||
|
||||
def _exception_chain_text(exc: BaseException) -> str:
|
||||
"""Concatenate messages from an exception __cause__/__context__ chain."""
|
||||
parts: list[str] = []
|
||||
cur: BaseException | None = exc
|
||||
seen: set[int] = set()
|
||||
while cur is not None and id(cur) not in seen:
|
||||
seen.add(id(cur))
|
||||
text = str(cur).strip()
|
||||
if text:
|
||||
parts.append(text)
|
||||
cur = cur.__cause__ or getattr(cur, '__context__', None)
|
||||
return ' | '.join(parts).lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_2_1_1_train_size_out_of_range(
|
||||
@@ -223,10 +237,12 @@ async def test_scenario_2_1_7_missing_experiment_run_id(
|
||||
scenario = load_scenario('01-linear-regression-basic.json')
|
||||
scenario = {k: v for k, v in scenario.items() if k != 'experiment_run_id'}
|
||||
|
||||
with pytest.raises(Exception, match='experiment_run_id'):
|
||||
with pytest.raises(WorkflowFailureError) as excinfo:
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s2-1-7'),
|
||||
)
|
||||
combined = _exception_chain_text(excinfo.value)
|
||||
assert 'experiment_run_id' in combined
|
||||
|
||||
@@ -7,7 +7,6 @@ Covers:
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from temporalio.testing import WorkflowEnvironment
|
||||
from temporalio.worker import Worker
|
||||
|
||||
@@ -15,6 +14,7 @@ from e2e.helpers import (
|
||||
assert_experiment_error,
|
||||
assert_experiment_run_name_set,
|
||||
assert_experiment_status,
|
||||
assert_no_experiment_row,
|
||||
insert_experiment_run,
|
||||
load_scenario,
|
||||
make_workflow_id,
|
||||
@@ -57,13 +57,13 @@ async def test_scenario_1_1_1_linear_regression_basic(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_2_polynomial_regression_degree2_with_scaler(
|
||||
async def test_scenario_1_1_2_linear_regression_with_scaler(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.2 – Polynomial Regression Degree 2 with Standard Scaler (cenário 03)."""
|
||||
scenario = load_scenario('03-polynomial-regression-degree2.json')
|
||||
"""Scenario 1.1.2 – Linear regression with Standard Scaler (cenário 02)."""
|
||||
scenario = load_scenario('02-linear-regression-with-scaler.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
@@ -80,13 +80,13 @@ async def test_scenario_1_1_2_polynomial_regression_degree2_with_scaler(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_3_linear_regression_with_lags(
|
||||
async def test_scenario_1_1_3_polynomial_regression_degree2_with_scaler(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.3 – Linear Regression with lag_train/lag_val per variable (cenário 05)."""
|
||||
scenario = load_scenario('05-linear-regression-with-lags.json')
|
||||
"""Scenario 1.1.3 – Polynomial Regression Degree 2 with Standard Scaler (cenário 03)."""
|
||||
scenario = load_scenario('03-polynomial-regression-degree2.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
@@ -103,13 +103,13 @@ async def test_scenario_1_1_3_linear_regression_with_lags(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_4_linear_regression_nan_interpolation(
|
||||
async def test_scenario_1_1_4_polynomial_regression_degree3_with_scaler(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.4 – nan_treatment='linear interpolation' (cenário 06)."""
|
||||
scenario = load_scenario('06-linear-regression-nan-interpolation.json')
|
||||
"""Scenario 1.1.4 – Polynomial regression degree 3 with Standard Scaler (cenário 04)."""
|
||||
scenario = load_scenario('04-polynomial-regression-degree3.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
@@ -121,17 +121,18 @@ async def test_scenario_1_1_4_linear_regression_nan_interpolation(
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_5_linear_regression_with_limits(
|
||||
async def test_scenario_1_1_5_linear_regression_with_lags(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.5 – support_filters with min/max limits per variable (cenário 08)."""
|
||||
scenario = load_scenario('08-linear-regression-with-limits.json')
|
||||
"""Scenario 1.1.5 – Linear Regression with lag_train/lag_val per variable (cenário 05)."""
|
||||
scenario = load_scenario('05-linear-regression-with-lags.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
@@ -143,17 +144,18 @@ async def test_scenario_1_1_5_linear_regression_with_limits(
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_6_polynomial_degree2_scaler_and_lags(
|
||||
async def test_scenario_1_1_6_linear_regression_nan_interpolation(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.6 – Polynomial degree 2, Standard Scaler and lags (cenário 09)."""
|
||||
scenario = load_scenario('09-polynomial-degree2-with-scaler-and-lags.json')
|
||||
"""Scenario 1.1.6 – nan_treatment='linear interpolation' (cenário 06)."""
|
||||
scenario = load_scenario('06-linear-regression-nan-interpolation.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
@@ -165,18 +167,17 @@ async def test_scenario_1_1_6_polynomial_degree2_scaler_and_lags(
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_7_static_window_removal(
|
||||
async def test_scenario_1_1_7_linear_regression_static_window_removal(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.7 – rem_static_win=true with window and static_threshold (cenário 11)."""
|
||||
scenario = load_scenario('11-linear-regression-static-threshold-custom.json')
|
||||
"""Scenario 1.1.7 – rem_static_win=true with default static_threshold (cenário 07)."""
|
||||
scenario = load_scenario('07-linear-regression-static-window-removal.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
@@ -192,16 +193,13 @@ async def test_scenario_1_1_7_static_window_removal(
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_8_polynomial_with_support_filters(
|
||||
async def test_scenario_1_1_8_linear_regression_with_limits(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.8 – Polynomial degree 4, Standard Scaler, upper/lower support filters (cenário 14)."""
|
||||
scenario = load_scenario('14-angular-test-polynomial-support-filters.json')
|
||||
# Override date range to match rows in our test CSV
|
||||
scenario['data_model_kwargs']['start_date'] = '2025-06-02 00:00:00'
|
||||
scenario['data_model_kwargs']['end_date'] = '2025-06-06 23:59:59'
|
||||
"""Scenario 1.1.8 – support_filters with min/max limits per variable (cenário 08)."""
|
||||
scenario = load_scenario('08-linear-regression-with-limits.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
@@ -215,6 +213,230 @@ async def test_scenario_1_1_8_polynomial_with_support_filters(
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_9_polynomial_degree2_scaler_and_lags(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.9 – Polynomial degree 2, Standard Scaler and lags (cenário 09)."""
|
||||
scenario = load_scenario('09-polynomial-degree2-with-scaler-and-lags.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s1-1-9'),
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_10_linear_regression_with_ar_opt_params(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.10 – Linear regression with include_ar in opt_params (cenário 10)."""
|
||||
scenario = load_scenario('10-linear-regression-with-ar.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s1-1-10'),
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_11_linear_regression_static_threshold_custom(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.11 – rem_static_win=true with custom static_threshold (cenário 11)."""
|
||||
scenario = load_scenario('11-linear-regression-static-threshold-custom.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s1-1-11'),
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_12_alternate_date_format_dd_mm_yyyy(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.12 – DATA column and dd/MM/yyyy format CSV (cenário 12)."""
|
||||
scenario = load_scenario('12-angular-test-date-format.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
file_name='training_data_dd_mm_yyyy.csv',
|
||||
)
|
||||
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s1-1-12'),
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_13_alternate_csv_narrow_date_window(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.13 – Same alternate CSV as 12 with date window (cenário 13)."""
|
||||
scenario = load_scenario('13-angular-test-double-date-column.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
file_name='training_data_dd_mm_yyyy.csv',
|
||||
)
|
||||
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s1-1-13'),
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_14_polynomial_with_support_filters(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.14 – Polynomial degree 4, Standard Scaler, line support filters (cenário 14)."""
|
||||
scenario = load_scenario('14-angular-test-polynomial-support-filters.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s1-1-14'),
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_15_linear_regression_custom_target_column_name(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.15 – Target column is not named ``target``; report path uses target_variable."""
|
||||
scenario = load_scenario('15-linear-regression-custom-target-column.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
file_name='training_data_custom_target.csv',
|
||||
)
|
||||
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s1-1-15'),
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_16_naive_timestamp_header_column(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.16 – CSV uses ``Timestamp`` header; snake_case date_column/date_format."""
|
||||
scenario = load_scenario('16-linear-regression-naive-timestamp-header.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
file_name='training_data_timestamp_naive.csv',
|
||||
)
|
||||
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s1-1-16'),
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_1_1_17_linear_regression_blank_timestamp_row_dropped(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 1.1.17 – CSV has one empty timestamp cell; row is dropped and training succeeds."""
|
||||
scenario = load_scenario('17-linear-regression-blank-timestamp-row.json')
|
||||
experiment_run_id = scenario['experiment_run_id']
|
||||
insert_experiment_run(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
file_name='training_data_blank_timestamp_row.csv',
|
||||
)
|
||||
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s1-1-17'),
|
||||
)
|
||||
|
||||
assert_experiment_status(postgres_engine, experiment_run_id, 'TRAINING_SUCCESS')
|
||||
assert_experiment_run_name_set(postgres_engine, experiment_run_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1.2 – Error paths
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -268,5 +490,4 @@ async def test_scenario_1_2_2_experiment_run_id_not_in_db(
|
||||
make_workflow_id('test-s1-2-2'),
|
||||
)
|
||||
|
||||
from e2e.helpers import assert_no_experiment_row
|
||||
assert_no_experiment_row(postgres_engine, 9999)
|
||||
|
||||
Reference in New Issue
Block a user