feat: enhance test scenarios and configuration for regression models
- Updated `pyproject.toml` to include new linting rules for end-to-end tests. - Modified `requirements-dev.txt` to add dependencies for E2E testing with `testcontainers` and `requests`. - Refactored multiple JSON test scenario files to standardize structure, including new fields for `experiment_run_id`, `bucket_name`, and `file_name`. - Improved model training parameters in `train_model_params.py` to use `experiment_name` directly. - Adjusted `data_manager_repository.py` to utilize the updated `experiment_name` for logging. These changes improve the organization and clarity of regression model tests and enhance the overall testing framework.
This commit is contained in:
232
e2e/test_train_model_validation.py
Normal file
232
e2e/test_train_model_validation.py
Normal file
@@ -0,0 +1,232 @@
|
||||
"""
|
||||
End-to-end tests for TrainModel parameter validation paths.
|
||||
|
||||
Covers scenarios 2.1.x: workflows that must terminate with
|
||||
ORCHESTRATOR_VALIDATION_ERROR due to invalid parameter values.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from temporalio.testing import WorkflowEnvironment
|
||||
from temporalio.worker import Worker
|
||||
|
||||
from e2e.helpers import (
|
||||
assert_experiment_error,
|
||||
insert_experiment_run,
|
||||
load_scenario,
|
||||
make_workflow_id,
|
||||
start_and_await_workflow,
|
||||
)
|
||||
from model_manager.workflows.train_model import TrainModel
|
||||
|
||||
# Base experiment_run ids for validation test scenarios (offset to avoid collision)
|
||||
_VALIDATION_ID_BASE = 3000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_2_1_1_train_size_out_of_range(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 2.1.1 – train_size=5 violates the 10–100 business rule.
|
||||
|
||||
Expected: workflow updates status → ORCHESTRATOR_VALIDATION_ERROR
|
||||
and error_message references 'train_size'.
|
||||
"""
|
||||
experiment_run_id = _VALIDATION_ID_BASE + 1
|
||||
scenario = load_scenario('01-linear-regression-basic.json')
|
||||
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'train_size': 5}
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s2-1-1'),
|
||||
)
|
||||
|
||||
assert_experiment_error(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
|
||||
error_substr='train_size',
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_2_1_2_empty_variable_columns(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 2.1.2 – variable_columns=[] → ORCHESTRATOR_VALIDATION_ERROR."""
|
||||
experiment_run_id = _VALIDATION_ID_BASE + 2
|
||||
scenario = load_scenario('01-linear-regression-basic.json')
|
||||
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'variable_columns': []}
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s2-1-2'),
|
||||
)
|
||||
|
||||
assert_experiment_error(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
|
||||
error_substr='variable_columns',
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_2_1_3_invalid_date_format(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 2.1.3 – date_format='INVALID' is not in the allowed list."""
|
||||
experiment_run_id = _VALIDATION_ID_BASE + 3
|
||||
scenario = load_scenario('01-linear-regression-basic.json')
|
||||
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'date_format': 'INVALID'}
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s2-1-3'),
|
||||
)
|
||||
|
||||
assert_experiment_error(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
|
||||
error_substr='date_format',
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_2_1_4_whitespace_only_model_name(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 2.1.4 – model_name=' ' (whitespace) → ORCHESTRATOR_VALIDATION_ERROR."""
|
||||
experiment_run_id = _VALIDATION_ID_BASE + 4
|
||||
scenario = load_scenario('01-linear-regression-basic.json')
|
||||
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'model_name': ' '}
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s2-1-4'),
|
||||
)
|
||||
|
||||
assert_experiment_error(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
|
||||
error_substr='model_name',
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_2_1_5_unknown_model_type(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 2.1.5 – model_type='totally_unknown' → ORCHESTRATOR_VALIDATION_ERROR.
|
||||
|
||||
The PluginStore will not find this model in the Gitea repo, causing
|
||||
load_model_metadata to fail before validate_train_params is even called.
|
||||
"""
|
||||
experiment_run_id = _VALIDATION_ID_BASE + 5
|
||||
scenario = load_scenario('01-linear-regression-basic.json')
|
||||
scenario = {
|
||||
**scenario,
|
||||
'experiment_run_id': experiment_run_id,
|
||||
'model_type': 'totally_unknown_model',
|
||||
}
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s2-1-5'),
|
||||
)
|
||||
|
||||
assert_experiment_error(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
|
||||
error_substr='totally_unknown_model',
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_2_1_6_missing_target_variable(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
postgres_engine,
|
||||
):
|
||||
"""Scenario 2.1.6 – target_variable='' (empty string) → ORCHESTRATOR_VALIDATION_ERROR."""
|
||||
experiment_run_id = _VALIDATION_ID_BASE + 6
|
||||
scenario = load_scenario('01-linear-regression-basic.json')
|
||||
scenario = {**scenario, 'experiment_run_id': experiment_run_id, 'target_variable': ''}
|
||||
insert_experiment_run(postgres_engine, experiment_run_id)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s2-1-6'),
|
||||
)
|
||||
|
||||
assert_experiment_error(
|
||||
postgres_engine,
|
||||
experiment_run_id,
|
||||
expected_status='ORCHESTRATOR_VALIDATION_ERROR',
|
||||
error_substr='target_variable',
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.integration
|
||||
async def test_scenario_2_1_7_missing_experiment_run_id(
|
||||
temporal_test_env: WorkflowEnvironment,
|
||||
temporal_worker: Worker,
|
||||
):
|
||||
"""Scenario 2.1.7 – experiment_run_id missing → workflow raises ValueError immediately.
|
||||
|
||||
No DB row is inserted because experiment_run_id is mandatory to even
|
||||
know which row to update. The workflow should raise before any DB call.
|
||||
"""
|
||||
scenario = load_scenario('01-linear-regression-basic.json')
|
||||
scenario = {k: v for k, v in scenario.items() if k != 'experiment_run_id'}
|
||||
|
||||
with pytest.raises(Exception, match='experiment_run_id'):
|
||||
await start_and_await_workflow(
|
||||
temporal_test_env.client,
|
||||
TrainModel.run,
|
||||
scenario,
|
||||
make_workflow_id('test-s2-1-7'),
|
||||
)
|
||||
Reference in New Issue
Block a user