6bd30e3328
- Updated `pyproject.toml` to include new linting rules for end-to-end tests. - Modified `requirements-dev.txt` to add dependencies for E2E testing with `testcontainers` and `requests`. - Refactored multiple JSON test scenario files to standardize structure, including new fields for `experiment_run_id`, `bucket_name`, and `file_name`. - Improved model training parameters in `train_model_params.py` to use `experiment_name` directly. - Adjusted `data_manager_repository.py` to utilize the updated `experiment_name` for logging. These changes improve the organization and clarity of regression model tests and enhance the overall testing framework.
110 lines
3.1 KiB
Python
110 lines
3.1 KiB
Python
"""
|
||
End-to-end tests for CleanupFiles workflow.
|
||
|
||
Covers scenarios 3.x: cleanup of temporary local directories.
|
||
"""
|
||
|
||
import os
|
||
import shutil
|
||
import tempfile
|
||
|
||
import pytest
|
||
import pytest_asyncio
|
||
from temporalio.testing import WorkflowEnvironment
|
||
from temporalio.worker import Worker
|
||
|
||
from e2e.helpers import make_workflow_id, start_and_await_workflow
|
||
from model_manager.workflows.cleanup_files import CleanupFiles
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.integration
|
||
async def test_scenario_3_1_1_cleanup_with_no_temp_dirs(
|
||
temporal_test_env: WorkflowEnvironment,
|
||
temporal_worker: Worker,
|
||
tmp_path,
|
||
):
|
||
"""Scenario 3.1.1 – Cleanup when the temp directory is empty.
|
||
|
||
The cleanup_temp_directories activity should complete without error
|
||
and the workflow should finish successfully.
|
||
"""
|
||
# Use an empty temp directory as the reports path
|
||
empty_dir = tmp_path / 'reports_temp'
|
||
empty_dir.mkdir()
|
||
|
||
result = await start_and_await_workflow(
|
||
temporal_test_env.client,
|
||
CleanupFiles.run,
|
||
{'temp_path': str(empty_dir)},
|
||
make_workflow_id('test-s3-1-1'),
|
||
)
|
||
|
||
# Workflow returns None on success
|
||
assert result is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.integration
|
||
async def test_scenario_3_1_2_cleanup_removes_old_temp_dirs(
|
||
temporal_test_env: WorkflowEnvironment,
|
||
temporal_worker: Worker,
|
||
tmp_path,
|
||
):
|
||
"""Scenario 3.1.2 – Cleanup removes stale subdirectories from the temp dir.
|
||
|
||
Creates two subdirectories with timestamp suffixes inside the reports
|
||
temp directory and verifies the activity removes them.
|
||
"""
|
||
reports_dir = tmp_path / 'reports_temp'
|
||
reports_dir.mkdir()
|
||
|
||
# Create two stale run directories
|
||
stale1 = reports_dir / 'run-1234567890'
|
||
stale2 = reports_dir / 'run-9876543210'
|
||
stale1.mkdir()
|
||
stale2.mkdir()
|
||
(stale1 / 'model.pkl').write_bytes(b'fake-model-data')
|
||
(stale2 / 'report.json').write_bytes(b'{"status": "old"}')
|
||
|
||
result = await start_and_await_workflow(
|
||
temporal_test_env.client,
|
||
CleanupFiles.run,
|
||
{'temp_path': str(reports_dir)},
|
||
make_workflow_id('test-s3-1-2'),
|
||
)
|
||
|
||
assert result is None
|
||
|
||
# The activity should have cleaned up the stale directories
|
||
remaining = list(reports_dir.iterdir())
|
||
assert len(remaining) == 0, (
|
||
f'Expected all stale dirs to be removed, but found: {remaining}'
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.integration
|
||
async def test_scenario_3_1_3_cleanup_nonexistent_temp_path(
|
||
temporal_test_env: WorkflowEnvironment,
|
||
temporal_worker: Worker,
|
||
tmp_path,
|
||
):
|
||
"""Scenario 3.1.3 – Cleanup with a temp_path that does not exist.
|
||
|
||
The activity must handle a missing directory gracefully without
|
||
raising an unhandled exception, since the directory may have already
|
||
been cleaned by a previous run.
|
||
"""
|
||
nonexistent = str(tmp_path / 'does_not_exist' / 'reports')
|
||
|
||
# Should not raise — the activity is expected to handle a missing path
|
||
result = await start_and_await_workflow(
|
||
temporal_test_env.client,
|
||
CleanupFiles.run,
|
||
{'temp_path': nonexistent},
|
||
make_workflow_id('test-s3-1-3'),
|
||
)
|
||
|
||
assert result is None
|