ba9eb3d7c7
- Made `date_column` a required field in `TrainModelParams`, ensuring it must be present in the input data. - Updated related documentation in `input-sample.md`, `README.md`, and various test scenarios to reflect the change in requirement. - Adjusted the handling of `date_format` to default to `yyyy-MM-dd HH:mm:ss` if omitted, enhancing usability. - Refined test scenarios to include new examples and ensure compliance with the updated parameter structure. These changes improve the robustness of the model training workflow and clarify the expectations for input data.
109 lines
3.3 KiB
Python
109 lines
3.3 KiB
Python
"""
|
||
End-to-end tests for CleanupFiles workflow.
|
||
|
||
Covers scenarios 3.x: cleanup of temporary local directories.
|
||
"""
|
||
|
||
import pytest
|
||
from temporalio.testing import WorkflowEnvironment
|
||
from temporalio.worker import Worker
|
||
|
||
from e2e.helpers import make_workflow_id, start_and_await_workflow
|
||
from model_manager.workflows.cleanup_files import CleanupFiles
|
||
|
||
# Matches Cleanup.dir_timestamp_pattern: name_YYYYMMDD_HHMMSS_microseconds
|
||
_STALE_DIR_OLD = 'stale_run_20200102_030405_000001'
|
||
_STALE_DIR_OLDER = 'stale_run_20191231_235959_999999'
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.integration
|
||
async def test_scenario_3_1_1_cleanup_with_no_temp_dirs(
|
||
temporal_test_env: WorkflowEnvironment,
|
||
temporal_worker: Worker,
|
||
tmp_path,
|
||
):
|
||
"""Scenario 3.1.1 – Cleanup when the temp directory is empty.
|
||
|
||
The cleanup_temp_directories activity should complete without error
|
||
and the workflow should finish successfully.
|
||
"""
|
||
# Use an empty temp directory as the reports path
|
||
empty_dir = tmp_path / 'reports_temp'
|
||
empty_dir.mkdir()
|
||
|
||
result = await start_and_await_workflow(
|
||
temporal_test_env.client,
|
||
CleanupFiles.run,
|
||
{'temp_path': str(empty_dir)},
|
||
make_workflow_id('test-s3-1-1'),
|
||
)
|
||
|
||
# Workflow returns None on success
|
||
assert result is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.integration
|
||
async def test_scenario_3_1_2_cleanup_removes_old_temp_dirs(
|
||
temporal_test_env: WorkflowEnvironment,
|
||
temporal_worker: Worker,
|
||
tmp_path,
|
||
):
|
||
"""Scenario 3.1.2 – Cleanup removes stale subdirectories from the temp dir.
|
||
|
||
Creates two subdirectories with timestamp suffixes inside the reports
|
||
temp directory and verifies the activity removes them.
|
||
"""
|
||
reports_dir = tmp_path / 'reports_temp'
|
||
reports_dir.mkdir()
|
||
|
||
# Create two stale run directories (names must match cleanup activity regex)
|
||
stale1 = reports_dir / _STALE_DIR_OLD
|
||
stale2 = reports_dir / _STALE_DIR_OLDER
|
||
stale1.mkdir()
|
||
stale2.mkdir()
|
||
(stale1 / 'model.pkl').write_bytes(b'fake-model-data')
|
||
(stale2 / 'report.json').write_bytes(b'{"status": "old"}')
|
||
|
||
result = await start_and_await_workflow(
|
||
temporal_test_env.client,
|
||
CleanupFiles.run,
|
||
{'temp_path': str(reports_dir)},
|
||
make_workflow_id('test-s3-1-2'),
|
||
)
|
||
|
||
assert result is None
|
||
|
||
# The activity should have cleaned up the stale directories
|
||
remaining = list(reports_dir.iterdir())
|
||
assert len(remaining) == 0, (
|
||
f'Expected all stale dirs to be removed, but found: {remaining}'
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
@pytest.mark.integration
|
||
async def test_scenario_3_1_3_cleanup_nonexistent_temp_path(
|
||
temporal_test_env: WorkflowEnvironment,
|
||
temporal_worker: Worker,
|
||
tmp_path,
|
||
):
|
||
"""Scenario 3.1.3 – Cleanup with a temp_path that does not exist.
|
||
|
||
The activity must handle a missing directory gracefully without
|
||
raising an unhandled exception, since the directory may have already
|
||
been cleaned by a previous run.
|
||
"""
|
||
nonexistent = str(tmp_path / 'does_not_exist' / 'reports')
|
||
|
||
# Should not raise — the activity is expected to handle a missing path
|
||
result = await start_and_await_workflow(
|
||
temporal_test_env.client,
|
||
CleanupFiles.run,
|
||
{'temp_path': nonexistent},
|
||
make_workflow_id('test-s3-1-3'),
|
||
)
|
||
|
||
assert result is None
|