d1f9394879
- Added a new fixture to manage runtime report artifacts in a writable temp directory during E2E tests, addressing permission issues in local CI/dev environments. - Updated `conftest.py` to include a requirements.txt file in the model packaging path for training activities. - Refactored existing fixtures to use `pytest.fixture` instead of `pytest_asyncio.fixture` for better compatibility. - Enhanced the `Reports` class to include a target alias for report metrics, ensuring compatibility with Evidently's reporting requirements. - Introduced new test scenarios to validate the handling of missing and whitespace-only `date_column` inputs in the training workflow. These changes improve the robustness of the E2E testing framework and enhance the clarity of model reporting metrics.
65 lines
1.9 KiB
Python
65 lines
1.9 KiB
Python
"""
|
|
Cleanup workflow for removing local filesystem.
|
|
|
|
This module provides a Temporal cron workflow that runs daily to clean up
|
|
temporary files and directories older than the configured retention period.
|
|
"""
|
|
|
|
from temporalio import workflow
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
import os
|
|
from datetime import timedelta
|
|
from typing import Any
|
|
|
|
from model_manager.activities.activities import Activities
|
|
from model_manager.runtime_paths import REPORTS_TEMP_DIR
|
|
from model_manager.workflows.train_model import no_retry_policy
|
|
|
|
TIMEOUT_CLEANUP_LOCAL = int(os.getenv('TIMEOUT_CLEANUP_LOCAL', '120'))
|
|
POD_ID = os.getenv('POD_ID')
|
|
|
|
|
|
@workflow.defn(name='cleanup_files')
|
|
class CleanupFiles:
|
|
"""
|
|
Cleanup workflow for removing stale files.
|
|
|
|
This workflow cleans up:
|
|
- Local temporary directories with timestamp suffixes
|
|
|
|
The workflow is designed to be simple and robust, with error handling
|
|
delegated to the individual activities.
|
|
"""
|
|
|
|
@workflow.run
|
|
async def run(self, input_data: dict[str, Any] | None = None) -> None:
|
|
"""
|
|
Execute the cleanup workflow.
|
|
|
|
This method orchestrates the cleanup of local directories
|
|
in sequence. No exception handling is needed as activities handle their
|
|
own errors and notifications.
|
|
"""
|
|
payload = input_data or {}
|
|
temp_path = payload.get('temp_path') or REPORTS_TEMP_DIR
|
|
|
|
# Metadata for tracking
|
|
metadata = {
|
|
'metadata': {
|
|
'pod_id': POD_ID,
|
|
'workflow_name': 'cleanup_files',
|
|
}
|
|
}
|
|
|
|
# Execute local directory cleanup
|
|
await workflow.execute_activity_method(
|
|
Activities.cleanup_temp_directories,
|
|
{
|
|
**metadata,
|
|
'temp_path': temp_path,
|
|
},
|
|
retry_policy=no_retry_policy,
|
|
start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_LOCAL),
|
|
)
|