ba9eb3d7c7
- Made `date_column` a required field in `TrainModelParams`, ensuring it must be present in the input data. - Updated related documentation in `input-sample.md`, `README.md`, and various test scenarios to reflect the change in requirement. - Adjusted the handling of `date_format` to default to `yyyy-MM-dd HH:mm:ss` if omitted, enhancing usability. - Refined test scenarios to include new examples and ensure compliance with the updated parameter structure. These changes improve the robustness of the model training workflow and clarify the expectations for input data.
64 lines
1.9 KiB
Python
64 lines
1.9 KiB
Python
"""
|
|
Cleanup workflow for removing local filesystem.
|
|
|
|
This module provides a Temporal cron workflow that runs daily to clean up
|
|
temporary files and directories older than the configured retention period.
|
|
"""
|
|
|
|
from temporalio import workflow
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
import os
|
|
from datetime import timedelta
|
|
from typing import Any
|
|
|
|
from model_manager.activities.activities import Activities
|
|
from model_manager.runtime_paths import REPORTS_TEMP_DIR
|
|
from model_manager.workflows.train_model import no_retry_policy
|
|
|
|
TIMEOUT_CLEANUP_LOCAL = int(os.getenv('TIMEOUT_CLEANUP_LOCAL', '120'))
|
|
|
|
|
|
@workflow.defn(name='cleanup_files')
|
|
class CleanupFiles:
|
|
"""
|
|
Cleanup workflow for removing stale files.
|
|
|
|
This workflow cleans up:
|
|
- Local temporary directories with timestamp suffixes
|
|
|
|
The workflow is designed to be simple and robust, with error handling
|
|
delegated to the individual activities.
|
|
"""
|
|
|
|
@workflow.run
|
|
async def run(self, input_data: dict[str, Any] | None = None) -> None:
|
|
"""
|
|
Execute the cleanup workflow.
|
|
|
|
This method orchestrates the cleanup of local directories
|
|
in sequence. No exception handling is needed as activities handle their
|
|
own errors and notifications.
|
|
"""
|
|
payload = input_data or {}
|
|
temp_path = payload.get('temp_path') or REPORTS_TEMP_DIR
|
|
|
|
# Metadata for tracking
|
|
metadata = {
|
|
'metadata': {
|
|
'pod_id': os.getenv('POD_ID'),
|
|
'workflow_name': 'cleanup_files',
|
|
}
|
|
}
|
|
|
|
# Execute local directory cleanup
|
|
await workflow.execute_activity_method(
|
|
Activities.cleanup_temp_directories,
|
|
{
|
|
**metadata,
|
|
'temp_path': temp_path,
|
|
},
|
|
retry_policy=no_retry_policy,
|
|
start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_LOCAL),
|
|
)
|