84 lines
2.7 KiB
Python
84 lines
2.7 KiB
Python
"""
|
|
Cleanup workflow for removing stale files from MinIO and local filesystem.
|
|
|
|
This module provides a Temporal cron workflow that runs daily to clean up
|
|
temporary files and directories older than the configured retention period.
|
|
"""
|
|
|
|
from temporalio import workflow
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
import os
|
|
from datetime import timedelta
|
|
from typing import Any
|
|
|
|
from model_manager.activities.activities import Activities
|
|
from model_manager.workflows.train_model import POD_ID, network_retry_policy, no_retry_policy
|
|
|
|
TIMEOUT_CLEANUP_MINIO = int(os.getenv('TIMEOUT_CLEANUP_MINIO', '300'))
|
|
TIMEOUT_CLEANUP_LOCAL = int(os.getenv('TIMEOUT_CLEANUP_LOCAL', '120'))
|
|
DEFAULT_CLEANUP_BUCKET = os.getenv('DEFAULT_CLEANUP_BUCKET', 'model-training')
|
|
|
|
|
|
@workflow.defn(name='cleanup_files')
|
|
class CleanupFiles:
|
|
"""
|
|
Cleanup workflow for removing stale files.
|
|
|
|
This workflow cleans up:
|
|
- MinIO files with timestamp prefixes
|
|
- Local temporary directories with timestamp suffixes
|
|
|
|
The workflow is designed to be simple and robust, with error handling
|
|
delegated to the individual activities.
|
|
"""
|
|
|
|
@workflow.run
|
|
async def run(self, input_data: dict[str, Any]) -> None:
|
|
"""
|
|
Execute the cleanup workflow.
|
|
|
|
This method orchestrates the cleanup of MinIO files and local directories
|
|
in sequence. No exception handling is needed as activities handle their
|
|
own errors and notifications.
|
|
|
|
Args:
|
|
input_data: Workflow configuration containing optional:
|
|
- bucket_name (str): Bucket to clean (defaults to environment variable)
|
|
"""
|
|
# Get bucket name from input or environment
|
|
bucket_name = input_data.get('bucket_name', DEFAULT_CLEANUP_BUCKET)
|
|
|
|
# Default temp path for local cleanup
|
|
temp_path = 'model_manager/reports/temp'
|
|
|
|
# Metadata for tracking
|
|
metadata = {
|
|
'metadata': {
|
|
'pod_id': POD_ID,
|
|
'workflow_name': 'cleanup_files',
|
|
}
|
|
}
|
|
|
|
# Execute MinIO cleanup
|
|
await workflow.execute_activity_method(
|
|
Activities.cleanup_minio_files,
|
|
{
|
|
**metadata,
|
|
'bucket_name': bucket_name,
|
|
},
|
|
retry_policy=network_retry_policy,
|
|
start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_MINIO),
|
|
)
|
|
|
|
# Execute local directory cleanup
|
|
await workflow.execute_activity_method(
|
|
Activities.cleanup_temp_directories,
|
|
{
|
|
**metadata,
|
|
'temp_path': temp_path,
|
|
},
|
|
retry_policy=no_retry_policy,
|
|
start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_LOCAL),
|
|
)
|