""" Cleanup workflow for removing stale files from MinIO and local filesystem. This module provides a Temporal cron workflow that runs daily to clean up temporary files and directories older than the configured retention period. """ from temporalio import workflow with workflow.unsafe.imports_passed_through(): import os from datetime import timedelta from typing import Any from model_manager.activities.activities import Activities from model_manager.workflows.train_model import POD_ID, network_retry_policy, no_retry_policy TIMEOUT_CLEANUP_MINIO = int(os.getenv('TIMEOUT_CLEANUP_MINIO', '300')) TIMEOUT_CLEANUP_LOCAL = int(os.getenv('TIMEOUT_CLEANUP_LOCAL', '120')) DEFAULT_CLEANUP_BUCKET = os.getenv('DEFAULT_CLEANUP_BUCKET', 'model-training') @workflow.defn(name='cleanup_files') class CleanupFiles: """ Cleanup workflow for removing stale files. This workflow cleans up: - MinIO files with timestamp prefixes - Local temporary directories with timestamp suffixes The workflow is designed to be simple and robust, with error handling delegated to the individual activities. """ @workflow.run async def run(self, input_data: dict[str, Any]) -> None: """ Execute the cleanup workflow. This method orchestrates the cleanup of MinIO files and local directories in sequence. No exception handling is needed as activities handle their own errors and notifications. Args: input_data: Workflow configuration containing optional: - bucket_name (str): Bucket to clean (defaults to environment variable) """ # Get bucket name from input or environment bucket_name = input_data.get('bucket_name', DEFAULT_CLEANUP_BUCKET) # Default temp path for local cleanup temp_path = 'model_manager/reports/temp' # Metadata for tracking metadata = { 'metadata': { 'pod_id': POD_ID, 'workflow_name': 'cleanup_files', } } # Execute MinIO cleanup await workflow.execute_activity_method( Activities.cleanup_minio_files, { **metadata, 'bucket_name': bucket_name, }, retry_policy=network_retry_policy, start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_MINIO), ) # Execute local directory cleanup await workflow.execute_activity_method( Activities.cleanup_temp_directories, { **metadata, 'temp_path': temp_path, }, retry_policy=no_retry_policy, start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_LOCAL), )