SIENTIAPDE-1350: Implement file cleanup workflow and activities, including MinIO and local directory cleanup, configuration, and metrics.

model_manager/workflows/cleanup_files.py

@@ -0,0 +1,83 @@
+"""
+Cleanup workflow for removing stale files from MinIO and local filesystem.
+
+This module provides a Temporal cron workflow that runs daily to clean up
+temporary files and directories older than the configured retention period.
+"""
+
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    import os
+    from datetime import timedelta
+    from typing import Any
+
+    from model_manager.activities.activities import Activities
+    from model_manager.workflows.train_model import POD_ID, network_retry_policy, no_retry_policy
+
+    TIMEOUT_CLEANUP_MINIO = int(os.getenv('TIMEOUT_CLEANUP_MINIO', '300'))
+    TIMEOUT_CLEANUP_LOCAL = int(os.getenv('TIMEOUT_CLEANUP_LOCAL', '120'))
+    DEFAULT_CLEANUP_BUCKET = os.getenv('DEFAULT_CLEANUP_BUCKET', 'model-training')
+
+
+@workflow.defn(name='cleanup_files')
+class CleanupFiles:
+    """
+    Cleanup workflow for removing stale files.
+
+    This workflow cleans up:
+    - MinIO files with timestamp prefixes
+    - Local temporary directories with timestamp suffixes
+
+    The workflow is designed to be simple and robust, with error handling
+    delegated to the individual activities.
+    """
+
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]) -> None:
+        """
+        Execute the cleanup workflow.
+
+        This method orchestrates the cleanup of MinIO files and local directories
+        in sequence. No exception handling is needed as activities handle their
+        own errors and notifications.
+
+        Args:
+            input_data: Workflow configuration containing optional:
+                - bucket_name (str): Bucket to clean (defaults to environment variable)
+        """
+        # Get bucket name from input or environment
+        bucket_name = input_data.get('bucket_name', DEFAULT_CLEANUP_BUCKET)
+
+        # Default temp path for local cleanup
+        temp_path = 'model_manager/reports/temp'
+
+        # Metadata for tracking
+        metadata = {
+            'metadata': {
+                'pod_id': POD_ID,
+                'workflow_name': 'cleanup_files',
+            }
+        }
+
+        # Execute MinIO cleanup
+        await workflow.execute_activity_method(
+            Activities.cleanup_minio_files,
+            {
+                **metadata,
+                'bucket_name': bucket_name,
+            },
+            retry_policy=network_retry_policy,
+            start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_MINIO),
+        )
+
+        # Execute local directory cleanup
+        await workflow.execute_activity_method(
+            Activities.cleanup_temp_directories,
+            {
+                **metadata,
+                'temp_path': temp_path,
+            },
+            retry_policy=no_retry_policy,
+            start_to_close_timeout=timedelta(seconds=TIMEOUT_CLEANUP_LOCAL),
+        )