SIENTIAPDE-1350: Implement file cleanup workflow and activities, including MinIO and local directory cleanup, configuration, and metrics.

model_manager/worker/worker.py

@@ -43,6 +43,7 @@ with workflow.unsafe.imports_passed_through():
         build_postgres_config,
     )
     from model_manager.utils.logger_helper import get_logger
+    from model_manager.workflows.cleanup_files import CleanupFiles
     from model_manager.workflows.train_model import TrainModel
 
 POD_ID = os.getenv('POD_ID')
@@ -73,7 +74,6 @@ async def main():
 
     metadata = {
         'pod_id': POD_ID,
-        'workflow_name': 'train_model',
     }
 
     logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata)
@@ -130,10 +130,25 @@ async def main():
                 activities.train_model,
                 activities.cleanup_resources,
             ],
-            max_concurrent_workflow_tasks=50,
-            max_concurrent_activities=50,
-            max_concurrent_local_activities=50,
-            max_cached_workflows=200,
+            max_concurrent_workflow_tasks=10,
+            max_concurrent_activities=10,
+            max_concurrent_local_activities=10,
+            max_cached_workflows=100,
+            workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
+            activity_task_poller_behavior=PollerBehaviorAutoscaling(),
+        ),
+        Worker(
+            temporal_client,
+            task_queue='cleanup-queue',
+            workflows=[CleanupFiles],
+            activities=[
+                activities.cleanup_minio_files,
+                activities.cleanup_temp_directories,
+            ],
+            max_concurrent_workflow_tasks=20,
+            max_concurrent_activities=20,
+            max_concurrent_local_activities=20,
+            max_cached_workflows=100,
             workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
             activity_task_poller_behavior=PollerBehaviorAutoscaling(),
         ),