feat: integrate PluginStore and MinIO repository into model manager activities

- Added PluginStore integration for model management.
- Replaced StorageRepository with MinIORepository in Activities, Cleanup, and Training classes.
- Updated training logic to handle validation files and improved data management.
- Enhanced configuration for MinIO and PluginStore in connectors.
- Removed deprecated model repository and storage repository files.
- Updated environment variable handling for new configurations.

model_manager/worker/worker.py

@@ -40,6 +40,9 @@ with workflow.unsafe.imports_passed_through():
     from prometheus_client import start_http_server
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.observability.logger import Logger as SientiaLogger
+    from sientia_model.model_repository.plugin_store import PluginStore
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.temporal.worker.prepare_worker import prepare_worker
 
     from model_manager import metrics
     from model_manager.activities.activities import Activities
@@ -48,6 +51,7 @@ with workflow.unsafe.imports_passed_through():
         build_minio_config,
         build_mlflow_config,
         build_mongodb_config,
+        build_plugin_store_config,
         build_postgres_config,
     )
     from model_manager.utils.logger_helper import get_logger
@@ -55,6 +59,7 @@ with workflow.unsafe.imports_passed_through():
     from model_manager.workflows.train_model import TrainModel
 
 POD_ID = os.getenv('POD_ID')
+RUNTIME = os.getenv('RUNTIME')
 SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091'))
 TRAIN_TASK_QUEUE = os.getenv('TRAIN_TASK_QUEUE', 'train_model-queue')
 CLEANUP_TASK_QUEUE = os.getenv('CLEANUP_TASK_QUEUE', 'cleanup-queue')
@@ -79,13 +84,19 @@ async def main():
         Exception: Any unhandled exception during worker execution
         SystemExit: On graceful shutdown or error conditions
     """
+
+    if not RUNTIME:
+        raise ValueError('RUNTIME environment variable is required')
+
     host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
     use_tls = os.getenv('TEMPORAL_USE_TLS', 'false').lower() == 'true'
     logger = get_logger(__name__)
 
     metadata = {
         'pod_id': POD_ID,
+        'runtime': RUNTIME,
     }
+    
 
     start_prometheus_server(logger, metadata)
     mongo_config = build_mongodb_config()
@@ -99,12 +110,41 @@ async def main():
 
     logger.custom_info(f'MongoDB client initialized at {mongo_config["uri"]}', metadata)
 
+    logger.custom_info(f'Initializing metrics controller', metadata)
+
+    metrics_controller = MetricsController(
+        logger=logger
+    )
+
+    logger.custom_info(f'Installing runtime {RUNTIME}', metadata)
+
+    plugin_store_parameters = build_plugin_store_config()
+    plugin_store = PluginStore(
+        base_url=plugin_store_parameters['base_url'],
+        owner=plugin_store_parameters['owner'],
+        repo=plugin_store_parameters['repo'],
+        username=plugin_store_parameters['username'],
+        password=plugin_store_parameters['password'],
+        branch=plugin_store_parameters['branch'],
+        cache_ttl_seconds=plugin_store_parameters['cache_ttl_seconds'],
+        pypi_index_url=plugin_store_parameters['pypi_index_url'],
+        pypi_username=plugin_store_parameters['pypi_username'],
+        pypi_password=plugin_store_parameters['pypi_password'],
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+
+    await plugin_store.install_runtime(runtime_name=RUNTIME)
+
     activities = Activities(
         postgres_config=build_postgres_config(),
         mlflow_config=build_mlflow_config(),
         minio_config=build_minio_config(),
+        plugin_store=plugin_store,
         logger=logger,
         notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
     )
 
     new_runtime = Runtime(
@@ -133,44 +173,33 @@ async def main():
         # The schedule can be created manually if needed
 
     workers = [
-        Worker(
-            temporal_client,
-            task_queue=TRAIN_TASK_QUEUE,
-            workflows=[TrainModel],
+        prepare_worker(
+            main_workflow=TrainModel,
+            other_workflows=[],
             activities=[
                 activities.update_experiment_run,
                 activities.validate_train_params,
                 activities.train_model,
                 activities.cleanup_resources,
             ],
-            max_concurrent_workflow_tasks=10,
-            max_concurrent_activities=10,
-            max_concurrent_local_activities=10,
-            max_cached_workflows=100,
-            workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
-            activity_task_poller_behavior=PollerBehaviorAutoscaling(),
+            temporal_client=temporal_client,
+            logger=logger,
         ),
-        Worker(
-            temporal_client,
-            task_queue=CLEANUP_TASK_QUEUE,
-            workflows=[CleanupFiles],
+        prepare_worker(
+            main_workflow=CleanupFiles,
+            other_workflows=[],
             activities=[
                 activities.cleanup_minio_files,
                 activities.cleanup_temp_directories,
             ],
-            max_concurrent_workflow_tasks=20,
-            max_concurrent_activities=20,
-            max_concurrent_local_activities=20,
-            max_cached_workflows=100,
-            workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
-            activity_task_poller_behavior=PollerBehaviorAutoscaling(),
+            temporal_client=temporal_client,
+            logger=logger,
         ),
     ]
 
-    handlers = []
-
-    for w in workers:
-        handlers.append(w.run())
+    handlers = [
+        w.run() for w in workers
+    ]
 
     logger.custom_info('Model manager workers initialized', metadata)
 
@@ -183,7 +212,7 @@ async def main():
     finally:
         notification_handler.shutdown()
         logger.custom_info('MongoDB client closed', metadata)
-        await activities.shutdown()
+        activities.shutdown()
         # Exit with a non-zero status code to indicate failure to Kubernetes
         metrics.APP_UP.labels(pod_id=POD_ID).set(0)  # Mark app as DOWN
         sys.exit(1)