SIENTIAPDE-1231

Enhance MinIO integration and update environment configurations

- Added MinIO configuration parameters to .env.example and values.yaml for improved storage management.
- Updated requirements.txt to include necessary libraries for MinIO support.
- Refactored Activities class to utilize Storage for MinIO interactions.
- Enhanced MLFlow class to integrate MinIO for data retrieval during model retraining.
- Introduced build_minio_config function to streamline MinIO configuration setup.
- Updated minimal_retrain workflow to support data storage in MinIO.

laborious/activities/mlflow.py

@@ -1,9 +1,9 @@
+from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, now
 from temporalio import activity, workflow
 
 
 with workflow.unsafe.imports_passed_through():
-    from datetime import datetime
-    from pandas import Timestamp, to_datetime
+    from pandas import to_datetime
     from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
     from sientia_do.temporal.activities.base import BaseActivity
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
@@ -15,6 +15,7 @@ with workflow.unsafe.imports_passed_through():
     import numpy as np
     from pandas import DataFrame
     import traceback
+    from laborious.utils.repository.minio_repository import MinioRepository
 
 
 class MLFlow(BaseActivity):
@@ -37,7 +38,8 @@ class MLFlow(BaseActivity):
     """
 
     def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str,
-                 mlflow_password: str, logger: Logger, notification_handler: NotificationHandler):
+                 minio_config: dict[str, Any], mlflow_password: str,
+                 logger: Logger, notification_handler: NotificationHandler):
         """
         Initialize MLFlow activities with server configuration.
 
@@ -63,6 +65,26 @@ class MLFlow(BaseActivity):
             f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger
         )
 
+        if not hasattr(self, 'minio_repository'):
+            self.minio_repository = MinioRepository(
+                logger=logger,
+                notification_handler=notification_handler,
+                minio_endpoint_url=minio_config['endpoint_url'],
+                minio_access_key=minio_config['access_key'],
+                minio_secret_key=minio_config['secret_key'],
+                minio_region_name=minio_config['region_name'],
+                minio_default_bucket=minio_config['default_bucket'])
+
+        if self.minio_repository is None:
+            self.minio_repository = MinioRepository(
+                logger=logger,
+                notification_handler=notification_handler,
+                minio_endpoint_url=minio_config['endpoint_url'],
+                minio_access_key=minio_config['access_key'],
+                minio_secret_key=minio_config['secret_key'],
+                minio_region_name=minio_config['region_name'],
+                minio_default_bucket=minio_config['default_bucket'])
+
     @activity.defn(name="request_transform")
     async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
@@ -223,7 +245,35 @@ class MLFlow(BaseActivity):
             Exception: If retraining fails or encounters critical errors
         """
         metadata = input_data['metadata']
-        data = DataFrame(input_data['data'])
+        object_key = input_data['object_key']
+
+        self.info(f'Loading retrain data from Key: {object_key}', metadata)
+
+        try:
+
+            data = self.minio_repository.get_parquet_as_dataframe(
+                object_key=object_key, metadata=metadata)
+        except Exception as e:
+            trace = traceback.format_exc()
+            self.send_notification(
+                metadata=metadata,
+                notification_id='ERROR_LOADING_RETRAIN_DATA',
+                message=f'Error loading retrain data: {e}',
+                block='retrain_model',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace
+            )
+            self.error(trace, metadata)
+            return {
+                'success': False,
+                'message': f'Error loading retrain data: {e}',
+                'traceback': trace,
+                'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ)
+            }
+
+        self.debug(
+            f'Retrain data loaded successfully: shape {data.shape}', metadata)
+
         model_name = input_data['model_name']
         model_config = input_data.get('model_config', {})