SIENTIAPDE-1712

Implement MinIO Offload and Retention Features

- Added configuration options for MinIO retention hours and offload threshold in README.
- Introduced MinIO payload offloading for large DataFrame-derived payloads, storing them as parquet files.
- Updated activities to utilize MinIO for data loading and cleanup, including new methods for offloading and retention management.
- Refactored existing activities to integrate MinIO functionality, ensuring compatibility with previous workflows.
- Removed the legacy MinioRepository class, consolidating MinIO operations under a new manager structure.
- Updated requirements to use the latest version of the sientia-dataops-library.

laborious/activities/activities.py

@@ -6,6 +6,7 @@ with workflow.unsafe.imports_passed_through():
 
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.observability.logger import Logger
+    from sientia_do.repository.minio_repository import MinioRepository
 
     from laborious.activities.api import API
     from laborious.activities.gates import Gates
@@ -13,6 +14,7 @@ with workflow.unsafe.imports_passed_through():
     from laborious.activities.model_metrics import ModelMetrics
     from laborious.activities.opc import OPC
     from laborious.activities.storage import Storage
+    
 
 
 class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
@@ -73,6 +75,16 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
         """
         metrics_controller = MetricsController(logger=logger)
 
+        minio_repository = MinioRepository(
+            endpoint_url=minio_config['endpoint_url'],
+            access_key=minio_config['access_key'],
+            secret_key=minio_config['secret_key'],
+            bucket=minio_config['default_bucket'],
+            logger=logger,
+            notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
+        )
+
         # Initialize parent classes
         Storage.__init__(
             self,
@@ -83,7 +95,8 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
             dbname=postgres_config['dbname'],
             min_connections=postgres_config['min_connections'],
             max_connections=postgres_config['max_connections'],
-            minio_config=minio_config,
+            retention_hours=minio_config['retention_hours'],
+            minio_repository=minio_repository,
             logger=logger,
             notification_handler=notification_handler,
             metrics_controller=metrics_controller,
@@ -95,7 +108,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
             mlflow_port=mlflow_config['port'],
             mlflow_username=mlflow_config['username'],
             mlflow_password=mlflow_config['password'],
-            minio_config=minio_config,
+            minio_repository=minio_repository,
             logger=logger,
             notification_handler=notification_handler,
             metrics_controller=metrics_controller,
@@ -103,6 +116,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
 
         Gates.__init__(
             self,
+            minio_repository=minio_repository,
             logger=logger,
             notification_handler=notification_handler,
             metrics_controller=metrics_controller,