SIENTIAPDE-1712

Implement MinIO Offload and Retention Features

- Added configuration options for MinIO retention hours and offload threshold in README.
- Introduced MinIO payload offloading for large DataFrame-derived payloads, storing them as parquet files.
- Updated activities to utilize MinIO for data loading and cleanup, including new methods for offloading and retention management.
- Refactored existing activities to integrate MinIO functionality, ensuring compatibility with previous workflows.
- Removed the legacy MinioRepository class, consolidating MinIO operations under a new manager structure.
- Updated requirements to use the latest version of the sientia-dataops-library.

laborious/workflows/minimal_retrain.py

@@ -73,26 +73,25 @@ class MinimalRetrain:
         model_config = input_data.get('model_config', {})
 
         storage_result = await workflow.execute_activity_method(
-            Activities.query_to_minio,
+            Activities.load_query_with_minio_offload,
             {
                 **metadata,
                 'query': input_data['query'],
                 'datetime_columns': input_data.get('datetime_columns', []),
                 'model_name': model_name,
-                'object_prefix': f'retrain_datasets/{model_name}/data',
             },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(seconds=600),
         )
 
-        if not storage_result['success']:
+        if isinstance(storage_result, dict) and storage_result.get('success') is False:
             return
 
         experiment_response = await workflow.execute_activity_method(
             Activities.retrain_model,
             {
                 **metadata,
-                'object_key': storage_result['object_key'],
+                'data': storage_result,
                 'model_name': model_name,
                 'model_config': model_config,
             },

laborious/workflows/predictions_batch.py

@@ -83,13 +83,14 @@ class PredictionsBatch:
             }
         }
 
-        # Load data using custom query
-        data = await workflow.execute_local_activity_method(
-            Activities.load_custom_query,
+        # Load data using custom query with optional MinIO offload for large frames
+        data = await workflow.execute_activity_method(
+            Activities.load_query_with_minio_offload,
             {
                 **metadata,
                 'query': input_data['query'],
                 'datetime_columns': input_data.get('datetime_columns', []),
+                'model_name': input_data['model_name'],
             },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(seconds=300),

laborious/workflows/sub_workflows/format_and_export_prediction.py

@@ -105,6 +105,7 @@ class FormatAndExportPrediction:
                     'model_id': input_data['model_id'],
                     'prediction_confidence': prediction_confidence,
                     'prediction_store_policy': input_data['prediction_store_policy'],
+                    'model_name': input_data['model_name'],
                 },
                 retry_policy=retry_policy,
                 start_to_close_timeout=timedelta(seconds=60),
@@ -118,13 +119,14 @@ class FormatAndExportPrediction:
                         **metadata,
                         'data': transformed_data,
                         'model_id': input_data['model_id'],
+                        'model_name': input_data['model_name'],
                     },
                     retry_policy=retry_policy,
                     start_to_close_timeout=timedelta(seconds=60),
                 )
 
                 write_transformed_handler = workflow.start_activity_method(
-                    Activities.export_data_to_postgres,
+                    Activities.export_payload_to_postgres,
                     {
                         **metadata,
                         'schema': input_data['schema'],

laborious/workflows/sub_workflows/prediction_process.py

@@ -2,11 +2,13 @@ from temporalio import workflow
 
 with workflow.unsafe.imports_passed_through():
     from datetime import timedelta
+    from collections.abc import Callable
     from typing import Any
 
     from sientia_do.temporal.policies import retry_policy
 
     from laborious.activities.activities import Activities
+    from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
 
 
 @workflow.defn(name='subworkflow.prediction_process')
@@ -37,6 +39,8 @@ class PredictionProcess:
         8. Export Delegation: Delegates to FormatAndExportPrediction workflow
     """
 
+    cleanup_prefixes: set[str] = set()
+
     @workflow.run
     async def run(self, input_data: dict[str, Any]):
         """
@@ -87,13 +91,39 @@ class PredictionProcess:
         model_config = input_data.get('model_config', {})
         save_transform = input_data.get('save_transform', True)
 
-        # Get last timestamp for incremental processing
-        last_timestamp = await workflow.execute_local_activity_method(
-            Activities.get_last_timestamp,
-            {**metadata, 'data': data},
-            retry_policy=retry_policy,
-            start_to_close_timeout=timedelta(minutes=1),
-        )
+        prefix = data.cleanup_prefix()
+
+        try:
+            await self._run_prediction_pipeline(
+                input_data,
+                metadata,
+                data,
+                model_id,
+                model_name,
+                model_config,
+                save_transform,
+            )
+        finally:
+            if self.cleanup_prefixes:
+                await workflow.execute_activity_method(
+                    Activities.cleanup_minio_objects_expired,
+                    {**metadata, 'prefix': prefix},
+                    retry_policy=retry_policy,
+                    start_to_close_timeout=timedelta(minutes=5),
+                )
+
+    async def _run_prediction_pipeline(
+        self,
+        input_data: dict[str, Any],
+        metadata: dict[str, Any],
+        data: Any,
+        model_id: Any,
+        model_name: str,
+        model_config: dict[str, Any],
+        save_transform: bool
+    ) -> None:
+
+        last_timestamp = data.last_timestamp
 
         # Apply input data quality gates
         gate_input = {
@@ -119,7 +149,12 @@ class PredictionProcess:
         # Request MLFlow model transformation
         response_data = await workflow.execute_local_activity_method(
             Activities.request_transform,
-            {**metadata, 'data': data, 'model_name': model_name, 'model_config': model_config},
+            {
+                **metadata,
+                'data': data,
+                'model_name': model_name,
+                'model_config': model_config
+            },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(minutes=5),
         )
@@ -221,7 +256,7 @@ class PredictionProcess:
 
     async def path_flag_handler(
         self,
-        data: dict,
+        data: MinioDataFramePayload,
         path_flag: str,
         input_data: dict,
         confidence: int,