SIENTIAPDE-1712

Implement MinIO Offload and Retention Features

- Added configuration options for MinIO retention hours and offload threshold in README.
- Introduced MinIO payload offloading for large DataFrame-derived payloads, storing them as parquet files.
- Updated activities to utilize MinIO for data loading and cleanup, including new methods for offloading and retention management.
- Refactored existing activities to integrate MinIO functionality, ensuring compatibility with previous workflows.
- Removed the legacy MinioRepository class, consolidating MinIO operations under a new manager structure.
- Updated requirements to use the latest version of the sientia-dataops-library.

e2e/conftest.py

@@ -3,6 +3,7 @@ Pytest configuration and fixtures for E2E tests.
 """
 
 from unittest.mock import AsyncMock, MagicMock, patch
+from io import BytesIO
 
 import pandas as pd
 import pytest
@@ -216,11 +217,28 @@ def metrics_controller(mock_logger):
 def mock_minio_repository():
     """Mock MinIO repository for object storage operations."""
     mock_repo = MagicMock()
+
+    # Provide at least valid parquet bytes so that MinioDataFramePayload.retrieve()
+    # can decode the payload if offloading is exercised in an integration scenario.
+    parquet_df = pd.DataFrame({'a': [1]})
+    parquet_buffer = BytesIO()
+    parquet_df.to_parquet(parquet_buffer, engine='pyarrow', index=True)
+    parquet_bytes = parquet_buffer.getvalue()
     
-    # Mock repository methods
-    mock_repo.put_parquet_from_dataframe = AsyncMock(return_value='test-object-key')
-    mock_repo.get_parquet_as_dataframe = AsyncMock(return_value=pd.DataFrame())
-    mock_repo.minio_bucket = 'test-bucket'
+    # sientia_do MinioRepository API
+    mock_repo.bucket = 'test-bucket'
+    mock_repo.upload_file = AsyncMock(
+        side_effect=lambda file_bytes, relative_key, content_type='application/octet-stream', bucket=None, metadata=None: {
+            'minio_object_name': f'sientia/streamlit-connectors/{relative_key}',
+            'original_filename': relative_key.rsplit('/', 1)[-1],
+            'uploaded_at': '2024-01-01T00:00:00Z',
+            'sha256_hash': 'deadbeef',
+        }
+    )
+    mock_repo.download_file = AsyncMock(return_value=parquet_bytes)
+    mock_repo.list_objects = AsyncMock(return_value=[])
+    mock_repo.delete_file = AsyncMock()
+    mock_repo.close = MagicMock()
     
     return mock_repo
 
@@ -261,7 +279,7 @@ def patch_create_engine(postgres_engine):
 @pytest_asyncio.fixture
 def patch_minio_repository(mock_minio_repository):
     """Patch MinioRepository to return mock."""
-    with patch('laborious.utils.repository.minio_repository.MinioRepository', return_value=mock_minio_repository):
+    with patch('sientia_do.repository.minio_repository.MinioRepository', return_value=mock_minio_repository):
         yield
 
 @pytest_asyncio.fixture
@@ -433,6 +451,8 @@ async def temporal_worker(temporal_test_env, test_activities):
         workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
         activities=[
             test_activities.load_custom_query,
+            test_activities.load_query_with_minio_offload,
+            test_activities.cleanup_minio_objects_expired,
             test_activities.get_last_timestamp,
             test_activities.input_gate,
             test_activities.request_transform,