SIENTIAPDE-1712

Enhance E2E testing with MinIO support and update documentation

- Updated `requirements-dev.txt` to include MinIO support in testcontainers.
- Added a new fixture for MinIO container setup in `conftest.py` to facilitate E2E tests involving S3-compatible storage.
- Introduced a new test fixture for activities using a real MinIO container in `conftest.py`.
- Updated E2E test scenarios and documentation to reflect the integration of MinIO for offload uploads and clarified error handling in workflows.
- Refactored existing tests to improve clarity and maintainability.

e2e/conftest.py

@@ -9,6 +9,7 @@ import pandas as pd
 import pytest
 import pytest_asyncio
 from sqlalchemy import create_engine, text
+from testcontainers.minio import MinioContainer
 from testcontainers.postgres import PostgresContainer
 from temporalio.testing import WorkflowEnvironment
 from temporalio.worker import Worker
@@ -26,6 +27,17 @@ TEST_MONGODB_CONNECTION_STRING = 'mongodb://localhost:27017'
 TEST_DATABASE_NAME = 'test_db'
 
 
+@pytest_asyncio.fixture(scope='session')
+def minio_container():
+    """
+    MinIO S3-compatible storage for E2E tests that exercise real offload uploads.
+    """
+    minio = MinioContainer()
+    minio.start()
+    yield minio
+    minio.stop()
+
+
 @pytest_asyncio.fixture(scope='session')
 def postgres_container():
     """
@@ -187,6 +199,20 @@ def mock_mongo_client():
     return mock_client
 
 
+@pytest.fixture
+def notification_inserts(mock_mongo_client):
+    """
+    Mongo insert_one mock used by CoreNotificationHandler for notification persistence.
+
+    Yields:
+        MagicMock for insert_one, reset before each test.
+    """
+    mock_db = mock_mongo_client.__getitem__.return_value
+    mock_collection = mock_db.__getitem__.return_value
+    mock_collection.insert_one.reset_mock()
+    yield mock_collection.insert_one
+
+
 @pytest_asyncio.fixture
 def notification_handler(mock_logger, mock_mongo_client):
     """
@@ -439,6 +465,88 @@ async def test_activities(
         await activities.shutdown()
 
 
+@pytest_asyncio.fixture(scope='function')
+async def test_activities_real_minio(
+    postgres_engine,
+    postgres_container,
+    minio_container,
+    mock_logger,
+    notification_handler,
+    metrics_controller,
+    patch_create_engine,
+    patch_mlflow,
+    patch_pi_web_api_repository,
+    mock_opc_repository,
+):
+    """
+    Activities with a real MinIO testcontainer (no MinioRepository patch) for offload tests.
+    """
+    minio_client = minio_container.get_client()
+    if not minio_client.bucket_exists('test-bucket'):
+        minio_client.make_bucket('test-bucket')
+    minio_port = minio_container.get_exposed_port(9000)
+    activities = Activities(
+        postgres_config={
+            'host': 'localhost',
+            'port': postgres_container.get_exposed_port(5432),
+            'user': 'test',
+            'password': 'test',
+            'dbname': 'test',
+            'min_connections': 1,
+            'max_connections': 5,
+        },
+        mlflow_config={
+            'host': 'http://localhost',
+            'port': '5000',
+            'username': 'test',
+            'password': 'test',
+        },
+        minio_config={
+            'endpoint_url': f'localhost:{minio_port}',
+            'access_key': 'minioadmin',
+            'secret_key': 'minioadmin',
+            'default_bucket': 'test-bucket',
+            'retention_hours': 24,
+            'secure': False,
+        },
+        opc_config={},
+        pi_web_api_config={
+            'base_url': 'http://localhost:8080',
+            'auth_type': 'bearer',
+            'auth_token': 'test_token',
+        },
+        logger=mock_logger,
+        notification_handler=notification_handler,
+    )
+    activities.opc_repository = {'1': mock_opc_repository}
+    try:
+        yield activities
+    finally:
+        await activities.shutdown()
+
+
+def _worker_activity_list(test_activities: Activities):
+    return [
+        test_activities.load_custom_query,
+        test_activities.load_query_with_minio_offload,
+        test_activities.cleanup_minio_objects_expired,
+        test_activities.input_gate,
+        test_activities.request_transform,
+        test_activities.mlflow_response_gate,
+        test_activities.mlflow_content_gate,
+        test_activities.request_predict,
+        test_activities.repeat_last_prediction,
+        test_activities.format_prediction,
+        test_activities.format_transformed_data,
+        test_activities.format_default_prediction,
+        test_activities.write_pi_web_api_data,
+        test_activities.write_opc_data,
+        test_activities.export_data_to_postgres,
+        test_activities.export_payload_to_postgres,
+        test_activities.write_metrics,
+    ]
+
+
 @pytest_asyncio.fixture(scope='function')
 async def temporal_test_env():
     """Create Temporal test environment."""
@@ -454,24 +562,18 @@ async def temporal_worker(temporal_test_env, test_activities):
         temporal_test_env.client,
         task_queue='test-queue',
         workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
-        activities=[
-            test_activities.load_custom_query,
-            test_activities.load_query_with_minio_offload,
-            test_activities.cleanup_minio_objects_expired,
-            test_activities.input_gate,
-            test_activities.request_transform,
-            test_activities.mlflow_response_gate,
-            test_activities.mlflow_content_gate,
-            test_activities.request_predict,
-            test_activities.repeat_last_prediction,
-            test_activities.format_prediction,
-            test_activities.format_transformed_data,
-            test_activities.format_default_prediction,
-            test_activities.write_pi_web_api_data,
-            test_activities.write_opc_data,
-            test_activities.export_data_to_postgres,
-            test_activities.export_payload_to_postgres,
-            test_activities.write_metrics,
-        ],
+        activities=_worker_activity_list(test_activities),
+    ) as worker:
+        yield worker
+
+
+@pytest_asyncio.fixture(scope='function')
+async def temporal_worker_real_minio(temporal_test_env, test_activities_real_minio):
+    """Temporal worker backed by Activities using real MinIO testcontainer."""
+    async with Worker(
+        temporal_test_env.client,
+        task_queue='test-queue',
+        workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
+        activities=_worker_activity_list(test_activities_real_minio),
     ) as worker:
         yield worker