SIENTIAPDE-1646

Update README, requirements, and E2E tests for improved configuration and functionality

- Enhanced the README with updated model configuration examples, including the addition of an alias for production.
- Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`.
- Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity.
- Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs.
- Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.

tests/laborious/utils/models/test_minio_dataframe_payload.py

@@ -1,8 +1,7 @@
 from datetime import datetime
 from io import BytesIO
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import MagicMock, patch
 
-import pytest
 from pandas import DataFrame
 
 from laborious.utils.models.minio_dataframe_payload import (
@@ -54,17 +53,15 @@ def test_has_data_true_when_object_key_set():
     assert payload.has_data() is True
 
 
-@pytest.mark.asyncio
-async def test_retrieve_inline_dict_as_dataframe():
+def test_retrieve_inline_dict_as_dataframe():
     payload = MinioDataFramePayload(last_timestamp='t', data={'a': [1, 2]})
-    minio = AsyncMock()
-    out = await payload.retrieve(minio, {'metadata': {}})
+    minio = MagicMock()
+    out = payload.retrieve(minio, {'metadata': {}})
     assert list(out.columns) == ['a']
     minio.download_file.assert_not_called()
 
 
-@pytest.mark.asyncio
-async def test_retrieve_downloads_parquet_when_offloaded():
+def test_retrieve_downloads_parquet_when_offloaded():
     source = DataFrame({'a': [1, 2]})
     buf = BytesIO()
     source.to_parquet(buf, engine='pyarrow', index=True)
@@ -76,12 +73,12 @@ async def test_retrieve_downloads_parquet_when_offloaded():
         object_key='training_datasets/m/f.parquet',
         object_prefix='training_datasets/m',
     )
-    minio = AsyncMock()
-    minio.download_file = AsyncMock(return_value=file_bytes)
+    minio = MagicMock()
+    minio.download_file = MagicMock(return_value=file_bytes)
 
-    out = await payload.retrieve(minio, {'metadata': {}})
+    out = payload.retrieve(minio, {'metadata': {}})
 
-    minio.download_file.assert_awaited_once_with(
+    minio.download_file.assert_called_once_with(
         object_name='training_datasets/m/f.parquet',
         metadata={'metadata': {}},
     )
@@ -113,21 +110,19 @@ def test_parse_object_timestamp_bad_datetime():
     assert MinioDataFramePayload.parse_object_timestamp(key) is None
 
 
-@pytest.mark.asyncio
-async def test_retrieve_empty_when_no_data():
+def test_retrieve_empty_when_no_data():
     payload = MinioDataFramePayload(last_timestamp='t', data=None, object_key=None)
-    minio = AsyncMock()
-    out = await payload.retrieve(minio, {})
+    minio = MagicMock()
+    out = payload.retrieve(minio, {})
     assert out.empty
     minio.download_file.assert_not_called()
 
 
-@pytest.mark.asyncio
 @patch('laborious.utils.models.minio_dataframe_payload.now')
-async def test_from_dataframe_none(mock_now):
+def test_from_dataframe_none(mock_now):
     mock_now.return_value = datetime(2024, 1, 1, 0, 0, 0)
-    minio = AsyncMock()
-    result = await MinioDataFramePayload.from_dataframe(
+    minio = MagicMock()
+    result = MinioDataFramePayload.from_dataframe(
         dataframe=None,
         minio_repo=minio,
         model_name='m',
@@ -139,15 +134,14 @@ async def test_from_dataframe_none(mock_now):
     assert result.object_key is None
 
 
-@pytest.mark.asyncio
 @patch('laborious.utils.models.minio_dataframe_payload.now')
-async def test_from_dataframe_empty(mock_now):
+def test_from_dataframe_empty(mock_now):
     mock_now.return_value = datetime(2024, 1, 1, 0, 0, 0)
-    minio = AsyncMock()
+    minio = MagicMock()
     mock_df = MagicMock()
     mock_df.__bool__ = MagicMock(return_value=True)
     mock_df.empty = True
-    result = await MinioDataFramePayload.from_dataframe(
+    result = MinioDataFramePayload.from_dataframe(
         dataframe=mock_df,
         minio_repo=minio,
         model_name='m',
@@ -174,12 +168,11 @@ def _mock_dataframe(data_dict, timestamp_values=None):
     return mock_df
 
 
-@pytest.mark.asyncio
 @patch('laborious.utils.models.minio_dataframe_payload.OFFLOAD_THRESHOLD_BYTES', 10**9)
-async def test_from_dataframe_inline():
-    minio = AsyncMock()
+def test_from_dataframe_inline():
+    minio = MagicMock()
     df = _mock_dataframe({'timestamp': ['2024-01-01'], 'value': [42]})
-    result = await MinioDataFramePayload.from_dataframe(
+    result = MinioDataFramePayload.from_dataframe(
         dataframe=df,
         minio_repo=minio,
         model_name='m',
@@ -190,12 +183,11 @@ async def test_from_dataframe_inline():
     assert result.last_timestamp == '2024-01-01'
 
 
-@pytest.mark.asyncio
 @patch('laborious.utils.models.minio_dataframe_payload.OFFLOAD_THRESHOLD_BYTES', 10**9)
-async def test_from_dataframe_inline_uses_provided_last_timestamp():
-    minio = AsyncMock()
+def test_from_dataframe_inline_uses_provided_last_timestamp():
+    minio = MagicMock()
     df = _mock_dataframe({'timestamp': ['2024-01-01'], 'value': [42]})
-    result = await MinioDataFramePayload.from_dataframe(
+    result = MinioDataFramePayload.from_dataframe(
         dataframe=df,
         minio_repo=minio,
         model_name='m',
@@ -205,17 +197,16 @@ async def test_from_dataframe_inline_uses_provided_last_timestamp():
     assert result.last_timestamp == '2024-01-02'
 
 
-@pytest.mark.asyncio
 @patch('laborious.utils.models.minio_dataframe_payload.now')
 @patch('laborious.utils.models.minio_dataframe_payload.OFFLOAD_THRESHOLD_BYTES', 0)
-async def test_from_dataframe_offloaded(mock_now):
+def test_from_dataframe_offloaded(mock_now):
     mock_now.return_value = datetime(2024, 1, 1, 0, 0, 0)
-    minio = AsyncMock()
-    minio.upload_file = AsyncMock(return_value={'minio_object_name': 'full/key.parquet'})
+    minio = MagicMock()
+    minio.upload_file = MagicMock(return_value={'minio_object_name': 'full/key.parquet'})
     minio.bucket = 'test-bucket'
 
     df = _mock_dataframe({'timestamp': ['2024-01-01'], 'value': [42]})
-    result = await MinioDataFramePayload.from_dataframe(
+    result = MinioDataFramePayload.from_dataframe(
         dataframe=df,
         minio_repo=minio,
         model_name='m',
@@ -226,7 +217,7 @@ async def test_from_dataframe_offloaded(mock_now):
     assert result.object_key == 'full/key.parquet'
     assert result.bucket == 'test-bucket'
     assert result.uri == 's3://test-bucket/full/key.parquet'
-    minio.upload_file.assert_awaited_once()
+    minio.upload_file.assert_called_once()
 
 
 def test_from_dict_inline():