SIENTIAPDE-1712

Update environment variables in values.yaml and enhance metric labels in metrics.py

- Changed POSTGRES_USER and POSTGRES_PASSWORD values in values.yaml for improved security.
- Added 'runtime' label to metrics in metrics.py for better environment identification.
- Updated CORE_LABELS to include 'runtime' for consistency across metrics.
- Modified type hint for data parameter in PredictionProcess to use a dictionary for better clarity.
- Adjusted tests to reflect changes in core labels and MinIO configuration.

tests/laborious/workflows/subworkflows/test_prediction_process.py

@@ -38,7 +38,7 @@ async def test_run(workflow_mock, prediction_process):
     # Arrange
     data_payload = MagicMock()
     data_payload.cleanup_prefix.return_value = 'training_datasets/test'
-    data_payload.last_timestamp = '2024-01-01'
+    data_payload.__getitem__ = lambda self, key: '2024-01-01' if key == 'last_timestamp' else MagicMock()
     input_data = {
         'metadata': metadata,
         'data': data_payload,
@@ -199,7 +199,7 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process):
     # Arrange
     data_payload = MagicMock()
     data_payload.cleanup_prefix.return_value = 'training_datasets/test'
-    data_payload.last_timestamp = '2024-01-01'
+    data_payload.__getitem__ = lambda self, key: '2024-01-01' if key == 'last_timestamp' else MagicMock()
     input_data = {
         'metadata': metadata,
         'data': data_payload,
@@ -251,7 +251,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_
     # Arrange
     data_payload = MagicMock()
     data_payload.cleanup_prefix.return_value = 'training_datasets/test'
-    data_payload.last_timestamp = '2024-01-01'
+    data_payload.__getitem__ = lambda self, key: '2024-01-01' if key == 'last_timestamp' else MagicMock()
     input_data = {
         'metadata': metadata,
         'data': data_payload,
@@ -336,7 +336,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process
     # Arrange
     data_payload = MagicMock()
     data_payload.cleanup_prefix.return_value = 'training_datasets/test'
-    data_payload.last_timestamp = '2024-01-01'
+    data_payload.__getitem__ = lambda self, key: '2024-01-01' if key == 'last_timestamp' else MagicMock()
     input_data = {
         'metadata': metadata,
         'data': data_payload,
@@ -441,7 +441,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p
     # Arrange
     data_payload = MagicMock()
     data_payload.cleanup_prefix.return_value = 'training_datasets/test'
-    data_payload.last_timestamp = '2024-01-01'
+    data_payload.__getitem__ = lambda self, key: '2024-01-01' if key == 'last_timestamp' else MagicMock()
     input_data = {
         'metadata': metadata,
         'data': data_payload,
@@ -784,7 +784,7 @@ async def test_run_with_cleanup_prefixes(workflow_mock, prediction_process):
 
     data_payload = MagicMock()
     data_payload.cleanup_prefix.return_value = 'training_datasets/test'
-    data_payload.last_timestamp = '2024-01-01'
+    data_payload.__getitem__ = lambda self, key: '2024-01-01' if key == 'last_timestamp' else MagicMock()
     input_data = {
         'metadata': metadata,
         'data': data_payload,
@@ -816,7 +816,7 @@ async def test_run_with_cleanup_prefixes(workflow_mock, prediction_process):
 
     workflow_mock.execute_activity_method.assert_any_call(
         Activities.cleanup_minio_objects_expired,
-        {**metadata, 'prefix': 'training_datasets/test'},
+        {**metadata, 'data': data_payload},
         retry_policy=ANY,
         start_to_close_timeout=ANY,
     )

tests/laborious/workflows/test_minimal_retrain.py

@@ -6,15 +6,6 @@ from laborious.activities.activities import Activities
 from laborious.workflows.minimal_retrain import MinimalRetrain
 
 
-@fixture(autouse=True)
-def _passthrough_from_dict():
-    with patch(
-        'laborious.workflows.minimal_retrain.MinioDataFramePayload.from_dict',
-        side_effect=lambda x: x,
-    ):
-        yield
-
-
 @fixture
 def minimal_retrain() -> MinimalRetrain:
     return MinimalRetrain()

tests/laborious/workflows/test_predictions_batch.py

@@ -22,13 +22,9 @@ metadata = {
 
 
 @mark.asyncio
-@patch(
-    'laborious.workflows.predictions_batch.MinioDataFramePayload.from_dict', side_effect=lambda x: x
-)
 @patch('laborious.workflows.predictions_batch.workflow', new_callable=AsyncMock)
-async def test_run(workflow_mock: AsyncMock, mock_from_dict, predictions_batch: PredictionsBatch):
+async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch):
     activity_return = MagicMock()
-    activity_return.cleanup_prefix.return_value = None
     workflow_mock.execute_activity_method.return_value = activity_return
 
     input_data = {
@@ -66,7 +62,6 @@ async def test_run(workflow_mock: AsyncMock, mock_from_dict, predictions_batch:
     prediction_input = {
         'metadata': metadata,
         'data': activity_return,
-        'cleanup_prefix': activity_return.cleanup_prefix(),
         'schema': input_data['schema'],
         'table_name': input_data['table_name'],
         'transform_table_name': input_data['transform_table_name'],