SIENTIAPDE-1172

feat: enhance orchestrator activities with new email and Postgres integrations

- Added Email and Postgres classes to the Activities class for improved functionality.
- Introduced new methods in MongoDB and SlotManager for loading and managing data.
- Updated requirements.txt to include jinja2.
- Added new formatting activity for log reports in Formatters class.
- Enhanced test coverage for MongoDB and SlotManager activities.

tests/orchestrator/activities/test_mongo_db.py

@@ -1,4 +1,5 @@
 from curses import meta
+from datetime import datetime
 from unittest.mock import MagicMock, patch, ANY
 from pytest import fixture, mark
 from orchestrator.activities.mongo_db import clear_mongo_id
@@ -518,3 +519,133 @@ async def test_create_collection_with_ttl_index_failure(mongo_db):
         )
     else:
         assert False, "Expected an exception to be raised"
+
+
+@mark.asyncio
+async def test_load_latest_data_none_last_data_timestamp(mongo_db):
+    """Test load_latest_data"""
+    collection = MagicMock()
+    mongo_db.database.__getitem__.return_value = collection
+
+    collection.find.return_value = [
+        {
+            'name': 'test1',
+            'value': 1,
+            'timestamp': datetime.strptime(
+                '2023-01-01 12:00:00.000000', '%Y-%m-%d %H:%M:%S.%f')
+        }
+    ]
+
+    result = await mongo_db.load_latest_data({
+        'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+        'collection_name': 'test_collection',
+        'last_data_timestamp': None,
+        'base_data_filter': {
+            'level': 'ERROR'
+        }
+    })
+
+    mongo_db.database.__getitem__.assert_called_once_with(
+        'test_collection')
+
+    collection.find.assert_called_once_with(
+        {
+            'level': 'ERROR'
+        },
+        {"_id": 0}
+    )
+
+    assert result == {
+        'name': {
+            0: 'test1'
+        },
+        'value': {
+            0: 1
+        },
+        'timestamp': {
+            0: '2023-01-01 12:00:00.000000'
+        }
+    }
+
+
+@mark.asyncio
+async def test_load_latest_data_not_none_last_data_timestamp(mongo_db):
+    """Test load_latest_data"""
+    collection = MagicMock()
+    mongo_db.database.__getitem__.return_value = collection
+
+    collection.find.return_value = [
+        {
+            'name': 'test1',
+            'value': 1,
+            'timestamp': datetime.strptime(
+                '2023-01-01 12:00:00.000000', '%Y-%m-%d %H:%M:%S.%f')
+        }
+    ]
+
+    result = await mongo_db.load_latest_data({
+        'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+        'collection_name': 'test_collection',
+        'last_data_timestamp': '2023-01-01 12:00:00.000000',
+        'base_data_filter': {
+            'level': 'ERROR'
+        }
+    })
+
+    mongo_db.database.__getitem__.assert_called_once_with(
+        'test_collection')
+
+    collection.find.assert_called_once_with(
+        {
+            'level': 'ERROR',
+            'timestamp': {
+                '$gt': datetime.strptime(
+                    '2023-01-01 12:00:00.000000', '%Y-%m-%d %H:%M:%S.%f')
+            }
+        },
+        {"_id": 0}
+    )
+
+    assert result == {
+        'name': {
+            0: 'test1'
+        },
+        'value': {
+            0: 1
+        },
+        'timestamp': {
+            0: '2023-01-01 12:00:00.000000'
+        }
+    }
+
+
+@mark.asyncio
+async def test_load_latest_data_error(mongo_db):
+    """Test load_latest_data"""
+    collection = MagicMock()
+    mongo_db.send_notification = MagicMock()
+    mongo_db.database.__getitem__.return_value = collection
+
+    collection.find.side_effect = Exception('test')
+
+    try:
+        await mongo_db.load_latest_data({
+            'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+            'collection_name': 'test_collection',
+            'last_data_timestamp': '2023-01-01 12:00:00.000000',
+            'base_data_filter': {
+                'level': 'ERROR'
+            }
+        })
+    except Exception as e:
+        assert str(e) == 'test'
+
+        mongo_db.send_notification.assert_called_once_with(
+            metadata={'workflow_name': 'test_pipeline',
+                      'schedule_name': 'test_schedule'},
+            notification_id='MONGO_LOAD_ERROR',
+            message='Error loading data from MongoDB: test',
+            block='load_latest_data',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY
+        )

tests/orchestrator/activities/test_slot_manager.py

@@ -1,4 +1,5 @@
 from unittest.mock import MagicMock, patch, call, ANY
+from pandas import DataFrame
 from pytest import mark, fixture
 from orchestrator.activities.slot_manager import SlotManager
 from sientia_do.notifications.models import NotificationLevel
@@ -206,3 +207,155 @@ async def test_delete_slots(slot_manager):
             "message": "Test exception"
         }
     }
+
+
+@mark.asyncio
+async def test_get_last_data_timestamp_none(slot_manager):
+    """Test get_last_data_timestamp"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule'
+    }
+
+    slot_manager.get = MagicMock(return_value=None)
+
+    result = await slot_manager.get_last_data_timestamp(test_data)
+
+    assert result is None
+
+
+@mark.asyncio
+async def test_get_last_data_timestamp_not_none(slot_manager):
+    """Test get_last_data_timestamp"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule'
+    }
+
+    slot_manager.get = MagicMock(return_value='2023-01-01 12:00:00')
+
+    result = await slot_manager.get_last_data_timestamp(test_data)
+
+    slot_manager.get.assert_called_once_with(
+        'notification_last_timestamp'
+    )
+
+    assert result == '2023-01-01 12:00:00'
+
+
+@mark.asyncio
+async def test_get_last_data_timestamp_error(slot_manager):
+    """Test get_last_data_timestamp error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule'
+    }
+
+    slot_manager.send_notification = MagicMock()
+    slot_manager.get = MagicMock(side_effect=Exception('test'))
+
+    try:
+
+        await slot_manager.get_last_data_timestamp(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        slot_manager.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id="REDIS_GET_ERROR",
+            message="Error getting last data timestamp: test",
+            block="get_last_data_timestamp",
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY
+        )
+
+    else:
+        assert False, "Expected exception"
+
+
+@mark.asyncio
+async def test_put_last_data_timestamp_empty_dataframe(slot_manager):
+    """Test put_last_data_timestamp with empty dataframe"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records')
+    }
+
+    slot_manager.set = MagicMock()
+
+    result = await slot_manager.put_last_data_timestamp(test_data)
+
+    assert result is None
+
+    slot_manager.set.assert_not_called()
+
+
+@mark.asyncio
+async def test_put_last_data_timestamp_not_empty_dataframe(slot_manager):
+    """Test put_last_data_timestamp with not empty dataframe"""
+
+    data = DataFrame({
+        'name': ['sensor1', 'sensor2'],
+        'value': [25.5, 30.0],
+        'timestamp': ['2023-01-01 12:00:00', '2023-01-01 12:00:01']
+    })
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': data.to_dict('records')
+    }
+
+    slot_manager.set = MagicMock()
+
+    result = await slot_manager.put_last_data_timestamp(test_data)
+
+    assert result == '2023-01-01 12:00:01'
+
+    slot_manager.set.assert_called_once_with(
+        'notification_last_timestamp',
+        '2023-01-01 12:00:01',
+        ttl=18000
+    )
+
+
+@mark.asyncio
+async def test_put_last_data_timestamp_error(slot_manager):
+    """Test put_last_data_timestamp error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': DataFrame({
+            'name': ['sensor1', 'sensor2'],
+            'value': [25.5, 30.0],
+            'timestamp': ['2023-01-01 12:00:00'] * 2
+        }).to_dict('records')
+    }
+
+    slot_manager.send_notification = MagicMock()
+    slot_manager.set = MagicMock(side_effect=Exception('test'))
+
+    try:
+        await slot_manager.put_last_data_timestamp(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        slot_manager.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id="REDIS_SET_ERROR",
+            message="Error setting last data timestamp: test",
+            block="put_last_data_timestamp",
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY
+        )
+
+    else:
+        assert False, "Expected exception"

tests/orchestrator/workflows/subworkflows/test_load_notification_package.py

@@ -0,0 +1,115 @@
+from unittest.mock import AsyncMock, patch, ANY, call
+from pytest import fixture, mark
+from orchestrator.workflows.subworkflows.load_notification_package import LoadNotificationPackage
+from orchestrator.activities.activities import Activities
+
+
+@fixture
+def load_notification_package():
+    return LoadNotificationPackage()
+
+
+metadata = {
+    'metadata': {
+        'schedule_name': 'test-schedule-name',
+        'workflow_name': 'test-workflow',
+        'model_name': '-',
+        'model_id': '-',
+    }
+}
+
+
+@mark.asyncio
+@patch("orchestrator.workflows.subworkflows.load_notification_package.workflow", new_callable=AsyncMock)
+async def test_run(workflow_mock, load_notification_package):
+    input_data = {
+        'metadata': metadata
+    }
+
+    workflow_mock.start_local_activity_method.side_effect = [
+        '2023-01-01 12:00:00',
+        [
+            {
+                'id': '1',
+            }
+        ],
+        [
+            {
+                'id_r': '1',
+            }
+        ]
+    ]
+
+    output = await load_notification_package.run(input_data)
+
+    assert output == {
+        'last_timestamp': '2023-01-01 12:00:00',
+        'notification_package': [
+            {
+                'id': '1',
+            }
+        ],
+        'sending_configs': [
+            {
+                'id_r': '1',
+            }
+        ]
+    }
+
+    workflow_mock.start_local_activity_method.assert_has_calls([
+        call(
+            Activities.get_last_data_timestamp,
+            input_data['metadata'],
+            start_to_close_timeout=ANY,
+            retry_policy=ANY
+        )
+    ])
+
+    workflow_mock.start_local_activity_method.assert_has_calls([
+        call(
+            Activities.find_documents_in_mongodb,
+            {
+                **input_data['metadata'],
+                'query': {
+                    'collection': 'receiver_groups',
+                    'filters': {
+                        'active': True
+                    }
+                }
+            },
+            start_to_close_timeout=ANY,
+            retry_policy=ANY
+        )
+    ])
+
+    workflow_mock.start_local_activity_method.assert_has_calls([
+        call(
+            Activities.load_latest_data,
+            {
+                **input_data['metadata'],
+                'collection_name': 'notification_queue',
+                'last_data_timestamp': '2023-01-01 12:00:00',
+                'base_data_filter': {
+                    'level': 'ERROR'
+                }
+            },
+            start_to_close_timeout=ANY,
+            retry_policy=ANY
+        )
+    ])
+
+    workflow_mock.start_activity_method.assert_has_calls([
+        call(
+            Activities.put_last_data_timestamp,
+            {
+                **input_data['metadata'],
+                'data': [
+                    {
+                        'id': '1',
+                    }
+                ]
+            },
+            start_to_close_timeout=ANY,
+            retry_policy=ANY
+        )
+    ])