SIENTIAPDE-988

Enhance ingestor and manager classes with notification handling

- Integrated NotificationHandler into Ingestor, DataManager, IngestorManager, and OpcManager for improved error reporting and monitoring.
- Updated methods to send notifications on critical events such as Kafka publishing errors, OPC connection issues, and cycle count warnings.
- Refactored related tests to ensure coverage of new notification functionalities and validate integration with existing components.
- Improved logging and error handling across the system to enhance traceability and operational insights.

tests/unit/test_ingestor.py

@@ -1,4 +1,4 @@
-from unittest.mock import MagicMock, patch
+from unittest.mock import ANY, MagicMock, patch
 
 from pytest import fixture
 from ingestor.ingestor import Ingestor
@@ -6,21 +6,27 @@ from ingestor.ingestor import Ingestor
 
 @patch("ingestor.ingestor.getenv")
 @patch("ingestor.ingestor.Ingestor.init_logger")
-def test___init__(init_logger, getenv):
+@patch("ingestor.ingestor.NotificationHandler")
+def test___init__(notification_handler, init_logger, getenv):
     getenv.side_effect = [
         "localhost:9092,localhost:35",  # KAFKA_SERVERS
         "localhost1",       # REDIS_HOST
         '63790',             # REDIS_PORT
+        "user",             # REDIS_USERNAME
+        "password",         # REDIS_PASSWORD
         '100',               # LEASE_TTL
         '200',               # HEARTBEAT_TTL
         "localhost1",      # HOSTNAME
         '50'                 # POLL_INTERVAL
     ]
+
     ingestor = Ingestor()
 
     getenv.assert_any_call("KAFKA_SERVERS", "localhost:9092")
     getenv.assert_any_call("REDIS_HOST", "localhost")
     getenv.assert_any_call("REDIS_PORT", 6379)
+    getenv.assert_any_call("REDIS_USERNAME", None)
+    getenv.assert_any_call("REDIS_PASSWORD", None)
     getenv.assert_any_call("LEASE_TTL", 10)
     getenv.assert_any_call("HEARTBEAT_TTL", 20)
     getenv.assert_any_call("HOSTNAME", "localhost")
@@ -29,18 +35,30 @@ def test___init__(init_logger, getenv):
     assert ingestor.kafka_servers == ["localhost:9092", "localhost:35"]
     assert ingestor.redis_host == "localhost1"
     assert ingestor.redis_port == 63790
+    assert ingestor.redis_username == "user"
+    assert ingestor.redis_password == "password"
     assert ingestor.lease_ttl == 100
     assert ingestor.heartbeat_ttl == 200
     assert ingestor.pod_id == "localhost1"
     assert ingestor.poll_interval == 50
 
     init_logger.assert_called_once()
+    notification_handler.assert_called_once_with(
+        servers=["localhost:9092", "localhost:35"],
+        logger=ingestor.logger,
+        project_name="OPC_INGESTOR",
+        pipeline_name="-",
+        trigger_name="-",
+        model_name="-",
+        model="-"
+    )
 
 
 @fixture
 @patch("ingestor.ingestor.getenv")
 @patch("ingestor.ingestor.Ingestor.init_logger")
-def ingestor(init_logger, getenv):
+@patch("ingestor.ingestor.NotificationHandler")
+def ingestor(notification_handler, init_logger, getenv):
     ing = Ingestor()
     ing.logger = MagicMock()
 
@@ -104,7 +122,10 @@ def test_prepare_ingestor(ingestor_manager_mock, ingestor):
         ingestor.heartbeat_ttl,
         ingestor.pod_id,
         ingestor.poll_interval,
-        ingestor.logger
+        ingestor.logger,
+        ingestor.redis_username,
+        ingestor.redis_password,
+        ingestor.notification_handler
     )
     ingestor_manager.declare_active.assert_called_once()
     ingestor_manager.get_slot_leases.assert_called_once()
@@ -206,6 +227,8 @@ def test_loop(ingestor_manager_started):
         return_value=["ingestor1", "ingestor2"])
     ingestor_manager_started.ingestor_manager.get_number_of_slots = MagicMock(
         return_value=5)
+    ingestor_manager_started.ingestor_manager.get_number_of_leases = MagicMock(
+        return_value=1)
 
     ingestor_manager_started.loop()
 
@@ -217,7 +240,7 @@ def test_loop(ingestor_manager_started):
         ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value)
     # Explanation: 5 - 2 = 3, 3 - 1 = 2
     ingestor_manager_started.manage_leases.assert_called_once_with(
-        3, 2)
+        4, 3, 2)
     ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once()
 
 
@@ -240,7 +263,7 @@ def test_loop_no_managed(ingestor_manager_started):
         ingestor_manager_started.ingestor_manager.get_number_of_slots.return_value)
     # Explanation: 5 - 2 = 3, 3 - 1 = 2
     ingestor_manager_started.manage_leases.assert_called_once_with(
-        3, -1)
+        ANY, 3, -1)
     ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once()
     ingestor_manager_started.logger.info.assert_any_call(
         "No slots acquired in this loop")