SIENTIAPDE-988

Enhance ingestor and manager classes with notification handling

- Integrated NotificationHandler into Ingestor, DataManager, IngestorManager, and OpcManager for improved error reporting and monitoring.
- Updated methods to send notifications on critical events such as Kafka publishing errors, OPC connection issues, and cycle count warnings.
- Refactored related tests to ensure coverage of new notification functionalities and validate integration with existing components.
- Improved logging and error handling across the system to enhance traceability and operational insights.

ingestor/managers/ingestor_manager.py

@@ -5,16 +5,19 @@ from copy import deepcopy
 from ingestor.managers.data_manager import DataManager
 from ingestor.managers.opc_manager import OpcManager
 from ingestor.managers.resource_manager import ResourceManager
+from sientia_do.notifications.models import Notification, NotificationLevel
+from sientia_do.notifications.handlers import NotificationHandler
 
 
 class IngestorManager():
     def __init__(self,
                  kafka_servers: str, redis_host: str, redis_port: int,
                  lease_ttl: int, heartbeat_ttl: int,  pod_id: str,
-                 poll_interval: int, logger: Logger,
+                 poll_interval: int, logger: Logger, notification_handler: NotificationHandler,
                  redis_username: str = None, redis_password: str = None):
 
-        self.data_manager = DataManager(kafka_servers, logger)
+        self.data_manager = DataManager(
+            kafka_servers, logger, notification_handler)
         self.opc_managers = {}
         self.resource_manager = ResourceManager(
             redis_host, redis_port, lease_ttl, heartbeat_ttl, pod_id, redis_username, redis_password
@@ -25,6 +28,8 @@ class IngestorManager():
         self.managed_tags = {}
         self.opc_servers = {}
 
+        self.notification_handler = notification_handler
+
     def initialize_opc_from_config(self, server_config: dict, data_manager: DataManager, logger: Logger) -> OpcManager | None:
         """
         Initializes an OPC Manager instance using the provided server configuration.
@@ -56,8 +61,17 @@ class IngestorManager():
             manager.config = server_config
             manager.connect()
         except Exception as e:
-            logger.error(f"Failed to initialize OpcManager: {e}")
-            traceback.print_exc()
+
+            trace = traceback.format_exc()
+            self.notification_handler.build_and_send_notification(
+                notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}',
+                message=f'Error initializing OPC manager: {e}',
+                block="opc_manager",
+                level=NotificationLevel.ERROR,
+                attachment_content=trace
+            )
+            self.logger.error(trace)
+
             return None
 
         return manager
@@ -100,6 +114,7 @@ class IngestorManager():
                     )
 
                 elif self.opc_managers[server].config != server_config:
+                    self.opc_managers[server].disconnect()
                     del self.opc_managers[server]
                     server_instance = self.initialize_opc_from_config(
                         server_config, self.data_manager, self.logger
@@ -117,6 +132,30 @@ class IngestorManager():
                 self.opc_managers[server].disconnect()
                 self.opc_managers.pop(server, None)
 
+    def check_opc_servers_integrity(self):
+        """
+        Checks the integrity of the OPC servers and updates the OPC servers if necessary.
+        """
+        for server, opc_manager in self.opc_managers.items():
+            opc_manager.check_cycles()
+
+            is_lost = opc_manager.check_opc_listenning()
+            if is_lost:
+                self.logger.warning(
+                    f"OPC server {server} is lost. "
+                    f"Desconnecting from server."
+                )
+                opc_manager.disconnect()
+                self.opc_managers[server] = self.initialize_opc_from_config(
+                    opc_manager.config, self.data_manager, self.logger
+                )
+                self.update_opc_servers()
+                for slot, slot_config in self.managed_tags.items():
+                    if server in slot_config:
+                        self.manage_server(
+                            slot, server, slot_config[server], slot_config[server]['tags']
+                        )
+
     def declare_active(self):
         """
         Declares the ingestor as active by sending a heartbeat signal to the resource manager.
@@ -342,10 +381,16 @@ class IngestorManager():
                 tags_to_sub
             )
         except Exception as e:
-            self.logger.error(
-                f"Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}"
+            trace = traceback.format_exc()
+            self.notification_handler.build_and_send_notification(
+                notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}',
+                message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}',
+                block="opc_manager",
+                level=NotificationLevel.ERROR,
+                attachment_content=trace
             )
-            self.logger.error(traceback.format_exc())
+            self.logger.error(trace)
+
             self.logger.warning(
                 "Removing subscription from server "
                 f"{server} for slot {slot}"