SIENTIAPDE-1325

SIENTIAPDE-1325: Refactor Ingestor and Manager Classes for Enhanced Asynchronous Operations

- Introduced asynchronous methods across Ingestor, IngestorManager, DataManager, and OpcManager to improve performance and responsiveness.
- Integrated MetricsController into various classes for better observability and monitoring.
- Updated Redis and MongoDB interactions to support asynchronous operations, enhancing data handling efficiency.
- Removed deprecated Redis metrics and streamlined resource management logic.
- Adjusted unit tests to accommodate the new asynchronous behavior and ensure proper mocking of async methods.

ingestor/ingestor.py

@@ -4,6 +4,7 @@ from typing import Any
 
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.observability.logger import get_logger
+from sientia_do.observability.metrics_controller import MetricsController
 
 import ingestor.metrics as metrics
 from ingestor.managers.ingestor_manager import IngestorManager
@@ -97,6 +98,7 @@ class Ingestor:
             logger=self.logger,
             project_name='opc_ingestor',
         )
+        self.metrics_controller = MetricsController(logger=self.logger)
 
         self.metadata = {
             'model_id': '-',
@@ -184,6 +186,7 @@ class Ingestor:
             metadata=self.metadata,
             logger=self.logger,
             notification_handler=self.notification_handler,
+            metrics_controller=self.metrics_controller,
             export_to_kafka=self.export_to_kafka,
         )
         assert self.ingestor_manager is not None
@@ -201,7 +204,7 @@ class Ingestor:
             len(self.ingestor_manager.managed_tags)
         )  # Set initial
 
-    def manage_no_slots(self, number_of_slots: int):
+    async def manage_no_slots(self, number_of_slots: int):
         """
         Manages the scenario where there are no slots assigned to the ingestor.
 
@@ -222,7 +225,7 @@ class Ingestor:
             # This ingestor is active and has no slots, so we need to try to
 
             # Get slot lease
-            self.ingestor_manager.get_slot_leases(1)
+            await self.ingestor_manager.get_slot_leases(1)
 
     async def manage_leases(self, available_slots: int, lacking_ingestors: int, slot_diff: int):
         """
@@ -255,7 +258,7 @@ class Ingestor:
             self.logger.info(f'Slots available: {available_slots}')
 
             # Get slot lease
-            self.ingestor_manager.get_slot_leases(available_slots)
+            await self.ingestor_manager.get_slot_leases(available_slots)
 
         elif lacking_ingestors <= 0 and slot_diff > 0:
             self.logger.info(f'Extra slots available: {slot_diff}')
@@ -264,7 +267,7 @@ class Ingestor:
 
             overleases = list(self.ingestor_manager.managed_tags.keys())[1:]
 
-            self.ingestor_manager.drop_slot_leases(overleases)
+            await self.ingestor_manager.drop_slot_leases(overleases)
 
             for lease in overleases:
                 await self.ingestor_manager.unsubscribe_slot(lease)
@@ -373,17 +376,17 @@ class Ingestor:
 
         current_managed_tags = deepcopy(self.ingestor_manager.managed_tags)
 
-        ingestors = self.ingestor_manager.get_active_ingestors()
+        ingestors = await self.ingestor_manager.get_active_ingestors()
         number_of_ingestors = len(ingestors)
-        number_of_leases = self.ingestor_manager.get_number_of_leases()
-        number_of_slots = self.ingestor_manager.get_number_of_slots()
+        number_of_leases = await self.ingestor_manager.get_number_of_leases()
+        number_of_slots = await self.ingestor_manager.get_number_of_slots()
 
         # Update active ingestors gauge
         metrics.ACTIVE_INGESTORS.set(number_of_ingestors)
 
         # Handle no slots
         self.logger.info('Managing no slots...')
-        self.manage_no_slots(number_of_slots)
+        await self.manage_no_slots(number_of_slots)
 
         available_slots = number_of_slots - number_of_leases
         lacking_ingestors = number_of_slots - number_of_ingestors
@@ -410,7 +413,7 @@ class Ingestor:
 
         # Update opc servers
         self.logger.info('Updating slot config...')
-        self.ingestor_manager.update_slot_config()
+        await self.ingestor_manager.update_slot_config()
 
         # Check OPC cycles
         self.logger.info('Checking OPC servers integrity...')