SIENTIAPDE-988

Add temporary functiontal tests, that must be removed soon.

Refactor tests: Remove outdated resource manager tests and add functional tests with Docker and Kafka integration

- Deleted existing unit tests for ResourceManager.
- Introduced new functional tests for single node operations with Redis and Kafka.
- Added Docker Compose setup for test environment.
- Implemented fixtures for Redis and Kafka consumers.
- Created comprehensive tests for data publishing and slot management.
- Added unit tests for DataManager and IngestorManager with mocked dependencies.
- Included tests for OpcManager covering connection, subscription, and data change notifications.

ingestor/managers/data_manager.py

@@ -1,27 +1,47 @@
 import json
 from logging import Logger
+from time import sleep
 from kafka import KafkaProducer
+from kafka.errors import NoBrokersAvailable
 
 
 class DataManager():
     def __init__(self, kafka_servers: str, logger: Logger) -> None:
-        self.kafka_producer = KafkaProducer(
-            bootstrap_servers=kafka_servers,
-            value_serializer=lambda v: json.dumps(v).encode(
-                'utf-8'),  # Serialize JSON messages
-            key_serializer=lambda k: str(k).encode('utf-8') if k else None,
-        )
+        for i in range(0, 3):
+            logger.info(
+                f"Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}")
+            try:
+                self.kafka_producer = KafkaProducer(
+                    bootstrap_servers=kafka_servers,
+                    value_serializer=lambda v: json.dumps(v).encode(
+                        'utf-8'),  # Serialize JSON messages
+                    key_serializer=lambda k: str(
+                        k).encode('utf-8') if k else None,
+                )
+                break
+            except NoBrokersAvailable:
+                logger.error(
+                    f"Kafka servers {kafka_servers} are not available. Retrying...")
+                sleep(5)
+        else:
+            logger.error(
+                f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts.")
+            raise NoBrokersAvailable(
+                f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts.")
+
+        logger.info(
+            f"DataManager initialized with Kafka servers: {kafka_servers}")
         self.logger = logger
 
     def __del__(self):
         """Destructor to close the producer connection."""
-        self.logger.info("Closing Kafka producer...")
+        print("Closing Kafka producer...")
         self.kafka_producer.flush()
         self.kafka_producer.close()
 
     def delivery_report(self, msg: str):
         """Callback for delivery reports from Kafka."""
-        self.logger.info(
+        self.logger.debug(
             f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}")
 
     def delivery_error(self, err: str):
@@ -45,7 +65,7 @@ class DataManager():
 
         try:
 
-            self.logger.info(
+            self.logger.debug(
                 f"Publishing message to topic {topic}: {data}")
             self.kafka_producer.send(
                 topic=topic, value=data).add_callback(

ingestor/managers/ingestor_manager.py

@@ -11,7 +11,6 @@ class IngestorManager():
     def __init__(self,
                  kafka_servers: str, redis_host: str, redis_port: int,
                  lease_ttl: int, heartbeat_ttl: int,  pod_id: str,
-                 number_of_ingestors: int,
                  poll_interval: int, logger: Logger):
 
         self.data_manager = DataManager(kafka_servers, logger)
@@ -19,7 +18,7 @@ class IngestorManager():
         self.resource_manager = ResourceManager(
             redis_host, redis_port, lease_ttl, heartbeat_ttl, pod_id
         )
-        self.number_of_ingestors = number_of_ingestors
+        self.number_of_slots = 0
         self.poll_interval = poll_interval
         self.logger = logger
         self.managed_tags = {}
@@ -98,9 +97,14 @@ class IngestorManager():
         ingestors = self.resource_manager.get_all_ingestors()
         return ingestors if ingestors else []
 
+    def get_number_of_slots(self) -> int:
+        slots = self.resource_manager.get_all_slots()
+        self.number_of_slots = len(slots) if slots else 0
+        return self.number_of_slots
+
     def get_slot_leases(self, max_slots: int = 1) -> Dict:
         acquired = {}
-        for i in range(1, self.number_of_ingestors + 1):
+        for i in range(1, self.number_of_slots + 1):
             if self.resource_manager.lease_tag(str(i)):
                 self.logger.info(f"Leased slot {i}")
                 slots = self.resource_manager.get_tag_slot(str(i))

ingestor/managers/opc_manager.py

@@ -27,6 +27,10 @@ class OpcManager():
         self.subscriptions = {}
         self.data_manager = data_manager
 
+    def __str__(self):
+        return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \
+            f"nodes={self.nodes}, subscriptions={self.subscriptions}"
+
     def initialize_from_config(self, server_config: dict, data_manager: DataManager, logger: Logger):
         """
         Initializes the OpcManager instance using a server configuration dictionary.

ingestor/managers/resource_manager.py

@@ -106,3 +106,14 @@ class ResourceManager:
         """
 
         return self.redis.keys("heartbeat:ingestor:*")
+
+    def get_all_slots(self) -> List[str]:
+        """
+        Retrieves the number of slots available in Redis.
+        This method counts the number of keys in Redis that match the pattern for OPC tag leases
+        and returns the count.
+        Returns:
+            int: The number of slots available.
+        """
+
+        return self.redis.keys("slot:opc_tags:*")