SIENTIAPDE-988

Implement initial structure for data ingestion with Kafka and OPC UA integration

- Added DataManager for Kafka message handling
- Introduced IngestorManager to manage data ingestion processes
- Created OpcManager for OPC UA client interactions
- Developed ResourceManager for Redis-based resource management
- Established testing framework with unit tests for DataManager and OpcManager
- Configured project settings for Python testing with pytest

ingestor/managers/data_manager.py

@@ -0,0 +1,51 @@
+import json
+from logging import Logger
+from kafka import KafkaProducer
+
+
+class DataManager():
+    def __init__(self, kafka_servers: str, logger: Logger) -> None:
+        self.kafka_producer = KafkaProducer(
+            bootstrap_servers=kafka_servers,
+            value_serializer=lambda v: json.dumps(v).encode(
+                'utf-8'),  # Serialize JSON messages
+            key_serializer=lambda k: str(k).encode('utf-8') if k else None,
+        )
+        self.logger = logger
+
+    def __del__(self):
+        """Destructor to close the producer connection."""
+        self.logger.info("Closing Kafka producer...")
+        self.kafka_producer.flush()
+        self.kafka_producer.close()
+
+    def delivery_report(self, msg: str):
+        """Callback for delivery reports from Kafka."""
+        self.logger.info(
+            f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}")
+
+    def delivery_error(self, err: str):
+        """Callback for delivery reports from Kafka."""
+        self.logger.error(f"Delivery failed for record : {err}")
+
+    def publish(self, topic: str, data: dict) -> None:
+        """
+        Publishes a message to a specified Kafka topic.
+
+        Args:
+            topic (str): The name of the Kafka topic to which the message will be published.
+            data (dict): The message data to be sent to the Kafka topic.
+
+        Returns:
+            None
+
+        Raises:
+            Exception: If there is an error during message delivery, it will be handled by the `delivery_error` callback.
+        """
+
+        self.kafka_producer.send(
+            topic=topic, value=data).add_callback(
+                self.delivery_report).add_errback(
+                    self.delivery_error)
+
+        self.kafka_producer.flush()