988ee65fbf
Implement initial structure for data ingestion with Kafka and OPC UA integration - Added DataManager for Kafka message handling - Introduced IngestorManager to manage data ingestion processes - Created OpcManager for OPC UA client interactions - Developed ResourceManager for Redis-based resource management - Established testing framework with unit tests for DataManager and OpcManager - Configured project settings for Python testing with pytest
52 lines
1.7 KiB
Python
52 lines
1.7 KiB
Python
import json
|
|
from logging import Logger
|
|
from kafka import KafkaProducer
|
|
|
|
|
|
class DataManager():
|
|
def __init__(self, kafka_servers: str, logger: Logger) -> None:
|
|
self.kafka_producer = KafkaProducer(
|
|
bootstrap_servers=kafka_servers,
|
|
value_serializer=lambda v: json.dumps(v).encode(
|
|
'utf-8'), # Serialize JSON messages
|
|
key_serializer=lambda k: str(k).encode('utf-8') if k else None,
|
|
)
|
|
self.logger = logger
|
|
|
|
def __del__(self):
|
|
"""Destructor to close the producer connection."""
|
|
self.logger.info("Closing Kafka producer...")
|
|
self.kafka_producer.flush()
|
|
self.kafka_producer.close()
|
|
|
|
def delivery_report(self, msg: str):
|
|
"""Callback for delivery reports from Kafka."""
|
|
self.logger.info(
|
|
f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}")
|
|
|
|
def delivery_error(self, err: str):
|
|
"""Callback for delivery reports from Kafka."""
|
|
self.logger.error(f"Delivery failed for record : {err}")
|
|
|
|
def publish(self, topic: str, data: dict) -> None:
|
|
"""
|
|
Publishes a message to a specified Kafka topic.
|
|
|
|
Args:
|
|
topic (str): The name of the Kafka topic to which the message will be published.
|
|
data (dict): The message data to be sent to the Kafka topic.
|
|
|
|
Returns:
|
|
None
|
|
|
|
Raises:
|
|
Exception: If there is an error during message delivery, it will be handled by the `delivery_error` callback.
|
|
"""
|
|
|
|
self.kafka_producer.send(
|
|
topic=topic, value=data).add_callback(
|
|
self.delivery_report).add_errback(
|
|
self.delivery_error)
|
|
|
|
self.kafka_producer.flush()
|