Add support for Kafka export in Ingestor and DataManager classes
- Introduced EXPORT_TO_KAFKA environment variable in Ingestor class. - Refactored DataManager to conditionally initialize Kafka producer based on export setting. - Updated IngestorManager to pass Redis configuration as a dictionary. - Enhanced error handling and logging for Kafka message publishing.
This commit is contained in:
vitor-aignosi
@@ -18,6 +18,7 @@ class DataManager:
|
||||
kafka_servers: str,
|
||||
mongo_connection_string: str,
|
||||
mongo_database: str,
|
||||
export_to_kafka: bool,
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
) -> None:
|
||||
@@ -35,40 +36,44 @@ class DataManager:
|
||||
|
||||
self.pod_id = os.getenv("HOSTNAME", "localhost")
|
||||
self.kafka_producer = None
|
||||
for i in range(0, 3):
|
||||
logger.info(
|
||||
f"Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}"
|
||||
)
|
||||
try:
|
||||
self.kafka_producer = KafkaProducer(
|
||||
bootstrap_servers=kafka_servers,
|
||||
value_serializer=lambda v: json.dumps(v).encode(
|
||||
"utf-8"
|
||||
), # Serialize JSON messages
|
||||
key_serializer=lambda k: str(
|
||||
k).encode("utf-8") if k else None,
|
||||
)
|
||||
# Kafka connected
|
||||
metrics.KAFKA_CONNECTION_STATUS.labels(
|
||||
pod_id=self.pod_id).set(1)
|
||||
break
|
||||
except NoBrokersAvailable:
|
||||
logger.error(
|
||||
f"Kafka servers {kafka_servers} are not available. Retrying..."
|
||||
)
|
||||
sleep(5)
|
||||
else:
|
||||
# Kafka not connected
|
||||
metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
|
||||
logger.error(
|
||||
f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts."
|
||||
)
|
||||
raise NoBrokersAvailable(
|
||||
f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts."
|
||||
)
|
||||
self.export_to_kafka = export_to_kafka
|
||||
|
||||
logger.info(
|
||||
f"DataManager initialized with Kafka servers: {kafka_servers}")
|
||||
if self.export_to_kafka:
|
||||
for i in range(0, 3):
|
||||
logger.info(
|
||||
f"Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}"
|
||||
)
|
||||
try:
|
||||
self.kafka_producer = KafkaProducer(
|
||||
bootstrap_servers=kafka_servers,
|
||||
value_serializer=lambda v: json.dumps(v).encode(
|
||||
"utf-8"
|
||||
), # Serialize JSON messages
|
||||
key_serializer=lambda k: str(
|
||||
k).encode("utf-8") if k else None,
|
||||
)
|
||||
# Kafka connected
|
||||
metrics.KAFKA_CONNECTION_STATUS.labels(
|
||||
pod_id=self.pod_id).set(1)
|
||||
break
|
||||
except NoBrokersAvailable:
|
||||
logger.error(
|
||||
f"Kafka servers {kafka_servers} are not available. Retrying..."
|
||||
)
|
||||
sleep(5)
|
||||
else:
|
||||
# Kafka not connected
|
||||
metrics.KAFKA_CONNECTION_STATUS.labels(
|
||||
pod_id=self.pod_id).set(0)
|
||||
logger.error(
|
||||
f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts."
|
||||
)
|
||||
raise NoBrokersAvailable(
|
||||
f"Failed to connect to Kafka servers {kafka_servers} after 3 attempts."
|
||||
)
|
||||
|
||||
logger.info(
|
||||
f"DataManager initialized with Kafka servers: {kafka_servers}")
|
||||
|
||||
self.connection_string = mongo_connection_string
|
||||
self.database = mongo_database
|
||||
@@ -147,29 +152,33 @@ class DataManager:
|
||||
Exception: If there is an error during message delivery, it will be handled by the `delivery_error` callback.
|
||||
"""
|
||||
|
||||
try:
|
||||
if self.export_to_kafka:
|
||||
try:
|
||||
|
||||
self.logger.debug(f"Publishing message to topic {topic}: {data}")
|
||||
self.kafka_producer.send(topic=topic, value=data).add_callback(
|
||||
self.delivery_report
|
||||
).add_errback(self.delivery_error)
|
||||
self.logger.debug(
|
||||
f"Publishing message to topic {topic}: {data}")
|
||||
self.kafka_producer.send(topic=topic, value=data).add_callback(
|
||||
self.delivery_report
|
||||
).add_errback(self.delivery_error)
|
||||
|
||||
self.kafka_producer.flush(timeout=10)
|
||||
metrics.KAFKA_MESSAGES_SENT.labels(
|
||||
pod_id=self.pod_id, topic=topic).inc()
|
||||
self.kafka_producer.flush(timeout=10)
|
||||
metrics.KAFKA_MESSAGES_SENT.labels(
|
||||
pod_id=self.pod_id, topic=topic).inc()
|
||||
|
||||
except Exception as e:
|
||||
metrics.KAFKA_MESSAGES_ERRORS.labels(
|
||||
pod_id=self.pod_id, topic=topic).inc()
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id=f"KAFKA_PRODUCER_ERROR_{topic}",
|
||||
message=f"Error publishing message to topic {topic}: {e}",
|
||||
block="kafka_producer",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace,
|
||||
)
|
||||
self.logger.error(trace)
|
||||
except Exception as e:
|
||||
metrics.KAFKA_MESSAGES_ERRORS.labels(
|
||||
pod_id=self.pod_id, topic=topic).inc()
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id=f"KAFKA_PRODUCER_ERROR_{topic}",
|
||||
message=f"Error publishing message to topic {topic}: {e}",
|
||||
block="kafka_producer",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace,
|
||||
)
|
||||
self.logger.error(trace)
|
||||
else:
|
||||
self.logger.info(f"Skipping message to topic {topic}: {data}")
|
||||
|
||||
try:
|
||||
collection = self.mongo_db[topic]
|
||||
|
||||
Reference in New Issue
Block a user