SIENTIAPDE-1325
SIENTIAPDE-1325: Refactor Ingestor and Manager Classes for Enhanced Asynchronous Operations - Introduced asynchronous methods across Ingestor, IngestorManager, DataManager, and OpcManager to improve performance and responsiveness. - Integrated MetricsController into various classes for better observability and monitoring. - Updated Redis and MongoDB interactions to support asynchronous operations, enhancing data handling efficiency. - Removed deprecated Redis metrics and streamlined resource management logic. - Adjusted unit tests to accommodate the new asynchronous behavior and ensure proper mocking of async methods.
This commit is contained in:
vitor-aignosi
@@ -5,17 +5,17 @@ from time import sleep
|
||||
|
||||
from kafka import KafkaProducer
|
||||
from kafka.errors import NoBrokersAvailable
|
||||
from pymongo import MongoClient
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from sientia_do.temporal.constants import now
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.repository.mongodb_repository import MongoDBRepository
|
||||
|
||||
import ingestor.metrics as metrics
|
||||
|
||||
|
||||
class DataManager(BaseActivity):
|
||||
class DataManager(SientiaMonitoring):
|
||||
"""
|
||||
Manages data persistence and export operations for the OPC Ingestor.
|
||||
|
||||
@@ -57,6 +57,7 @@ class DataManager(BaseActivity):
|
||||
metadata: dict,
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
) -> None:
|
||||
"""
|
||||
Initializes the DataManager instance with Kafka and MongoDB connections.
|
||||
@@ -91,6 +92,13 @@ class DataManager(BaseActivity):
|
||||
self.kafka_producer = None
|
||||
self.export_to_kafka = export_to_kafka
|
||||
|
||||
SientiaMonitoring.__init__(
|
||||
self,
|
||||
logger=logger,
|
||||
metrics_controller=metrics_controller,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
if self.export_to_kafka:
|
||||
for i in range(0, 3):
|
||||
logger.info(
|
||||
@@ -129,19 +137,18 @@ class DataManager(BaseActivity):
|
||||
self.connection_string = mongo_connection_string
|
||||
self.database = mongo_database
|
||||
|
||||
self.mongo_client: MongoClient = MongoClient(self.connection_string)
|
||||
self.mongo_client.server_info()
|
||||
self.mongo_repository = MongoDBRepository(
|
||||
connection_string=self.connection_string,
|
||||
database_name=self.database,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
self.metadata = metadata
|
||||
|
||||
self.mongo_db = self.mongo_client[self.database]
|
||||
|
||||
logger.info(f'DataManager initialized with MongoDB servers: {self.connection_string}')
|
||||
|
||||
BaseActivity.__init__(
|
||||
self, logger=logger, notification_handler=notification_handler, set_error_counter=True
|
||||
)
|
||||
|
||||
def shutdown(self):
|
||||
"""
|
||||
Gracefully shuts down the DataManager and closes all connections.
|
||||
@@ -165,13 +172,10 @@ class DataManager(BaseActivity):
|
||||
else:
|
||||
self.logger.warning('Kafka producer is already closed or not initialized.')
|
||||
|
||||
if self.mongo_client:
|
||||
try:
|
||||
self.mongo_client.close()
|
||||
except Exception as e:
|
||||
self.logger.error(f'Error closing MongoDB client: {e}')
|
||||
else:
|
||||
self.logger.warning('MongoDB client is already closed or not initialized.')
|
||||
try:
|
||||
self.mongo_repository.close()
|
||||
except Exception as e:
|
||||
self.logger.error(f'Error closing MongoDB client: {e}')
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
@@ -203,7 +207,7 @@ class DataManager(BaseActivity):
|
||||
"""
|
||||
self.logger.error(f'Delivery failed for record : {err}')
|
||||
|
||||
def publish(self, topic: str, data: dict) -> None:
|
||||
async def publish(self, topic: str, data: dict) -> None:
|
||||
"""
|
||||
Publishes a message to a specified Kafka topic.
|
||||
|
||||
@@ -226,12 +230,24 @@ class DataManager(BaseActivity):
|
||||
).add_errback(self.delivery_error)
|
||||
|
||||
self.kafka_producer.flush(timeout=10)
|
||||
metrics.KAFKA_MESSAGES_SENT.labels(pod_id=self.pod_id, topic=topic).inc()
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.KAFKA_MESSAGES_SENT,
|
||||
tags={
|
||||
'pod_id': self.pod_id,
|
||||
'topic': topic,
|
||||
},
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
metrics.KAFKA_MESSAGES_ERRORS.labels(pod_id=self.pod_id, topic=topic).inc()
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.KAFKA_MESSAGES_ERRORS,
|
||||
tags={
|
||||
'pod_id': self.pod_id,
|
||||
'topic': topic,
|
||||
},
|
||||
)
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
await self.send_notification_async(
|
||||
metadata=self.metadata,
|
||||
notification_id=f'KAFKA_PRODUCER_ERROR_{topic}',
|
||||
message=f'Error publishing message to topic {topic}: {e}',
|
||||
@@ -242,23 +258,25 @@ class DataManager(BaseActivity):
|
||||
self.logger.error(trace)
|
||||
|
||||
try:
|
||||
collection = self.mongo_db[topic]
|
||||
|
||||
collection.insert_one(
|
||||
{
|
||||
**data,
|
||||
'inserted_at': now(),
|
||||
}
|
||||
await self.mongo_repository.insert(
|
||||
collection_name=topic,
|
||||
document=data,
|
||||
metadata=self.metadata,
|
||||
)
|
||||
self.logger.debug(f'Message inserted into MongoDB collection {topic}: {data}')
|
||||
|
||||
metrics.TAG_WRITTEN_COUNT.labels(
|
||||
pod_id=self.pod_id, tag_name=data['name'], collection_name=topic
|
||||
).inc()
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.TAG_WRITTEN_COUNT,
|
||||
tags={
|
||||
'pod_id': self.pod_id,
|
||||
'tag_name': data['name'],
|
||||
'collection_name': topic,
|
||||
},
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
await self.send_notification_async(
|
||||
metadata=self.metadata,
|
||||
notification_id=f'MONGO_PRODUCER_ERROR_{topic}',
|
||||
message=f'Error inserting message to MongoDB: {e}',
|
||||
|
||||
Reference in New Issue
Block a user