gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SAM0123-231: Remove Kafka integration from Ingestor service.

Browse Source
This commit is contained in:
vitor-aignosi
2026-07-23 12:25:29 -03:00
parent c157d1649a
commit b10c226e4e
8 changed files with 18 additions and 414 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
ingestor/ingestor.py
View File

@@ -19,7 +19,7 @@ class Ingestor(SientiaMonitoring):
- Managing slot leases for load balancing across multiple instances
- Connecting to and monitoring OPC UA servers
- Subscribing to OPC tags and collecting real-time data
- Distributing data to Kafka and MongoDB
- Distributing data to MongoDB
- Providing health monitoring and metrics collection
The ingestor uses a slot-based architecture where each slot represents
@@ -27,8 +27,6 @@ class Ingestor(SientiaMonitoring):
This allows for horizontal scaling and load distribution.
Environment Variables:
KAFKA_SERVERS: Comma-separated list of Kafka server addresses (default: "localhost:9092")
EXPORT_TO_KAFKA: Enable/disable Kafka export (default: "false")
REDIS_HOST: Redis server hostname (default: "localhost")
REDIS_PORT: Redis server port (default: 6379)
REDIS_USERNAME: Redis username (optional)
@@ -43,7 +41,6 @@ class Ingestor(SientiaMonitoring):
MONGODB_DATABASE: MongoDB database name (default: "sientia")
Attributes:
export_to_kafka (bool): Whether to export data to Kafka
redis_host (str): Redis server hostname
redis_port (int): Redis server port
redis_username (str): Redis username (optional)
@@ -54,7 +51,6 @@ class Ingestor(SientiaMonitoring):
poll_interval (int): Interval in seconds for polling operations
mongo_database (str): MongoDB database name
mongo_connection_string (str): Complete MongoDB connection string
kafka_servers (list): List of Kafka server addresses
logger: Logger instance for application logging
notification_handler: Handler for sending notifications
metadata (dict): Application metadata for notifications and tracking
@@ -66,17 +62,12 @@ class Ingestor(SientiaMonitoring):
Initializes the ingestor with configuration values retrieved from environment variables.
Sets up all necessary connections and configurations for:
- Kafka connectivity (if enabled)
- Redis for slot management and coordination
- MongoDB for data persistence and notifications
- OPC UA server management
- Metrics collection and monitoring
"""
kafka_servers = getenv('KAFKA_SERVERS', 'localhost:9092')
export_to_kafka: bool = getenv('EXPORT_TO_KAFKA', 'false') == 'true'
self.export_to_kafka = export_to_kafka
self.redis_host = getenv('REDIS_HOST', 'localhost')
self.redis_port = int(getenv('REDIS_PORT', '6379'))
self.redis_username = getenv('REDIS_USERNAME', None)
@@ -91,7 +82,6 @@ class Ingestor(SientiaMonitoring):
self.mongo_database = getenv('MONGODB_DATABASE', 'sientia')
self.mongo_connection_string = f'mongodb://{mongo_username}:{mongo_password}@{mongo_url}'
self.kafka_servers = kafka_servers.split(',')
self.logger = get_logger(__name__)
self.notification_handler = NotificationHandler(
connection_string=self.mongo_connection_string,
@@ -179,7 +169,6 @@ class Ingestor(SientiaMonitoring):
"""
self.ingestor_manager = IngestorManager(
kafka_servers=','.join(self.kafka_servers),
redis_data={
'host': self.redis_host,
'port': self.redis_port,
@@ -195,7 +184,6 @@ class Ingestor(SientiaMonitoring):
logger=self.logger,
notification_handler=self.notification_handler,
metrics_controller=self.metrics_controller,
export_to_kafka=self.export_to_kafka,
)
assert self.ingestor_manager is not None

160
ingestor/managers/data_manager.py
View File

@@ -1,10 +1,6 @@
import json
import os
import traceback
from time import sleep
from kafka import KafkaProducer
from kafka.errors import NoBrokersAvailable
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
@@ -18,31 +14,22 @@ import ingestor.metrics as metrics
class DataManager(SientiaMonitoring):
"""
Manages data persistence and export operations for the OPC Ingestor.
Manages data persistence operations for the OPC Ingestor.
The DataManager is responsible for:
- Storing OPC data in MongoDB for historical analysis and persistence
- Exporting data to Kafka for real-time streaming and downstream processing
- Managing database connections and ensuring data integrity
- Providing data access interfaces for other components
The manager supports both MongoDB and Kafka operations, with Kafka export
being optional and configurable. It implements retry logic for connection
failures and provides comprehensive error handling and notification.
Args:
kafka_servers (str): Comma-separated string of Kafka server addresses
mongo_connection_string (str): MongoDB connection string
mongo_database (str): MongoDB database name
export_to_kafka (bool): Whether to enable Kafka export functionality
metadata (dict): Application metadata for notifications and tracking
logger (Logger): Logger instance for application logging
notification_handler (NotificationHandler): Handler for sending notifications
Attributes:
pod_id (str): Pod identifier for metrics labeling
kafka_producer (KafkaProducer): Kafka producer instance for data export
export_to_kafka (bool): Whether Kafka export is enabled
connection_string (str): MongoDB connection string
database (str): MongoDB database name
mongo_client (MongoClient): MongoDB client instance
@@ -51,47 +38,25 @@ class DataManager(SientiaMonitoring):
def __init__(
self,
kafka_servers: str,
mongo_connection_string: str,
mongo_database: str,
export_to_kafka: bool,
metadata: dict,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
) -> None:
"""
Initializes the DataManager instance with Kafka and MongoDB connections.
This constructor attempts to establish connections to the specified services:
1. Kafka: Initializes producer with retry logic (up to 3 attempts)
2. MongoDB: Establishes connection and verifies server availability
The initialization process includes:
- Kafka producer setup with JSON serialization
- MongoDB client initialization and connection testing
- Metrics recording for connection status
- Error handling with notifications
Initializes the DataManager instance with a MongoDB connection.
Args:
kafka_servers (str): Comma-separated string of Kafka server addresses
mongo_connection_string (str): MongoDB connection string
mongo_database (str): MongoDB database name
export_to_kafka (bool): Whether to enable Kafka export
metadata (dict): Application metadata
logger (Logger): Logger instance
notification_handler (NotificationHandler): Notification handler
Raises:
NoBrokersAvailable: If the connection to Kafka servers fails after 3 attempts.
Metrics:
- KAFKA_CONNECTION_STATUS: Set to 1 on successful connection, 0 on failure
"""
self.pod_id = os.getenv('HOSTNAME', 'localhost')
self.kafka_producer = None
self.export_to_kafka = export_to_kafka
SientiaMonitoring.__init__(
self,
@@ -100,37 +65,6 @@ class DataManager(SientiaMonitoring):
notification_handler=notification_handler,
)
if self.export_to_kafka:
for i in range(0, 3):
logger.info(
f'Trying ({i}) to initializing DataManager with Kafka servers: {kafka_servers}'
)
try:
self.kafka_producer = KafkaProducer(
bootstrap_servers=kafka_servers,
value_serializer=lambda v: json.dumps(v).encode(
'utf-8'
), # Serialize JSON messages
key_serializer=lambda k: str(k).encode('utf-8') if k else None,
)
# Kafka connected
metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1)
break
except NoBrokersAvailable:
logger.error(f'Kafka servers {kafka_servers} are not available. Retrying...')
sleep(5)
else:
# Kafka not connected
metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
logger.error(
f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.'
)
raise NoBrokersAvailable(
f'Failed to connect to Kafka servers {kafka_servers} after 3 attempts.'
)
logger.info(f'DataManager initialized with Kafka servers: {kafka_servers}')
logger.info(
f'Trying to initializing DataManager with MongoDB servers: {mongo_connection_string}'
)
@@ -153,26 +87,7 @@ class DataManager(SientiaMonitoring):
def shutdown(self):
"""
Gracefully shuts down the DataManager and closes all connections.
This method ensures proper cleanup of:
- Kafka producer connection with message flushing
- MongoDB client connection
- Metrics recording for connection status
The method handles connection closure gracefully, logging any errors
that occur during shutdown while ensuring all resources are properly released.
"""
if self.kafka_producer:
try:
self.kafka_producer.flush(timeout=10)
self.kafka_producer.close()
# Mark as disconnected
metrics.KAFKA_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
except Exception as e:
self.logger.error(f'Error closing Kafka producer: {e}')
else:
self.logger.warning('Kafka producer is already closed or not initialized.')
try:
self.mongo_repository.close()
except Exception as e:
@@ -181,83 +96,18 @@ class DataManager(SientiaMonitoring):
def __del__(self):
self.shutdown()
def delivery_report(self, msg):
"""
Callback for successful Kafka message delivery reports.
This method is called by the Kafka producer when a message is successfully
delivered to a topic. It logs the delivery details including topic, partition,
and offset information for debugging and monitoring purposes.
Args:
msg: Kafka message object containing delivery details
"""
self.logger.debug(
f'Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}'
)
def delivery_error(self, err):
"""
Callback for Kafka message delivery error reports.
This method is called by the Kafka producer when a message delivery fails.
It logs the error details for debugging and monitoring purposes.
Args:
err: Error information from the failed delivery attempt
"""
self.logger.error(f'Delivery failed for record : {err}')
async def publish(self, topic: str, data: dict) -> None:
"""
Publishes a message to a specified Kafka topic.
Persists a message to MongoDB.
Args:
topic (str): The name of the Kafka topic to which the message will be published.
data (dict): The message data to be sent to the Kafka topic.
topic (str): The name of the MongoDB collection to which the message will be written.
data (dict): The message data to be stored.
Returns:
None
Raises:
Exception: If there is an error during message delivery, it will be handled by the `delivery_error` callback.
"""
if self.export_to_kafka and self.kafka_producer:
try:
self.logger.debug(f'Publishing message to topic {topic}: {data}')
self.kafka_producer.send(topic=topic, value=data).add_callback(
self.delivery_report
).add_errback(self.delivery_error)
self.kafka_producer.flush(timeout=10)
await self.emit_metric(
metric_object=metrics.KAFKA_MESSAGES_SENT,
tags={
'pod_id': self.pod_id,
'topic': topic,
},
)
except Exception as e:
await self.emit_metric(
metric_object=metrics.KAFKA_MESSAGES_ERRORS,
tags={
'pod_id': self.pod_id,
'topic': topic,
},
)
trace = traceback.format_exc()
await self.send_notification_async(
metadata=self.metadata,
notification_id=f'KAFKA_PRODUCER_ERROR_{topic}',
message=f'Error publishing message to topic {topic}: {e}',
block='kafka_producer',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
self.logger.error(trace)
try:
await self.mongo_repository.insert(
collection_name=topic,

10
ingestor/managers/ingestor_manager.py
View File

@@ -19,7 +19,7 @@ class IngestorManager(SientiaMonitoring):
Central coordinator for managing OPC data ingestion operations.
The IngestorManager orchestrates the interaction between different components:
- DataManager: Handles data persistence and Kafka export
- DataManager: Handles data persistence
- OPC Managers: Manage individual OPC UA server connections
- ResourceManager: Coordinates slot leasing and load balancing
@@ -36,7 +36,6 @@ class IngestorManager(SientiaMonitoring):
- Load balancing across multiple ingestor instances
Args:
kafka_servers (str): Comma-separated list of Kafka server addresses
redis_data (dict): Redis connection parameters (host, port, username, password)
lease_ttl (int): Time-to-live for slot leases in seconds
heartbeat_ttl (int): Time-to-live for heartbeat signals in seconds
@@ -46,10 +45,9 @@ class IngestorManager(SientiaMonitoring):
metadata (dict): Application metadata for notifications and tracking
logger (Logger): Logger instance for application logging
notification_handler (NotificationHandler): Handler for sending notifications
export_to_kafka (bool): Whether to export data to Kafka
Attributes:
data_manager (DataManager): Manages data persistence and Kafka export
data_manager (DataManager): Manages data persistence
opc_managers (dict): Dictionary of OPC managers keyed by server name
resource_manager (ResourceManager): Manages Redis-based resource coordination
number_of_slots (int): Total number of slots configured in the system
@@ -61,7 +59,6 @@ class IngestorManager(SientiaMonitoring):
def __init__(
self,
kafka_servers: str,
redis_data: dict,
lease_ttl: int,
heartbeat_ttl: int,
@@ -72,7 +69,6 @@ class IngestorManager(SientiaMonitoring):
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
export_to_kafka: bool = False,
):
redis_host: str = redis_data['host']
redis_port: int = int(redis_data['port'])
@@ -87,10 +83,8 @@ class IngestorManager(SientiaMonitoring):
)
self.data_manager = DataManager(
kafka_servers=kafka_servers,
mongo_connection_string=mongo_connection_string,
mongo_database=mongo_database,
export_to_kafka=export_to_kafka,
metadata=metadata,
logger=logger,
notification_handler=notification_handler,

17
ingestor/metrics.py
View File

@@ -19,7 +19,6 @@ from prometheus_client import Counter, Gauge, Histogram
# Metric label definitions for consistent labeling across all metrics
POD_ID_LABEL = ['pod_id']
SERVER_LABELS = ['pod_id', 'server_name', 'server_url']
KAFKA_LABELS = ['pod_id', 'topic']
REDIS_LABELS = ['pod_id', 'operation']
NOTIFICATION_LABELS = ['pod_id', 'level', 'block']
@@ -134,22 +133,6 @@ OPC_RECONNECTIONS_TOTAL = Counter(
['pod_id', 'server_name'],
)
# --- Data Manager (Kafka) Metrics ---
KAFKA_MESSAGES_SENT = Counter(
'kafka_messages_sent_total', 'Total messages sent to Kafka', KAFKA_LABELS
)
KAFKA_MESSAGES_ERRORS = Counter(
'kafka_messages_errors_total',
'Total errors sending messages to Kafka',
KAFKA_LABELS,
)
KAFKA_CONNECTION_STATUS = Gauge(
'kafka_connection_status',
'Connection status with Kafka (1=connected, 0=disconnected)',
POD_ID_LABEL,
)
# --- Notification Metrics ---
NOTIFICATIONS_SENT = Counter(
'notifications_sent_total',