gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-988

Browse Source 
Enhance ingestor and manager classes with notification handling

- Integrated NotificationHandler into Ingestor, DataManager, IngestorManager, and OpcManager for improved error reporting and monitoring.
- Updated methods to send notifications on critical events such as Kafka publishing errors, OPC connection issues, and cycle count warnings.
- Refactored related tests to ensure coverage of new notification functionalities and validate integration with existing components.
- Improved logging and error handling across the system to enhance traceability and operational insights.
This commit is contained in:
vitor-aignosi
2025-04-30 19:28:43 -03:00
parent 3fc3753636
commit d00078d323
8 changed files with 423 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
ingestor/managers/data_manager.py
View File

@@ -3,10 +3,14 @@ from logging import Logger
from time import sleep
from kafka import KafkaProducer
from kafka.errors import NoBrokersAvailable
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.notifications.models import NotificationLevel
import traceback
class DataManager():
def __init__(self, kafka_servers: str, logger: Logger) -> None:
def __init__(self, kafka_servers: str, logger: Logger,
notification_handler: NotificationHandler) -> None:
"""
Initializes the DataManager instance with a Kafka producer.
This constructor attempts to establish a connection to the specified Kafka servers
@@ -45,6 +49,7 @@ class DataManager():
logger.info(
f"DataManager initialized with Kafka servers: {kafka_servers}")
self.logger = logger
self.notification_handler = notification_handler
def __del__(self):
"""Destructor to close the producer connection."""
@@ -91,4 +96,12 @@ class DataManager():
self.kafka_producer.flush(timeout=10)
except Exception as e:
self.logger.error(f"Failed to publish message: {e}")
trace = traceback.format_exc()
self.notification_handler.build_and_send_notification(
notification_id=f"KAFKA_PRODUCER_ERROR_{topic}",
message=f"Error publishing message to topic {topic}: {e}",
block="kafka_producer",
level=NotificationLevel.ERROR,
attachment_content=trace
)
self.logger.error(trace)

59
ingestor/managers/ingestor_manager.py
View File

@@ -5,16 +5,19 @@ from copy import deepcopy
from ingestor.managers.data_manager import DataManager
from ingestor.managers.opc_manager import OpcManager
from ingestor.managers.resource_manager import ResourceManager
from sientia_do.notifications.models import Notification, NotificationLevel
from sientia_do.notifications.handlers import NotificationHandler
class IngestorManager():
def __init__(self,
kafka_servers: str, redis_host: str, redis_port: int,
lease_ttl: int, heartbeat_ttl: int, pod_id: str,
poll_interval: int, logger: Logger,
poll_interval: int, logger: Logger, notification_handler: NotificationHandler,
redis_username: str = None, redis_password: str = None):
self.data_manager = DataManager(kafka_servers, logger)
self.data_manager = DataManager(
kafka_servers, logger, notification_handler)
self.opc_managers = {}
self.resource_manager = ResourceManager(
redis_host, redis_port, lease_ttl, heartbeat_ttl, pod_id, redis_username, redis_password
@@ -25,6 +28,8 @@ class IngestorManager():
self.managed_tags = {}
self.opc_servers = {}
self.notification_handler = notification_handler
def initialize_opc_from_config(self, server_config: dict, data_manager: DataManager, logger: Logger) -> OpcManager | None:
"""
Initializes an OPC Manager instance using the provided server configuration.
@@ -56,8 +61,17 @@ class IngestorManager():
manager.config = server_config
manager.connect()
except Exception as e:
logger.error(f"Failed to initialize OpcManager: {e}")
traceback.print_exc()
trace = traceback.format_exc()
self.notification_handler.build_and_send_notification(
notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}',
message=f'Error initializing OPC manager: {e}',
block="opc_manager",
level=NotificationLevel.ERROR,
attachment_content=trace
)
self.logger.error(trace)
return None
return manager
@@ -100,6 +114,7 @@ class IngestorManager():
)
elif self.opc_managers[server].config != server_config:
self.opc_managers[server].disconnect()
del self.opc_managers[server]
server_instance = self.initialize_opc_from_config(
server_config, self.data_manager, self.logger
@@ -117,6 +132,30 @@ class IngestorManager():
self.opc_managers[server].disconnect()
self.opc_managers.pop(server, None)
def check_opc_servers_integrity(self):
"""
Checks the integrity of the OPC servers and updates the OPC servers if necessary.
"""
for server, opc_manager in self.opc_managers.items():
opc_manager.check_cycles()
is_lost = opc_manager.check_opc_listenning()
if is_lost:
self.logger.warning(
f"OPC server {server} is lost. "
f"Desconnecting from server."
)
opc_manager.disconnect()
self.opc_managers[server] = self.initialize_opc_from_config(
opc_manager.config, self.data_manager, self.logger
)
self.update_opc_servers()
for slot, slot_config in self.managed_tags.items():
if server in slot_config:
self.manage_server(
slot, server, slot_config[server], slot_config[server]['tags']
)
def declare_active(self):
"""
Declares the ingestor as active by sending a heartbeat signal to the resource manager.
@@ -342,10 +381,16 @@ class IngestorManager():
tags_to_sub
)
except Exception as e:
self.logger.error(
f"Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}"
trace = traceback.format_exc()
self.notification_handler.build_and_send_notification(
notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}',
message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}',
block="opc_manager",
level=NotificationLevel.ERROR,
attachment_content=trace
)
self.logger.error(traceback.format_exc())
self.logger.error(trace)
self.logger.warning(
"Removing subscription from server "
f"{server} for slot {slot}"

104
ingestor/managers/opc_manager.py
View File

@@ -5,14 +5,14 @@ from typing import Callable
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.sync import Client
from sientia_do.notifications.models import NotificationLevel
from sientia_do.notifications.handlers import NotificationHandler
from ingestor.managers.data_manager import DataManager
class OpcManager():
def __init__(self, name: str, url: str, data_manager: DataManager,
logger: Logger, server_uri: str, cert_path: str = None,
private_key_path: str = None, server_cert_path: str = None):
logger: Logger, server_uri: str, notification_handler: NotificationHandler,
cert_path: str = None, private_key_path: str = None, server_cert_path: str = None):
self.url = url
self.name = name
self.server_uri = server_uri
@@ -27,6 +27,8 @@ class OpcManager():
self.subscriptions = {}
self.data_manager = data_manager
self.notification_handler = notification_handler
def __str__(self):
return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \
f"nodes={self.nodes}, subscriptions={self.subscriptions}"
@@ -233,70 +235,48 @@ class OpcManager():
[self.data_manager.publish(e, data)
for e in self.nodes[tag]['topics']]
def check_cycles(self, removed_data: dict, node: str, config: dict,
handle_listen_events: Callable[[str, str, NotificationLevel], None]):
"""
Checks the cycle count for a specific node and triggers a notification if the cycle count exceeds a threshold.
Args:
removed_data (dict): A dictionary containing data that has been removed.
Used to check if the node is present.
node (str): The identifier of the node being checked.
config (dict): Configuration dictionary containing metadata such as the tag name.
handle_listen_events (Callable[[str, str, NotificationLevel], None]):
A callback function to handle notification events. It takes three arguments:
- A message string describing the event.
- A tag string identifying the event.
- A NotificationLevel enum indicating the severity of the event.
Behavior:
- If the node is not in `removed_data`, the cycle count for the node is incremented.
- If the cycle count reaches or exceeds 5, the `handle_listen_events` callback is invoked
with a warning message, a tag, and a notification level.
Notification Example:
If the cycle count exceeds the threshold, a warning message is generated in the format:
"{cycles} cycles without receive from {node}:{name}"
where `cycles` is the current cycle count, `node` is the node identifier, and `name` is the tag name
from the `config` dictionary.
def check_cycles(self):
"""
Checks the cycle counts for all monitored nodes and sends notifications if thresholds are exceeded.
if node not in removed_data.keys():
This method iterates through all monitored nodes and updates their cycle counts based on
configured increments. If a node's cycle count exceeds a threshold (5 cycles), it triggers
a warning notification.
"""
for node, config in self.nodes.items():
self.nodes[node]['cycle_rule']['cycle_count'] += self.nodes[node]['cycle_rule']['cycle_increment']
if self.nodes[node]['cycle_rule']['cycle_count'] >= 5 and handle_listen_events:
name = config['tag_name']
cycles = self.nodes[node]['cycle_rule']['cycle_count']
handle_listen_events(
f'{cycles} cycles without receive from {node}:{name}',
f'TAG_{node}:{name}_LISTENNING_STOPPED',
NotificationLevel.WARNING
)
if self.nodes[node]['cycle_rule']['cycle_count'] >= 5:
name = config['tag_name']
cycles = self.nodes[node]['cycle_rule']['cycle_count']
self.notification_handler.build_and_send_notification(
notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED',
message=f'{cycles} cycles without receive from {node}:{name}',
block="opc_manager",
level=NotificationLevel.WARNING
)
def check_opc_listenning(self, handle_listen_events: Callable[[str, str, NotificationLevel], None]) -> None:
def check_opc_listenning(self) -> bool:
"""
Monitors the OPC connection and triggers events based on the number of cycles
without receiving data from the OPC server.
Args:
handle_listen_events (Callable[[str, str, NotificationLevel], None]):
A callback function to handle notification events. It takes three arguments:
- A message string describing the event.
- An event code string.
- A NotificationLevel indicating the severity of the event.
Behavior:
- Increments the non-receive count each time the method is called.
- If the non-receive count reaches 5, triggers a notification event indicating
that the OPC server has stopped sending data.
- If the non-receive count reaches 15, triggers a notification event indicating
a retry to connect to the OPC server, disconnects the current session, and
reinitializes the collector with the existing configuration.
Checks the OPC connection and triggers notifications if the connection is lost.
Returns:
bool: True if the connection is lost, False otherwise.
"""
self.non_receive_count += 1
if self.non_receive_count >= 5 and handle_listen_events:
handle_listen_events(
f'{self.non_receive_count} cycles without receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}',
f'OPC_LISTENNING_STOPPED__{self.name}', NotificationLevel.ERROR)
if self.non_receive_count >= 15 and handle_listen_events:
handle_listen_events(
f'Retrying to connect to server {self.name}',
f'OPC_CONNECTION_RETRY__{self.name}', NotificationLevel.ERROR)
self.disconnect()
self.init_collector(
self.nodes, self.collect_period, self.period)
if self.non_receive_count >= 5:
self.notification_handler.build_and_send_notification(
notification_id=f'OPC_LISTENNING_STOPPED__{self.name}',
message=f'{self.non_receive_count} cycles without receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}',
block="opc_manager",
level=NotificationLevel.ERROR
)
if self.non_receive_count >= 15:
self.notification_handler.build_and_send_notification(
notification_id=f'OPC_CONNECTION_RETRY__{self.name}',
message=f'Retrying to connect to server {self.name}',
block="opc_manager",
level=NotificationLevel.ERROR
)
return True
return False