gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-988

Browse Source 
Enhance ingestor and manager classes with notification handling

- Integrated NotificationHandler into Ingestor, DataManager, IngestorManager, and OpcManager for improved error reporting and monitoring.
- Updated methods to send notifications on critical events such as Kafka publishing errors, OPC connection issues, and cycle count warnings.
- Refactored related tests to ensure coverage of new notification functionalities and validate integration with existing components.
- Improved logging and error handling across the system to enhance traceability and operational insights.
This commit is contained in:
Vitor Pimentel dos Santos
2025-04-30 19:28:43 -03:00
parent 3fc3753636
commit d00078d323
8 changed files with 423 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
ingestor/ingestor.py
View File

@@ -2,6 +2,7 @@ from logging import Formatter, StreamHandler, getLogger
from os import getenv
from ingestor.managers.ingestor_manager import IngestorManager
from sientia_do.notifications.handlers import NotificationHandler
class Ingestor:
@@ -37,14 +38,27 @@ class Ingestor:
self.poll_interval = int(getenv("POLL_INTERVAL", 5))
self.kafka_servers = kafka_servers.split(",")
self.logger = None
self.init_logger()
self.notification_handler = NotificationHandler(
servers=self.kafka_servers,
logger=self.logger,
project_name="OPC_INGESTOR",
pipeline_name="-",
trigger_name="-",
model_name="-",
model="-"
)
# build args for build notificarions components
# call build notifications components
def init_logger(self):
"""
Initializes a logger instance for the class.
This method sets up a logger with a specified log level, a stream handler,
and a formatter. The log level is determined by the environment variable
"LOG_LEVEL", defaulting to "INFO" if not set. The logger is then attached
This method sets up a logger with a specified log level, a stream handler,
and a formatter. The log level is determined by the environment variable
"LOG_LEVEL", defaulting to "INFO" if not set. The logger is then attached
to the instance for use throughout the class.
Attributes:
self.logger (logging.Logger): The configured logger instance.
@@ -105,7 +119,8 @@ class Ingestor:
self.ingestor_manager = IngestorManager(
self.kafka_servers, self.redis_host, self.redis_port, self.lease_ttl,
self.heartbeat_ttl, self.pod_id, self.poll_interval, self.logger, self.redis_username, self.redis_password
self.heartbeat_ttl, self.pod_id, self.poll_interval, self.logger,
self.notification_handler, self.redis_username, self.redis_password
)
# Declare ingestor ative
@@ -120,9 +135,9 @@ class Ingestor:
def manage_no_slots(self, number_of_slots: int):
"""
Manages the scenario where there are no slots assigned to the ingestor.
This method checks if the ingestor is active (i.e., has no managed tags)
and if the number of available slots is greater than zero. If both
conditions are met, it attempts to acquire a slot lease and handles
This method checks if the ingestor is active (i.e., has no managed tags)
and if the number of available slots is greater than zero. If both
conditions are met, it attempts to acquire a slot lease and handles
the acquired tags accordingly.
Args:
number_of_slots (int): The number of available slots.
@@ -138,16 +153,16 @@ class Ingestor:
def manage_leases(self, available_slots: int, lacking_ingestors: int, slot_diff: int):
"""
Manages the allocation and deallocation of slot leases for ingestors based on
Manages the allocation and deallocation of slot leases for ingestors based on
the number of available slots, lacking ingestors, and slot differences.
Args:
available_slots (int): The number of slots currently available for allocation.
lacking_ingestors (int): The number of ingestors that are active and without slots.
slot_diff (int): The difference between the total slots and the required slots.
Behavior:
- If there are available slots and lacking ingestors, attempts to acquire slot leases
- If there are available slots and lacking ingestors, attempts to acquire slot leases
for the available slots and processes the acquired tags.
- If there are no lacking ingestors but there are extra slots (slot_diff > 0),
- If there are no lacking ingestors but there are extra slots (slot_diff > 0),
releases the extra slot leases to ensure proper allocation.
Logs:
- Logs the number of available slots when attempting to acquire leases.
@@ -223,3 +238,6 @@ class Ingestor:
# Update opc servers
self.ingestor_manager.update_slot_config()
# Check OPC cycles
self.ingestor_manager.check_opc_servers_integrity()

17
ingestor/managers/data_manager.py
View File

@@ -3,10 +3,14 @@ from logging import Logger
from time import sleep
from kafka import KafkaProducer
from kafka.errors import NoBrokersAvailable
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.notifications.models import NotificationLevel
import traceback
class DataManager():
def __init__(self, kafka_servers: str, logger: Logger) -> None:
def __init__(self, kafka_servers: str, logger: Logger,
notification_handler: NotificationHandler) -> None:
"""
Initializes the DataManager instance with a Kafka producer.
This constructor attempts to establish a connection to the specified Kafka servers
@@ -45,6 +49,7 @@ class DataManager():
logger.info(
f"DataManager initialized with Kafka servers: {kafka_servers}")
self.logger = logger
self.notification_handler = notification_handler
def __del__(self):
"""Destructor to close the producer connection."""
@@ -91,4 +96,12 @@ class DataManager():
self.kafka_producer.flush(timeout=10)
except Exception as e:
self.logger.error(f"Failed to publish message: {e}")
trace = traceback.format_exc()
self.notification_handler.build_and_send_notification(
notification_id=f"KAFKA_PRODUCER_ERROR_{topic}",
message=f"Error publishing message to topic {topic}: {e}",
block="kafka_producer",
level=NotificationLevel.ERROR,
attachment_content=trace
)
self.logger.error(trace)

59
ingestor/managers/ingestor_manager.py
View File

@@ -5,16 +5,19 @@ from copy import deepcopy
from ingestor.managers.data_manager import DataManager
from ingestor.managers.opc_manager import OpcManager
from ingestor.managers.resource_manager import ResourceManager
from sientia_do.notifications.models import Notification, NotificationLevel
from sientia_do.notifications.handlers import NotificationHandler
class IngestorManager():
def __init__(self,
kafka_servers: str, redis_host: str, redis_port: int,
lease_ttl: int, heartbeat_ttl: int, pod_id: str,
poll_interval: int, logger: Logger,
poll_interval: int, logger: Logger, notification_handler: NotificationHandler,
redis_username: str = None, redis_password: str = None):
self.data_manager = DataManager(kafka_servers, logger)
self.data_manager = DataManager(
kafka_servers, logger, notification_handler)
self.opc_managers = {}
self.resource_manager = ResourceManager(
redis_host, redis_port, lease_ttl, heartbeat_ttl, pod_id, redis_username, redis_password
@@ -25,6 +28,8 @@ class IngestorManager():
self.managed_tags = {}
self.opc_servers = {}
self.notification_handler = notification_handler
def initialize_opc_from_config(self, server_config: dict, data_manager: DataManager, logger: Logger) -> OpcManager | None:
"""
Initializes an OPC Manager instance using the provided server configuration.
@@ -56,8 +61,17 @@ class IngestorManager():
manager.config = server_config
manager.connect()
except Exception as e:
logger.error(f"Failed to initialize OpcManager: {e}")
traceback.print_exc()
trace = traceback.format_exc()
self.notification_handler.build_and_send_notification(
notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}',
message=f'Error initializing OPC manager: {e}',
block="opc_manager",
level=NotificationLevel.ERROR,
attachment_content=trace
)
self.logger.error(trace)
return None
return manager
@@ -100,6 +114,7 @@ class IngestorManager():
)
elif self.opc_managers[server].config != server_config:
self.opc_managers[server].disconnect()
del self.opc_managers[server]
server_instance = self.initialize_opc_from_config(
server_config, self.data_manager, self.logger
@@ -117,6 +132,30 @@ class IngestorManager():
self.opc_managers[server].disconnect()
self.opc_managers.pop(server, None)
def check_opc_servers_integrity(self):
"""
Checks the integrity of the OPC servers and updates the OPC servers if necessary.
"""
for server, opc_manager in self.opc_managers.items():
opc_manager.check_cycles()
is_lost = opc_manager.check_opc_listenning()
if is_lost:
self.logger.warning(
f"OPC server {server} is lost. "
f"Desconnecting from server."
)
opc_manager.disconnect()
self.opc_managers[server] = self.initialize_opc_from_config(
opc_manager.config, self.data_manager, self.logger
)
self.update_opc_servers()
for slot, slot_config in self.managed_tags.items():
if server in slot_config:
self.manage_server(
slot, server, slot_config[server], slot_config[server]['tags']
)
def declare_active(self):
"""
Declares the ingestor as active by sending a heartbeat signal to the resource manager.
@@ -342,10 +381,16 @@ class IngestorManager():
tags_to_sub
)
except Exception as e:
self.logger.error(
f"Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}"
trace = traceback.format_exc()
self.notification_handler.build_and_send_notification(
notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}',
message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}',
block="opc_manager",
level=NotificationLevel.ERROR,
attachment_content=trace
)
self.logger.error(traceback.format_exc())
self.logger.error(trace)
self.logger.warning(
"Removing subscription from server "
f"{server} for slot {slot}"

104
ingestor/managers/opc_manager.py
View File

@@ -5,14 +5,14 @@ from typing import Callable
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.sync import Client
from sientia_do.notifications.models import NotificationLevel
from sientia_do.notifications.handlers import NotificationHandler
from ingestor.managers.data_manager import DataManager
class OpcManager():
def __init__(self, name: str, url: str, data_manager: DataManager,
logger: Logger, server_uri: str, cert_path: str = None,
private_key_path: str = None, server_cert_path: str = None):
logger: Logger, server_uri: str, notification_handler: NotificationHandler,
cert_path: str = None, private_key_path: str = None, server_cert_path: str = None):
self.url = url
self.name = name
self.server_uri = server_uri
@@ -27,6 +27,8 @@ class OpcManager():
self.subscriptions = {}
self.data_manager = data_manager
self.notification_handler = notification_handler
def __str__(self):
return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \
f"nodes={self.nodes}, subscriptions={self.subscriptions}"
@@ -233,70 +235,48 @@ class OpcManager():
[self.data_manager.publish(e, data)
for e in self.nodes[tag]['topics']]
def check_cycles(self, removed_data: dict, node: str, config: dict,
handle_listen_events: Callable[[str, str, NotificationLevel], None]):
"""
Checks the cycle count for a specific node and triggers a notification if the cycle count exceeds a threshold.
Args:
removed_data (dict): A dictionary containing data that has been removed.
Used to check if the node is present.
node (str): The identifier of the node being checked.
config (dict): Configuration dictionary containing metadata such as the tag name.
handle_listen_events (Callable[[str, str, NotificationLevel], None]):
A callback function to handle notification events. It takes three arguments:
- A message string describing the event.
- A tag string identifying the event.
- A NotificationLevel enum indicating the severity of the event.
Behavior:
- If the node is not in `removed_data`, the cycle count for the node is incremented.
- If the cycle count reaches or exceeds 5, the `handle_listen_events` callback is invoked
with a warning message, a tag, and a notification level.
Notification Example:
If the cycle count exceeds the threshold, a warning message is generated in the format:
"{cycles} cycles without receive from {node}:{name}"
where `cycles` is the current cycle count, `node` is the node identifier, and `name` is the tag name
from the `config` dictionary.
def check_cycles(self):
"""
Checks the cycle counts for all monitored nodes and sends notifications if thresholds are exceeded.
if node not in removed_data.keys():
This method iterates through all monitored nodes and updates their cycle counts based on
configured increments. If a node's cycle count exceeds a threshold (5 cycles), it triggers
a warning notification.
"""
for node, config in self.nodes.items():
self.nodes[node]['cycle_rule']['cycle_count'] += self.nodes[node]['cycle_rule']['cycle_increment']
if self.nodes[node]['cycle_rule']['cycle_count'] >= 5 and handle_listen_events:
name = config['tag_name']
cycles = self.nodes[node]['cycle_rule']['cycle_count']
handle_listen_events(
f'{cycles} cycles without receive from {node}:{name}',
f'TAG_{node}:{name}_LISTENNING_STOPPED',
NotificationLevel.WARNING
)
if self.nodes[node]['cycle_rule']['cycle_count'] >= 5:
name = config['tag_name']
cycles = self.nodes[node]['cycle_rule']['cycle_count']
self.notification_handler.build_and_send_notification(
notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED',
message=f'{cycles} cycles without receive from {node}:{name}',
block="opc_manager",
level=NotificationLevel.WARNING
)
def check_opc_listenning(self, handle_listen_events: Callable[[str, str, NotificationLevel], None]) -> None:
def check_opc_listenning(self) -> bool:
"""
Monitors the OPC connection and triggers events based on the number of cycles
without receiving data from the OPC server.
Args:
handle_listen_events (Callable[[str, str, NotificationLevel], None]):
A callback function to handle notification events. It takes three arguments:
- A message string describing the event.
- An event code string.
- A NotificationLevel indicating the severity of the event.
Behavior:
- Increments the non-receive count each time the method is called.
- If the non-receive count reaches 5, triggers a notification event indicating
that the OPC server has stopped sending data.
- If the non-receive count reaches 15, triggers a notification event indicating
a retry to connect to the OPC server, disconnects the current session, and
reinitializes the collector with the existing configuration.
Checks the OPC connection and triggers notifications if the connection is lost.
Returns:
bool: True if the connection is lost, False otherwise.
"""
self.non_receive_count += 1
if self.non_receive_count >= 5 and handle_listen_events:
handle_listen_events(
f'{self.non_receive_count} cycles without receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}',
f'OPC_LISTENNING_STOPPED__{self.name}', NotificationLevel.ERROR)
if self.non_receive_count >= 15 and handle_listen_events:
handle_listen_events(
f'Retrying to connect to server {self.name}',
f'OPC_CONNECTION_RETRY__{self.name}', NotificationLevel.ERROR)
self.disconnect()
self.init_collector(
self.nodes, self.collect_period, self.period)
if self.non_receive_count >= 5:
self.notification_handler.build_and_send_notification(
notification_id=f'OPC_LISTENNING_STOPPED__{self.name}',
message=f'{self.non_receive_count} cycles without receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}',
block="opc_manager",
level=NotificationLevel.ERROR
)
if self.non_receive_count >= 15:
self.notification_handler.build_and_send_notification(
notification_id=f'OPC_CONNECTION_RETRY__{self.name}',
message=f'Retrying to connect to server {self.name}',
block="opc_manager",
level=NotificationLevel.ERROR
)
return True
return False