gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update dependencies, improve CI workflow, and enhance code formatting

Browse Source 
- Updated the `sientia-dataops-library` dependency version from 1.4.3 to 1.4.6 in `requirements.txt`.
- Modified the GitHub Actions workflow to install development and runtime dependencies separately, improving clarity and organization.
- Added code formatting and linting checks using Ruff, along with type checking using mypy, to ensure code quality.
- Updated `.gitignore` to include additional cache directories and log files.
- Refactored code in various files for consistency in string formatting and improved logging messages.
This commit is contained in:
vitor-aignosi
2025-10-17 12:58:24 -03:00
parent 464c9aae9b
commit e2462af31c
21 changed files with 1536 additions and 1368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
ingestor/managers/opc_manager.py
View File

@@ -1,15 +1,16 @@
import json
import asyncio
from pathlib import Path
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua import Client
from sientia_do.notifications.models import NotificationLevel
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.temporal.constants import OPC_TIMEZONE, DATETIME_FORMAT_WITH_TZ
from ingestor.managers.data_manager import DataManager
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, OPC_TIMEZONE
import ingestor.metrics as metrics
from ingestor.managers.data_manager import DataManager
class OpcManager(BaseActivity):
@@ -54,9 +55,19 @@ class OpcManager(BaseActivity):
metadata (dict): Application metadata
"""
def __init__(self, name: str, url: str, data_manager: DataManager, logger: Logger,
server_uri: str, notification_handler: NotificationHandler, metadata: dict,
cert_path: str = None, private_key_path: str = None, server_cert_path: str = None):
def __init__(
self,
name: str,
url: str,
data_manager: DataManager,
logger: Logger,
server_uri: str,
notification_handler: NotificationHandler,
metadata: dict,
cert_path: str = None,
private_key_path: str = None,
server_cert_path: str = None,
):
self.url = url
self.name = name
self.server_uri = server_uri
@@ -71,14 +82,14 @@ class OpcManager(BaseActivity):
self.data_manager = data_manager
self.metadata = metadata
BaseActivity.__init__(self, logger=logger,
notification_handler=notification_handler,
set_error_counter=True)
BaseActivity.__init__(
self, logger=logger, notification_handler=notification_handler, set_error_counter=True
)
metrics.OPC_CONNECTION_STATUS.labels(
pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0)
metrics.OPC_TAGS_SUBSCRIBED.labels(
pod_id=self.pod_id, server_name=self.name).set(0)
pod_id=self.pod_id, server_name=self.name, server_url=self.url
).set(0)
metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0)
def __str__(self):
"""
@@ -87,8 +98,10 @@ class OpcManager(BaseActivity):
Returns:
str: Human-readable representation showing server details and current state.
"""
return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \
f"nodes={self.nodes}, subscriptions={self.subscriptions}"
return (
f'OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n'
f'nodes={self.nodes}, subscriptions={self.subscriptions}'
)
async def shutdown(self):
"""
@@ -103,10 +116,9 @@ class OpcManager(BaseActivity):
and ensure clean disconnection from OPC servers.
"""
try:
await self.disconnect()
except Exception as e:
self.logger.error(f"Error during cleanup: {e}")
self.logger.error(f'Error during cleanup: {e}')
async def set_security(self):
"""
@@ -133,11 +145,11 @@ class OpcManager(BaseActivity):
if not all([self.cert_path, self.private_key_path]):
raise ValueError(
"Certificate and private key paths must be provided for secure connection.")
'Certificate and private key paths must be provided for secure connection.'
)
cert = Path(self.cert_path)
private_key = Path(self.private_key_path)
server_cert = Path(
self.server_cert_path) if self.server_cert_path else None
server_cert = Path(self.server_cert_path) if self.server_cert_path else None
await self.client.set_application_uri(self.server_uri)
self.logger.info('Setting security...')
@@ -145,7 +157,7 @@ class OpcManager(BaseActivity):
SecurityPolicyBasic256,
certificate=str(cert),
private_key=str(private_key),
server_certificate=str(server_cert)
server_certificate=str(server_cert),
)
await self.client.set_secure_channel_timeout(10000000)
await self.client.set_session_timeout(10000000)
@@ -172,8 +184,7 @@ class OpcManager(BaseActivity):
- OPC_CONNECTION_STATUS: Set to 1 on successful connection
"""
metrics.OPC_CONNECTIONS_TOTAL.labels(
pod_id=self.pod_id, server_name=self.name).inc()
metrics.OPC_CONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc()
try:
self.client = Client(self.url, watchdog_intervall=3600000)
self.client.name = self.pod_id
@@ -182,14 +193,15 @@ class OpcManager(BaseActivity):
self.logger.info(f'Starting connection to {self.name}...')
await self.client.connect()
metrics.OPC_CONNECTION_STATUS.labels(
pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(1)
pod_id=self.pod_id, server_name=self.name, server_url=self.url
).set(1)
self.logger.info(f'Connection to {self.name} successful.')
except Exception as e:
metrics.OPC_CONNECTION_STATUS.labels(
pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0)
metrics.OPC_CONNECTIONS_FAILED.labels(
pod_id=self.pod_id, server_name=self.name).inc()
self.logger.error(f"Failed to connect to {self.name}: {e}")
pod_id=self.pod_id, server_name=self.name, server_url=self.url
).set(0)
metrics.OPC_CONNECTIONS_FAILED.labels(pod_id=self.pod_id, server_name=self.name).inc()
self.logger.error(f'Failed to connect to {self.name}: {e}')
raise
async def create_subscription(self, name: str, period: int = 500):
@@ -214,16 +226,16 @@ class OpcManager(BaseActivity):
"""
if not self.client:
raise ValueError("Client not connected. Call connect first.")
raise ValueError('Client not connected. Call connect first.')
try:
p = period if period is not None else 500
self.subscriptions[name] = await self.client.create_subscription(p, self)
self.logger.info(f'Subscription {name} created on {self.name}.')
metrics.OPC_SUBSCRIPTIONS_CREATED.labels(
pod_id=self.pod_id, server_name=self.name, slot_name=name).inc()
pod_id=self.pod_id, server_name=self.name, slot_name=name
).inc()
except Exception as e:
self.logger.error(
f"Failed to create subscription {name} on {self.name}: {e}")
self.logger.error(f'Failed to create subscription {name} on {self.name}: {e}')
raise
async def subscribe(self, subscription: str, nodes: dict, collect_period: int):
@@ -252,23 +264,23 @@ class OpcManager(BaseActivity):
"""
if not self.subscriptions.get(subscription):
raise ValueError(
"Subscription not created. Call create_subscription first.")
raise ValueError('Subscription not created. Call create_subscription first.')
self.logger.info(f"Subscribing to {subscription} on {self.name}...")
self.logger.info(f"Subscribing to nodes: {nodes}")
self.logger.info(f'Subscribing to {subscription} on {self.name}...')
self.logger.info(f'Subscribing to nodes: {nodes}')
addr_nodes = [self.client.get_node(n) for n in nodes]
self.logger.debug(f"Addr nodes: {addr_nodes}")
self.logger.debug(f'Addr nodes: {addr_nodes}')
self.nodes.update(nodes)
self.logger.debug(f"Nodes: {self.nodes}")
metrics.OPC_TAGS_SUBSCRIBED.labels(
pod_id=self.pod_id, server_name=self.name).set(len(self.nodes))
self.logger.debug(f'Nodes: {self.nodes}')
metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(
len(self.nodes)
)
self.collect_period = collect_period
for node, config in self.nodes.items():
self.nodes[node]['cycle_rule'] = {
'cycle_increment': collect_period*1000/float(config['frequency']),
'cycle_count': 0
'cycle_increment': collect_period * 1000 / float(config['frequency']),
'cycle_count': 0,
}
await self.subscriptions[subscription].subscribe_data_change(addr_nodes)
@@ -288,18 +300,17 @@ class OpcManager(BaseActivity):
- An info message upon successful unsubscription.
Behavior:
- If the subscription exists, it is deleted and removed from the
- If the subscription exists, it is deleted and removed from the
subscriptions dictionary.
- If the subscription does not exist, no action is taken.
"""
if not self.subscriptions.get(subscription):
self.logger.warning(
f"Subscription '{subscription}' not found. Cannot unsubscribe.")
self.logger.warning(f"Subscription '{subscription}' not found. Cannot unsubscribe.")
return
await self.subscriptions[subscription].delete()
del self.subscriptions[subscription]
self.logger.info(f"Unsubscribed from {subscription}.")
self.logger.info(f'Unsubscribed from {subscription}.')
async def disconnect(self):
"""
@@ -322,28 +333,27 @@ class OpcManager(BaseActivity):
self.logger.warning('Disconnecting from OPC server')
if self.client is None:
self.logger.warning("Client already disconnected.")
self.logger.warning('Client already disconnected.')
return
try:
for sub in self.subscriptions:
await self.subscriptions[sub].delete()
self.logger.warning("Deleted all subscriptions.")
self.logger.warning('Deleted all subscriptions.')
except Exception as sub_error:
self.logger.error(f"Failed to clean up subscription: {sub_error}")
self.logger.error(f'Failed to clean up subscription: {sub_error}')
try:
await self.client.disconnect()
except Exception as conn_error:
self.logger.error(
f"Failed to disconnect from OPC UA server: {conn_error}")
self.logger.error(f'Failed to disconnect from OPC UA server: {conn_error}')
finally:
del self.client
self.client = None
metrics.OPC_CONNECTION_STATUS.labels(
pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(0)
metrics.OPC_TAGS_SUBSCRIBED.labels(
pod_id=self.pod_id, server_name=self.name).set(0)
self.logger.warning("Disconnected from OPC UA server.")
pod_id=self.pod_id, server_name=self.name, server_url=self.url
).set(0)
metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0)
self.logger.warning('Disconnected from OPC UA server.')
async def datachange_notification(self, node, _val, data):
"""
@@ -371,35 +381,33 @@ class OpcManager(BaseActivity):
monitored_item = data.monitored_item
value = monitored_item.Value.Value.Value
# source_timestamp
source_timestamp = monitored_item.Value.SourceTimestamp.replace(
tzinfo=OPC_TIMEZONE)
source_timestamp = monitored_item.Value.SourceTimestamp.replace(tzinfo=OPC_TIMEZONE)
tag = str(node)
self.logger.debug(
f"Data change notification received for tag:"
f"{tag} after {self.nodes[tag]['cycle_rule']['cycle_count']} cycles")
f'Data change notification received for tag:'
f'{tag} after {self.nodes[tag]["cycle_rule"]["cycle_count"]} cycles'
)
self.nodes[tag]['cycle_rule']['cycle_count'] = 0
self.non_receive_count = 0
metrics.OPC_CYCLES_WITHOUT_DATA.labels(
pod_id=self.pod_id, server_name=self.name).set(0)
metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set(0)
data = {
'tag': tag,
'name': self.nodes[str(node)]['tag_name'],
'timestamp': source_timestamp.strftime(DATETIME_FORMAT_WITH_TZ),
'value': value
'value': value,
}
_a = [self.data_manager.publish(e, data)
for e in self.nodes[tag]['topics']]
_a = [self.data_manager.publish(e, data) for e in self.nodes[tag]['topics']]
def check_cycles(self):
"""
Checks the cycle counts for all monitored nodes and sends
notifications if thresholds are exceeded.
This method iterates through all monitored nodes and updates their cycle counts based on
This method iterates through all monitored nodes and updates their cycle counts based on
configured increments. If a node's cycle count exceeds a threshold (5 cycles), it triggers
a warning notification.
@@ -408,8 +416,7 @@ class OpcManager(BaseActivity):
- Sends warning notifications for nodes exceeding cycle thresholds
"""
for node, config in self.nodes.items():
self.nodes[node]['cycle_rule']['cycle_count'] += config[
'cycle_rule']['cycle_increment']
self.nodes[node]['cycle_rule']['cycle_count'] += config['cycle_rule']['cycle_increment']
if self.nodes[node]['cycle_rule']['cycle_count'] >= 5:
name = config['tag_name']
cycles = self.nodes[node]['cycle_rule']['cycle_count']
@@ -417,8 +424,8 @@ class OpcManager(BaseActivity):
metadata=self.metadata,
notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED',
message=f'{cycles} cycles without receive from {node}:{name}',
block="opc_manager",
level=NotificationLevel.WARNING
block='opc_manager',
level=NotificationLevel.WARNING,
)
def check_opc_listenning(self) -> bool:
@@ -441,26 +448,26 @@ class OpcManager(BaseActivity):
"""
self.non_receive_count += 1
metrics.OPC_CYCLES_WITHOUT_DATA.labels(
pod_id=self.pod_id, server_name=self.name).set(self.non_receive_count)
metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set(
self.non_receive_count
)
if self.non_receive_count >= 5:
self.send_notification(
metadata=self.metadata,
notification_id=f'OPC_LISTENNING_STOPPED__{self.name}',
message=f'{self.non_receive_count} cycles without '
f'receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}',
block="opc_manager",
level=NotificationLevel.ERROR
block='opc_manager',
level=NotificationLevel.ERROR,
)
if self.non_receive_count >= 15:
metrics.OPC_RECONNECTIONS_TOTAL.labels(
pod_id=self.pod_id, server_name=self.name).inc()
metrics.OPC_RECONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc()
self.send_notification(
metadata=self.metadata,
notification_id=f'OPC_CONNECTION_RETRY__{self.name}',
message=f'Retrying to connect to server {self.name}',
block="opc_manager",
level=NotificationLevel.ERROR
block='opc_manager',
level=NotificationLevel.ERROR,
)
return True
return False