gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
SIENTIAPDE-1325: Refactor Ingestor and Manager Classes for Enhanced Asynchronous Operations

- Introduced asynchronous methods across Ingestor, IngestorManager, DataManager, and OpcManager to improve performance and responsiveness.
- Integrated MetricsController into various classes for better observability and monitoring.
- Updated Redis and MongoDB interactions to support asynchronous operations, enhancing data handling efficiency.
- Removed deprecated Redis metrics and streamlined resource management logic.
- Adjusted unit tests to accommodate the new asynchronous behavior and ensure proper mocking of async methods.
This commit is contained in:
vitor-aignosi
2025-10-31 16:32:45 -03:00
parent 4158a74cac
commit 85a371a38b
7 changed files with 336 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
ingestor/managers/opc_manager.py
View File

@@ -8,14 +8,15 @@ from asyncua.crypto.security_policies import SecurityPolicyBasic256
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, OPC_TIMEZONE
import ingestor.metrics as metrics
from ingestor.managers.data_manager import DataManager
class OpcManager(BaseActivity):
class OpcManager(SientiaMonitoring):
"""
Manages OPC UA server connections and tag subscriptions.
@@ -66,6 +67,7 @@ class OpcManager(BaseActivity):
logger: Logger,
server_uri: str,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
metadata: dict,
cert_path: str | None = None,
private_key_path: str | None = None,
@@ -86,8 +88,12 @@ class OpcManager(BaseActivity):
self.data_manager = data_manager
self.metadata = metadata
BaseActivity.__init__(
self, logger=logger, notification_handler=notification_handler, set_error_counter=True
SientiaMonitoring.__init__(
self,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
set_error_counter=True,
)
metrics.OPC_CONNECTION_STATUS.labels(
@@ -192,7 +198,15 @@ class OpcManager(BaseActivity):
- OPC_CONNECTION_STATUS: Set to 1 on successful connection
"""
metrics.OPC_CONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc()
await self.emit_metric(
metric_object=metrics.OPC_CONNECTIONS_TOTAL,
method='inc',
value=1,
tags={
'pod_id': self.pod_id,
'server_name': self.name,
},
)
try:
self.client = Client(self.url, timeout=10, watchdog_intervall=3600000)
assert self.client is not None # Informa ao mypy que client não é None
@@ -207,9 +221,16 @@ class OpcManager(BaseActivity):
await self.set_security()
self.logger.info(f'Starting connection to {self.name}...')
await self.client.connect()
metrics.OPC_CONNECTION_STATUS.labels(
pod_id=self.pod_id, server_name=self.name, server_url=self.url
).set(1)
await self.emit_metric(
metric_object=metrics.OPC_CONNECTION_STATUS,
method='set',
value=1,
tags={
'pod_id': self.pod_id,
'server_name': self.name,
'server_url': self.url,
},
)
self.logger.info(f'Connection to {self.name} successful.')
except Exception:
await self.disconnect()
@@ -244,9 +265,16 @@ class OpcManager(BaseActivity):
self.subscription_period_ms, self
)
self.logger.info(f'Subscription {name} created on {self.name}.')
metrics.OPC_SUBSCRIPTIONS_CREATED.labels(
pod_id=self.pod_id, server_name=self.name, slot_name=name
).inc()
await self.emit_metric(
metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED,
method='inc',
value=1,
tags={
'pod_id': self.pod_id,
'server_name': self.name,
'slot_name': name,
},
)
except Exception as e:
self.logger.error(f'Failed to create subscription {name} on {self.name}: {e}')
raise
@@ -297,8 +325,14 @@ class OpcManager(BaseActivity):
await self.subscriptions[subscription].subscribe_data_change(addr_nodes)
metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(
len(self.nodes)
await self.emit_metric(
metric_object=metrics.OPC_TAGS_SUBSCRIBED,
method='set',
value=len(self.nodes),
tags={
'pod_id': self.pod_id,
'server_name': self.name,
},
)
async def unsubscribe(self, subscription: str):
@@ -387,7 +421,7 @@ class OpcManager(BaseActivity):
errors = await self.disconnection_fallback()
if errors:
self.send_notification(
await self.send_notification_async(
metadata=self.metadata,
notification_id=f'OPC_DISCONNECTION_ERROR_{self.name}',
message=f'Failed to disconnect from OPC UA server {self.name} after 5 attempts',
@@ -400,10 +434,25 @@ class OpcManager(BaseActivity):
del self.client
self.client = None
metrics.OPC_CONNECTION_STATUS.labels(
pod_id=self.pod_id, server_name=self.name, server_url=self.url
).set(0)
metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0)
await self.emit_metric(
metric_object=metrics.OPC_CONNECTION_STATUS,
method='set',
value=0,
tags={
'pod_id': self.pod_id,
'server_name': self.name,
'server_url': self.url,
},
)
await self.emit_metric(
metric_object=metrics.OPC_TAGS_SUBSCRIBED,
method='set',
value=0,
tags={
'pod_id': self.pod_id,
'server_name': self.name,
},
)
async def datachange_notification(self, node, _val, data):
"""
@@ -447,13 +496,21 @@ class OpcManager(BaseActivity):
}
for topic in self.nodes[tag]['topics']:
self.data_manager.publish(topic, data)
await self.data_manager.publish(topic, data)
self.nodes[tag]['cycle_rule']['cycle_count'] = 0
self.non_receive_count = 0
metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set(0)
await self.emit_metric(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=0,
tags={
'pod_id': self.pod_id,
'server_name': self.name,
},
)
def check_cycles(self):
async def check_cycles(self):
"""
Checks the cycle counts for all monitored nodes and sends
notifications if thresholds are exceeded.
@@ -471,7 +528,7 @@ class OpcManager(BaseActivity):
if self.nodes[node]['cycle_rule']['cycle_count'] >= 5:
name = config['tag_name']
cycles = self.nodes[node]['cycle_rule']['cycle_count']
self.send_notification(
await self.send_notification_async(
metadata=self.metadata,
notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED',
message=f'{cycles} cycles without receive from {node}:{name}',
@@ -479,7 +536,7 @@ class OpcManager(BaseActivity):
level=NotificationLevel.WARNING,
)
def check_opc_listenning(self) -> bool:
async def check_opc_listenning(self) -> bool:
"""
Checks the OPC connection and triggers notifications if the connection is lost.
@@ -499,11 +556,17 @@ class OpcManager(BaseActivity):
"""
self.non_receive_count += 1
metrics.OPC_CYCLES_WITHOUT_DATA.labels(pod_id=self.pod_id, server_name=self.name).set(
self.non_receive_count
await self.emit_metric(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=self.non_receive_count,
tags={
'pod_id': self.pod_id,
'server_name': self.name,
},
)
if self.non_receive_count >= 5:
self.send_notification(
await self.send_notification_async(
metadata=self.metadata,
notification_id=f'OPC_LISTENNING_STOPPED__{self.name}',
message=f'{self.non_receive_count} cycles without '
@@ -512,8 +575,16 @@ class OpcManager(BaseActivity):
level=NotificationLevel.ERROR,
)
if self.non_receive_count >= 15:
metrics.OPC_RECONNECTIONS_TOTAL.labels(pod_id=self.pod_id, server_name=self.name).inc()
self.send_notification(
await self.emit_metric(
metric_object=metrics.OPC_RECONNECTIONS_TOTAL,
method='inc',
value=1,
tags={
'pod_id': self.pod_id,
'server_name': self.name,
},
)
await self.send_notification_async(
metadata=self.metadata,
notification_id=f'OPC_CONNECTION_RETRY__{self.name}',
message=f'Retrying to connect to server {self.name}',