gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
SIENTIAPDE-1325: Refactor Ingestor and Manager Classes for Enhanced Asynchronous Operations

- Introduced asynchronous methods across Ingestor, IngestorManager, DataManager, and OpcManager to improve performance and responsiveness.
- Integrated MetricsController into various classes for better observability and monitoring.
- Updated Redis and MongoDB interactions to support asynchronous operations, enhancing data handling efficiency.
- Removed deprecated Redis metrics and streamlined resource management logic.
- Adjusted unit tests to accommodate the new asynchronous behavior and ensure proper mocking of async methods.
This commit is contained in:
vitor-aignosi
2025-10-31 16:32:45 -03:00
parent 4158a74cac
commit 85a371a38b
7 changed files with 336 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
ingestor/managers/ingestor_manager.py
View File

@@ -5,7 +5,8 @@ from copy import deepcopy
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
import ingestor.metrics as metrics
from ingestor.managers.data_manager import DataManager
@@ -13,7 +14,7 @@ from ingestor.managers.opc_manager import OpcManager
from ingestor.managers.resource_manager import ResourceManager
class IngestorManager(BaseActivity):
class IngestorManager(SientiaMonitoring):
"""
Central coordinator for managing OPC data ingestion operations.
@@ -70,6 +71,7 @@ class IngestorManager(BaseActivity):
metadata: dict,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
export_to_kafka: bool = False,
):
redis_host: str = redis_data['host']
@@ -85,6 +87,7 @@ class IngestorManager(BaseActivity):
metadata=metadata,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
self.opc_managers: dict = {}
self.resource_manager = ResourceManager(
@@ -97,6 +100,7 @@ class IngestorManager(BaseActivity):
notification_handler=notification_handler,
username=redis_username,
password=redis_password,
metrics_controller=metrics_controller,
)
self.number_of_slots = 0
self.poll_interval = poll_interval
@@ -105,8 +109,12 @@ class IngestorManager(BaseActivity):
self.metadata = metadata
BaseActivity.__init__(
self, logger=logger, notification_handler=notification_handler, set_error_counter=True
SientiaMonitoring.__init__(
self,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
set_error_counter=True,
)
async def initialize_opc_from_config(self, server_config: dict) -> OpcManager | None:
@@ -150,6 +158,7 @@ class IngestorManager(BaseActivity):
cert_path=server_config.get('cert_path'),
private_key_path=server_config.get('private_key_path'),
server_cert_path=server_config.get('server_cert_path'),
metrics_controller=self.metrics_controller,
)
manager.config = server_config
@@ -264,7 +273,14 @@ class IngestorManager(BaseActivity):
)
await self.remove_server(server)
metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers))
await self.emit_metric(
metric_object=metrics.OPC_MANAGERS_ACTIVE,
method='set',
value=len(self.opc_managers),
tags={
'pod_id': self.pod_id,
},
)
async def check_opc_servers_integrity(self):
"""
@@ -294,9 +310,16 @@ class IngestorManager(BaseActivity):
for server in to_disconnect:
await self.remove_server(server)
metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers))
await self.emit_metric(
metric_object=metrics.OPC_MANAGERS_ACTIVE,
method='set',
value=len(self.opc_managers),
tags={
'pod_id': self.pod_id,
},
)
def declare_active(self):
async def declare_active(self):
"""
Declares the ingestor as active by sending a heartbeat signal to the resource manager.
@@ -309,9 +332,9 @@ class IngestorManager(BaseActivity):
- Enables load balancing and health monitoring
"""
self.resource_manager.ingestor_heartbeat()
await self.resource_manager.ingestor_heartbeat()
def get_active_ingestors(self) -> list[str]:
async def get_active_ingestors(self) -> list[str]:
"""
Retrieve a list of active ingestors.
@@ -325,10 +348,10 @@ class IngestorManager(BaseActivity):
the pod identifiers for load balancing and coordination purposes.
"""
ingestors = self.resource_manager.get_all_ingestors()
ingestors = await self.resource_manager.get_all_ingestors()
return ingestors if ingestors else []
def get_number_of_leases(self) -> int:
async def get_number_of_leases(self) -> int:
"""
Retrieves the number of leases managed by the resource manager.
@@ -343,12 +366,19 @@ class IngestorManager(BaseActivity):
- Updates Prometheus metrics for total leases
"""
leases = self.resource_manager.get_all_leases()
leases = await self.resource_manager.get_all_leases()
self.number_of_slots = len(leases) if leases else 0
metrics.LEASES_TOTAL.set(self.number_of_slots)
await self.emit_metric(
metric_object=metrics.LEASES_TOTAL,
method='set',
value=self.number_of_slots,
tags={
'pod_id': self.pod_id,
},
)
return self.number_of_slots
def get_number_of_slots(self) -> int:
async def get_number_of_slots(self) -> int:
"""
Retrieves the number of slots managed by the resource manager.
@@ -363,12 +393,19 @@ class IngestorManager(BaseActivity):
- Updates Prometheus metrics for total slots
"""
slots = self.resource_manager.get_all_slots()
slots = await self.resource_manager.get_all_slots()
self.number_of_slots = len(slots) if slots else 0
metrics.SLOTS_TOTAL.set(self.number_of_slots)
await self.emit_metric(
metric_object=metrics.SLOTS_TOTAL,
method='set',
value=self.number_of_slots,
tags={
'pod_id': self.pod_id,
},
)
return self.number_of_slots
def get_slot_leases(self, max_slots: int = 1) -> dict:
async def get_slot_leases(self, max_slots: int = 1) -> dict:
"""
Acquires a specified number of resource slots by leasing them from the resource manager.
@@ -398,24 +435,45 @@ class IngestorManager(BaseActivity):
acquired = {}
for i in range(1, self.number_of_slots + 1):
if self.resource_manager.lease_tag(str(i)):
if await self.resource_manager.lease_tag(str(i)):
self.logger.info(f'Leased slot {i}')
slots = self.resource_manager.get_tag_slot(str(i))
slots = await self.resource_manager.get_tag_slot(str(i))
if slots is None:
continue
acquired[str(i)] = slots
metrics.SLOTS_ACQUIRED.labels(pod_id=self.pod_id).inc()
await self.emit_metric(
metric_object=metrics.SLOTS_ACQUIRED,
method='inc',
value=1,
tags={
'pod_id': self.pod_id,
},
)
if len(acquired) >= max_slots:
self.managed_tags.update(acquired)
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags))
await self.emit_metric(
metric_object=metrics.SLOTS_MANAGED,
method='set',
value=len(self.managed_tags),
tags={
'pod_id': self.pod_id,
},
)
return acquired
self.logger.warning(
f'Unable to acquire {max_slots} slots. Only {acquired} slots were leased.'
)
self.managed_tags.update(acquired)
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags))
await self.emit_metric(
metric_object=metrics.SLOTS_MANAGED,
method='set',
value=len(self.managed_tags),
tags={
'pod_id': self.pod_id,
},
)
return acquired
async def unsubscribe_slot(self, slot: str):
@@ -441,7 +499,7 @@ class IngestorManager(BaseActivity):
if server in self.opc_managers:
await self.opc_managers[server].unsubscribe(slot)
def update_slot_config(self):
async def update_slot_config(self):
"""
Updates the configuration of managed slots by renewing their leases,
fetching the latest configurations, and handling any changes or removals.
@@ -470,8 +528,8 @@ class IngestorManager(BaseActivity):
removed_slots: list[str] = []
for slot, _slot_config in self.managed_tags.items():
self.resource_manager.renew_tag_lease(slot)
update = self.resource_manager.get_tag_slot(slot)
await self.resource_manager.renew_tag_lease(slot)
update = await self.resource_manager.get_tag_slot(slot)
if update is None:
removed_slots.append(slot)
continue
@@ -481,7 +539,7 @@ class IngestorManager(BaseActivity):
for slot in removed_slots:
self.managed_tags.pop(slot, None)
def drop_slot_leases(self, ids: list[str]) -> None:
async def drop_slot_leases(self, ids: list[str]) -> None:
"""
Releases the leases associated with the specified slot IDs.
@@ -502,8 +560,15 @@ class IngestorManager(BaseActivity):
- Updates metrics for released slots count
"""
for lease_id in ids:
self.resource_manager.drop_tag_lease(lease_id)
metrics.SLOTS_RELEASED.labels(pod_id=self.pod_id).inc()
await self.resource_manager.drop_tag_lease(lease_id)
await self.emit_metric(
metric_object=metrics.SLOTS_RELEASED,
method='inc',
value=1,
tags={
'pod_id': self.pod_id,
},
)
async def manage_server(self, slot: str, server: str, server_config: dict, tags: dict) -> int:
"""
@@ -560,11 +625,16 @@ class IngestorManager(BaseActivity):
)
self.logger.info(tags_to_sub)
except Exception as e:
metrics.OPC_SUBSCRIPTION_ERRORS.labels(
pod_id=self.pod_id, server=server, slot=slot
).inc()
await self.emit_metric(
metric_object=metrics.OPC_SUBSCRIPTION_ERRORS,
tags={
'pod_id': self.pod_id,
'server': server,
'slot': slot,
},
)
trace = traceback.format_exc()
self.send_notification(
await self.send_notification_async(
metadata=self.metadata,
notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}',
message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}',