gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Update dependencies, improve CI workflow, and enhance code formatting

Browse Source 
- Updated the `sientia-dataops-library` dependency version from 1.4.3 to 1.4.6 in `requirements.txt`.
- Modified the GitHub Actions workflow to install development and runtime dependencies separately, improving clarity and organization.
- Added code formatting and linting checks using Ruff, along with type checking using mypy, to ensure code quality.
- Updated `.gitignore` to include additional cache directories and log files.
- Refactored code in various files for consistency in string formatting and improved logging messages.
This commit is contained in:
Vitor Pimentel dos Santos
2025-10-17 12:58:24 -03:00
parent 464c9aae9b
commit e2462af31c
21 changed files with 1536 additions and 1368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
ingestor/managers/ingestor_manager.py
View File

@@ -1,15 +1,16 @@
import asyncio
import traceback
from typing import Dict, List
from copy import deepcopy
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
import ingestor.metrics as metrics
from ingestor.managers.data_manager import DataManager
from ingestor.managers.opc_manager import OpcManager
from ingestor.managers.resource_manager import ResourceManager
import ingestor.metrics as metrics
class IngestorManager(BaseActivity):
@@ -57,14 +58,20 @@ class IngestorManager(BaseActivity):
metadata (dict): Application metadata
"""
def __init__(self,
kafka_servers: str, redis_data: dict,
lease_ttl: int, heartbeat_ttl: int,
poll_interval: int, mongo_connection_string: str, mongo_database: str,
metadata: dict,
logger: Logger, notification_handler: NotificationHandler,
export_to_kafka: bool = False):
def __init__(
self,
kafka_servers: str,
redis_data: dict,
lease_ttl: int,
heartbeat_ttl: int,
poll_interval: int,
mongo_connection_string: str,
mongo_database: str,
metadata: dict,
logger: Logger,
notification_handler: NotificationHandler,
export_to_kafka: bool = False,
):
redis_host = redis_data.get('host')
redis_port = redis_data.get('port')
redis_username = redis_data.get('username', None)
@@ -77,7 +84,7 @@ class IngestorManager(BaseActivity):
export_to_kafka=export_to_kafka,
metadata=metadata,
logger=logger,
notification_handler=notification_handler
notification_handler=notification_handler,
)
self.opc_managers = {}
self.resource_manager = ResourceManager(
@@ -98,9 +105,9 @@ class IngestorManager(BaseActivity):
self.metadata = metadata
BaseActivity.__init__(self, logger=logger,
notification_handler=notification_handler,
set_error_counter=True)
BaseActivity.__init__(
self, logger=logger, notification_handler=notification_handler, set_error_counter=True
)
async def initialize_opc_from_config(self, server_config: dict) -> OpcManager | None:
"""
@@ -130,8 +137,7 @@ class IngestorManager(BaseActivity):
"""
try:
self.logger.info(
f"Initializing OpcManager at {server_config['url']}")
self.logger.info(f'Initializing OpcManager at {server_config["url"]}')
manager = OpcManager(
name=server_config['name'],
url=server_config['url'],
@@ -142,7 +148,7 @@ class IngestorManager(BaseActivity):
metadata=self.metadata,
cert_path=server_config.get('cert_path'),
private_key_path=server_config.get('private_key_path'),
server_cert_path=server_config.get('server_cert_path')
server_cert_path=server_config.get('server_cert_path'),
)
manager.config = server_config
@@ -153,9 +159,9 @@ class IngestorManager(BaseActivity):
metadata=self.metadata,
notification_id=f'OPC_CONNECTION_ERROR_{server_config["name"]}',
message=f'Error initializing OPC manager: {e}',
block="opc_manager",
block='opc_manager',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
self.logger.error(trace)
@@ -216,49 +222,41 @@ class IngestorManager(BaseActivity):
server_config.pop('tags', None)
server_instance = self.opc_managers.get(server, None)
if server_instance is None:
self.logger.info(
f"Initializing OPC manager for server {server}"
)
server_instance = await self.initialize_opc_from_config(
server_config
)
self.logger.info(f'Initializing OPC manager for server {server}')
server_instance = await self.initialize_opc_from_config(server_config)
elif server_instance.config != server_config:
self.logger.warning(
f"Reinitializing OPC manager for server {server}"
)
self.logger.warning(f'Reinitializing OPC manager for server {server}')
server_instance.disconnect()
del self.opc_managers[server]
server_instance = await self.initialize_opc_from_config(
server_config
)
server_instance = await self.initialize_opc_from_config(server_config)
else:
self.logger.debug(
f"OPC manager for server {server} is already initialized and up to date"
f'OPC manager for server {server} is already initialized and up to date'
)
if server_instance is not None:
self.opc_managers[server] = server_instance
else:
self.logger.warning(
f"Failed to initialize OPC manager for server {server}, "
f"removing server from managed tags."
f'Failed to initialize OPC manager for server {server}, '
f'removing server from managed tags.'
)
_a = [self.managed_tags[slot].pop(server, None)
for slot, _value in current_managed_tags.items()]
_a = [
self.managed_tags[slot].pop(server, None)
for slot, _value in current_managed_tags.items()
]
servers = list(self.opc_managers.keys())
for server in servers:
if server not in registered_servers:
self.logger.warning(
f"Server {server} not found in managed tags. "
f"Desconnecting from server."
f'Server {server} not found in managed tags. Desconnecting from server.'
)
await self.opc_managers[server].disconnect()
self.opc_managers.pop(server, None)
metrics.OPC_MANAGERS_ACTIVE.labels(
pod_id=self.pod_id).set(len(self.opc_managers))
metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers))
def check_opc_servers_integrity(self):
"""
@@ -280,16 +278,12 @@ class IngestorManager(BaseActivity):
is_lost = opc_manager.check_opc_listenning()
if is_lost:
self.logger.warning(
f"OPC server {server} is lost. "
f"Server will be disconnected."
)
self.logger.warning(f'OPC server {server} is lost. Server will be disconnected.')
for slot, _config in self.managed_tags.items():
self.managed_tags[slot].pop(server, None)
metrics.OPC_MANAGERS_ACTIVE.labels(
pod_id=self.pod_id).set(len(self.opc_managers))
metrics.OPC_MANAGERS_ACTIVE.labels(pod_id=self.pod_id).set(len(self.opc_managers))
def declare_active(self):
"""
@@ -306,7 +300,7 @@ class IngestorManager(BaseActivity):
self.resource_manager.ingestor_heartbeat()
def get_active_ingestors(self) -> List[str]:
def get_active_ingestors(self) -> list[str]:
"""
Retrieve a list of active ingestors.
@@ -363,7 +357,7 @@ class IngestorManager(BaseActivity):
metrics.SLOTS_TOTAL.set(self.number_of_slots)
return self.number_of_slots
def get_slot_leases(self, max_slots: int = 1) -> Dict:
def get_slot_leases(self, max_slots: int = 1) -> dict:
"""
Acquires a specified number of resource slots by leasing them from the resource manager.
@@ -377,7 +371,7 @@ class IngestorManager(BaseActivity):
max_slots (int): The maximum number of slots to lease. Defaults to 1.
Returns:
Dict: A dictionary where the keys are the slot identifiers (as strings)
Dict: A dictionary where the keys are the slot identifiers (as strings)
and the values are the leased slot details.
Behavior:
@@ -394,7 +388,7 @@ class IngestorManager(BaseActivity):
acquired = {}
for i in range(1, self.number_of_slots + 1):
if self.resource_manager.lease_tag(str(i)):
self.logger.info(f"Leased slot {i}")
self.logger.info(f'Leased slot {i}')
slots = self.resource_manager.get_tag_slot(str(i))
if slots is None:
continue
@@ -403,17 +397,14 @@ class IngestorManager(BaseActivity):
if len(acquired) >= max_slots:
self.managed_tags.update(acquired)
metrics.SLOTS_MANAGED.labels(
pod_id=self.pod_id).set(len(self.managed_tags))
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags))
return acquired
self.logger.warning(
f"Unable to acquire {max_slots} slots. "
f"Only {acquired} slots were leased."
f'Unable to acquire {max_slots} slots. Only {acquired} slots were leased.'
)
self.managed_tags.update(acquired)
metrics.SLOTS_MANAGED.labels(
pod_id=self.pod_id).set(len(self.managed_tags))
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(len(self.managed_tags))
return acquired
async def unsubscribe_slot(self, slot: str):
@@ -441,7 +432,7 @@ class IngestorManager(BaseActivity):
def update_slot_config(self):
"""
Updates the configuration of managed slots by renewing their leases,
Updates the configuration of managed slots by renewing their leases,
fetching the latest configurations, and handling any changes or removals.
This method performs the following steps:
@@ -458,7 +449,7 @@ class IngestorManager(BaseActivity):
- Updates OPC server subscriptions based on the current state of managed slots.
Raises:
None explicitly, but relies on the behavior of `resource_manager` and
None explicitly, but relies on the behavior of `resource_manager` and
other dependencies for error handling.
Logging:
@@ -480,17 +471,17 @@ class IngestorManager(BaseActivity):
for slot in removed_slots:
self.managed_tags.pop(slot, None)
def drop_slot_leases(self, ids: List[str]) -> None:
def drop_slot_leases(self, ids: list[str]) -> None:
"""
Releases the leases associated with the specified slot IDs.
This method iterates through a list of slot IDs and calls the
`drop_tag_lease` method of the `resource_manager` to release
This method iterates through a list of slot IDs and calls the
`drop_tag_lease` method of the `resource_manager` to release
the lease for each ID. It's used during load balancing and
graceful shutdown scenarios.
Args:
ids (List[str]): A list of slot IDs for which the leases
ids (List[str]): A list of slot IDs for which the leases
should be released.
Returns:
@@ -509,7 +500,7 @@ class IngestorManager(BaseActivity):
Manages the subscription of tags to a specified OPC server and slot.
This method ensures that the specified server and slot have an active subscription
for the provided tags. If the server or slot is not properly configured, or if
for the provided tags. If the server or slot is not properly configured, or if
subscription fails, appropriate error handling is performed.
Args:
@@ -527,12 +518,12 @@ class IngestorManager(BaseActivity):
Logs:
- Logs informational messages about the subscription process.
- Logs errors if the server is not found, subscription creation fails, or
- Logs errors if the server is not found, subscription creation fails, or
tag subscription fails.
- Logs a warning if a subscription is removed due to failure.
Raises:
Exception: Any unexpected exceptions during subscription creation or tag
Exception: Any unexpected exceptions during subscription creation or tag
subscription are logged but not propagated.
Side Effects:
@@ -541,77 +532,63 @@ class IngestorManager(BaseActivity):
- Updates error metrics and notifications
"""
self.logger.info(
f"Subscribing to tags from {slot}:{server}"
)
self.logger.info(f'Subscribing to tags from {slot}:{server}')
tags_to_sub = server_config.get('tags')
if server not in self.opc_managers:
self.logger.error(
f"Server {server} not found in opc_managers."
)
self.logger.error(f'Server {server} not found in opc_managers.')
return 1
if slot not in self.opc_managers[server].subscriptions:
try:
await self.opc_managers[server].create_subscription(
slot
)
await self.opc_managers[server].create_subscription(slot)
except Exception as e:
self.logger.error(
f"Failed to create subscription for slot {slot}: {e}"
)
self.logger.error(f'Failed to create subscription for slot {slot}: {e}')
return 2
try:
self.logger.info(
tags_to_sub
)
self.logger.info(tags_to_sub)
await self.opc_managers[server].subscribe(
slot, deepcopy(tags_to_sub), self.poll_interval
)
self.logger.info(
tags_to_sub
)
self.logger.info(tags_to_sub)
except Exception as e:
metrics.OPC_SUBSCRIPTION_ERRORS.labels(
pod_id=self.pod_id, server=server, slot=slot).inc()
pod_id=self.pod_id, server=server, slot=slot
).inc()
trace = traceback.format_exc()
self.send_notification(
metadata=self.metadata,
notification_id=f'OPC_SUBSCRIPTION_ERROR_{slot}:{server}',
message=f'Failed to subscribe to tags from {slot}:{server}\n{tags}: {e}',
block="opc_manager",
block='opc_manager',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
self.logger.error(trace)
self.logger.warning(
"Removing subscription from server "
f"{server} for slot {slot}"
)
self.logger.warning(f'Removing subscription from server {server} for slot {slot}')
await self.opc_managers[server].unsubscribe(slot)
return 2
return 0
async def subscribe_to_tags(self, tags: Dict) -> None:
async def subscribe_to_tags(self, tags: dict) -> None:
"""
Subscribes to a set of tags and manages their configurations.
This method processes a dictionary of tags, iterating through each slot and server
configuration. It attempts to manage the server configurations and removes any
configuration. It attempts to manage the server configurations and removes any
servers that return a specific response code.
Args:
tags (Dict): A dictionary containing tag configurations. The structure is
tags (Dict): A dictionary containing tag configurations. The structure is
expected to be {slot: {server: server_config}}.
Side Effects:
- Logs the provided tags for debugging purposes.
- Updates the `managed_tags` attribute by removing servers that meet the
- Updates the `managed_tags` attribute by removing servers that meet the
removal criteria.
- Establishes OPC subscriptions for all configured tags.
Removal Criteria:
- If the `manage_server` method returns a response code of 2 for a given
- If the `manage_server` method returns a response code of 2 for a given
slot and server, that server is removed from the `managed_tags` attribute.
The method ensures that only successfully configured servers remain in the
@@ -622,9 +599,7 @@ class IngestorManager(BaseActivity):
self.logger.info(tags)
for slot, slot_config in tags.items():
for server, server_config in slot_config.items():
response = await self.manage_server(
slot, server, server_config, tags
)
response = await self.manage_server(slot, server, server_config, tags)
if response == 2:
to_remove.append([slot, server])