gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1205

Browse Source 
Refactor Ingestor and OPC Manager for Asynchronous Operations

- Updated main function to be asynchronous and integrated asyncio for better concurrency.
- Refactored Ingestor methods to support async operations, including prepare_ingestor, loop, and shutdown.
- Enhanced IngestorManager and OpcManager with async methods for improved performance and responsiveness.
- Replaced blocking calls with await statements to ensure non-blocking behavior during operations.
- Added a new run_async_main function to handle the async event loop setup.
This commit is contained in:
vitor-aignosi
2025-08-26 16:27:35 -03:00
parent ec3c738519
commit d4aa44d774
5 changed files with 111 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
ingestor/app.py
View File

@@ -1,4 +1,5 @@
import os
import asyncio
import signal
import traceback
from threading import Event
@@ -13,11 +14,11 @@ exit_signal = Event()
POD_ID = os.getenv("HOSTNAME", "localhost")
def main():
async def main():
start_prometheus_server()
ingestor = Ingestor()
try:
ingestor.prepare_ingestor()
await ingestor.prepare_ingestor()
except Exception as e:
metrics.APP_ERRORS_TOTAL.labels(
pod_id=POD_ID).inc() # Increment errors
@@ -28,11 +29,12 @@ def main():
while not exit_signal.is_set():
start_time = time() # Start loop timer
try:
ingestor.loop()
await ingestor.loop()
metrics.APP_LOOP_COUNT.labels(
pod_id=POD_ID).inc() # Increment loop counter
exit_signal.wait(ingestor.poll_interval)
# Use asyncio.sleep instead of exit_signal.wait for better async compatibility
await asyncio.sleep(ingestor.poll_interval)
except KeyboardInterrupt: # Handle Ctrl+C gracefully
print("KeyboardInterrupt received. Setting exit_signal flag.")
@@ -48,13 +50,13 @@ def main():
duration = time() - start_time
metrics.APP_LOOP_DURATION.labels(pod_id=POD_ID).observe(duration)
ingestor.shutdown()
await ingestor.shutdown()
metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN
ingestor.logger.info("Main loop exit_signaled.")
# Give Prometheus a chance to scrape one last time before exiting (optional)
sleep(5)
await asyncio.sleep(5)
os._exit(0)
@@ -75,8 +77,22 @@ def start_prometheus_server():
os._exit(1)
def run_async_main():
"""Run the async main function with proper event loop setup"""
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(main())
except KeyboardInterrupt:
print("KeyboardInterrupt received in main thread.")
exit_signal.set()
finally:
loop.close()
if __name__ == "__main__":
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
signal.signal(signal.SIGHUP, signal_handler)
main()
run_async_main()

37
ingestor/ingestor.py
View File

@@ -1,3 +1,4 @@
import asyncio
from os import getenv
from copy import deepcopy
from typing import Dict, Any
@@ -78,14 +79,14 @@ class Ingestor:
}
self.ingestor_manager = None
def shutdown(self):
async def shutdown(self):
if self.ingestor_manager:
self.ingestor_manager.shutdown()
await self.ingestor_manager.shutdown()
def __del__(self):
self.shutdown()
asyncio.run(self.shutdown())
def handle_acquired_tags(self, acquired):
async def handle_acquired_tags(self, acquired):
"""
Handles the acquired tags by subscribing to them if available.
This method checks if there are any acquired tags. If no tags are acquired,
@@ -103,9 +104,9 @@ class Ingestor:
else:
# Subscribe to acquired slots
self.ingestor_manager.update_opc_servers()
self.ingestor_manager.subscribe_to_tags(acquired)
await self.ingestor_manager.subscribe_to_tags(acquired)
def prepare_ingestor(self):
async def prepare_ingestor(self):
"""
Prepares the ingestor by initializing the IngestorManager, declaring the ingestor as active,
acquiring slot leases, and handling the acquired tags.
@@ -153,7 +154,7 @@ class Ingestor:
acquired = self.ingestor_manager.get_slot_leases()
self.logger.info(f"Acquired slots: {acquired}")
self.handle_acquired_tags(acquired)
await self.handle_acquired_tags(acquired)
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(
len(self.ingestor_manager.managed_tags)
@@ -176,7 +177,7 @@ class Ingestor:
# Get slot lease
self.ingestor_manager.get_slot_leases(1)
def manage_leases(
async def manage_leases(
self, available_slots: int, lacking_ingestors: int, slot_diff: int
):
"""
@@ -214,7 +215,7 @@ class Ingestor:
self.ingestor_manager.drop_slot_leases(overleases)
for lease in overleases:
self.ingestor_manager.unsubscribe_slot(lease)
await self.ingestor_manager.unsubscribe_slot(lease)
self.ingestor_manager.managed_tags.pop(lease)
# Update metric after removal
@@ -222,7 +223,7 @@ class Ingestor:
len(self.ingestor_manager.managed_tags)
)
def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]):
async def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]):
"""
Updates the ingestor manager with the new managed tags.
Args:
@@ -232,7 +233,7 @@ class Ingestor:
self.logger.debug(
f"Current managed tags: {self.ingestor_manager.managed_tags}")
self.ingestor_manager.update_opc_servers()
await self.ingestor_manager.update_opc_servers()
new_managed_tags = deepcopy(self.ingestor_manager.managed_tags)
self.logger.debug(
@@ -249,25 +250,25 @@ class Ingestor:
for slot, config in new_managed_tags.items():
if slot not in old_managed_tags:
self.logger.info(f"Subscribing to new slot {slot}")
self.ingestor_manager.subscribe_to_tags({slot: config})
await self.ingestor_manager.subscribe_to_tags({slot: config})
continue
if config != old_managed_tags[slot]:
self.logger.info(f"Resubscribing to slot {slot}")
self.ingestor_manager.unsubscribe_slot(slot)
self.ingestor_manager.subscribe_to_tags({slot: config})
await self.ingestor_manager.unsubscribe_slot(slot)
await self.ingestor_manager.subscribe_to_tags({slot: config})
for slot in old_managed_tags.keys():
if slot not in new_managed_tags:
self.logger.info(f"Unsubscribing from slot {slot}")
self.ingestor_manager.unsubscribe_slot(slot)
await self.ingestor_manager.unsubscribe_slot(slot)
# Ensure the gauge is updated after any potential changes here
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(
len(self.ingestor_manager.managed_tags)
)
def loop(self):
async def loop(self):
"""
Executes the main loop for managing ingestors and slots.
This method performs the following tasks:
@@ -310,7 +311,7 @@ class Ingestor:
slot_diff = len(self.ingestor_manager.managed_tags) - 1
self.logger.info("Managing leases...")
self.manage_leases(available_slots, lacking_ingestors, slot_diff)
await self.manage_leases(available_slots, lacking_ingestors, slot_diff)
# Update managed slots gauge
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(
@@ -337,4 +338,4 @@ class Ingestor:
self.ingestor_manager.check_opc_servers_integrity()
self.logger.info("Updating managed tags...")
self.update_ingestor_manager(current_managed_tags)
await self.update_ingestor_manager(current_managed_tags)

35
ingestor/managers/ingestor_manager.py
View File

@@ -1,3 +1,4 @@
import asyncio
import traceback
from typing import Dict, List
from copy import deepcopy
@@ -57,7 +58,7 @@ class IngestorManager(BaseActivity):
notification_handler=notification_handler,
set_error_counter=True)
def initialize_opc_from_config(self, server_config: dict) -> OpcManager | None:
async def initialize_opc_from_config(self, server_config: dict) -> OpcManager | None:
"""
Initializes an OPC Manager instance using the provided server configuration.
Args:
@@ -93,7 +94,7 @@ class IngestorManager(BaseActivity):
)
manager.config = server_config
manager.connect()
await manager.connect()
except Exception as e:
trace = traceback.format_exc()
@@ -111,15 +112,15 @@ class IngestorManager(BaseActivity):
return manager
def shutdown(self):
async def shutdown(self):
for _server_name, server in self.opc_managers.items():
server.disconnect()
await server.disconnect()
self.data_manager.shutdown()
def __del__(self):
self.shutdown()
asyncio.run(self.shutdown())
def update_opc_servers(self):
async def update_opc_servers(self):
"""
Updates the OPC (OLE for Process Control) server connections managed by the ingestor.
This method ensures that the OPC servers defined in `self.managed_tags` are properly
@@ -156,7 +157,7 @@ class IngestorManager(BaseActivity):
self.logger.info(
f"Initializing OPC manager for server {server}"
)
server_instance = self.initialize_opc_from_config(
server_instance = await self.initialize_opc_from_config(
server_config
)
@@ -166,7 +167,7 @@ class IngestorManager(BaseActivity):
)
server_instance.disconnect()
del self.opc_managers[server]
server_instance = self.initialize_opc_from_config(
server_instance = await self.initialize_opc_from_config(
server_config
)
else:
@@ -191,7 +192,7 @@ class IngestorManager(BaseActivity):
f"Server {server} not found in managed tags. "
f"Desconnecting from server."
)
self.opc_managers[server].disconnect()
await self.opc_managers[server].disconnect()
self.opc_managers.pop(server, None)
metrics.OPC_MANAGERS_ACTIVE.labels(
@@ -312,7 +313,7 @@ class IngestorManager(BaseActivity):
pod_id=self.pod_id).set(len(self.managed_tags))
return acquired
def unsubscribe_slot(self, slot: str):
async def unsubscribe_slot(self, slot: str):
"""
Unsubscribes a specific slot from all associated OPC servers.
Args:
@@ -323,7 +324,7 @@ class IngestorManager(BaseActivity):
for server in self.managed_tags[slot].keys():
if server in self.opc_managers:
self.opc_managers[server].unsubscribe(slot)
await self.opc_managers[server].unsubscribe(slot)
def update_slot_config(self):
"""
@@ -378,7 +379,7 @@ class IngestorManager(BaseActivity):
self.resource_manager.drop_tag_lease(lease_id)
metrics.SLOTS_RELEASED.labels(pod_id=self.pod_id).inc()
def manage_server(self, slot: str, server: str, server_config: dict, tags: dict) -> int:
async def manage_server(self, slot: str, server: str, server_config: dict, tags: dict) -> int:
"""
Manages the subscription of tags to a specified OPC server and slot.
This method ensures that the specified server and slot have an active subscription
@@ -416,7 +417,7 @@ class IngestorManager(BaseActivity):
return 1
if slot not in self.opc_managers[server].subscriptions:
try:
self.opc_managers[server].create_subscription(
await self.opc_managers[server].create_subscription(
slot
)
except Exception as e:
@@ -428,7 +429,7 @@ class IngestorManager(BaseActivity):
self.logger.info(
tags_to_sub
)
self.opc_managers[server].subscribe(
await self.opc_managers[server].subscribe(
slot, deepcopy(tags_to_sub), self.poll_interval
)
self.logger.info(
@@ -452,11 +453,11 @@ class IngestorManager(BaseActivity):
"Removing subscription from server "
f"{server} for slot {slot}"
)
self.opc_managers[server].unsubscribe(slot)
await self.opc_managers[server].unsubscribe(slot)
return 2
return 0
def subscribe_to_tags(self, tags: Dict) -> None:
async def subscribe_to_tags(self, tags: Dict) -> None:
"""
Subscribes to a set of tags and manages their configurations.
This method processes a dictionary of tags, iterating through each slot and server
@@ -478,7 +479,7 @@ class IngestorManager(BaseActivity):
self.logger.info(tags)
for slot, slot_config in tags.items():
for server, server_config in slot_config.items():
response = self.manage_server(
response = await self.manage_server(
slot, server, server_config, tags
)
if response == 2:

58
ingestor/managers/opc_manager.py
View File

@@ -1,7 +1,8 @@
import json
import asyncio
from pathlib import Path
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.sync import Client
from asyncua import Client
from sientia_do.notifications.models import NotificationLevel
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.temporal.activities.base import BaseActivity
@@ -42,7 +43,18 @@ class OpcManager(BaseActivity):
return f"OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n" \
f"nodes={self.nodes}, subscriptions={self.subscriptions}"
def set_security(self):
async def shutdown(self):
"""Comprehensive cleanup method"""
try:
await self.disconnect()
except Exception as e:
self.logger.error(f"Error during cleanup: {e}")
def __del__(self):
asyncio.run(self.shutdown())
async def set_security(self):
"""
Configures the security settings for the OPC UA client.
This method sets up the security policy, certificates, and timeouts
@@ -70,18 +82,18 @@ class OpcManager(BaseActivity):
server_cert = Path(
self.server_cert_path) if self.server_cert_path else None
self.client.application_uri = self.server_uri
await self.client.set_application_uri(self.server_uri)
self.logger.info('Setting security...')
self.client.set_security(
await self.client.set_security(
SecurityPolicyBasic256,
certificate=str(cert),
private_key=str(private_key),
server_certificate=str(server_cert)
)
self.client.secure_channel_timeout = 10000000
self.client.session_timeout = 10000000
await self.client.set_secure_channel_timeout(10000000)
await self.client.set_session_timeout(10000000)
def connect(self):
async def connect(self):
"""
Establishes a connection to the OPC server.
This method initializes the OPC client using the provided URL and
@@ -96,9 +108,9 @@ class OpcManager(BaseActivity):
try:
self.client = Client(self.url)
if self.cert_path:
self.set_security()
await self.set_security()
self.logger.info(f'Starting connection to {self.name}...')
self.client.connect()
await self.client.connect()
metrics.OPC_CONNECTION_STATUS.labels(
pod_id=self.pod_id, server_name=self.name, server_url=self.url).set(1)
self.logger.info(f'Connection to {self.name} successful.')
@@ -110,7 +122,7 @@ class OpcManager(BaseActivity):
self.logger.error(f"Failed to connect to {self.name}: {e}")
raise
def create_subscription(self, name: str, period: int = 500):
async def create_subscription(self, name: str, period: int = 500):
"""
Creates a subscription with the specified monitoring period.
This method establishes a subscription to monitor data changes or events
@@ -129,7 +141,7 @@ class OpcManager(BaseActivity):
raise ValueError("Client not connected. Call connect first.")
try:
p = period if period is not None else 500
self.subscriptions[name] = self.client.create_subscription(p, self)
self.subscriptions[name] = await self.client.create_subscription(p, self)
self.logger.info(f'Subscription {name} created on {self.name}.')
metrics.OPC_SUBSCRIPTIONS_CREATED.labels(
pod_id=self.pod_id, server_name=self.name, slot_name=name).inc()
@@ -138,7 +150,7 @@ class OpcManager(BaseActivity):
f"Failed to create subscription {name} on {self.name}: {e}")
raise
def subscribe(self, subscription: str, nodes: dict, collect_period: int):
async def subscribe(self, subscription: str, nodes: dict, collect_period: int):
"""
Subscribes to a set of OPC UA nodes for data change notifications.
This method adds the specified nodes to the subscription and configures
@@ -161,8 +173,7 @@ class OpcManager(BaseActivity):
self.logger.info(f"Subscribing to {subscription} on {self.name}...")
self.logger.info(f"Subscribing to nodes: {nodes}")
addr_nodes = [self.client.get_node(
n) for n in nodes]
addr_nodes = [self.client.get_node(n) for n in nodes]
self.logger.debug(f"Addr nodes: {addr_nodes}")
self.nodes.update(nodes)
self.logger.debug(f"Nodes: {self.nodes}")
@@ -176,9 +187,9 @@ class OpcManager(BaseActivity):
'cycle_count': 0
}
self.subscriptions[subscription].subscribe_data_change(addr_nodes)
await self.subscriptions[subscription].subscribe_data_change(addr_nodes)
def unsubscribe(self, subscription: str):
async def unsubscribe(self, subscription: str):
"""
Unsubscribes from a given subscription.
Args:
@@ -196,14 +207,11 @@ class OpcManager(BaseActivity):
self.logger.warning(
f"Subscription '{subscription}' not found. Cannot unsubscribe.")
return
self.subscriptions[subscription].delete()
await self.subscriptions[subscription].delete()
del self.subscriptions[subscription]
self.logger.info(f"Unsubscribed from {subscription}.")
def __del__(self):
self.disconnect()
def disconnect(self):
async def disconnect(self):
"""
Disconnects from the OPC UA server.
This method handles the disconnection process by deleting the subscription
@@ -219,14 +227,14 @@ class OpcManager(BaseActivity):
self.logger.warning("Client already disconnected.")
return
try:
_a = [self.subscriptions[sub].delete()
for sub in self.subscriptions]
for sub in self.subscriptions:
await self.subscriptions[sub].delete()
self.logger.warning("Deleted all subscriptions.")
except Exception as sub_error:
self.logger.error(f"Failed to clean up subscription: {sub_error}")
try:
self.client.disconnect()
await self.client.disconnect()
except Exception as conn_error:
self.logger.error(
f"Failed to disconnect from OPC UA server: {conn_error}")
@@ -239,7 +247,7 @@ class OpcManager(BaseActivity):
pod_id=self.pod_id, server_name=self.name).set(0)
self.logger.warning("Disconnected from OPC UA server.")
def datachange_notification(self, node, _val, data):
async def datachange_notification(self, node, _val, data):
"""
Handles data change notifications for monitored OPC UA nodes.
This method is triggered when a monitored node's value changes. It processes

18
run_local.sh Executable file
View File

@@ -0,0 +1,18 @@
#!/bin/bash
# Exit on any error
set -e
echo "Activating virtual environment..."
source ./venv/bin/activate
echo "Loading environment variables from .env..."
if [ -f .env ]; then
export $(cat .env | grep -v '^#' | xargs)
echo "Environment variables loaded from .env"
else
echo "Warning: .env file not found. Continuing without environment variables."
fi
echo "Starting ingestor application..."
python -m ingestor.app