SIENTIAPDE-1205
Refactor Ingestor and OPC Manager for Asynchronous Operations - Updated main function to be asynchronous and integrated asyncio for better concurrency. - Refactored Ingestor methods to support async operations, including prepare_ingestor, loop, and shutdown. - Enhanced IngestorManager and OpcManager with async methods for improved performance and responsiveness. - Replaced blocking calls with await statements to ensure non-blocking behavior during operations. - Added a new run_async_main function to handle the async event loop setup.
This commit is contained in:
vitor-aignosi
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
from os import getenv
|
||||
from copy import deepcopy
|
||||
from typing import Dict, Any
|
||||
@@ -78,14 +79,14 @@ class Ingestor:
|
||||
}
|
||||
self.ingestor_manager = None
|
||||
|
||||
def shutdown(self):
|
||||
async def shutdown(self):
|
||||
if self.ingestor_manager:
|
||||
self.ingestor_manager.shutdown()
|
||||
await self.ingestor_manager.shutdown()
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
asyncio.run(self.shutdown())
|
||||
|
||||
def handle_acquired_tags(self, acquired):
|
||||
async def handle_acquired_tags(self, acquired):
|
||||
"""
|
||||
Handles the acquired tags by subscribing to them if available.
|
||||
This method checks if there are any acquired tags. If no tags are acquired,
|
||||
@@ -103,9 +104,9 @@ class Ingestor:
|
||||
else:
|
||||
# Subscribe to acquired slots
|
||||
self.ingestor_manager.update_opc_servers()
|
||||
self.ingestor_manager.subscribe_to_tags(acquired)
|
||||
await self.ingestor_manager.subscribe_to_tags(acquired)
|
||||
|
||||
def prepare_ingestor(self):
|
||||
async def prepare_ingestor(self):
|
||||
"""
|
||||
Prepares the ingestor by initializing the IngestorManager, declaring the ingestor as active,
|
||||
acquiring slot leases, and handling the acquired tags.
|
||||
@@ -153,7 +154,7 @@ class Ingestor:
|
||||
acquired = self.ingestor_manager.get_slot_leases()
|
||||
self.logger.info(f"Acquired slots: {acquired}")
|
||||
|
||||
self.handle_acquired_tags(acquired)
|
||||
await self.handle_acquired_tags(acquired)
|
||||
|
||||
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(
|
||||
len(self.ingestor_manager.managed_tags)
|
||||
@@ -176,7 +177,7 @@ class Ingestor:
|
||||
# Get slot lease
|
||||
self.ingestor_manager.get_slot_leases(1)
|
||||
|
||||
def manage_leases(
|
||||
async def manage_leases(
|
||||
self, available_slots: int, lacking_ingestors: int, slot_diff: int
|
||||
):
|
||||
"""
|
||||
@@ -214,7 +215,7 @@ class Ingestor:
|
||||
self.ingestor_manager.drop_slot_leases(overleases)
|
||||
|
||||
for lease in overleases:
|
||||
self.ingestor_manager.unsubscribe_slot(lease)
|
||||
await self.ingestor_manager.unsubscribe_slot(lease)
|
||||
self.ingestor_manager.managed_tags.pop(lease)
|
||||
|
||||
# Update metric after removal
|
||||
@@ -222,7 +223,7 @@ class Ingestor:
|
||||
len(self.ingestor_manager.managed_tags)
|
||||
)
|
||||
|
||||
def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]):
|
||||
async def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]):
|
||||
"""
|
||||
Updates the ingestor manager with the new managed tags.
|
||||
Args:
|
||||
@@ -232,7 +233,7 @@ class Ingestor:
|
||||
self.logger.debug(
|
||||
f"Current managed tags: {self.ingestor_manager.managed_tags}")
|
||||
|
||||
self.ingestor_manager.update_opc_servers()
|
||||
await self.ingestor_manager.update_opc_servers()
|
||||
new_managed_tags = deepcopy(self.ingestor_manager.managed_tags)
|
||||
|
||||
self.logger.debug(
|
||||
@@ -249,25 +250,25 @@ class Ingestor:
|
||||
for slot, config in new_managed_tags.items():
|
||||
if slot not in old_managed_tags:
|
||||
self.logger.info(f"Subscribing to new slot {slot}")
|
||||
self.ingestor_manager.subscribe_to_tags({slot: config})
|
||||
await self.ingestor_manager.subscribe_to_tags({slot: config})
|
||||
continue
|
||||
|
||||
if config != old_managed_tags[slot]:
|
||||
self.logger.info(f"Resubscribing to slot {slot}")
|
||||
self.ingestor_manager.unsubscribe_slot(slot)
|
||||
self.ingestor_manager.subscribe_to_tags({slot: config})
|
||||
await self.ingestor_manager.unsubscribe_slot(slot)
|
||||
await self.ingestor_manager.subscribe_to_tags({slot: config})
|
||||
|
||||
for slot in old_managed_tags.keys():
|
||||
if slot not in new_managed_tags:
|
||||
self.logger.info(f"Unsubscribing from slot {slot}")
|
||||
self.ingestor_manager.unsubscribe_slot(slot)
|
||||
await self.ingestor_manager.unsubscribe_slot(slot)
|
||||
|
||||
# Ensure the gauge is updated after any potential changes here
|
||||
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(
|
||||
len(self.ingestor_manager.managed_tags)
|
||||
)
|
||||
|
||||
def loop(self):
|
||||
async def loop(self):
|
||||
"""
|
||||
Executes the main loop for managing ingestors and slots.
|
||||
This method performs the following tasks:
|
||||
@@ -310,7 +311,7 @@ class Ingestor:
|
||||
slot_diff = len(self.ingestor_manager.managed_tags) - 1
|
||||
|
||||
self.logger.info("Managing leases...")
|
||||
self.manage_leases(available_slots, lacking_ingestors, slot_diff)
|
||||
await self.manage_leases(available_slots, lacking_ingestors, slot_diff)
|
||||
|
||||
# Update managed slots gauge
|
||||
metrics.SLOTS_MANAGED.labels(pod_id=self.pod_id).set(
|
||||
@@ -337,4 +338,4 @@ class Ingestor:
|
||||
self.ingestor_manager.check_opc_servers_integrity()
|
||||
|
||||
self.logger.info("Updating managed tags...")
|
||||
self.update_ingestor_manager(current_managed_tags)
|
||||
await self.update_ingestor_manager(current_managed_tags)
|
||||
|
||||
Reference in New Issue
Block a user