gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update dependencies, improve CI workflow, and enhance code formatting

Browse Source 
- Updated the `sientia-dataops-library` dependency version from 1.4.3 to 1.4.6 in `requirements.txt`.
- Modified the GitHub Actions workflow to install development and runtime dependencies separately, improving clarity and organization.
- Added code formatting and linting checks using Ruff, along with type checking using mypy, to ensure code quality.
- Updated `.gitignore` to include additional cache directories and log files.
- Refactored code in various files for consistency in string formatting and improved logging messages.
This commit is contained in:
vitor-aignosi
2025-10-17 12:58:24 -03:00
parent 464c9aae9b
commit e2462af31c
21 changed files with 1536 additions and 1368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
ingestor/ingestor.py
View File

@@ -1,14 +1,12 @@
import asyncio
from os import getenv
from copy import deepcopy
from typing import Dict, Any
from os import getenv
from typing import Any
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import get_logger
from ingestor.managers.ingestor_manager import IngestorManager
import ingestor.metrics as metrics
from ingestor.managers.ingestor_manager import IngestorManager
class Ingestor:
@@ -73,36 +71,36 @@ class Ingestor:
- Metrics collection and monitoring
"""
kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092")
export_to_kafka = getenv("EXPORT_TO_KAFKA", "false")
kafka_servers = getenv('KAFKA_SERVERS', 'localhost:9092')
export_to_kafka = getenv('EXPORT_TO_KAFKA', 'false')
if export_to_kafka and export_to_kafka == "true":
if export_to_kafka and export_to_kafka == 'true':
export_to_kafka = True
else:
export_to_kafka = False
self.export_to_kafka = export_to_kafka
self.redis_host = getenv("REDIS_HOST", "localhost")
self.redis_port = int(getenv("REDIS_PORT", "6379"))
self.redis_username = getenv("REDIS_USERNAME", None)
self.redis_password = getenv("REDIS_PASSWORD", None)
self.lease_ttl = int(getenv("LEASE_TTL", "10"))
self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", "20"))
self.pod_id = getenv("HOSTNAME", "localhost")
self.poll_interval = int(getenv("POLL_INTERVAL", "5"))
mongo_url = getenv("MONGODB_URL", "localhost:27017")
mongo_username = getenv("MONGODB_USERNAME", "sientia")
mongo_password = getenv("MONGODB_PASSWORD", "sientia")
self.mongo_database = getenv("MONGODB_DATABASE", "sientia")
self.mongo_connection_string = f"mongodb://{mongo_username}:{mongo_password}@{mongo_url}"
self.redis_host = getenv('REDIS_HOST', 'localhost')
self.redis_port = int(getenv('REDIS_PORT', '6379'))
self.redis_username = getenv('REDIS_USERNAME', None)
self.redis_password = getenv('REDIS_PASSWORD', None)
self.lease_ttl = int(getenv('LEASE_TTL', '10'))
self.heartbeat_ttl = int(getenv('HEARTBEAT_TTL', '20'))
self.pod_id = getenv('HOSTNAME', 'localhost')
self.poll_interval = int(getenv('POLL_INTERVAL', '5'))
mongo_url = getenv('MONGODB_URL', 'localhost:27017')
mongo_username = getenv('MONGODB_USERNAME', 'sientia')
mongo_password = getenv('MONGODB_PASSWORD', 'sientia')
self.mongo_database = getenv('MONGODB_DATABASE', 'sientia')
self.mongo_connection_string = f'mongodb://{mongo_username}:{mongo_password}@{mongo_url}'
self.kafka_servers = kafka_servers.split(",")
self.kafka_servers = kafka_servers.split(',')
self.logger = get_logger(__name__)
self.notification_handler = NotificationHandler(
connection_string=self.mongo_connection_string,
database=self.mongo_database,
logger=self.logger,
project_name="opc_ingestor"
project_name='opc_ingestor',
)
self.metadata = {
@@ -148,7 +146,7 @@ class Ingestor:
"""
if not acquired:
self.logger.warning("No slots available")
self.logger.warning('No slots available')
else:
# Subscribe to acquired slots
@@ -198,7 +196,7 @@ class Ingestor:
# Get slot lease
acquired = self.ingestor_manager.get_slot_leases()
self.logger.info(f"Acquired slots: {acquired}")
self.logger.info(f'Acquired slots: {acquired}')
await self.handle_acquired_tags(acquired)
@@ -229,9 +227,7 @@ class Ingestor:
# Get slot lease
self.ingestor_manager.get_slot_leases(1)
async def manage_leases(
self, available_slots: int, lacking_ingestors: int, slot_diff: int
):
async def manage_leases(self, available_slots: int, lacking_ingestors: int, slot_diff: int):
"""
Manages the allocation and deallocation of slot leases for ingestors.
@@ -257,14 +253,13 @@ class Ingestor:
if available_slots > 0 and lacking_ingestors > 0:
# Some ingestors are inactive, so there are "available_slots" slots available
self.logger.info(f"Slots available: {available_slots}")
self.logger.info(f'Slots available: {available_slots}')
# Get slot lease
self.ingestor_manager.get_slot_leases(available_slots)
elif lacking_ingestors <= 0 and slot_diff > 0:
self.logger.info(f"Extra slots available: {slot_diff}")
self.logger.info(f'Extra slots available: {slot_diff}')
# There's enough slots for all ingestors, but this ingestor has more than one slot
# So we need to drop the extra leases
@@ -281,7 +276,7 @@ class Ingestor:
len(self.ingestor_manager.managed_tags)
)
async def update_ingestor_manager(self, old_managed_tags: Dict[str, Any]):
async def update_ingestor_manager(self, old_managed_tags: dict[str, Any]):
"""
Updates the ingestor manager with new managed tags and handles configuration changes.
@@ -302,37 +297,39 @@ class Ingestor:
- Updates metrics to reflect current state
"""
self.logger.debug(
f"Current managed tags: {self.ingestor_manager.managed_tags}")
self.logger.debug(f'Current managed tags: {self.ingestor_manager.managed_tags}')
await self.ingestor_manager.update_opc_servers()
new_managed_tags = deepcopy(self.ingestor_manager.managed_tags)
self.logger.debug(
f"Comparing new managed tags {new_managed_tags} with old managed tags {old_managed_tags}"
f'Comparing new managed tags {new_managed_tags} with old managed tags {old_managed_tags}'
)
keys = set(new_managed_tags) | set(old_managed_tags)
changes = {k: (new_managed_tags.get(k), old_managed_tags.get(k))
for k in keys if new_managed_tags.get(k) != old_managed_tags.get(k)}
changes = {
k: (new_managed_tags.get(k), old_managed_tags.get(k))
for k in keys
if new_managed_tags.get(k) != old_managed_tags.get(k)
}
self.logger.debug(f"Changes: {changes}")
self.logger.debug(f'Changes: {changes}')
for slot, config in new_managed_tags.items():
if slot not in old_managed_tags:
self.logger.info(f"Subscribing to new slot {slot}")
self.logger.info(f'Subscribing to new slot {slot}')
await self.ingestor_manager.subscribe_to_tags({slot: config})
continue
if config != old_managed_tags[slot]:
self.logger.info(f"Resubscribing to slot {slot}")
self.logger.info(f'Resubscribing to slot {slot}')
await self.ingestor_manager.unsubscribe_slot(slot)
await self.ingestor_manager.subscribe_to_tags({slot: config})
for slot in old_managed_tags.keys():
if slot not in new_managed_tags:
self.logger.info(f"Unsubscribing from slot {slot}")
self.logger.info(f'Unsubscribing from slot {slot}')
await self.ingestor_manager.unsubscribe_slot(slot)
# Ensure the gauge is updated after any potential changes here
@@ -366,7 +363,7 @@ class Ingestor:
self.ingestor_manager.declare_active()
self.logger.info("Polling for slot updates...")
self.logger.info('Polling for slot updates...')
# Get active ingestors
current_managed_tags = deepcopy(self.ingestor_manager.managed_tags)
@@ -380,14 +377,14 @@ class Ingestor:
metrics.ACTIVE_INGESTORS.set(number_of_ingestors)
# Handle no slots
self.logger.info("Managing no slots...")
self.logger.info('Managing no slots...')
self.manage_no_slots(number_of_slots)
available_slots = number_of_slots - number_of_leases
lacking_ingestors = number_of_slots - number_of_ingestors
slot_diff = len(self.ingestor_manager.managed_tags) - 1
self.logger.info("Managing leases...")
self.logger.info('Managing leases...')
await self.manage_leases(available_slots, lacking_ingestors, slot_diff)
# Update managed slots gauge
@@ -396,23 +393,23 @@ class Ingestor:
)
self.logger.debug(
f"Active ingestors: {ingestors}, "
f"Number of slots: {number_of_slots}, "
f"Number of leases: {number_of_leases}, "
f"Managed tags: {self.ingestor_manager.managed_tags}, "
f"Managed servers: {self.ingestor_manager.opc_managers}"
f'Active ingestors: {ingestors}, '
f'Number of slots: {number_of_slots}, '
f'Number of leases: {number_of_leases}, '
f'Managed tags: {self.ingestor_manager.managed_tags}, '
f'Managed servers: {self.ingestor_manager.opc_managers}'
)
if not self.ingestor_manager.managed_tags:
# No slots acquired
self.logger.info("No slots acquired in this loop")
self.logger.info('No slots acquired in this loop')
# Update opc servers
self.logger.info("Updating slot config...")
self.logger.info('Updating slot config...')
self.ingestor_manager.update_slot_config()
# Check OPC cycles
self.logger.info("Checking OPC servers integrity...")
self.logger.info('Checking OPC servers integrity...')
self.ingestor_manager.check_opc_servers_integrity()
self.logger.info("Updating managed tags...")
self.logger.info('Updating managed tags...')
await self.update_ingestor_manager(current_managed_tags)