gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1049

Browse Source 
Refactor logging statements for consistency and clarity; update environment variable parsing to handle string inputs.
This commit is contained in:
vitor-aignosi
2025-05-20 09:12:49 -03:00
parent 9196861033
commit bb4ab7d209
2 changed files with 50 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
ingestor/ingestor.py
View File

@@ -1,9 +1,10 @@
from logging import Formatter, StreamHandler, getLogger
from os import getenv
from ingestor.managers.ingestor_manager import IngestorManager
from sientia_do.notifications.handlers import NotificationHandler
from ingestor.managers.ingestor_manager import IngestorManager
class Ingestor:
def __init__(self):
@@ -29,13 +30,13 @@ class Ingestor:
kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092")
self.redis_host = getenv("REDIS_HOST", "localhost")
self.redis_port = int(getenv("REDIS_PORT", 6379))
self.redis_port = int(getenv("REDIS_PORT", '6379'))
self.redis_username = getenv("REDIS_USERNAME", None)
self.redis_password = getenv("REDIS_PASSWORD", None)
self.lease_ttl = int(getenv("LEASE_TTL", 10))
self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", 20))
self.lease_ttl = int(getenv("LEASE_TTL", '10'))
self.heartbeat_ttl = int(getenv("HEARTBEAT_TTL", '20'))
self.pod_id = getenv("HOSTNAME", "localhost")
self.poll_interval = int(getenv("POLL_INTERVAL", 5))
self.poll_interval = int(getenv("POLL_INTERVAL", '5'))
self.kafka_servers = kafka_servers.split(",")
self.logger = None
@@ -49,9 +50,8 @@ class Ingestor:
model_name="-",
model="-"
)
# build args for build notificarions components
# call build notifications components
self.ingestor_manager = None
def init_logger(self):
"""
@@ -128,7 +128,7 @@ class Ingestor:
# Get slot lease
acquired = self.ingestor_manager.get_slot_leases()
self.logger.info(f"Acquired slots: {acquired}")
self.logger.info("Acquired slots: %s", acquired)
self.handle_acquired_tags(acquired)
@@ -171,7 +171,7 @@ class Ingestor:
if available_slots > 0 and lacking_ingestors > 0:
# Some ingestors are innactive, so theres "available_slots" slots available
self.logger.info(f"Slots available: {available_slots}")
self.logger.info("Slots available: %s", available_slots)
# Get slot lease
acquired = self.ingestor_manager.get_slot_leases(available_slots)
@@ -180,7 +180,7 @@ class Ingestor:
elif lacking_ingestors == 0 and slot_diff > 0:
self.logger.info(f"Extra slots available: {slot_diff}")
self.logger.info("Extra slots available: %s", slot_diff)
# There's enough slots for all ingestors, but this ingestor has more than one slot
# So we need to drop the extra leases
@@ -210,37 +210,35 @@ class Ingestor:
self.ingestor_manager.declare_active()
self.logger.info("Polling for slot updates...")
# Get active ingestors[
# Get active ingestors
ingestors = self.ingestor_manager.get_active_ingestors()
self.logger.debug(f"Active ingestors: {ingestors}")
number_of_ingestors = len(ingestors)
self.logger.debug(f"Number of ingestors: {number_of_ingestors}")
number_of_leases = self.ingestor_manager.get_number_of_leases()
self.logger.debug(f"Number of leases: {number_of_leases}")
number_of_slots = self.ingestor_manager.get_number_of_slots()
self.logger.debug(f"Number of slots: {number_of_slots}")
# Handle no slots
self.logger.debug("Managing no slots...")
self.manage_no_slots(number_of_slots)
available_slots = number_of_slots - number_of_leases
self.logger.debug(f"Available slots: {available_slots}")
lacking_ingestors = number_of_slots - number_of_ingestors
self.logger.debug(f"Lacking ingestors: {lacking_ingestors}")
slot_diff = len(self.ingestor_manager.managed_tags) - 1
self.logger.debug(f"Slot diff: {slot_diff}")
self.logger.debug("Managing leases...")
self.manage_leases(available_slots, lacking_ingestors, slot_diff)
self.logger.debug(
f"Active ingestors: {ingestors}, "
f"Number of slots: {number_of_slots}, "
f"Number of leases: {number_of_leases}, "
f"Managed tags: {self.ingestor_manager.managed_tags}"
f"Managed servers: {self.ingestor_manager.opc_managers}"
"Active ingestors: %s, "
"Number of slots: %s, "
"Number of leases: %s, "
"Managed tags: %s, "
"Managed servers: %s",
ingestors,
number_of_slots,
number_of_leases,
self.ingestor_manager.managed_tags,
self.ingestor_manager.opc_managers
)
if not self.ingestor_manager.managed_tags:
# No slots acquired