fa51d142e8
Enhance README and Docker setup; improve IngestorManager and DataManager functionality, add error handling, and implement unit tests for OPC initialization and subscription management.
94 lines
2.9 KiB
Python
94 lines
2.9 KiB
Python
from logging import Formatter, StreamHandler, getLogger
|
|
from ingestor.managers.ingestor_manager import IngestorManager
|
|
from os import getenv
|
|
from time import sleep
|
|
|
|
|
|
def main():
|
|
# Get os parameters
|
|
kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092")
|
|
redis_host = getenv("REDIS_HOST", "localhost")
|
|
redis_port = int(getenv("REDIS_PORT", 6379))
|
|
lease_ttl = int(getenv("LEASE_TTL", 20))
|
|
heartbeat_ttl = int(getenv("HEARTBEAT_TTL", 10))
|
|
pod_id = getenv("HOSTNAME", "localhost")
|
|
number_of_ingestors = int(getenv("REPLICA_COUNT", 1))
|
|
poll_interval = int(getenv("POLL_INTERVAL", 5))
|
|
|
|
logger = getLogger(__name__)
|
|
logger.setLevel(getenv("LOG_LEVEL", "INFO"))
|
|
handler = StreamHandler()
|
|
formatter = Formatter(
|
|
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
handler.setFormatter(formatter)
|
|
|
|
logger.addHandler(handler)
|
|
|
|
ingestor_manager = IngestorManager(
|
|
kafka_servers, redis_host, redis_port,
|
|
lease_ttl, heartbeat_ttl, pod_id,
|
|
number_of_ingestors, poll_interval, logger
|
|
)
|
|
|
|
# Get slot lease
|
|
acquired = ingestor_manager.get_slot_leases()
|
|
logger.info(f"Acquired slots: {acquired}")
|
|
|
|
if not acquired:
|
|
logger.warning("No slots available")
|
|
|
|
else:
|
|
# Declare ingestor ative
|
|
ingestor_manager.declare_active()
|
|
# Subscribe to acquired slots
|
|
ingestor_manager.update_opc_servers()
|
|
ingestor_manager.subscribe_to_tags(acquired)
|
|
|
|
while True:
|
|
logger.info("Polling for slot updates...")
|
|
# Get active ingestors
|
|
ingestors = ingestor_manager.get_active_ingestors()
|
|
|
|
ingestor_diff = number_of_ingestors - len(ingestors)
|
|
slot_diff = len(ingestor_manager.managed_tags) - 1
|
|
|
|
if ingestor_diff > 0:
|
|
# Some ingestors are innactive, so theres "ingestor_diff" slots available
|
|
logger.info(f"Slots available: {ingestor_diff}")
|
|
|
|
# Get slot lease
|
|
acquired = ingestor_manager.get_slot_leases(ingestor_diff)
|
|
|
|
if not acquired:
|
|
logger.info("No slots acquired")
|
|
|
|
else:
|
|
# Subscribe to acquired slots
|
|
ingestor_manager.update_opc_servers()
|
|
ingestor_manager.subscribe_to_tags(acquired)
|
|
|
|
elif slot_diff > 0:
|
|
# Some ingestors are active and without slots, so we need to drop
|
|
|
|
overleases = list(ingestor_manager.managed_tags.keys())[1:]
|
|
|
|
ingestor_manager.drop_slot_leases(overleases)
|
|
|
|
# Update opc servers
|
|
ingestor_manager.update_slot_config()
|
|
|
|
if not ingestor_manager.managed_tags:
|
|
logger.info("No managed tags found")
|
|
sleep(poll_interval)
|
|
continue
|
|
|
|
# Declare ingestor as active
|
|
ingestor_manager.declare_active()
|
|
|
|
# Sleep for poll interval
|
|
sleep(poll_interval)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|