from logging import Formatter, StreamHandler, getLogger from ingestor.managers.ingestor_manager import IngestorManager from os import getenv from time import sleep def main(): # Get os parameters kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092") redis_host = getenv("REDIS_HOST", "localhost") redis_port = int(getenv("REDIS_PORT", 6379)) lease_ttl = int(getenv("LEASE_TTL", 20)) heartbeat_ttl = int(getenv("HEARTBEAT_TTL", 10)) pod_id = getenv("HOSTNAME", "localhost") number_of_ingestors = int(getenv("REPLICA_COUNT", 1)) poll_interval = int(getenv("POLL_INTERVAL", 5)) logger = getLogger(__name__) logger.setLevel(getenv("LOG_LEVEL", "INFO")) handler = StreamHandler() formatter = Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) logger.addHandler(handler) ingestor_manager = IngestorManager( kafka_servers, redis_host, redis_port, lease_ttl, heartbeat_ttl, pod_id, number_of_ingestors, poll_interval, logger ) # Get slot lease acquired = ingestor_manager.get_slot_leases() logger.info(f"Acquired slots: {acquired}") if not acquired: logger.warning("No slots available") else: # Declare ingestor ative ingestor_manager.declare_active() # Subscribe to acquired slots ingestor_manager.update_opc_servers() ingestor_manager.subscribe_to_tags(acquired) while True: logger.info("Polling for slot updates...") # Get active ingestors ingestors = ingestor_manager.get_active_ingestors() ingestor_diff = number_of_ingestors - len(ingestors) slot_diff = len(ingestor_manager.managed_tags) - 1 if ingestor_diff > 0: # Some ingestors are innactive, so theres "ingestor_diff" slots available logger.info(f"Slots available: {ingestor_diff}") # Get slot lease acquired = ingestor_manager.get_slot_leases(ingestor_diff) if not acquired: logger.info("No slots acquired") else: # Subscribe to acquired slots ingestor_manager.update_opc_servers() ingestor_manager.subscribe_to_tags(acquired) elif slot_diff > 0: # Some ingestors are active and without slots, so we need to drop overleases = list(ingestor_manager.managed_tags.keys())[1:] ingestor_manager.drop_slot_leases(overleases) # Update opc servers ingestor_manager.update_slot_config() if not ingestor_manager.managed_tags: logger.info("No managed tags found") sleep(poll_interval) continue # Declare ingestor as active ingestor_manager.declare_active() # Sleep for poll interval sleep(poll_interval) if __name__ == "__main__": main()