SIENTIAPDE-988

Enhance README and Docker setup; improve IngestorManager and DataManager functionality, add error handling, and implement unit tests for OPC initialization and subscription management.

ingestor/ingestor.py

@@ -6,14 +6,14 @@ from time import sleep
 
 def main():
     # Get os parameters
-    kafka_servers = getenv("KAFKA_SERVERS")
-    redis_host = getenv("REDIS_HOST")
-    redis_port = int(getenv("REDIS_PORT"))
-    lease_ttl = int(getenv("LEASE_TTL"))
-    heartbeat_ttl = int(getenv("HEARTBEAT_TTL"))
-    pod_id = getenv("HOSTNAME")
-    number_of_ingestors = int(getenv("REPLICA_COUNT"))
-    poll_interval = int(getenv("POLL_INTERVAL"))
+    kafka_servers = getenv("KAFKA_SERVERS", "localhost:9092")
+    redis_host = getenv("REDIS_HOST", "localhost")
+    redis_port = int(getenv("REDIS_PORT", 6379))
+    lease_ttl = int(getenv("LEASE_TTL", 20))
+    heartbeat_ttl = int(getenv("HEARTBEAT_TTL", 10))
+    pod_id = getenv("HOSTNAME", "localhost")
+    number_of_ingestors = int(getenv("REPLICA_COUNT", 1))
+    poll_interval = int(getenv("POLL_INTERVAL", 5))
 
     logger = getLogger(__name__)
     logger.setLevel(getenv("LOG_LEVEL", "INFO"))
@@ -29,22 +29,23 @@ def main():
         lease_ttl, heartbeat_ttl, pod_id,
         number_of_ingestors, poll_interval, logger
     )
-    # Declare ingestor ative
-    ingestor_manager.declare_active()
+
     # Get slot lease
     acquired = ingestor_manager.get_slot_leases()
+    logger.info(f"Acquired slots: {acquired}")
 
-    # Subscribe to acquired slots
-    ingestor_manager.update_opc_servers()
-    ingestor_manager.subscribe_to_tags(acquired)
+    if not acquired:
+        logger.warning("No slots available")
+
+    else:
+        # Declare ingestor ative
+        ingestor_manager.declare_active()
+        # Subscribe to acquired slots
+        ingestor_manager.update_opc_servers()
+        ingestor_manager.subscribe_to_tags(acquired)
 
     while True:
-        # Declare ingestor as active
-        ingestor_manager.declare_active()
-
-        # Update opc servers
-        ingestor_manager.update_slot_config()
-
+        logger.info("Polling for slot updates...")
         # Get active ingestors
         ingestors = ingestor_manager.get_active_ingestors()
 
@@ -53,13 +54,18 @@ def main():
 
         if ingestor_diff > 0:
             # Some ingestors are innactive, so theres "ingestor_diff" slots available
+            logger.info(f"Slots available: {ingestor_diff}")
 
             # Get slot lease
             acquired = ingestor_manager.get_slot_leases(ingestor_diff)
 
-            # Subscribe to acquired slots
-            ingestor_manager.update_opc_servers()
-            ingestor_manager.subscribe_to_tags(acquired)
+            if not acquired:
+                logger.info("No slots acquired")
+
+            else:
+                # Subscribe to acquired slots
+                ingestor_manager.update_opc_servers()
+                ingestor_manager.subscribe_to_tags(acquired)
 
         elif slot_diff > 0:
             # Some ingestors are active and without slots, so we need to drop
@@ -68,6 +74,17 @@ def main():
 
             ingestor_manager.drop_slot_leases(overleases)
 
+        # Update opc servers
+        ingestor_manager.update_slot_config()
+
+        if not ingestor_manager.managed_tags:
+            logger.info("No managed tags found")
+            sleep(poll_interval)
+            continue
+
+        # Declare ingestor as active
+        ingestor_manager.declare_active()
+
         # Sleep for poll interval
         sleep(poll_interval)