Add support for Kafka export in Ingestor and DataManager classes

- Introduced EXPORT_TO_KAFKA environment variable in Ingestor class.
- Refactored DataManager to conditionally initialize Kafka producer based on export setting.
- Updated IngestorManager to pass Redis configuration as a dictionary.
- Enhanced error handling and logging for Kafka message publishing.

ingestor/managers/ingestor_manager.py

@@ -12,14 +12,20 @@ import ingestor.metrics as metrics
 
 class IngestorManager():
     def __init__(self,
-                 kafka_servers: str, redis_host: str, redis_port: int,
+                 kafka_servers: str, redis_data: dict,
                  lease_ttl: int, heartbeat_ttl: int,  pod_id: str,
                  poll_interval: int, mongo_connection_string: str, mongo_database: str,
                  logger: Logger, notification_handler: NotificationHandler,
-                 redis_username: str = None, redis_password: str = None):
+                 export_to_kafka: bool = False):
+
+        redis_host = redis_data.get('host')
+        redis_port = redis_data.get('port')
+        redis_username = redis_data.get('username', None)
+        redis_password = redis_data.get('password', None)
 
         self.data_manager = DataManager(
-            kafka_servers, mongo_connection_string, mongo_database, logger, notification_handler)
+            kafka_servers, mongo_connection_string, mongo_database,
+            export_to_kafka, logger, notification_handler)
         self.opc_managers = {}
         self.resource_manager = ResourceManager(
             redis_host, redis_port, lease_ttl, heartbeat_ttl, pod_id, redis_username, redis_password
@@ -156,7 +162,7 @@ class IngestorManager():
                     _a = [self.managed_tags[slot].pop(server, None)
                           for slot, _value in current_managed_tags.items()]
 
-        for server in list(self.opc_managers.keys()):
+        for server in self.opc_managers.keys():
             if server not in registered_servers:
                 self.logger.warning(
                     f"Server {server} not found in managed tags. "