SIENTIAPDE-1084

Enhance documentation and improve shutdown methods for DataManager and IngestorManager

- Updated README.md to clarify persistent storage capabilities.
- Enhanced docstrings in DataManager and IngestorManager for shutdown processes, detailing resource cleanup and error handling.
- Improved comments in various methods to provide clearer explanations of functionality and side effects.

ingestor/managers/data_manager.py

@@ -149,7 +149,17 @@ class DataManager(BaseActivity):
                               set_error_counter=True)
 
     def shutdown(self):
-        """Closes the Kafka producer connection."""
+        """
+        Gracefully shuts down the DataManager and closes all connections.
+
+        This method ensures proper cleanup of:
+        - Kafka producer connection with message flushing
+        - MongoDB client connection
+        - Metrics recording for connection status
+
+        The method handles connection closure gracefully, logging any errors
+        that occur during shutdown while ensuring all resources are properly released.
+        """
         if self.kafka_producer:
             try:
                 self.kafka_producer.flush(timeout=10)
@@ -176,13 +186,30 @@ class DataManager(BaseActivity):
         self.shutdown()
 
     def delivery_report(self, msg: str):
-        """Callback for delivery reports from Kafka."""
+        """
+        Callback for successful Kafka message delivery reports.
+
+        This method is called by the Kafka producer when a message is successfully
+        delivered to a topic. It logs the delivery details including topic, partition,
+        and offset information for debugging and monitoring purposes.
+
+        Args:
+            msg: Kafka message object containing delivery details
+        """
         self.logger.debug(
             f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}"
         )
 
     def delivery_error(self, err: str):
-        """Callback for delivery reports from Kafka."""
+        """
+        Callback for Kafka message delivery error reports.
+
+        This method is called by the Kafka producer when a message delivery fails.
+        It logs the error details for debugging and monitoring purposes.
+
+        Args:
+            err: Error information from the failed delivery attempt
+        """
         self.logger.error(f"Delivery failed for record : {err}")
 
     def publish(self, topic: str, data: dict) -> None: