Update MongoDB connection handling in DataManager and enhance IngestorManager configuration

- Updated DataManager to simplify MongoDB connection logic by removing retry loop and adding logging for connection attempts.
- Enhanced IngestorManager to accept MongoDB connection parameters and export_to_kafka flag as part of its initialization.
- Updated requirements.txt to use the latest version of the sientia-dataops-library.

tests/unit/managers/test_data_manager.py

@@ -7,20 +7,28 @@ from ingestor.managers.data_manager import DataManager
 
 @fixture
 @patch("ingestor.managers.data_manager.KafkaProducer")
-def data_manager(kafka):
+@patch("ingestor.managers.data_manager.MongoClient")
+def data_manager(mongo, kafka):
     return DataManager(
         kafka_servers="localhost:9092",
+        mongo_connection_string="mongodb://localhost:27017",
+        mongo_database="sientia",
+        export_to_kafka=True,
         logger=MagicMock(),
         notification_handler=MagicMock()
     )
 
 
 @patch("ingestor.managers.data_manager.KafkaProducer")
-def test___init___success(kafka):
+@patch("ingestor.managers.data_manager.MongoClient")
+def test___init___success(mongo, kafka):
     logger_mock = MagicMock()
 
     data_manager = DataManager(
         kafka_servers="localhost:9092",
+        mongo_connection_string="mongodb://localhost:27017",
+        mongo_database="sientia",
+        export_to_kafka=True,
         logger=logger_mock,
         notification_handler=MagicMock()
     )
@@ -38,16 +46,20 @@ def test___init___success(kafka):
         "DataManager initialized with Kafka servers: localhost:9092"
     )
     logger_mock.error.assert_not_called()
-    assert logger_mock.info.call_count == 2
+    assert logger_mock.info.call_count == 4
 
 
 @patch("ingestor.managers.data_manager.KafkaProducer")
-def test___init___second_attempt(kafka):
+@patch("ingestor.managers.data_manager.MongoClient")
+def test___init___second_attempt(mongo, kafka):
     kafka.side_effect = [NoBrokersAvailable, MagicMock()]
     logger_mock = MagicMock()
 
     data_manager = DataManager(
         kafka_servers="localhost:9092",
+        mongo_connection_string="mongodb://localhost:27017",
+        mongo_database="sientia",
+        export_to_kafka=True,
         logger=logger_mock,
         notification_handler=MagicMock()
     )
@@ -71,17 +83,21 @@ def test___init___second_attempt(kafka):
     logger_mock.error.assert_called_once_with(
         "Kafka servers localhost:9092 are not available. Retrying..."
     )
-    assert logger_mock.info.call_count == 3
+    assert logger_mock.info.call_count == 5
 
 
 @patch("ingestor.managers.data_manager.KafkaProducer")
-def test___init___failure_max_attempts(kafka):
+@patch("ingestor.managers.data_manager.MongoClient")
+def test___init___failure_max_attempts(mongo, kafka):
     kafka.side_effect = NoBrokersAvailable
     logger_mock = MagicMock()
 
     try:
         DataManager(
             kafka_servers="localhost:9092",
+            mongo_connection_string="mongodb://localhost:27017",
+            mongo_database="sientia",
+            export_to_kafka=True,
             logger=logger_mock,
             notification_handler=MagicMock()
         )
@@ -130,6 +146,16 @@ def test_shutdown_no_producer(data_manager):
     )
 
 
+def test_shutdown_no_mongo_client(data_manager):
+    data_manager.mongo_client = None
+
+    data_manager.shutdown()
+
+    data_manager.logger.warning.assert_any_call(
+        "MongoDB client is already closed or not initialized."
+    )
+
+
 def test_shutdown_exception(data_manager):
     data_manager.kafka_producer.flush = MagicMock(
         side_effect=Exception("Test error"))
@@ -141,6 +167,17 @@ def test_shutdown_exception(data_manager):
     )
 
 
+def test_shutdown_exception_mongo(data_manager):
+    data_manager.mongo_client.close = MagicMock(
+        side_effect=Exception("Test error"))
+
+    data_manager.shutdown()
+
+    data_manager.logger.error.assert_called_once_with(
+        "Error closing MongoDB client: Test error"
+    )
+
+
 def test___del__(data_manager):
     data_manager.shutdown = MagicMock()
     data_manager.__del__()
@@ -190,6 +227,18 @@ def test_publish(data_manager):
     data_manager.kafka_producer.flush.assert_called_once()
 
 
+def test_publish_no_kafka(data_manager):
+    data_manager.export_to_kafka = False
+    topic = "test_topic"
+    data = {"key": "value"}
+
+    data_manager.publish(topic, data)
+
+    data_manager.logger.debug.assert_any_call(
+        f"Skipping message to topic {topic}: {data}"
+    )
+
+
 @patch("ingestor.managers.data_manager.traceback")
 def test_publish_error(traceback, data_manager):
     topic = "test_topic"
@@ -215,3 +264,19 @@ def test_publish_error(traceback, data_manager):
         level=NotificationLevel.ERROR,
         attachment_content=traceback.format_exc.return_value
     )
+
+
+def test_publish_error_mongo(data_manager):
+    data_manager.export_to_kafka = False
+    data_manager.mongo_db.__getitem__.return_value.insert_one = MagicMock(
+        side_effect=Exception("Test error"))
+
+    data_manager.publish("test_topic", {"key": "value"})
+
+    data_manager.notification_handler.build_and_send_notification.assert_called_once_with(
+        notification_id="MONGO_PRODUCER_ERROR_test_topic",
+        message="Error inserting message to MongoDB: Test error",
+        block="mongo_producer",
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY
+    )