SIENTIAPDE-1166

feat: add TTL index functionality to MongoDB activities and update requirements

- Updated MongoDB class to accept TTL index duration in seconds.
- Implemented create_collection_with_ttl_index method to create collections with TTL indexes.
- Added environment variable for TTL index duration in connectors_config.
- Updated tests to cover new TTL index functionality and ensure proper behavior.
- Bumped sientia-dataops-library version to 1.3.4 in requirements.txt.

tests/orchestrator/activities/test_activities.py

@@ -18,7 +18,8 @@ def test___init__(mock_formatters_init, mock_slot_manager_init,
 
     mongo_db_config = {
         'connection_string': 'mongodb://localhost:27017',
-        'database_name': 'test_db'
+        'database_name': 'test_db',
+        'ttl_index_seconds': 3600
     }
 
     redis_config = {
@@ -65,6 +66,7 @@ def test___init__(mock_formatters_init, mock_slot_manager_init,
         ANY,
         connection_string='mongodb://localhost:27017',
         database_name='test_db',
+        ttl_index_seconds=3600,
         logger=logger,
         notification_handler=notification_handler
     )

tests/orchestrator/activities/test_mongo_db.py

@@ -55,6 +55,7 @@ def mongo_db(mongo_mock):
         MongoDB(
             connection_string="mongodb://localhost:27017",
             database_name="test_db",
+            ttl_index_seconds=3600,
             logger=MagicMock(),
             notification_handler=MagicMock()
         )
@@ -68,6 +69,7 @@ def test___init__(mongo_mock):
     mongo_db = MongoDB(
         connection_string="mongodb://localhost:27017",
         database_name="test_db",
+        ttl_index_seconds=3600,
         logger=MagicMock(),
         notification_handler=MagicMock()
     )
@@ -398,3 +400,121 @@ async def test_delete_pipelines_timestamps_failure(datetime_mock, mongo_db):
 
     else:
         assert False, "Expected an exception to be raised"
+
+
+@mark.asyncio
+async def test_create_collection_with_ttl_index_success(mongo_db):
+    input_data = {
+        **metadata,
+        "pipelines": {
+            "scouter-pipeline": {
+                "topic": "raw_scouter_pipeline"
+            },
+            "scouter-pipeline-2": {
+                "topic": "raw_scouter_pipeline_2"
+            },
+            "scouter-pipeline-3": {
+                "topic": "raw_scouter_pipeline_3"
+            }
+        }
+    }
+
+    mongo_db.database.list_collection_names.return_value = [
+        "raw_scouter_pipeline_2",
+        "raw_scouter_pipeline_3"
+    ]
+
+    collection_1 = MagicMock(
+        list_indexes=MagicMock(
+            return_value=[
+                {
+                    "key": "asdad",
+                }
+            ]
+        )
+    )
+
+    collection_2 = MagicMock(
+        list_indexes=MagicMock(
+            return_value=[
+                {
+                    "key": "inserted_at",
+                    "expireAfterSeconds": None
+                }
+            ]
+        )
+    )
+    collection_3 = MagicMock(
+        list_indexes=MagicMock(
+            return_value=[
+                {
+                    "key": "inserted_at",
+                    "expireAfterSeconds": 3600
+                }
+            ]
+        )
+    )
+
+    mongo_db.database.__getitem__ = MagicMock(
+        side_effect=[
+            collection_1,
+            collection_2,
+            collection_3
+        ]
+    )
+
+    await mongo_db.create_collection_with_ttl_index(input_data)
+
+    mongo_db.database.list_collection_names.assert_called_once_with()
+
+    mongo_db.database.create_collection.assert_called_once_with(
+        "raw_scouter_pipeline"
+    )
+
+    collection_1.list_indexes.assert_called_once()
+    collection_1.create_index.assert_called_once_with(
+        "inserted_at",
+        expireAfterSeconds=3600,
+        background=True
+    )
+
+    collection_2.list_indexes.assert_called_once()
+    collection_2.create_index.assert_called_once_with(
+        "inserted_at",
+        expireAfterSeconds=3600,
+        background=True
+    )
+
+    collection_3.list_indexes.assert_called_once()
+    collection_3.create_index.assert_not_called()
+
+
+@mark.asyncio
+async def test_create_collection_with_ttl_index_failure(mongo_db):
+    input_data = {
+        **metadata,
+        "pipelines": {
+            "scouter-pipeline": {
+                "topic": "raw_scouter_pipeline"
+            }
+        }
+    }
+
+    mongo_db.database.list_collection_names.return_value = []
+
+    mongo_db.database.create_collection.side_effect = Exception("Error")
+
+    try:
+        await mongo_db.create_collection_with_ttl_index(input_data)
+    except Exception as e:
+        assert str(e) == "Error"
+        mongo_db.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id="MONGODB_CREATE_COLLECTION_ERROR",
+            message="Failed to create collection raw_scouter_pipeline with TTL index: Error",
+            block="create_collection_with_ttl_index",
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY
+        )
+    else:
+        assert False, "Expected an exception to be raised"

tests/orchestrator/utils/test_connectors_config.py

@@ -57,10 +57,11 @@ def test_build_mongo_db_config_with_env_vars():
     environ['MONGODB_PASSWORD'] = 'sientia1'
     environ['MONGODB_URL'] = 'localhost:27018'
     environ['MONGODB_DATABASE_NAME'] = 'test_db'
-
+    environ['MONGODB_TTL_INDEX_HOURS'] = '2'
     assert build_mongodb_config() == {
         'connection_string': 'mongodb://sientia1:sientia1@localhost:27018',
-        'database_name': 'test_db'
+        'database_name': 'test_db',
+        'ttl_index_seconds': 7200
     }
 
 
@@ -69,10 +70,11 @@ def test_build_mongo_db_config_with_defaults():
     environ.pop('MONGODB_PASSWORD', None)
     environ.pop('MONGODB_DATABASE_NAME', None)
     environ.pop('MONGODB_URL', None)
-
+    environ.pop('MONGODB_TTL_INDEX_HOURS', None)
     assert build_mongodb_config() == {
         'connection_string': 'mongodb://root:wKZDbMNU1c@localhost:27018',
-        'database_name': 'sientia'
+        'database_name': 'sientia',
+        'ttl_index_seconds': 3600
     }
 
 

tests/orchestrator/workflows/test_orchestrator.py

@@ -166,6 +166,18 @@ async def test_run(workflow_mock, orchestrator):
         )
     ])
 
+    workflow_mock.start_activity_method.assert_has_calls([
+        call(
+            Activities.create_collection_with_ttl_index,
+            {
+                **metadata,
+                'pipelines': workflow_mock.start_local_activity_method.return_value['scouter']
+            },
+            retry_policy=ANY,
+            start_to_close_timeout=ANY
+        )
+    ])
+
     workflow_mock.start_activity_method.assert_has_calls([
         call(
             Activities.delete_slots,