SIENTIAPDE-1325

Refactor MongoDB, Redis, and Gates classes to extend SientiaMonitoring instead of BaseActivity. Update requirements to point to local dataops library path. Implement repository pattern for MongoDB and Redis operations, enhancing code organization and maintainability.

scouter/activities/mongodb.py

@@ -13,45 +13,13 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.notifications.models import NotificationLevel
     from sientia_do.observability.logger import Logger
-    from sientia_do.temporal.activities.base import BaseActivity
+    from sientia_do.repository.mongodb_repository import MongoDBRepository
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
     from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
 
 
-def clear_mongo_id(docs: list) -> list:
-    """
-    Remove MongoDB internal `_id` fields from documents.
 
-    This utility function recursively removes the MongoDB `_id` field from
-    documents and nested structures. It's used to clean data before
-    processing or export operations.
-
-    Args:
-        docs (list): List of documents to clean
-
-    Returns:
-        list: Documents with `_id` fields removed
-
-    Note:
-        This function modifies the input list in-place and returns the same reference
-    """
-    for doc in docs:
-        if isinstance(doc, list):
-            clear_mongo_id(doc)
-
-        elif isinstance(doc, dict):
-            if '_id' in doc:
-                del doc['_id']
-
-            for _key, value in doc.items():
-                if isinstance(value, list):
-                    clear_mongo_id(value)
-                elif isinstance(value, dict):
-                    clear_mongo_id([value])
-
-    return docs
-
-
-class MongoDB(BaseActivity):
+class MongoDB(SientiaMonitoring):
     """
     MongoDB operations for data retrieval and storage.
 
@@ -85,37 +53,22 @@ class MongoDB(BaseActivity):
         Raises:
             ConnectionError: If MongoDB connection fails
         """
-        self.connection_string = connection_string
-        self.database_name = database_name
-
-        self.client: MongoClient = MongoClient(
-            self.connection_string, serverSelectionTimeoutMS=5000
-        )
-        self.client.server_info()  # Trigger an exception if connection fails
-
-        self.database = self.client[self.database_name]
-
-        # Initialize MongoDB client here (omitted for brevity)
-        logger.info('MongoDB connection initialized')
-
-        BaseActivity.__init__(
-            self, logger=logger, notification_handler=notification_handler, set_error_counter=True
+        
+        self.mongodb_repository = MongoDBRepository(
+            connection_string=connection_string,
+            database_name=database_name,
+            logger=logger,
+            notification_handler=notification_handler,
         )
 
-    def shutdown(self):
-        """
-        Gracefully close MongoDB client connection.
+        SientiaMonitoring.__init__(self, logger=logger, notification_handler=notification_handler)
 
-        This method ensures proper cleanup of MongoDB connections to prevent
-        connection leaks and ensure graceful application termination.
+    def close(self):
         """
-        try:
-            if self.client:
-                self.logger.info('Closing MongoDB connection...')
-                self.client.close()
-                self.logger.info('MongoDB connection closed successfully')
-        except Exception as e:
-            self.logger.error(f'Failed to close MongoDB connection: {e}')
+        Close the MongoDB connection.
+        """
+        self.mongodb_repository.close()
+        SientiaMonitoring.shutdown(self)
 
     def __del__(self):
         """
@@ -124,7 +77,7 @@ class MongoDB(BaseActivity):
         This destructor ensures that MongoDB connections are properly closed
         when the object is garbage collected, preventing resource leaks.
         """
-        self.shutdown()
+        self.close()
 
     @activity.defn(name='load_latest_data')
     async def load_latest_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
@@ -166,9 +119,11 @@ class MongoDB(BaseActivity):
 
             self.debug(f'Data filter: {data_filter}', metadata=metadata)
 
-            data = list(self.database[collection_name].find(data_filter, {'_id': 0}))
-
-            data = clear_mongo_id(data)
+            data = self.mongodb_repository.find(
+                collection_name=collection_name,
+                filters=data_filter,
+                metadata=metadata,
+            )
 
             self.debug(f'Collected: {data}', metadata=metadata)