SIENTIAPDE-1084

Remove deprecated files and configurations, including .env, Dockerfile, docker-compose.yml, and client-schedule.py. Update README.md to reflect new architecture and features, enhancing clarity on system capabilities and workflows. Adjust values.yaml for image tag and replica count, and improve code documentation across various modules for better maintainability.

scouter/activities/mongodb.py

@@ -15,13 +15,20 @@ with workflow.unsafe.imports_passed_through():
 
 def clear_mongo_id(docs: list) -> list:
     """
-    Remove the MongoDB internal `_id` field from the document.
+    Remove MongoDB internal `_id` fields from documents.
+
+    This utility function recursively removes the MongoDB `_id` field from
+    documents and nested structures. It's used to clean data before
+    processing or export operations.
 
     Args:
-        docs (list): The document to clear.
+        docs (list): List of documents to clean
 
     Returns:
-        list: The documents without the `_id` field.
+        list: Documents with `_id` fields removed
+
+    Note:
+        This function modifies the input list in-place and returns the same reference
     """
     for doc in docs:
         if isinstance(doc, list):
@@ -41,9 +48,35 @@ def clear_mongo_id(docs: list) -> list:
 
 
 class MongoDB(BaseActivity):
+    """
+    MongoDB operations for data retrieval and storage.
+
+    This class provides MongoDB connectivity and operations for the Scouter system.
+    It handles:
+    - Connection management with automatic reconnection
+    - Data retrieval with timestamp-based filtering
+    - Document cleaning and preprocessing
+    - Error handling and notification integration
+
+    The class implements Temporal activities for MongoDB operations, enabling
+    distributed data processing with fault tolerance and monitoring.
+    """
+
     def __init__(self, connection_string: str, database_name: str,
                  logger: Logger,
                  notification_handler: NotificationHandler):
+        """
+        Initialize MongoDB connection and services.
+
+        Args:
+            connection_string (str): MongoDB connection URI string
+            database_name (str): Name of the target database
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+
+        Raises:
+            ConnectionError: If MongoDB connection fails
+        """
         self.connection_string = connection_string
         self.database_name = database_name
 
@@ -63,7 +96,10 @@ class MongoDB(BaseActivity):
 
     def shutdown(self):
         """
-        Close the MongoDB client connection.
+        Gracefully close MongoDB client connection.
+
+        This method ensures proper cleanup of MongoDB connections to prevent
+        connection leaks and ensure graceful application termination.
         """
         try:
             if self.client:
@@ -75,14 +111,34 @@ class MongoDB(BaseActivity):
 
     def __del__(self):
         """
-        Destructor to ensure MongoDB client is closed when the object is deleted.
+        Destructor to ensure MongoDB client is closed.
+
+        This destructor ensures that MongoDB connections are properly closed
+        when the object is garbage collected, preventing resource leaks.
         """
         self.shutdown()
 
     @activity.defn(name="load_latest_data")
     async def load_latest_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
-        Loads the latest data from MongoDB.
+        Load the latest data from MongoDB collection since a specified timestamp.
+
+        This activity retrieves data from a MongoDB collection, optionally
+        filtering by timestamp to enable incremental data processing. It
+        handles connection management and provides comprehensive error reporting.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - collection_name (str): Name of the MongoDB collection
+                - last_data_timestamp (str | None): Last processed timestamp for filtering
+
+        Returns:
+            dict[str, Any]: Retrieved data, or empty dict if no data found
+
+        Raises:
+            Exception: If MongoDB operation fails
         """
         metadata = input_data['metadata']
         collection_name = input_data['collection_name']