SIENTIAPDE-1084

Remove deprecated files and configurations, including .env, Dockerfile, docker-compose.yml, and client-schedule.py. Update README.md to reflect new architecture and features, enhancing clarity on system capabilities and workflows. Adjust values.yaml for image tag and replica count, and improve code documentation across various modules for better maintainability.

scouter/activities/redis.py

@@ -14,17 +14,58 @@ with workflow.unsafe.imports_passed_through():
 
 
 class Redis(RedisBase):
+    """
+    Redis operations for data caching and temporary storage.
+
+    This class extends the base Redis functionality to provide specialized
+    operations for the Scouter system, including:
+    - Data timestamp management for incremental processing
+    - Temporary data storage with configurable TTL
+    - Data grouping and holding for batch processing
+    - Error handling and notification integration
+
+    The class implements Temporal activities for Redis operations, enabling
+    distributed data processing with fault tolerance and monitoring.
+    """
+
     def __init__(self, host: str, port: int,
                  username: str, password: str,
                  logger: Logger, notification_handler: NotificationHandler):
+        """
+        Initialize Redis connection and services.
 
+        Args:
+            host (str): Redis server hostname or IP address
+            port (int): Redis server port number
+            username (str): Redis authentication username
+            password (str): Redis authentication password
+            logger (Logger): Logger instance for operation logging
+            notification_handler (NotificationHandler): Handler for system notifications
+        """
         RedisBase.__init__(self, host, port, username,
                            password, logger, notification_handler)
 
     @activity.defn(name="get_last_data_timestamp")
     async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
         """
-        Gets the last data timestamp from redis.
+        Retrieve the last processed data timestamp from Redis.
+
+        This activity retrieves the timestamp of the last successfully processed
+        data point for a specific workflow and schedule combination. It's used
+        for incremental data processing to avoid reprocessing the same data.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - workflow_name (str): Name of the workflow
+                - schedule_name (str): Name of the data collection schedule
+
+        Returns:
+            str | None: Last processed timestamp string, or None if no previous data exists
+
+        Raises:
+            Exception: If Redis operation fails
         """
         metadata = input_data['metadata']
         key = f"last_data_timestamp:{input_data['workflow_name']}:{input_data['schedule_name']}"
@@ -55,9 +96,27 @@ class Redis(RedisBase):
         return data_hold
 
     @activity.defn(name="put_last_data_timestamp")
-    async def put_last_data_timestamp(self, input_data: dict[str, Any]):
+    async def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
         """
-        Puts the last data timestamp into redis.
+        Store the last processed data timestamp in Redis.
+
+        This activity stores the timestamp of the most recent data point that
+        has been successfully processed. The timestamp is used for incremental
+        data loading in subsequent workflow executions.
+
+        Args:
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - data (dict[str, Any]): Processed data to extract timestamp from
+                - workflow_name (str): Name of the workflow
+                - schedule_name (str): Name of the data collection schedule
+
+        Returns:
+            str | None: The timestamp that was stored, or None if no data was processed
+
+        Raises:
+            Exception: If Redis operation fails
         """
         metadata = input_data['metadata']
         key = f"last_data_timestamp:{input_data['workflow_name']}:{input_data['schedule_name']}"
@@ -96,18 +155,30 @@ class Redis(RedisBase):
         return last_data_timestamp
 
     @activity.defn(name="group_and_hold_data")
-    async def group_and_hold_data(self, input_data: dict[str, Any]):
+    async def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
-        Groups and holds data in redis. Keep a copy of the most recent
-        received data for a given pipeline and schedule. This activity updates
-        the data in redis and return the full keeped data.
+        Group data by tags and store temporarily in Redis with TTL.
+
+        This activity organizes processed data by tag names and stores it in Redis
+        with a configurable retention period. The data is grouped to enable
+        efficient batch processing and export operations.
 
         Args:
-            input_data (dict[str, Any]): The data to group and hold.
-                workflow_name (str): The name of the workflow.
-                schedule_name (str): The name of the schedule.
-                data (dict[str, Any]): The data to group and hold.
-                retention_time (int): The retention time for data in redis in seconds.
+            input_data (dict[str, Any]): Activity input parameters.
+                Required fields:
+                - metadata (dict[str, Any]): Workflow execution metadata
+                - schedule_name (str): Name of the data collection schedule
+                - workflow_name (str): Name of the workflow
+                - data (dict[str, Any]): Data to group and store
+                - model_id (str): Unique model identifier
+                - model_tags (dict[str, Any]): Tag configuration
+                - retention_time (int): Data retention period in seconds
+
+        Returns:
+            dict[str, Any]: Grouped data organized by tag names
+
+        Raises:
+            Exception: If Redis operation fails
         """
         metadata = input_data['metadata']