SIENTIAPDE-1316

Update .gitignore and refactor metrics.py, activities.py, and gates.py for improved clarity and consistency. Added coverage.xml and cache directories to .gitignore. Standardized string formatting and parameter handling in metrics and activities classes, enhancing code readability. Removed the deprecated faker.py file and adjusted related tests accordingly.

scouter/activities/redis.py

@@ -1,17 +1,20 @@
-from temporalio import workflow, activity
+from collections.abc import Hashable
+
+from temporalio import activity, workflow
 
 with workflow.unsafe.imports_passed_through():
-    from logging import Logger
     import traceback
+    from typing import Any
+
+    from pandas import DataFrame
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.notifications.models import NotificationLevel
-    from sientia_do.temporal.activities.redis_base import Redis as RedisBase
     from sientia_do.observability.logger import Logger
-    from typing import Any
-    from pandas import DataFrame
-    from scouter import metrics
+    from sientia_do.temporal.activities.redis_base import Redis as RedisBase
     from sientia_do.temporal.constants import DATETIME_FORMAT, now
 
+    from scouter import metrics
+
 
 class Redis(RedisBase):
     """
@@ -28,9 +31,15 @@ class Redis(RedisBase):
     distributed data processing with fault tolerance and monitoring.
     """
 
-    def __init__(self, host: str, port: int,
-                 username: str, password: str,
-                 logger: Logger, notification_handler: NotificationHandler):
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        username: str,
+        password: str,
+        logger: Logger,
+        notification_handler: NotificationHandler,
+    ):
         """
         Initialize Redis connection and services.
 
@@ -42,10 +51,9 @@ class Redis(RedisBase):
             logger (Logger): Logger instance for operation logging
             notification_handler (NotificationHandler): Handler for system notifications
         """
-        RedisBase.__init__(self, host, port, username,
-                           password, logger, notification_handler)
+        RedisBase.__init__(self, host, port, username, password, logger, notification_handler)
 
-    @activity.defn(name="get_last_data_timestamp")
+    @activity.defn(name='get_last_data_timestamp')
     async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
         """
         Retrieve the last processed data timestamp from Redis.
@@ -68,34 +76,31 @@ class Redis(RedisBase):
             Exception: If Redis operation fails
         """
         metadata = input_data['metadata']
-        key = f"last_data_timestamp:{input_data['workflow_name']}:{input_data['schedule_name']}"
+        key = f'last_data_timestamp:{input_data["workflow_name"]}:{input_data["schedule_name"]}'
 
-        self.info(f"Getting last data timestamp for {key}")
+        self.info(f'Getting last data timestamp for {key}')
 
         try:
             data_hold = self.get(key)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,
-                notification_id="REDIS_GET_ERROR",
-                message=f"Error getting last data timestamp: {e}",
-                block="get_last_data_timestamp",
+                notification_id='REDIS_GET_ERROR',
+                message=f'Error getting last data timestamp: {e}',
+                block='get_last_data_timestamp',
                 level=NotificationLevel.ERROR,
-                attachment_content=traceback.format_exc()
+                attachment_content=traceback.format_exc(),
             )
             raise e
 
-        self.info(
-            f"Last collected timestamp: {data_hold}",
-            metadata=metadata
-        )
+        self.info(f'Last collected timestamp: {data_hold}', metadata=metadata)
 
         if not data_hold:
             return None
 
         return data_hold
 
-    @activity.defn(name="put_last_data_timestamp")
+    @activity.defn(name='put_last_data_timestamp')
     async def put_last_data_timestamp(self, input_data: dict[str, Any]) -> str | None:
         """
         Store the last processed data timestamp in Redis.
@@ -119,43 +124,37 @@ class Redis(RedisBase):
             Exception: If Redis operation fails
         """
         metadata = input_data['metadata']
-        key = f"last_data_timestamp:{input_data['workflow_name']}:{input_data['schedule_name']}"
+        key = f'last_data_timestamp:{input_data["workflow_name"]}:{input_data["schedule_name"]}'
 
-        self.info(f"Putting last data timestamp for {key}")
+        self.info(f'Putting last data timestamp for {key}')
 
         data = DataFrame(input_data['data'])
 
         if data.empty:
-            self.warning("No data to insert",
-                         metadata=metadata
-                         )
+            self.warning('No data to insert', metadata=metadata)
             return None
 
         last_data_timestamp = data['inserted_at'].max()
 
-        self.info(
-            f"Last collected timestamp to insert: {last_data_timestamp}",
-            metadata=metadata
-        )
+        self.info(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata)
 
         try:
-            self.set(key, last_data_timestamp, ttl=60*60*5)
+            self.set(key, last_data_timestamp, ttl=60 * 60 * 5)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,
-                notification_id="REDIS_SET_ERROR",
-                message=f"Error setting last data timestamp: {e}",
-
-                block="put_last_data_timestamp",
+                notification_id='REDIS_SET_ERROR',
+                message=f'Error setting last data timestamp: {e}',
+                block='put_last_data_timestamp',
                 level=NotificationLevel.ERROR,
-                attachment_content=traceback.format_exc()
+                attachment_content=traceback.format_exc(),
             )
             raise e
 
         return last_data_timestamp
 
-    @activity.defn(name="group_and_hold_data")
-    async def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
+    @activity.defn(name='group_and_hold_data')
+    async def group_and_hold_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
         """
         Group data by tags and store temporarily in Redis with TTL.
 
@@ -182,109 +181,92 @@ class Redis(RedisBase):
         """
         metadata = input_data['metadata']
 
-        self.debug("Grouping and holding data...",
-                   metadata=metadata
-                   )
+        self.debug('Grouping and holding data...', metadata=metadata)
         data = DataFrame(input_data['data'])
         model_tags = input_data['model_tags']
         retention_time = input_data['retention_time']
 
-        key = f"held_data_{input_data['workflow_name']}_{input_data['schedule_name']}"
+        key = f'held_data_{input_data["workflow_name"]}_{input_data["schedule_name"]}'
 
-        self.info(f"Getting held data for {key}")
+        self.info(f'Getting held data for {key}')
 
         try:
             data_hold = self.get(key)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,
-                notification_id="REDIS_GET_ERROR",
-                message=f"Error getting held data: {e}",
-                block="group_and_hold_data",
+                notification_id='REDIS_GET_ERROR',
+                message=f'Error getting held data: {e}',
+                block='group_and_hold_data',
                 level=NotificationLevel.ERROR,
-                attachment_content=traceback.format_exc()
+                attachment_content=traceback.format_exc(),
             )
             raise e
 
         if not data_hold:
             data_hold = {}
             if data.empty:
-                self.warning("No data to export",
-                             metadata=metadata
-                             )
+                self.warning('No data to export', metadata=metadata)
                 return data_hold
 
-        self.info(f"Grouping and holding data for {len(data)} rows")
+        self.info(f'Grouping and holding data for {len(data)} rows')
 
         try:
-
             # Remove possibly removed tags
             tags = list(model_tags.keys())
             tags.append('timestamp')
-            self.debug(
-                f"Tags to keep: {tags}",
-                metadata=metadata
-            )
-            data_hold = {tag: content for tag,
-                         content in data_hold.items() if tag in tags}
-            self.debug(
-                f"Data hold after removing removed tags: {data_hold}",
-                metadata=metadata
-            )
+            self.debug(f'Tags to keep: {tags}', metadata=metadata)
+            data_hold = {tag: content for tag, content in data_hold.items() if tag in tags}
+            self.debug(f'Data hold after removing removed tags: {data_hold}', metadata=metadata)
 
             to_register_metrics = []
             for _, row in data.iterrows():
                 value = row['value']
 
                 data_hold[row['name']] = value
-                to_register_metrics.append(
-                    (row['name'], value))
+                to_register_metrics.append((row['name'], value))
 
-            data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \
-                data_hold['timestamp']
+            data_hold['timestamp'] = (
+                data['timestamp'].max() if not data.empty else data_hold['timestamp']
+            )
 
             self.set(key, data_hold, ttl=retention_time)
 
             # Register metrics
-            self.debug(
-                f"Metrics to register: {to_register_metrics}",
-                metadata=metadata
-            )
+            self.debug(f'Metrics to register: {to_register_metrics}', metadata=metadata)
             for metric in to_register_metrics:
                 metrics.TAG_CHANGES_MONITOR.labels(
                     pod_id=self.pod_id,
                     model_name=metadata['model_name'],
                     pipeline_name=metadata['workflow_name'],
-                    tag_name=metric[0]
+                    tag_name=metric[0],
                 ).set(metric[1])
 
             data_hold_df = DataFrame(data_hold, index=[0])
             data_hold_melted = data_hold_df.melt(
-                id_vars='timestamp', var_name='variable', value_name='value')
+                id_vars='timestamp', var_name='variable', value_name='value'
+            )
             data_hold_melted['model_id'] = input_data['model_id']
 
             data_hold_melted.reset_index(drop=True, inplace=True)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,
-                notification_id="REDIS_SET_ERROR",
-                message=f"Error setting held data: {e}",
-                block="group_and_hold_data",
+                notification_id='REDIS_SET_ERROR',
+                message=f'Error setting held data: {e}',
+                block='group_and_hold_data',
                 level=NotificationLevel.ERROR,
-                attachment_content=traceback.format_exc()
+                attachment_content=traceback.format_exc(),
             )
             raise e
 
-        self.info(f"Data held and melted has {len(data_hold_melted)} rows")
+        self.info(f'Data held and melted has {len(data_hold_melted)} rows')
 
-        self.debug(
-            f"Data held and melted:\n {data_hold_melted.to_string()}",
-            metadata=metadata
-        )
+        self.debug(f'Data held and melted:\n {data_hold_melted.to_string()}', metadata=metadata)
 
         return data_hold_melted.to_dict()
 
-    @activity.defn(name="store_data_package")
+    @activity.defn(name='store_data_package')
     async def store_data_package(self, input_data: dict[str, Any]):
         """
         Stores the data package in redis. It's a debug feature and must be toggled on.
@@ -296,25 +278,22 @@ class Redis(RedisBase):
             data: The data used to collect the data.
         """
         metadata = input_data['metadata']
-        key = f"data_package_{input_data['workflow_name']}_{input_data['schedule_name']}_{now().strftime(DATETIME_FORMAT)}"
+        key = f'data_package_{input_data["workflow_name"]}_{input_data["schedule_name"]}_{now().strftime(DATETIME_FORMAT)}'
 
         data = DataFrame(input_data['data'])
         held_data = DataFrame(input_data['held_data'])
 
-        cache = {
-            'data': data.to_dict(),
-            'held_data': held_data.to_dict()
-        }
+        cache = {'data': data.to_dict(), 'held_data': held_data.to_dict()}
 
         try:
             self.set(key, cache, ttl=120)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,
-                notification_id="REDIS_SET_ERROR",
-                message=f"Error setting data package: {e}",
-                block="store_data_package",
+                notification_id='REDIS_SET_ERROR',
+                message=f'Error setting data package: {e}',
+                block='store_data_package',
                 level=NotificationLevel.ERROR,
-                attachment_content=traceback.format_exc()
+                attachment_content=traceback.format_exc(),
             )
             raise e