SIENTIAPDE-1005

Implement workflows for fake data generation, scouter processing, and core scouter operations

- Added `FakeData` workflow to generate random data and send it to a Kafka topic.
- Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow.
- Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL.
- Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling.
- Enhanced Redis and Postgres activities with robust testing for data handling and error notifications.
- Introduced quality filters for data validation and implemented tests to verify their functionality.

scouter/activities/redis.py

@@ -4,15 +4,85 @@ with workflow.unsafe.imports_passed_through():
     from logging import Logger
     from sientia_do.notifications.handlers import NotificationHandler
     from scouter.activities.base import BaseActivity
+    import redis
+    import json
     from typing import Any
-    from kafka import KafkaConsumer
     from pandas import DataFrame
+    import numpy as np
+    from datetime import datetime
 
 
 class Redis(BaseActivity):
-    def __init__(self, host: str, port: int, db: int, logger: Logger, notification_handler: NotificationHandler):
+    def __init__(self, host: str, port: int,
+                 logger: Logger, notification_handler: NotificationHandler):
         self.host = host
         self.port = port
-        self.db = db
 
-        super().__init__(logger, notification_handler)f
+        self.redis_client = redis.Redis(
+            host=self.host,
+            port=self.port,
+            decode_responses=True
+        )
+
+        BaseActivity.__init__(self, logger, notification_handler)
+
+    def get(self, key: str):
+        history = self.redis_client.get(key)
+        return json.loads(history) if history else None
+
+    def set(self, key: str, data: dict, ttl=600):
+        self.redis_client.set(key, json.dumps(data), ex=ttl)
+
+    @activity.defn(name="group_and_hold_data")
+    async def group_and_hold_data(self, input_data: dict[str, Any]):
+        """
+        Groups and holds data in redis. Keep a copy of the most recent 
+        received data for a given pipeline and schedule. This activity updates
+        the data in redis and return the full keeped data.
+
+        Args:
+            input_data (dict[str, Any]): The data to group and hold.
+                workflow_name (str): The name of the workflow.
+                schedule_name (str): The name of the schedule.
+                data (dict[str, Any]): The data to group and hold.
+                retention_time (int): The retention time for data in redis in seconds.
+        """
+        self.logger.debug("Grouping and holding data...")
+        data = DataFrame(input_data['data'])
+        retention_time = input_data['retention_time']
+
+        key = f"{input_data['workflow_name']}_{input_data['schedule_name']}"
+
+        data_hold = self.get(key)
+
+        if not data_hold:
+            data_hold = {}
+            if data.empty:
+                self.logger.warning("No data to export")
+                return data_hold
+
+        for _, row in data.iterrows():
+            value = row['value']
+
+            if value is None:
+                data_hold[row['name']] = np.nan
+
+            else:
+                data_hold[row['name']] = value
+
+        data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \
+            datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+        self.set(key, data_hold, ttl=retention_time)
+
+        data_hold_df = DataFrame(data_hold, index=[0])
+        data_hold_melted = data_hold_df.melt(
+            id_vars='timestamp', var_name='variable', value_name='value')
+        data_hold_melted['model_id'] = input_data['model_id']
+
+        data_hold_melted.reset_index(drop=True, inplace=True)
+
+        self.logger.debug(
+            f"Data grouped and held successfully:\n {data_hold_melted.to_string()}")
+
+        return data_hold_melted.to_dict()