SIENTIAPDE-1005

Implement workflows for fake data generation, scouter processing, and core scouter operations

- Added `FakeData` workflow to generate random data and send it to a Kafka topic.
- Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow.
- Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL.
- Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling.
- Enhanced Redis and Postgres activities with robust testing for data handling and error notifications.
- Introduced quality filters for data validation and implemented tests to verify their functionality.

scouter/activities/faker.py

@@ -0,0 +1,80 @@
+import random
+from datetime import datetime, timezone
+from typing import Any
+import json
+from logging import Logger
+from kafka import KafkaProducer
+from temporalio import activity
+
+from sientia_do.notifications.handlers import NotificationHandler
+from scouter.activities.base import BaseActivity
+
+
+class Faker(BaseActivity):
+    def __init__(self, bootstrap_servers: str, logger: Logger,
+                 notification_handler: NotificationHandler):
+        self.producer = KafkaProducer(
+            bootstrap_servers=bootstrap_servers,
+            value_serializer=lambda v: json.dumps(v).encode('utf-8')
+        )
+
+        # Predefined lists for tag and name
+        self.tags = {
+            'ns=1;i=1001': 'Temperature Sensor',
+            'ns=1;i=1002': 'Vibration Meter',
+            'ns=1;i=1003': 'Pressure Gauge',
+            'ns=1;i=1004': 'Flow Meter',
+            'ns=1;i=1005': 'Voltage Sensor',
+            'ns=1;i=1006': 'Current Sensor'
+        }
+
+        BaseActivity.__init__(self, logger, notification_handler)
+
+    @activity.defn(name="generate_and_send_data")
+    async def generate_and_send_data(self, input_data: dict[str, Any]):
+        """
+        Generates random data and sends it to a Kafka topic.
+
+        Args:
+            input_data (dict[str, Any]): The input data containing:
+                topic (str): The Kafka topic to send data to
+                num_messages (int, optional): Number of messages to generate. 
+                Defaults to random.randint(1, len(self.tags)).
+
+        """
+        topic = input_data.get('topic')
+        num_messages = input_data.get(
+            'num_messages', random.randint(1, len(self.tags)))
+
+        if not topic:
+            raise ValueError("Topic must be specified in input_data")
+
+        self.logger.info(
+            f"Generating {num_messages} messages for topic {topic}")
+
+        for _ in range(num_messages):
+            # Select random tag and name
+            tag = random.choice(list(self.tags.keys()))
+            name = self.tags[tag]
+
+            # Generate random value between 0 and 100
+            if random.random() < 0.1:
+                value = None
+            else:
+                value = round(random.uniform(0, 100), 2)
+
+            # Create data dictionary
+            data = {
+                'tag': tag,
+                'name': name,
+                'timestamp': datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
+                'value': value
+            }
+
+            # Send to Kafka
+            self.producer.send(topic, value=data)
+
+        # Ensure all messages are sent
+        self.producer.flush()
+
+        self.logger.info("Success")