SIENTIAPDE-1005

Implement workflows for fake data generation, scouter processing, and core scouter operations

- Added `FakeData` workflow to generate random data and send it to a Kafka topic.
- Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow.
- Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL.
- Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling.
- Enhanced Redis and Postgres activities with robust testing for data handling and error notifications.
- Introduced quality filters for data validation and implemented tests to verify their functionality.

scouter/worker/worker.py

@@ -0,0 +1,136 @@
+from temporalio import workflow, client
+from temporalio.worker import Worker
+
+with workflow.unsafe.imports_passed_through():
+    import os
+    import logging
+    from sientia_do.notifications.handlers import NotificationHandler
+    from scouter.activities.activities import Activities
+    from scouter.workflow.scouter import Scouter
+    from scouter.workflow.sub_workflows.core_scouter import CoreScouter
+    from scouter.workflow.fake_data import FakeData
+    from scouter.activities.faker import Faker
+    import asyncio
+    import sys
+
+
+def build_postgres_config():
+    return {
+        'host': os.getenv('POSTGRES_HOST', 'localhost'),
+        'port': int(os.getenv('POSTGRES_PORT', '5432')),
+        'user': os.getenv('POSTGRES_USER', 'sientia'),
+        'password': os.getenv('POSTGRES_PASSWORD', 'sientia'),
+        'dbname': os.getenv('POSTGRES_DBNAME', 'sientia'),
+        'min_connections': int(os.getenv('POSTGRES_MIN_CONNECTIONS', '5')),
+        'max_connections': int(os.getenv('POSTGRES_MAX_CONNECTIONS', '20'))
+    }
+
+
+def build_kafka_config():
+    return {
+        'bootstrap_servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'),
+        'polling_time': int(os.getenv('KAFKA_POLLING_TIME', '1000')),
+        'group_id': 'scouter-group'
+    }
+
+
+def build_redis_config():
+    return {
+        'host': os.getenv('REDIS_HOST', 'localhost'),
+        'port': int(os.getenv('REDIS_PORT', '6379')),
+    }
+
+
+async def main():
+
+    log_level = os.getenv('LOG_LEVEL', 'INFO').upper()
+
+    host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
+    logger = logging.getLogger(__name__)
+    logger.setLevel(log_level)
+    stream_handler = logging.StreamHandler(sys.stdout)
+    stream_handler.setLevel(log_level)
+
+    stream_handler.setFormatter(
+        logging.Formatter(
+            '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+        )
+    )
+
+    logger.addHandler(stream_handler)
+
+    logger.info('Starting Worker...')
+
+    logger.info('Starting Notification Handler...')
+
+    notification_handler = NotificationHandler(
+        servers=os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'http://localhost:9092'),
+        logger=logger,
+        project_name=os.getenv('PROJECT_NAME', 'scouter'),
+        pipeline_name='-',
+        trigger_name='-',
+        model_name='-',
+        model='-'
+    )
+
+    logger.info('Starting Activities...')
+
+    activities = Activities(
+        logger=logger,
+        notification_handler=notification_handler,
+        postgres_config=build_postgres_config(),
+        kafka_config=build_kafka_config(),
+        redis_config=build_redis_config()
+    )
+
+    logger.info('Starting Faker Activities...')
+
+    faker_activities = Faker(
+        logger=logger,
+        notification_handler=notification_handler,
+        bootstrap_servers=os.getenv(
+            'KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092')
+    )
+
+    logger.info('Starting Temporal Client...')
+
+    temporal_client = await client.Client.connect(
+        target_host=host,
+        namespace=os.getenv('TEMPORAL_NAMESPACE', 'default')
+    )
+
+    logger.info('Starting Workers...')
+
+    workers = [
+        Worker(
+            temporal_client,
+            task_queue='scouter-queue',
+            workflows=[Scouter, CoreScouter],
+            activities=[
+                activities.load_from_kafka,
+                activities.data_quality_gate,
+                activities.aggregate_data,
+                activities.group_and_hold_data,
+                activities.export_data_to_postgres,
+            ]
+        ),
+        Worker(
+            temporal_client,
+            task_queue='fake_data-queue',
+            workflows=[FakeData],
+            activities=[
+                faker_activities.generate_and_send_data,
+            ]
+        )
+    ]
+
+    handlers = []
+    for w in workers:
+        handlers.append(w.run())
+
+    logger.info('Workers started successfully')
+
+    await asyncio.gather(*handlers)
+
+if __name__ == '__main__':
+    asyncio.run(main())