b203b7d22c
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
66 lines
2.2 KiB
Python
66 lines
2.2 KiB
Python
from temporalio import activity, workflow
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
from scouter.activities.postgres import Postgres
|
|
from scouter.activities.redis import Redis
|
|
from scouter.activities.kafka import Kafka
|
|
from scouter.activities.gates import Gates
|
|
from logging import Logger
|
|
from sientia_do.notifications.handlers import NotificationHandler
|
|
from typing import Any
|
|
|
|
|
|
class Activities(Postgres, Redis, Kafka, Gates):
|
|
"""Activities class that combines multiple services with proper initialization."""
|
|
|
|
def __init__(self,
|
|
postgres_config: dict[str, Any],
|
|
redis_config: dict[str, Any],
|
|
kafka_config: dict[str, Any],
|
|
logger: Logger,
|
|
notification_handler: NotificationHandler):
|
|
|
|
# Initialize Postgres
|
|
Postgres.__init__(
|
|
self,
|
|
host=postgres_config['host'],
|
|
port=postgres_config['port'],
|
|
user=postgres_config['user'],
|
|
password=postgres_config['password'],
|
|
dbname=postgres_config['dbname'],
|
|
min_connections=postgres_config['min_connections'],
|
|
max_connections=postgres_config['max_connections'],
|
|
logger=logger,
|
|
notification_handler=notification_handler
|
|
)
|
|
|
|
# Initialize Redis
|
|
Redis.__init__(
|
|
self,
|
|
host=redis_config['host'],
|
|
port=redis_config['port'],
|
|
logger=logger,
|
|
notification_handler=notification_handler
|
|
)
|
|
|
|
# Initialize Kafka
|
|
Kafka.__init__(
|
|
self,
|
|
bootstrap_servers=kafka_config['bootstrap_servers'],
|
|
polling_time=kafka_config['polling_time'],
|
|
group_id=kafka_config['group_id'],
|
|
logger=logger,
|
|
notification_handler=notification_handler
|
|
)
|
|
|
|
# Initialize Gates
|
|
Gates.__init__(
|
|
self,
|
|
logger=logger,
|
|
notification_handler=notification_handler
|
|
)
|
|
|
|
@activity.defn(name="prepare_activity")
|
|
def prepare_activity(self, input_data: dict[str, Any]):
|
|
super().prepare_activity(input_data)
|