SIENTIAPDE-1005
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
This commit is contained in:
65
scouter/activities/activities.py
Normal file
65
scouter/activities/activities.py
Normal file
@@ -0,0 +1,65 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from scouter.activities.postgres import Postgres
|
||||
from scouter.activities.redis import Redis
|
||||
from scouter.activities.kafka import Kafka
|
||||
from scouter.activities.gates import Gates
|
||||
from logging import Logger
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Activities(Postgres, Redis, Kafka, Gates):
|
||||
"""Activities class that combines multiple services with proper initialization."""
|
||||
|
||||
def __init__(self,
|
||||
postgres_config: dict[str, Any],
|
||||
redis_config: dict[str, Any],
|
||||
kafka_config: dict[str, Any],
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler):
|
||||
|
||||
# Initialize Postgres
|
||||
Postgres.__init__(
|
||||
self,
|
||||
host=postgres_config['host'],
|
||||
port=postgres_config['port'],
|
||||
user=postgres_config['user'],
|
||||
password=postgres_config['password'],
|
||||
dbname=postgres_config['dbname'],
|
||||
min_connections=postgres_config['min_connections'],
|
||||
max_connections=postgres_config['max_connections'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler
|
||||
)
|
||||
|
||||
# Initialize Redis
|
||||
Redis.__init__(
|
||||
self,
|
||||
host=redis_config['host'],
|
||||
port=redis_config['port'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler
|
||||
)
|
||||
|
||||
# Initialize Kafka
|
||||
Kafka.__init__(
|
||||
self,
|
||||
bootstrap_servers=kafka_config['bootstrap_servers'],
|
||||
polling_time=kafka_config['polling_time'],
|
||||
group_id=kafka_config['group_id'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler
|
||||
)
|
||||
|
||||
# Initialize Gates
|
||||
Gates.__init__(
|
||||
self,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler
|
||||
)
|
||||
|
||||
@activity.defn(name="prepare_activity")
|
||||
def prepare_activity(self, input_data: dict[str, Any]):
|
||||
super().prepare_activity(input_data)
|
||||
Reference in New Issue
Block a user