SIENTIAPDE-1005

Implement workflows for fake data generation, scouter processing, and core scouter operations

- Added `FakeData` workflow to generate random data and send it to a Kafka topic.
- Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow.
- Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL.
- Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling.
- Enhanced Redis and Postgres activities with robust testing for data handling and error notifications.
- Introduced quality filters for data validation and implemented tests to verify their functionality.
This commit is contained in:
vitor-aignosi
2025-05-15 16:53:24 -03:00
parent 4e579dd5bd
commit b203b7d22c
29 changed files with 2070 additions and 49 deletions

View File

@@ -0,0 +1,65 @@
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
from scouter.activities.postgres import Postgres
from scouter.activities.redis import Redis
from scouter.activities.kafka import Kafka
from scouter.activities.gates import Gates
from logging import Logger
from sientia_do.notifications.handlers import NotificationHandler
from typing import Any
class Activities(Postgres, Redis, Kafka, Gates):
"""Activities class that combines multiple services with proper initialization."""
def __init__(self,
postgres_config: dict[str, Any],
redis_config: dict[str, Any],
kafka_config: dict[str, Any],
logger: Logger,
notification_handler: NotificationHandler):
# Initialize Postgres
Postgres.__init__(
self,
host=postgres_config['host'],
port=postgres_config['port'],
user=postgres_config['user'],
password=postgres_config['password'],
dbname=postgres_config['dbname'],
min_connections=postgres_config['min_connections'],
max_connections=postgres_config['max_connections'],
logger=logger,
notification_handler=notification_handler
)
# Initialize Redis
Redis.__init__(
self,
host=redis_config['host'],
port=redis_config['port'],
logger=logger,
notification_handler=notification_handler
)
# Initialize Kafka
Kafka.__init__(
self,
bootstrap_servers=kafka_config['bootstrap_servers'],
polling_time=kafka_config['polling_time'],
group_id=kafka_config['group_id'],
logger=logger,
notification_handler=notification_handler
)
# Initialize Gates
Gates.__init__(
self,
logger=logger,
notification_handler=notification_handler
)
@activity.defn(name="prepare_activity")
def prepare_activity(self, input_data: dict[str, Any]):
super().prepare_activity(input_data)