b203b7d22c
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
31 lines
1.0 KiB
Python
31 lines
1.0 KiB
Python
from temporalio import workflow
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
from scouter.activities.faker import Faker
|
|
from datetime import timedelta
|
|
from typing import Dict, Any
|
|
from scouter.utils.policies import retry_policy
|
|
|
|
|
|
@workflow.defn(name="fake_data")
|
|
class FakeData:
|
|
@workflow.run
|
|
async def run(self, workflow_input: Dict[str, Any]) -> str:
|
|
"""
|
|
Generates random data and sends it to a Kafka topic.
|
|
|
|
Args:
|
|
workflow_input (dict[str, Any]): The input data containing:
|
|
topic (str): The Kafka topic to send data to
|
|
num_messages (int, optional): Number of messages to generate.
|
|
Defaults to random.randint(1, len(self.tags)).
|
|
"""
|
|
await workflow.execute_activity_method(
|
|
Faker.generate_and_send_data,
|
|
{
|
|
'topic': workflow_input['topic']
|
|
},
|
|
retry_policy=retry_policy,
|
|
start_to_close_timeout=timedelta(seconds=60)
|
|
)
|