b203b7d22c
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
137 lines
3.9 KiB
Python
137 lines
3.9 KiB
Python
from temporalio import workflow, client
|
|
from temporalio.worker import Worker
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
import os
|
|
import logging
|
|
from sientia_do.notifications.handlers import NotificationHandler
|
|
from scouter.activities.activities import Activities
|
|
from scouter.workflow.scouter import Scouter
|
|
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
|
|
from scouter.workflow.fake_data import FakeData
|
|
from scouter.activities.faker import Faker
|
|
import asyncio
|
|
import sys
|
|
|
|
|
|
def build_postgres_config():
|
|
return {
|
|
'host': os.getenv('POSTGRES_HOST', 'localhost'),
|
|
'port': int(os.getenv('POSTGRES_PORT', '5432')),
|
|
'user': os.getenv('POSTGRES_USER', 'sientia'),
|
|
'password': os.getenv('POSTGRES_PASSWORD', 'sientia'),
|
|
'dbname': os.getenv('POSTGRES_DBNAME', 'sientia'),
|
|
'min_connections': int(os.getenv('POSTGRES_MIN_CONNECTIONS', '5')),
|
|
'max_connections': int(os.getenv('POSTGRES_MAX_CONNECTIONS', '20'))
|
|
}
|
|
|
|
|
|
def build_kafka_config():
|
|
return {
|
|
'bootstrap_servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'),
|
|
'polling_time': int(os.getenv('KAFKA_POLLING_TIME', '1000')),
|
|
'group_id': 'scouter-group'
|
|
}
|
|
|
|
|
|
def build_redis_config():
|
|
return {
|
|
'host': os.getenv('REDIS_HOST', 'localhost'),
|
|
'port': int(os.getenv('REDIS_PORT', '6379')),
|
|
}
|
|
|
|
|
|
async def main():
|
|
|
|
log_level = os.getenv('LOG_LEVEL', 'INFO').upper()
|
|
|
|
host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
|
|
logger = logging.getLogger(__name__)
|
|
logger.setLevel(log_level)
|
|
stream_handler = logging.StreamHandler(sys.stdout)
|
|
stream_handler.setLevel(log_level)
|
|
|
|
stream_handler.setFormatter(
|
|
logging.Formatter(
|
|
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
)
|
|
)
|
|
|
|
logger.addHandler(stream_handler)
|
|
|
|
logger.info('Starting Worker...')
|
|
|
|
logger.info('Starting Notification Handler...')
|
|
|
|
notification_handler = NotificationHandler(
|
|
servers=os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'http://localhost:9092'),
|
|
logger=logger,
|
|
project_name=os.getenv('PROJECT_NAME', 'scouter'),
|
|
pipeline_name='-',
|
|
trigger_name='-',
|
|
model_name='-',
|
|
model='-'
|
|
)
|
|
|
|
logger.info('Starting Activities...')
|
|
|
|
activities = Activities(
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
postgres_config=build_postgres_config(),
|
|
kafka_config=build_kafka_config(),
|
|
redis_config=build_redis_config()
|
|
)
|
|
|
|
logger.info('Starting Faker Activities...')
|
|
|
|
faker_activities = Faker(
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
bootstrap_servers=os.getenv(
|
|
'KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092')
|
|
)
|
|
|
|
logger.info('Starting Temporal Client...')
|
|
|
|
temporal_client = await client.Client.connect(
|
|
target_host=host,
|
|
namespace=os.getenv('TEMPORAL_NAMESPACE', 'default')
|
|
)
|
|
|
|
logger.info('Starting Workers...')
|
|
|
|
workers = [
|
|
Worker(
|
|
temporal_client,
|
|
task_queue='scouter-queue',
|
|
workflows=[Scouter, CoreScouter],
|
|
activities=[
|
|
activities.load_from_kafka,
|
|
activities.data_quality_gate,
|
|
activities.aggregate_data,
|
|
activities.group_and_hold_data,
|
|
activities.export_data_to_postgres,
|
|
]
|
|
),
|
|
Worker(
|
|
temporal_client,
|
|
task_queue='fake_data-queue',
|
|
workflows=[FakeData],
|
|
activities=[
|
|
faker_activities.generate_and_send_data,
|
|
]
|
|
)
|
|
]
|
|
|
|
handlers = []
|
|
for w in workers:
|
|
handlers.append(w.run())
|
|
|
|
logger.info('Workers started successfully')
|
|
|
|
await asyncio.gather(*handlers)
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(main())
|