from temporalio import workflow, client from temporalio.worker import Worker with workflow.unsafe.imports_passed_through(): import os import logging from sientia_do.notifications.handlers import NotificationHandler from scouter.activities.activities import Activities from scouter.workflow.scouter import Scouter from scouter.workflow.sub_workflows.core_scouter import CoreScouter from scouter.workflow.fake_data import FakeData from scouter.activities.faker import Faker import asyncio import sys def build_postgres_config(): return { 'host': os.getenv('POSTGRES_HOST', 'localhost'), 'port': int(os.getenv('POSTGRES_PORT', '5432')), 'user': os.getenv('POSTGRES_USER', 'sientia'), 'password': os.getenv('POSTGRES_PASSWORD', 'sientia'), 'dbname': os.getenv('POSTGRES_DBNAME', 'sientia'), 'min_connections': int(os.getenv('POSTGRES_MIN_CONNECTIONS', '5')), 'max_connections': int(os.getenv('POSTGRES_MAX_CONNECTIONS', '20')) } def build_kafka_config(): return { 'bootstrap_servers': os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092'), 'polling_time': int(os.getenv('KAFKA_POLLING_TIME', '1000')), 'group_id': 'scouter-group' } def build_redis_config(): return { 'host': os.getenv('REDIS_HOST', 'localhost'), 'port': int(os.getenv('REDIS_PORT', '6379')), } async def main(): log_level = os.getenv('LOG_LEVEL', 'INFO').upper() host = os.getenv('TEMPORAL_HOST', 'localhost:7233') logger = logging.getLogger(__name__) logger.setLevel(log_level) stream_handler = logging.StreamHandler(sys.stdout) stream_handler.setLevel(log_level) stream_handler.setFormatter( logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s' ) ) logger.addHandler(stream_handler) logger.info('Starting Worker...') logger.info('Starting Notification Handler...') notification_handler = NotificationHandler( servers=os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'http://localhost:9092'), logger=logger, project_name=os.getenv('PROJECT_NAME', 'scouter'), pipeline_name='-', trigger_name='-', model_name='-', model='-' ) logger.info('Starting Activities...') activities = Activities( logger=logger, notification_handler=notification_handler, postgres_config=build_postgres_config(), kafka_config=build_kafka_config(), redis_config=build_redis_config() ) logger.info('Starting Faker Activities...') faker_activities = Faker( logger=logger, notification_handler=notification_handler, bootstrap_servers=os.getenv( 'KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092') ) logger.info('Starting Temporal Client...') temporal_client = await client.Client.connect( target_host=host, namespace=os.getenv('TEMPORAL_NAMESPACE', 'default') ) logger.info('Starting Workers...') workers = [ Worker( temporal_client, task_queue='scouter-queue', workflows=[Scouter, CoreScouter], activities=[ activities.load_from_kafka, activities.data_quality_gate, activities.aggregate_data, activities.group_and_hold_data, activities.export_data_to_postgres, ] ), Worker( temporal_client, task_queue='fake_data-queue', workflows=[FakeData], activities=[ faker_activities.generate_and_send_data, ] ) ] handlers = [] for w in workers: handlers.append(w.run()) logger.info('Workers started successfully') await asyncio.gather(*handlers) if __name__ == '__main__': asyncio.run(main())