95defccbf6
Refactor notification handler imports across multiple files to use CoreNotificationHandler instead of NotificationHandler for consistency and clarity.
119 lines
3.6 KiB
Python
119 lines
3.6 KiB
Python
from temporalio import workflow, client
|
|
from temporalio.worker import Worker
|
|
|
|
with workflow.unsafe.imports_passed_through():
|
|
import sys
|
|
import os
|
|
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
|
from sientia_do.temporal.utils.logger import get_logger
|
|
from scouter.activities.activities import Activities
|
|
from scouter.workflow.scouter import Scouter
|
|
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
|
|
from scouter.workflow.fake_data import FakeData
|
|
from scouter.activities.faker import Faker
|
|
import asyncio
|
|
from scouter.utils.connectors_config import (
|
|
build_postgres_config,
|
|
build_redis_config,
|
|
build_mongodb_config
|
|
)
|
|
|
|
|
|
async def main():
|
|
host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
logger.info('Starting Worker...')
|
|
|
|
logger.info('Starting Notification Handler...')
|
|
|
|
mongo_config = build_mongodb_config()
|
|
notification_handler = NotificationHandler(
|
|
connection_string=mongo_config['connection_string'],
|
|
database=mongo_config['database_name'],
|
|
logger=logger,
|
|
project_name=os.getenv('PROJECT_NAME', 'scouter')
|
|
)
|
|
|
|
logger.info('Starting Activities...')
|
|
|
|
activities = Activities(
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
postgres_config=build_postgres_config(),
|
|
redis_config=build_redis_config(),
|
|
mongodb_config=build_mongodb_config()
|
|
)
|
|
|
|
logger.info('Starting Faker Activities...')
|
|
|
|
faker_activities = Faker(
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
bootstrap_servers=os.getenv(
|
|
'KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092')
|
|
)
|
|
|
|
logger.info('Starting Temporal Client...')
|
|
|
|
temporal_client = await client.Client.connect(
|
|
target_host=host,
|
|
namespace=os.getenv('TEMPORAL_NAMESPACE', 'default')
|
|
)
|
|
|
|
logger.info('Starting Workers...')
|
|
|
|
workers = [
|
|
Worker(
|
|
temporal_client,
|
|
task_queue='scouter-queue',
|
|
workflows=[Scouter, CoreScouter],
|
|
activities=[
|
|
activities.load_latest_data,
|
|
activities.get_last_data_timestamp,
|
|
activities.put_last_data_timestamp,
|
|
activities.data_quality_gate,
|
|
activities.aggregate_data,
|
|
activities.group_and_hold_data,
|
|
activities.export_data_to_postgres,
|
|
activities.store_data_package
|
|
],
|
|
max_concurrent_workflow_tasks=100,
|
|
max_concurrent_activities=100,
|
|
max_concurrent_local_activities=100,
|
|
max_concurrent_workflow_task_polls=100,
|
|
max_cached_workflows=50,
|
|
),
|
|
Worker(
|
|
temporal_client,
|
|
task_queue='fake_data-queue',
|
|
workflows=[FakeData],
|
|
activities=[
|
|
faker_activities.generate_and_send_data,
|
|
]
|
|
)
|
|
]
|
|
|
|
handlers = []
|
|
for w in workers:
|
|
handlers.append(w.run())
|
|
|
|
logger.info('Workers started successfully')
|
|
|
|
try:
|
|
await asyncio.gather(*handlers)
|
|
|
|
except BaseException as e: # NOSONAR
|
|
logger.error("An unhandled exception occurred: %s", e, exc_info=True)
|
|
finally:
|
|
if notification_handler:
|
|
notification_handler.shutdown()
|
|
if activities:
|
|
activities.shutdown()
|
|
# Exit with a non-zero status code to indicate failure to Kubernetes
|
|
sys.exit(1)
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(main())
|