gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1316

Browse Source 
Update .gitignore and refactor metrics.py, activities.py, and gates.py for improved clarity and consistency. Added coverage.xml and cache directories to .gitignore. Standardized string formatting and parameter handling in metrics and activities classes, enhancing code readability. Removed the deprecated faker.py file and adjusted related tests accordingly.
This commit is contained in:
vitor-aignosi
2025-10-16 13:31:12 -03:00
parent 8bdbf049b8
commit 97eb5bc904
27 changed files with 1101 additions and 1336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
scouter/worker/worker.py
View File

@@ -1,29 +1,29 @@
from temporalio import workflow, client
from temporalio.worker import Worker, PollerBehaviorAutoscaling
from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig
from temporalio import client, workflow
from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig
from temporalio.worker import PollerBehaviorAutoscaling, Worker
with workflow.unsafe.imports_passed_through():
import sys
import asyncio
import os
import sys
from prometheus_client import start_http_server
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import get_logger
from scouter.activities.activities import Activities
from scouter.workflow.scouter import Scouter
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
from scouter.workflow.fake_data import FakeData
from scouter.activities.faker import Faker
import asyncio
from prometheus_client import start_http_server
from scouter import metrics
from scouter.activities.activities import Activities
from scouter.utils.connectors_config import (
build_mongodb_config,
build_postgres_config,
build_redis_config,
build_mongodb_config
)
from scouter.workflow.scouter import Scouter
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
# Environment configuration
POD_ID = os.getenv("HOSTNAME", "localhost")
SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091"))
POD_ID = os.getenv('HOSTNAME', 'localhost')
SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091'))
async def main():
@@ -59,9 +59,9 @@ async def main():
'schedule_name': '-',
}
logger.custom_info(f"Starting Worker with pod_id: {POD_ID}", metadata)
logger.custom_info(f'Starting Worker with pod_id: {POD_ID}', metadata)
logger.custom_info("Starting prometheus client...", metadata)
logger.custom_info('Starting prometheus client...', metadata)
start_prometheus_server()
logger.custom_info('Starting Notification Handler...', metadata)
@@ -71,7 +71,7 @@ async def main():
connection_string=mongo_config['connection_string'],
database=mongo_config['database_name'],
logger=logger,
project_name=os.getenv('PROJECT_NAME', 'scouter')
project_name=os.getenv('PROJECT_NAME', 'scouter'),
)
logger.custom_info('Starting Activities...', metadata)
@@ -81,34 +81,21 @@ async def main():
notification_handler=notification_handler,
postgres_config=build_postgres_config(),
redis_config=build_redis_config(),
mongodb_config=build_mongodb_config()
mongodb_config=build_mongodb_config(),
)
logger.custom_info('Starting Faker Activities...', metadata)
faker_activities = Faker(
logger=logger,
notification_handler=notification_handler,
bootstrap_servers=os.getenv(
'KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092')
)
logger.custom_info(
f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
new_runtime = Runtime(
telemetry=TelemetryConfig(
metrics=PrometheusConfig(
bind_address=f"0.0.0.0:{SDK_METRICS_PORT}")
metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}')
)
)
logger.custom_info('Starting Temporal Client...', metadata)
temporal_client = await client.Client.connect(
target_host=host,
namespace=os.getenv('TEMPORAL_NAMESPACE', 'scouter'),
runtime=new_runtime
target_host=host, namespace=os.getenv('TEMPORAL_NAMESPACE', 'scouter'), runtime=new_runtime
)
logger.custom_info('Starting Workers...', metadata)
@@ -134,15 +121,7 @@ async def main():
max_concurrent_local_activities=50,
max_cached_workflows=200,
workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
activity_task_poller_behavior=PollerBehaviorAutoscaling()
),
Worker(
temporal_client,
task_queue='fake_data-queue',
workflows=[FakeData],
activities=[
faker_activities.generate_and_send_data,
]
activity_task_poller_behavior=PollerBehaviorAutoscaling(),
)
]
@@ -156,8 +135,9 @@ async def main():
await asyncio.gather(*handlers)
except BaseException as e: # NOSONAR
logger.custom_error("An unhandled exception occurred: %s",
e, exc_info=True, metadata=metadata)
logger.custom_error(
'An unhandled exception occurred: %s', e, exc_info=True, metadata=metadata
)
finally:
if notification_handler:
notification_handler.shutdown()
@@ -183,12 +163,12 @@ def start_prometheus_server():
SystemExit: If metrics server fails to start
"""
try:
port = int(os.getenv("HTTP_METRICS_PORT", 9090))
port = int(os.getenv('HTTP_METRICS_PORT', 9090))
start_http_server(port)
print(f"Prometheus server started on port {port}.")
print(f'Prometheus server started on port {port}.')
metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP
except Exception as e:
print(f"Failed to start Prometheus server: {e}")
print(f'Failed to start Prometheus server: {e}')
os._exit(1)