gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
Update environment configuration and refactor activities to include metrics controller. Remove Kafka settings and adjust Redis and MongoDB initialization. Update tests to reflect changes in initialization and metrics tracking.
This commit is contained in:
vitor-aignosi
2025-10-30 16:49:55 -03:00
parent 0c16075823
commit d4a74ec5eb
16 changed files with 225 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
scouter/activities/activities.py
View File

@@ -1,3 +1,4 @@
from sientia_do.observability.metrics_controller import MetricsController
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
@@ -50,6 +51,10 @@ class Activities(Postgres, Redis, Gates, MongoDB):
logger (Logger): Logger instance for application logging
notification_handler (NotificationHandler): Handler for system notifications
"""
metrics_controller = MetricsController(
logger=logger,
)
# Initialize Postgres
Postgres.__init__(
self,
@@ -62,6 +67,7 @@ class Activities(Postgres, Redis, Gates, MongoDB):
max_connections=postgres_config['max_connections'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
# Initialize Redis
@@ -73,10 +79,11 @@ class Activities(Postgres, Redis, Gates, MongoDB):
notification_handler=notification_handler,
username=redis_config['username'],
password=redis_config['password'],
metrics_controller=metrics_controller,
)
# Initialize Gates
Gates.__init__(self, logger=logger, notification_handler=notification_handler)
Gates.__init__(self, logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller)
# Initialize MongoDB
MongoDB.__init__(
@@ -85,6 +92,7 @@ class Activities(Postgres, Redis, Gates, MongoDB):
database_name=mongodb_config['database_name'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
self.pod_id = getenv('HOSTNAME', 'localhost')

8
scouter/activities/gates.py
View File

@@ -1,5 +1,6 @@
from collections.abc import Hashable
from sientia_do.observability.metrics_controller import MetricsController
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
@@ -36,21 +37,22 @@ class Gates(SientiaMonitoring):
ensure data integrity and enable flexible data processing workflows.
"""
def __init__(self, logger: Logger, notification_handler: NotificationHandler):
def __init__(self, logger: Logger, notification_handler: NotificationHandler, metrics_controller: MetricsController):
"""
Initialize the Gates class with logging and notification services.
Args:
logger (Logger): Logger instance for operation logging
notification_handler (NotificationHandler): Handler for system notifications
metrics_controller (MetricsController): Metrics controller instance
"""
SientiaMonitoring.__init__(self, logger, notification_handler)
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
def close(self):
"""
Close the Gates class.
"""
SientiaMonitoring.close(self)
SientiaMonitoring.shutdown(self)
def apply_aggregation(
self, values: DataFrame, aggr_function: str, metadata: dict[str, Any]

10
scouter/activities/mongodb.py
View File

@@ -1,6 +1,6 @@
from collections.abc import Hashable
from datetime import UTC
from sientia_do.observability.metrics_controller import MetricsController
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
@@ -37,6 +37,7 @@ class MongoDB(SientiaMonitoring):
database_name: str,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
"""
Initialize MongoDB connection and services.
@@ -56,9 +57,10 @@ class MongoDB(SientiaMonitoring):
database_name=database_name,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
SientiaMonitoring.__init__(self, logger=logger, notification_handler=notification_handler)
SientiaMonitoring.__init__(self, logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller)
def close(self):
"""
@@ -77,7 +79,7 @@ class MongoDB(SientiaMonitoring):
self.close()
@activity.defn(name='load_latest_data')
async def load_latest_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
async def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
"""
Load the latest data from MongoDB collection since a specified timestamp.
@@ -116,7 +118,7 @@ class MongoDB(SientiaMonitoring):
self.debug(f'Data filter: {data_filter}', metadata=metadata)
data = self.mongodb_repository.find(
data = await self.mongodb_repository.find(
collection_name=collection_name,
filters=data_filter,
metadata=metadata,

18
scouter/activities/redis.py
View File

@@ -1,9 +1,9 @@
from collections.abc import Hashable
from sientia_do.observability.metrics_controller import MetricsController
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import traceback
from collections.abc import Hashable
from typing import Any
from pandas import DataFrame
@@ -38,6 +38,7 @@ class Redis(SientiaMonitoring):
password: str,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
"""
Initialize Redis connection and services.
@@ -50,7 +51,7 @@ class Redis(SientiaMonitoring):
logger (Logger): Logger instance for operation logging
notification_handler (NotificationHandler): Handler for system notifications
"""
SientiaMonitoring.__init__(self, logger, notification_handler)
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
self.redis_repository = RedisRepository(
host=host,
port=port,
@@ -58,6 +59,7 @@ class Redis(SientiaMonitoring):
password=password,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
def close(self):
@@ -95,7 +97,7 @@ class Redis(SientiaMonitoring):
self.info(f'Getting last data timestamp for {key}')
try:
data_hold = self.redis_repository.get(key)
data_hold = await self.redis_repository.get(key)
except Exception as e:
self.send_notification(
metadata=metadata,
@@ -153,7 +155,7 @@ class Redis(SientiaMonitoring):
self.info(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata)
try:
self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5)
await self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5)
except Exception as e:
self.send_notification(
metadata=metadata,
@@ -206,7 +208,7 @@ class Redis(SientiaMonitoring):
self.info(f'Getting held data for {key}')
try:
data_hold = self.redis_repository.get(key)
data_hold = await self.redis_repository.get(key)
except Exception as e:
self.send_notification(
metadata=metadata,
@@ -250,7 +252,7 @@ class Redis(SientiaMonitoring):
data['timestamp'].max() if not data.empty else data_hold['timestamp']
)
self.redis_repository.set(key, data_hold, ttl=retention_time)
await self.redis_repository.set(key, data_hold, ttl=retention_time)
data_hold_df = DataFrame(data_hold, index=[0])
data_hold_melted = data_hold_df.melt(
@@ -296,7 +298,7 @@ class Redis(SientiaMonitoring):
cache = {'data': data.to_dict(), 'held_data': held_data.to_dict()}
try:
self.redis_repository.set(key, cache, ttl=120)
await self.redis_repository.set(key, cache, ttl=120)
except Exception as e:
self.send_notification(
metadata=metadata,

2
scouter/worker/worker.py
View File

@@ -136,7 +136,7 @@ async def main():
except BaseException as e: # NOSONAR
logger.custom_error(
'An unhandled exception occurred: %s', e, exc_info=True, metadata=metadata
'An unhandled exception occurred: %s', metadata=metadata
)
finally:
if notification_handler:

2
scouter/workflow/scouter.py
View File

@@ -95,7 +95,7 @@ class Scouter:
retry_policy=retry_policy,
)
if data == {}:
if not data:
return
await workflow.execute_activity_method(