gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #16 from Aignosi/SIENTIAPDE-1199-revisar-e-testar-observabilidade

Browse Source 
Sientiapde 1199 revisar e testar observabilidade
This commit is contained in:
Matheus Demoner
2025-08-25 10:14:52 -03:00
committed by GitHub
parent 2637e7dc97 799aa5af3e
commit aad0921957
11 changed files with 77 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@@ -5,7 +5,7 @@ asyncua
redis
aiokafka
pymongo
git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.7
git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1
git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5
pydruid[pandas]
prometheus-client

2
scouter/activities/activities.py
View File

@@ -3,7 +3,7 @@ from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.temporal.utils.logger import Logger
from sientia_do.observability.logger import Logger
from scouter.activities.redis import Redis
from scouter.activities.gates import Gates
from scouter.activities.mongodb import MongoDB

2
scouter/activities/faker.py
View File

@@ -7,7 +7,7 @@ from temporalio import activity
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.temporal.utils.logger import Logger
from sientia_do.observability.logger import Logger
class Faker(BaseActivity):

34
scouter/activities/gates.py
View File

@@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.temporal.utils.logger import Logger
from sientia_do.observability.logger import Logger
from scouter.utils.quality.filters import null_values_filter, out_of_bounds_filter
from typing import Any
import traceback
@@ -89,8 +89,8 @@ class Gates(BaseActivity):
# Convert input data to DataFrame
df = DataFrame(input_data['data'])
self.debug(
f"Aggregating time series data: {df.to_string()}",
self.info(
f"Aggregating time series data for {len(df)} rows",
metadata=metadata
)
@@ -147,10 +147,16 @@ class Gates(BaseActivity):
}
result_df = DataFrame(list(result.values()))
self.debug(
f"Aggregated data:\n {result_df.to_string()}",
self.info(
f"Aggregated data has {len(result_df)} rows",
metadata=metadata
)
self.debug(
f"Aggregated data: {result_df.to_string()}",
metadata=metadata
)
return result_df.to_dict()
except Exception as e:
@@ -195,8 +201,8 @@ class Gates(BaseActivity):
data = DataFrame(input_data['data'])
model_tags = input_data['model_tags']
self.debug(
f"Applying quality gate to data: {data.to_string()}",
self.info(
f"Applying quality gate to data to {len(data)} rows",
metadata=metadata
)
@@ -249,8 +255,8 @@ class Gates(BaseActivity):
if policy == "DISCARD":
data = data[~data.index.isin(filtered_data.index)]
self.debug(
"Data quality gate applied",
self.info(
f"Data quality gate applied, final data has {len(data)} rows",
metadata=metadata
)
@@ -265,8 +271,18 @@ class Gates(BaseActivity):
"""
metadata = input_data['metadata']
self.info(
f"Writing metrics for {metadata['model_name']}",
metadata=metadata
)
metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name']
).inc()
self.info(
f"Metrics written for {metadata['model_name']}",
metadata=metadata
)

4
scouter/activities/mongodb.py
View File

@@ -9,7 +9,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.temporal.utils.logger import Logger
from sientia_do.observability.logger import Logger
def clear_mongo_id(docs: list) -> list:
@@ -87,7 +87,7 @@ class MongoDB(BaseActivity):
collection_name = input_data['collection_name']
last_data_timestamp = input_data['last_data_timestamp']
self.debug(
self.info(
f"Loading data from MongoDB: {input_data}",
metadata=metadata
)

18
scouter/activities/redis.py
View File

@@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.activities.redis_base import Redis as RedisBase
from sientia_do.temporal.utils.logger import Logger
from sientia_do.observability.logger import Logger
from typing import Any
from pandas import DataFrame
from datetime import datetime
@@ -29,6 +29,8 @@ class Redis(RedisBase):
metadata = input_data['metadata']
key = f"last_data_timestamp_{input_data['workflow_name']}_{input_data['schedule_name']}"
self.info(f"Getting last data timestamp for {key}")
try:
data_hold = self.get(key)
except Exception as e:
@@ -42,7 +44,7 @@ class Redis(RedisBase):
)
raise e
self.debug(
self.info(
f"Last collected timestamp: {data_hold}",
metadata=metadata
)
@@ -60,6 +62,8 @@ class Redis(RedisBase):
metadata = input_data['metadata']
key = f"last_data_timestamp_{input_data['workflow_name']}_{input_data['schedule_name']}"
self.info(f"Putting last data timestamp for {key}")
data = DataFrame(input_data['data'])
if data.empty:
@@ -70,7 +74,7 @@ class Redis(RedisBase):
last_data_timestamp = data['inserted_at'].max()
self.debug(
self.info(
f"Last collected timestamp to insert: {last_data_timestamp}",
metadata=metadata
)
@@ -116,6 +120,8 @@ class Redis(RedisBase):
key = f"held_data_{input_data['workflow_name']}_{input_data['schedule_name']}"
self.info(f"Getting held data for {key}")
try:
data_hold = self.get(key)
except Exception as e:
@@ -137,6 +143,8 @@ class Redis(RedisBase):
)
return data_hold
self.info(f"Grouping and holding data for {len(data)} rows")
try:
# Remove possibly removed tags
@@ -195,8 +203,10 @@ class Redis(RedisBase):
)
raise e
self.info(f"Data held and melted has {len(data_hold_melted)} rows")
self.debug(
f"Data grouped and held successfully:\n {data_hold_melted.to_string()}",
f"Data held and melted:\n {data_hold_melted.to_string()}",
metadata=metadata
)

45
scouter/worker/worker.py
View File

@@ -1,12 +1,12 @@
from temporalio import workflow, client
from temporalio.worker import Worker
from temporalio.worker import Worker, PollerBehaviorAutoscaling
from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig
with workflow.unsafe.imports_passed_through():
import sys
import os
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.temporal.utils.logger import get_logger
from sientia_do.observability.logger import get_logger
from scouter.activities.activities import Activities
from scouter.workflow.scouter import Scouter
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
@@ -30,12 +30,20 @@ async def main():
logger = get_logger(__name__)
logger.info(f"Starting Worker with pod_id: {POD_ID}")
metadata = {
'pod_id': POD_ID,
'model_name': '-',
'model_id': '-',
'workflow_name': '-',
'schedule_name': '-',
}
logger.info("Starting prometheus client...")
logger.custom_info(f"Starting Worker with pod_id: {POD_ID}", metadata)
logger.custom_info("Starting prometheus client...", metadata)
start_prometheus_server()
logger.info('Starting Notification Handler...')
logger.custom_info('Starting Notification Handler...', metadata)
mongo_config = build_mongodb_config()
notification_handler = NotificationHandler(
@@ -45,7 +53,7 @@ async def main():
project_name=os.getenv('PROJECT_NAME', 'scouter')
)
logger.info('Starting Activities...')
logger.custom_info('Starting Activities...', metadata)
activities = Activities(
logger=logger,
@@ -55,7 +63,7 @@ async def main():
mongodb_config=build_mongodb_config()
)
logger.info('Starting Faker Activities...')
logger.custom_info('Starting Faker Activities...', metadata)
faker_activities = Faker(
logger=logger,
@@ -64,7 +72,8 @@ async def main():
'KAFKA_BOOTSTRAP_SERVERS', 'localhost:9092')
)
logger.info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...')
logger.custom_info(
f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
new_runtime = Runtime(
telemetry=TelemetryConfig(
@@ -73,7 +82,7 @@ async def main():
)
)
logger.info('Starting Temporal Client...')
logger.custom_info('Starting Temporal Client...', metadata)
temporal_client = await client.Client.connect(
target_host=host,
@@ -81,7 +90,7 @@ async def main():
runtime=new_runtime
)
logger.info('Starting Workers...')
logger.custom_info('Starting Workers...', metadata)
workers = [
Worker(
@@ -99,11 +108,12 @@ async def main():
activities.write_metrics,
activities.store_data_package,
],
max_concurrent_workflow_tasks=100,
max_concurrent_activities=100,
max_concurrent_local_activities=100,
max_concurrent_workflow_task_polls=100,
max_cached_workflows=50,
max_concurrent_workflow_tasks=50,
max_concurrent_activities=50,
max_concurrent_local_activities=50,
max_cached_workflows=200,
workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
activity_task_poller_behavior=PollerBehaviorAutoscaling()
),
Worker(
temporal_client,
@@ -119,13 +129,14 @@ async def main():
for w in workers:
handlers.append(w.run())
logger.info('Workers started successfully')
logger.custom_info('Workers started successfully', metadata)
try:
await asyncio.gather(*handlers)
except BaseException as e: # NOSONAR
logger.error("An unhandled exception occurred: %s", e, exc_info=True)
logger.custom_error("An unhandled exception occurred: %s",
e, exc_info=True, metadata=metadata)
finally:
if notification_handler:
notification_handler.shutdown()

2
scouter/workflow/fake_data.py
View File

@@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through():
from scouter.activities.faker import Faker
from datetime import timedelta
from typing import Dict, Any
from sientia_do.temporal.utils.policies import retry_policy
from sientia_do.temporal.policies import retry_policy
@workflow.defn(name="fake_data")

2
scouter/workflow/scouter.py
View File

@@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through():
from scouter.activities.activities import Activities
from typing import Any
from datetime import timedelta
from sientia_do.temporal.utils.policies import retry_policy
from sientia_do.temporal.policies import retry_policy
@workflow.defn(name="scouter")

2
scouter/workflow/sub_workflows/core_scouter.py
View File

@@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through():
from scouter.activities.activities import Activities
from typing import Any
from datetime import timedelta
from sientia_do.temporal.utils.policies import retry_policy
from sientia_do.temporal.policies import retry_policy
@workflow.defn(name="core_scouter")

4
values.yaml
View File

@@ -11,7 +11,7 @@ image:
# This sets the pull policy for images.
pullPolicy: Always
# Overrides the image tag whose default is the chart appVersion.
tag: "0.4.0"
tag: "0.4.2"
# This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
imagePullSecrets:
@@ -150,7 +150,7 @@ env:
- name: GITHUB_REPO_URL
value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git"
- name: GITHUB_BRANCH
value: "SIENTIAPDE-1169-pensar-e-projetar-testes-de-breakdown-e-performance"
value: "SIENTIAPDE-1199-revisar-e-testar-observabilidade"
- name: PYTHON_APP
value: "scouter.worker.worker"