From 2efad30304092885e2e6e3bdbd976cee463240af Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 12:52:07 -0300 Subject: [PATCH 01/16] SIENTIAPDE-1174 SIENTIAPDE-1174 Update dependencies, modify image tag, and enhance metrics tracking in various activities for improved observability and performance. --- requirements.txt | 5 ++-- scouter/activities/gates.py | 6 +++++ scouter/activities/mongodb.py | 3 ++- scouter/activities/redis.py | 16 +++++++++++- scouter/metrics.py | 15 +++++++++++ scouter/workflow/scouter.py | 4 ++- .../workflow/sub_workflows/core_scouter.py | 8 ++++++ tests/test_metrics.py | 25 +++++++++++++++++++ values.yaml | 14 ++++++++--- 9 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 scouter/metrics.py create mode 100644 tests/test_metrics.py diff --git a/requirements.txt b/requirements.txt index 6feb3ea..1affea9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ asyncua redis aiokafka pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.5 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.6 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5 -pydruid[pandas] \ No newline at end of file +pydruid[pandas] +prometheus-client \ No newline at end of file diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index ea9d1bf..3c68aaf 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -3,6 +3,8 @@ from temporalio import workflow, activity with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.base import BaseActivity + from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.temporal.utils.logger import Logger from scouter.utils.quality.filters import null_values_filter, out_of_bounds_filter from typing import Any import traceback @@ -16,6 +18,10 @@ quality_gate_filters = { class Gates(BaseActivity): + def __init__(self, logger: Logger, notification_handler: NotificationHandler): + BaseActivity.__init__( + self, logger, notification_handler, set_error_counter=True) + def apply_aggregation(self, group: DataFrame, aggr_function: str, metadata: dict[str, Any]) -> float | None | str: """ diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index c4c6fe4..3a80436 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -57,7 +57,8 @@ class MongoDB(BaseActivity): BaseActivity.__init__(self, logger=logger, - notification_handler=notification_handler) + notification_handler=notification_handler, + set_error_counter=True) def shutdown(self): """ diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 5174d87..33d3170 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -1,8 +1,8 @@ -import traceback from temporalio import workflow, activity with workflow.unsafe.imports_passed_through(): from logging import Logger + import traceback from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.redis_base import Redis as RedisBase @@ -10,6 +10,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any from pandas import DataFrame from datetime import datetime + from scouter import metrics class Redis(RedisBase): @@ -136,16 +137,29 @@ class Redis(RedisBase): return data_hold try: + + to_register_metrics = [] for _, row in data.iterrows(): value = row['value'] data_hold[row['name']] = value + to_register_metrics.append( + (row['name'], value)) data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \ datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.set(key, data_hold, ttl=retention_time) + # Register metrics + for metric in to_register_metrics: + metrics.TAG_CHANGES_MONITOR.labels( + pod_id=metadata['pod_id'], + model_name=input_data['model_name'], + pipeline_name=input_data['schedule_name'], + tag_name=metric[0] + ).set(metric[1]) + data_hold_df = DataFrame(data_hold, index=[0]) data_hold_melted = data_hold_df.melt( id_vars='timestamp', var_name='variable', value_name='value') diff --git a/scouter/metrics.py b/scouter/metrics.py new file mode 100644 index 0000000..3800279 --- /dev/null +++ b/scouter/metrics.py @@ -0,0 +1,15 @@ +from prometheus_client import Gauge, Counter + +CORE_LABELS = ["pod_id", "model_name", "pipeline_name"] + +LABORIOUS_DATA_WRITTEN_COUNT = Counter( + "scouter_laborious_data_written_count", + "Number of writings to the database table laborious_data", + CORE_LABELS, +) + +TAG_CHANGES_MONITOR = Gauge( + "scouter_tag_changes_monitor", + "Current value change of each tag", + [*CORE_LABELS, "tag_name"], +) diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index ffcb625..c2afbff 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -5,6 +5,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any from datetime import timedelta from sientia_do.temporal.utils.policies import retry_policy + from os import getenv @workflow.defn(name="scouter") @@ -37,7 +38,8 @@ class Scouter: 'model_id': input_data['model_id'], 'model_name': input_data['model_name'], 'schedule_name': input_data['schedule_name'], - 'workflow_name': input_data['workflow_name'] + 'workflow_name': input_data['workflow_name'], + 'pod_id': getenv("HOSTNAME", "localhost") } } diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index 21a6120..c71f10b 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -1,5 +1,7 @@ from temporalio import workflow +from scouter import metrics + with workflow.unsafe.imports_passed_through(): from scouter.activities.activities import Activities from typing import Any @@ -86,6 +88,12 @@ class CoreScouter: start_to_close_timeout=timedelta(seconds=60) ) + metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels( + pod_id=metadata['pod_id'], + model_name=input_data['model_name'], + pipeline_name=input_data['workflow_name'] + ).inc() + if input_data.get('debug_data_package', False): await workflow.execute_activity_method( Activities.store_data_package, diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..875113c --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,25 @@ +# tests/unit/test_metrics.py + +import pytest +from prometheus_client import Counter, Gauge, Histogram +import scouter.metrics as metrics + +# --- Test Functions for Each Metric (Corrected for v0.22.0 _name behavior) --- + + +def test_scouter_laborious_data_written_count(): + """Verify the definition of LABORIOUS_DATA_WRITTEN_COUNT.""" + assert metrics.LABORIOUS_DATA_WRITTEN_COUNT is not None + assert isinstance(metrics.LABORIOUS_DATA_WRITTEN_COUNT, Counter) + assert metrics.LABORIOUS_DATA_WRITTEN_COUNT._name == "scouter_laborious_data_written_count" + assert set(metrics.LABORIOUS_DATA_WRITTEN_COUNT._labelnames) == { + "pod_id", "model_name", "pipeline_name"} + + +def test_scouter_tag_changes_monitor(): + """Verify the definition of TAG_CHANGES_MONITOR.""" + assert metrics.TAG_CHANGES_MONITOR is not None + assert isinstance(metrics.TAG_CHANGES_MONITOR, Gauge) + assert metrics.TAG_CHANGES_MONITOR._name == "scouter_tag_changes_monitor" + assert set(metrics.TAG_CHANGES_MONITOR._labelnames) == { + "pod_id", "model_name", "pipeline_name", "tag_name"} diff --git a/values.yaml b/values.yaml index 3e73d24..d7fb472 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.2.7" + tag: "0.3.0" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: @@ -111,14 +111,20 @@ tolerations: [] affinity: {} -service: {} +services: + metrics: + enabled: true + type: ClusterIP + port: 9090 + targetPort: 9090 + name: metrics env: # Entrypoint variables - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1172-criar-pipeline-de-alertas-orquestrador" + value: "SIENTIAPDE-1174-mapear-e-implementar-metricas-a-serem-criadas" - name: PYTHON_APP value: "scouter.worker.worker" @@ -160,6 +166,8 @@ env: - name: LOG_LEVEL value: "DEBUG" + - name: HTTP_METRICS_PORT + value: "9090" - name: PROJECT_NAME value: "sientia-scouter" From 7925b8a2ae0f2b3fe2d483970d84627ee8abf761 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 13:01:39 -0300 Subject: [PATCH 02/16] SIENTIAPDE-1174 Update replica count in values.yaml, add application status metrics in metrics.py, and implement Prometheus server in worker.py for enhanced observability. --- scouter/metrics.py | 6 ++++++ scouter/worker/worker.py | 22 +++++++++++++++++++++- values.yaml | 2 +- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/scouter/metrics.py b/scouter/metrics.py index 3800279..a3114be 100644 --- a/scouter/metrics.py +++ b/scouter/metrics.py @@ -1,5 +1,11 @@ from prometheus_client import Gauge, Counter +APP_UP = Gauge( + "app_up", + "Indicates if the application is running (1) or shutting down (0)", + ["pod_id"], +) + CORE_LABELS = ["pod_id", "model_name", "pipeline_name"] LABORIOUS_DATA_WRITTEN_COUNT = Counter( diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 239ecd8..62d2752 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -12,19 +12,26 @@ with workflow.unsafe.imports_passed_through(): from scouter.workflow.fake_data import FakeData from scouter.activities.faker import Faker import asyncio + from prometheus_client import start_http_server + from scouter import metrics from scouter.utils.connectors_config import ( build_postgres_config, build_redis_config, build_mongodb_config ) +POD_ID = os.getenv("HOSTNAME", "localhost") + async def main(): host = os.getenv('TEMPORAL_HOST', 'localhost:7233') logger = get_logger(__name__) - logger.info('Starting Worker...') + logger.info(f"Starting Worker with pod_id: {POD_ID}") + + logger.info("Starting prometheus client...") + start_prometheus_server() logger.info('Starting Notification Handler...') @@ -112,7 +119,20 @@ async def main(): if activities: activities.shutdown() # Exit with a non-zero status code to indicate failure to Kubernetes + metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN sys.exit(1) + +def start_prometheus_server(): + try: + port = int(os.getenv("HTTP_METRICS_PORT", 9090)) + start_http_server(port) + print(f"Prometheus server started on port {port}.") + metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP + except Exception as e: + print(f"Failed to start Prometheus server: {e}") + os._exit(1) + + if __name__ == '__main__': asyncio.run(main()) diff --git a/values.yaml b/values.yaml index d7fb472..c55fa07 100644 --- a/values.yaml +++ b/values.yaml @@ -3,7 +3,7 @@ # Declare variables to be passed into your templates. # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ -replicaCount: 5 +replicaCount: 3 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: From 1c78cfb558135d8334e49f146fa5de05a0b08e20 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 14:39:58 -0300 Subject: [PATCH 03/16] SIENTIAPDE-1174 Add write_metrics activity to Gates for database metric logging and import metrics module --- scouter/activities/gates.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index 3c68aaf..48b6ef4 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -1,5 +1,7 @@ from temporalio import workflow, activity +from scouter import metrics + with workflow.unsafe.imports_passed_through(): from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.base import BaseActivity @@ -74,7 +76,7 @@ class Gates(BaseActivity): Args: input_data (dict[str, Any]): The data to aggregate. Contains: data (dict[str, Any]): The time series data. - model_tags (dict[str, Any]): The tags configuration + model_tags (dict[str, Any]): The tags configuration containing aggregation functions. Returns: @@ -253,3 +255,18 @@ class Gates(BaseActivity): ) return data.to_dict() + + @activity.defn(name="write_metrics") + async def write_metrics(self, input_data: dict[str, Any]) -> dict[str, Any]: + """ + Write metrics to the database. + input_data: + metadata: dict[str, Any] + """ + metadata = input_data['metadata'] + + metrics.LABORIOUS_DATA_WRITTEN_COUNT( + pod_id=metadata['pod_id'], + model_name=metadata['model_name'], + pipeline_name=metadata['pipeline_name'] + ).inc() From b4827935a7775381872e8dd99401aea7a3cc65f6 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 14:43:34 -0300 Subject: [PATCH 04/16] SIENTIAPDE-1174 Integrate write_metrics activity into CoreScouter for enhanced metric logging and update worker.py to include write_metrics in workflow tasks. --- scouter/worker/worker.py | 3 ++- scouter/workflow/sub_workflows/core_scouter.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 62d2752..aff8341 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -84,7 +84,8 @@ async def main(): activities.aggregate_data, activities.group_and_hold_data, activities.export_data_to_postgres, - activities.store_data_package + activities.write_metrics, + activities.store_data_package, ], max_concurrent_workflow_tasks=100, max_concurrent_activities=100, diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index c71f10b..3a9c1e6 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -88,11 +88,14 @@ class CoreScouter: start_to_close_timeout=timedelta(seconds=60) ) - metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels( - pod_id=metadata['pod_id'], - model_name=input_data['model_name'], - pipeline_name=input_data['workflow_name'] - ).inc() + await workflow.execute_activity_method( + Activities.write_metrics, + { + **metadata, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60) + ) if input_data.get('debug_data_package', False): await workflow.execute_activity_method( From ce20d863289620ff9e1193ffec57acbfc877a0b2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 14:48:00 -0300 Subject: [PATCH 05/16] SIENTIAPDE-1174 Remove unused metrics import from core_scouter.py to streamline code and improve clarity. --- scouter/workflow/sub_workflows/core_scouter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index 3a9c1e6..b9f4a35 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -1,7 +1,5 @@ from temporalio import workflow -from scouter import metrics - with workflow.unsafe.imports_passed_through(): from scouter.activities.activities import Activities from typing import Any From 5d3eb740224be79b2af8bdf507ca1c26146510c2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 14:57:22 -0300 Subject: [PATCH 06/16] SIENTIAPDE-1174 SIENTIAPDE-1174 Add pod_id initialization in Activities class and update Scouter to reference it for improved consistency in environment variable handling. --- scouter/activities/activities.py | 3 +++ scouter/workflow/scouter.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scouter/activities/activities.py b/scouter/activities/activities.py index 775f958..9ada6d3 100644 --- a/scouter/activities/activities.py +++ b/scouter/activities/activities.py @@ -8,6 +8,7 @@ with workflow.unsafe.imports_passed_through(): from scouter.activities.gates import Gates from scouter.activities.mongodb import MongoDB from typing import Any + from os import getenv class Activities(Postgres, Redis, Gates, MongoDB,): @@ -61,6 +62,8 @@ class Activities(Postgres, Redis, Gates, MongoDB,): notification_handler=notification_handler ) + self.pod_id = getenv("HOSTNAME", "localhost") + def shutdown(self): Postgres.close(self) MongoDB.shutdown(self) diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index c2afbff..3f15e13 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -39,7 +39,7 @@ class Scouter: 'model_name': input_data['model_name'], 'schedule_name': input_data['schedule_name'], 'workflow_name': input_data['workflow_name'], - 'pod_id': getenv("HOSTNAME", "localhost") + 'pod_id': Activities.pod_id } } From c69ae5df032ff6f20ade28fefe6398ba515aaa9d Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 14:59:12 -0300 Subject: [PATCH 07/16] SIENTIAPDE-1174 Remove unused getenv import from scouter.py to enhance code clarity and maintainability. --- scouter/workflow/scouter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index 3f15e13..a8f43a7 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -5,7 +5,6 @@ with workflow.unsafe.imports_passed_through(): from typing import Any from datetime import timedelta from sientia_do.temporal.utils.policies import retry_policy - from os import getenv @workflow.defn(name="scouter") From 205810e882ddc3886212ca8a96133e577dc7c9ad Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 15:45:18 -0300 Subject: [PATCH 08/16] SIENTIAPDE-1174 Update requirements for sientia-dataops-library to version 1.3.7, add ServiceMonitor configuration for Prometheus in values.yaml, and refactor pod_id usage in Gates and Redis activities for improved consistency in metric tracking. --- requirements.txt | 2 +- scouter/activities/gates.py | 2 +- scouter/activities/redis.py | 2 +- values.yaml | 19 +++++++++++++++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1affea9..1b3009f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ asyncua redis aiokafka pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.6 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.7 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5 pydruid[pandas] prometheus-client \ No newline at end of file diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index 48b6ef4..846df94 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -266,7 +266,7 @@ class Gates(BaseActivity): metadata = input_data['metadata'] metrics.LABORIOUS_DATA_WRITTEN_COUNT( - pod_id=metadata['pod_id'], + pod_id=self.pod_id, model_name=metadata['model_name'], pipeline_name=metadata['pipeline_name'] ).inc() diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 33d3170..4bbfef0 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -154,7 +154,7 @@ class Redis(RedisBase): # Register metrics for metric in to_register_metrics: metrics.TAG_CHANGES_MONITOR.labels( - pod_id=metadata['pod_id'], + pod_id=self.pod_id, model_name=input_data['model_name'], pipeline_name=input_data['schedule_name'], tag_name=metric[0] diff --git a/values.yaml b/values.yaml index c55fa07..f089441 100644 --- a/values.yaml +++ b/values.yaml @@ -119,6 +119,25 @@ services: targetPort: 9090 name: metrics +# Configuração do ServiceMonitor para o Prometheus Operator +# ref: https://github.com/prometheus-operator/prometheus-operator +serviceMonitor: + # Se true, um recurso ServiceMonitor será criado. + enabled: true + # O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m). + interval: 30s + # O path do endpoint de métricas na sua aplicação. + path: /metrics + # Labels adicionais para o recurso ServiceMonitor. + # Essencial para que o Prometheus Operator o descubra. Se você usa o helm chart kube-prometheus-stack, + # ele procura por ServiceMonitors com o label "release: kube-prometheus-stack". + additionalLabels: + release: kube-prometheus-stack + # Configurações de relabeling adicionais, se necessário. + # ref: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + relabelings: [] + port: metrics + env: # Entrypoint variables - name: GITHUB_REPO_URL From a8222359cc7ffd24b8f9b5d9d8e8ed8c7b343b1b Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 16:25:02 -0300 Subject: [PATCH 09/16] SIENTIAPDE-1174 Refactor Scouter class to remove pod_id reference, enhancing code clarity and consistency in data handling. --- scouter/workflow/scouter.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index a8f43a7..ffcb625 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -37,8 +37,7 @@ class Scouter: 'model_id': input_data['model_id'], 'model_name': input_data['model_name'], 'schedule_name': input_data['schedule_name'], - 'workflow_name': input_data['workflow_name'], - 'pod_id': Activities.pod_id + 'workflow_name': input_data['workflow_name'] } } From b9c004827fba477f1aa32e78d8584c50d5aabaab Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 16:27:18 -0300 Subject: [PATCH 10/16] SIENTIAPDE-1174 Update Redis activity to use metadata for model and pipeline names, improving consistency in metric tracking. --- scouter/activities/redis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 4bbfef0..49c324f 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -155,8 +155,8 @@ class Redis(RedisBase): for metric in to_register_metrics: metrics.TAG_CHANGES_MONITOR.labels( pod_id=self.pod_id, - model_name=input_data['model_name'], - pipeline_name=input_data['schedule_name'], + model_name=metadata['model_name'], + pipeline_name=metadata['schedule_name'], tag_name=metric[0] ).set(metric[1]) From 7ea9c1ce6552d3af73c0d7c654b0f97fddf399d4 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 16:31:38 -0300 Subject: [PATCH 11/16] SIENTIAPDE-1174 Update Gates and Redis activities to use 'workflow_name' instead of 'pipeline_name' in metric tracking, ensuring consistency across the codebase. --- scouter/activities/gates.py | 2 +- scouter/activities/redis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index 846df94..cd61625 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -268,5 +268,5 @@ class Gates(BaseActivity): metrics.LABORIOUS_DATA_WRITTEN_COUNT( pod_id=self.pod_id, model_name=metadata['model_name'], - pipeline_name=metadata['pipeline_name'] + pipeline_name=metadata['workflow_name'] ).inc() diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 49c324f..bffc478 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -156,7 +156,7 @@ class Redis(RedisBase): metrics.TAG_CHANGES_MONITOR.labels( pod_id=self.pod_id, model_name=metadata['model_name'], - pipeline_name=metadata['schedule_name'], + pipeline_name=metadata['workflow_name'], tag_name=metric[0] ).set(metric[1]) From fd740ca4512a9ea1cf614586f3cd8fa0e67cc006 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 31 Jul 2025 16:34:55 -0300 Subject: [PATCH 12/16] SIENTIAPDE-1174 Update Gates activity to use labels in LABORIOUS_DATA_WRITTEN_COUNT metric, improving clarity in metric tracking. --- scouter/activities/gates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index cd61625..3930542 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -265,7 +265,7 @@ class Gates(BaseActivity): """ metadata = input_data['metadata'] - metrics.LABORIOUS_DATA_WRITTEN_COUNT( + metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels( pod_id=self.pod_id, model_name=metadata['model_name'], pipeline_name=metadata['workflow_name'] From 40c57ed23dd302129a8c1455b344a1f12f1df575 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 1 Aug 2025 07:40:06 -0300 Subject: [PATCH 13/16] SIENTIAPDE-1174 Implement async test for write_metrics method in Gates activity, enhancing test coverage for metric logging functionality. --- tests/activities/test_gates.py | 15 +++++++++++++++ tests/activities/test_redis.py | 1 + 2 files changed, 16 insertions(+) diff --git a/tests/activities/test_gates.py b/tests/activities/test_gates.py index dec6b9d..f0057fd 100644 --- a/tests/activities/test_gates.py +++ b/tests/activities/test_gates.py @@ -402,3 +402,18 @@ async def test_aggregate_data_raise_exception(gates_fixture): ) else: assert False + + +@pytest.mark.asyncio +@patch('scouter.activities.gates.metrics') +async def test_write_metrics(mock_metrics, gates_fixture): + """Test write_metrics method.""" + input_data = { + 'metadata': metadata['metadata'] + } + await gates_fixture.write_metrics(input_data) + mock_metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels.assert_called_once_with( + pod_id=gates_fixture.pod_id, + model_name=metadata['metadata']['model_name'], + pipeline_name=metadata['metadata']['workflow_name'] + ) diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index ce3215e..bed4c10 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -20,6 +20,7 @@ def redis_activity(_mock_redis_init): activity.redis_client = MagicMock() activity.logger = logger activity.notification_handler = notification_handler + activity.pod_id = 'test_pod_id' return activity From 25f9d8931786a8461947683c1733d5804a39af78 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Fri, 1 Aug 2025 08:36:08 -0300 Subject: [PATCH 14/16] SIENTIAPDE-1174 Update image tag to 0.3.1 in values.yaml and modify write_metrics method signature in Gates activity for improved type clarity. --- scouter/activities/gates.py | 2 +- values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index 3930542..c7840f4 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -257,7 +257,7 @@ class Gates(BaseActivity): return data.to_dict() @activity.defn(name="write_metrics") - async def write_metrics(self, input_data: dict[str, Any]) -> dict[str, Any]: + async def write_metrics(self, input_data: dict[str, Any]): """ Write metrics to the database. input_data: diff --git a/values.yaml b/values.yaml index f089441..6726d60 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.3.0" + tag: "0.3.1" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: From d4718c7d95afcd53b2002f855f07162bef78aa96 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 4 Aug 2025 10:04:01 -0300 Subject: [PATCH 15/16] SIENTIAPDE-1174 Update Helm upgrade command in values.yaml to use version 0.4.0, ensuring consistency in deployment instructions. --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 6726d60..1d9dd60 100644 --- a/values.yaml +++ b/values.yaml @@ -217,7 +217,7 @@ ssh: # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-scouter-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0-uat +# helm upgrade --install sientia-scouter-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0 # kubectl create secret generic git-ssh-key-sientia-scouter-worker \ # --namespace sientia \ From c6fca2a8c2700445693a34b23b82e809554a0a36 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Mon, 4 Aug 2025 15:26:21 -0300 Subject: [PATCH 16/16] SIENTIAPDE-1174 Update image tag to 0.3.2 in values.yaml for version consistency in deployment. --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 1d9dd60..a729358 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.3.1" + tag: "0.3.2" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: