From ac55ed8aff0ef95e77c988ba3b0368f817ff707a Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 19 Aug 2025 16:48:41 -0300 Subject: [PATCH 1/4] Update dependencies and refactor logger imports for observability - Updated sientia-dataops-library version to 1.4.1 in requirements.txt. - Changed image tag from 0.3.2 to 0.4.2 in values.yaml. - Refactored logger imports from sientia_do.temporal.utils.logger to sientia_do.observability.logger in multiple manager files for improved observability. --- ingestor/ingestor.py | 2 +- ingestor/managers/data_manager.py | 2 +- ingestor/managers/ingestor_manager.py | 2 +- ingestor/managers/opc_manager.py | 2 +- ingestor/managers/resource_manager.py | 2 +- requirements.txt | 2 +- values.yaml | 4 ++-- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index f39746b..c8c6ce3 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -3,7 +3,7 @@ from copy import deepcopy from typing import Dict, Any from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler -from sientia_do.temporal.utils.logger import get_logger +from sientia_do.observability.logger import get_logger from ingestor.managers.ingestor_manager import IngestorManager diff --git a/ingestor/managers/data_manager.py b/ingestor/managers/data_manager.py index 5d58c51..ad65367 100644 --- a/ingestor/managers/data_manager.py +++ b/ingestor/managers/data_manager.py @@ -7,7 +7,7 @@ from kafka.errors import NoBrokersAvailable from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.base import BaseActivity -from sientia_do.temporal.utils.logger import Logger +from sientia_do.observability.logger import Logger import traceback import ingestor.metrics as metrics import os diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index eec7116..64c3aca 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -4,7 +4,7 @@ from copy import deepcopy from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.base import BaseActivity -from sientia_do.temporal.utils.logger import Logger +from sientia_do.observability.logger import Logger from ingestor.managers.data_manager import DataManager from ingestor.managers.opc_manager import OpcManager from ingestor.managers.resource_manager import ResourceManager diff --git a/ingestor/managers/opc_manager.py b/ingestor/managers/opc_manager.py index b5fa903..918a96c 100644 --- a/ingestor/managers/opc_manager.py +++ b/ingestor/managers/opc_manager.py @@ -5,7 +5,7 @@ from asyncua.sync import Client from sientia_do.notifications.models import NotificationLevel from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.temporal.activities.base import BaseActivity -from sientia_do.temporal.utils.logger import Logger +from sientia_do.observability.logger import Logger from ingestor.managers.data_manager import DataManager import ingestor.metrics as metrics diff --git a/ingestor/managers/resource_manager.py b/ingestor/managers/resource_manager.py index d2db351..5288d69 100644 --- a/ingestor/managers/resource_manager.py +++ b/ingestor/managers/resource_manager.py @@ -5,7 +5,7 @@ from time import time import ingestor.metrics as metrics from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel -from sientia_do.temporal.utils.logger import Logger +from sientia_do.observability.logger import Logger from sientia_do.temporal.activities.base import BaseActivity diff --git a/requirements.txt b/requirements.txt index 5687b65..98ca99a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ asyncua==1.1.5 redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.8 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1 prometheus_client pymongo \ No newline at end of file diff --git a/values.yaml b/values.yaml index 1372684..93c7049 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.3.2" + tag: "0.4.2" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: @@ -158,7 +158,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-opc-ingestor.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1174-mapear-e-implementar-metricas-a-serem-criadas" + value: "SIENTIAPDE-1199-revisar-e-testar-observabilidade" - name: PYTHON_APP value: "ingestor.app" From 2fa07728f96b20d7cf4ec94a727e35f0a5a6c6cb Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 19 Aug 2025 16:50:12 -0300 Subject: [PATCH 2/4] Refactor values.yaml to enable metrics service and remove unused API and OPC services - Enabled metrics service with updated configuration for Prometheus ServiceMonitor. - Removed disabled API and OPC service configurations to streamline values.yaml. - Updated helm upgrade command version to 0.5.0. --- values.yaml | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/values.yaml b/values.yaml index 93c7049..2c2fa76 100644 --- a/values.yaml +++ b/values.yaml @@ -112,27 +112,12 @@ tolerations: [] affinity: {} services: - api: - enabled: false - type: ClusterIP - port: 4841 - targetPort: 4841 - name: api - - opc: - enabled: false - type: ClusterIP - port: 4840 - targetPort: 4840 - name: server - metrics: enabled: true type: ClusterIP port: 9090 targetPort: 9090 name: metrics - # Configuração do ServiceMonitor para o Prometheus Operator # ref: https://github.com/prometheus-operator/prometheus-operator @@ -140,18 +125,14 @@ serviceMonitor: # Se true, um recurso ServiceMonitor será criado. enabled: true # O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m). - interval: 30s - # O path do endpoint de métricas na sua aplicação. - path: /metrics - # Labels adicionais para o recurso ServiceMonitor. - # Essencial para que o Prometheus Operator o descubra. Se você usa o helm chart kube-prometheus-stack, - # ele procura por ServiceMonitors com o label "release: kube-prometheus-stack". + endpoints: + - port: metrics + path: /metrics + interval: 30s + relabelings: [] + additionalLabels: release: kube-prometheus-stack - # Configurações de relabeling adicionais, se necessário. - # ref: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config - relabelings: [] - port: metrics env: # Entrypoint variables @@ -212,7 +193,7 @@ ssh: # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-opc-ingestor sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0 +# helm upgrade --install sientia-opc-ingestor sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.5.0 # kubectl create secret generic git-ssh-key-sientia-opc-ingestor \ # --namespace sientia \ From a19ec20c12ea7b8659d1405bb2e293094584ba2e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 20 Aug 2025 12:16:38 -0300 Subject: [PATCH 3/4] SIENTIAPDE-1199 Update Ingestor project name and metadata keys to lowercase for consistency - Changed project_name, workflow_name, and schema_name from uppercase to lowercase in Ingestor class and corresponding unit tests. --- ingestor/ingestor.py | 6 +++--- tests/unit/test_ingestor.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index c8c6ce3..27f1819 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -66,14 +66,14 @@ class Ingestor: connection_string=self.mongo_connection_string, database=self.mongo_database, logger=self.logger, - project_name="OPC_INGESTOR" + project_name="opc_ingestor" ) self.metadata = { 'model_id': '-', 'model_name': '-', - 'workflow_name': 'OPC_INGESTOR', - 'schema_name': 'OPC_INGESTOR', + 'workflow_name': 'opc_ingestor', + 'schema_name': 'opc_ingestor', 'pod_id': self.pod_id, } self.ingestor_manager = None diff --git a/tests/unit/test_ingestor.py b/tests/unit/test_ingestor.py index 215e0fc..5faccfa 100644 --- a/tests/unit/test_ingestor.py +++ b/tests/unit/test_ingestor.py @@ -49,8 +49,8 @@ def test___init__(notification_handler, getenv): assert ingestor.metadata == { "model_id": "-", "model_name": "-", - "workflow_name": "OPC_INGESTOR", - "schema_name": "OPC_INGESTOR", + "workflow_name": "opc_ingestor", + "schema_name": "opc_ingestor", "pod_id": "localhost1" } @@ -58,7 +58,7 @@ def test___init__(notification_handler, getenv): connection_string="mongodb://sientia:sientia@localhost:27017", database="sientia", logger=ingestor.logger, - project_name="OPC_INGESTOR" + project_name="opc_ingestor" ) From f90a261d7c9b424f71cc47fc84c55dcb4cfd6ca3 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 20 Aug 2025 12:20:15 -0300 Subject: [PATCH 4/4] SIENTIAPDE-1199 Refactor logging levels in Ingestor and IngestorManager for improved clarity - Changed debug log statements to info level in Ingestor class for better visibility of subscription and management actions. - Updated log level for initializing OPC manager in IngestorManager to info for consistent logging practices. --- ingestor/ingestor.py | 16 ++++++++-------- ingestor/managers/ingestor_manager.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ingestor/ingestor.py b/ingestor/ingestor.py index 27f1819..0d68d2f 100644 --- a/ingestor/ingestor.py +++ b/ingestor/ingestor.py @@ -248,18 +248,18 @@ class Ingestor: for slot, config in new_managed_tags.items(): if slot not in old_managed_tags: - self.logger.debug(f"Subscribing to new slot {slot}") + self.logger.info(f"Subscribing to new slot {slot}") self.ingestor_manager.subscribe_to_tags({slot: config}) continue if config != old_managed_tags[slot]: - self.logger.debug(f"Resubscribing to slot {slot}") + self.logger.info(f"Resubscribing to slot {slot}") self.ingestor_manager.unsubscribe_slot(slot) self.ingestor_manager.subscribe_to_tags({slot: config}) for slot in old_managed_tags.keys(): if slot not in new_managed_tags: - self.logger.debug(f"Unsubscribing from slot {slot}") + self.logger.info(f"Unsubscribing from slot {slot}") self.ingestor_manager.unsubscribe_slot(slot) # Ensure the gauge is updated after any potential changes here @@ -302,14 +302,14 @@ class Ingestor: metrics.ACTIVE_INGESTORS.set(number_of_ingestors) # Handle no slots - self.logger.debug("Managing no slots...") + self.logger.info("Managing no slots...") self.manage_no_slots(number_of_slots) available_slots = number_of_slots - number_of_leases lacking_ingestors = number_of_slots - number_of_ingestors slot_diff = len(self.ingestor_manager.managed_tags) - 1 - self.logger.debug("Managing leases...") + self.logger.info("Managing leases...") self.manage_leases(available_slots, lacking_ingestors, slot_diff) # Update managed slots gauge @@ -329,12 +329,12 @@ class Ingestor: self.logger.info("No slots acquired in this loop") # Update opc servers - self.logger.debug("Updating slot config...") + self.logger.info("Updating slot config...") self.ingestor_manager.update_slot_config() # Check OPC cycles - self.logger.debug("Checking OPC servers integrity...") + self.logger.info("Checking OPC servers integrity...") self.ingestor_manager.check_opc_servers_integrity() - self.logger.debug("Updating managed tags...") + self.logger.info("Updating managed tags...") self.update_ingestor_manager(current_managed_tags) diff --git a/ingestor/managers/ingestor_manager.py b/ingestor/managers/ingestor_manager.py index 64c3aca..860321a 100644 --- a/ingestor/managers/ingestor_manager.py +++ b/ingestor/managers/ingestor_manager.py @@ -153,7 +153,7 @@ class IngestorManager(BaseActivity): server_config.pop('tags', None) server_instance = self.opc_managers.get(server, None) if server_instance is None: - self.logger.debug( + self.logger.info( f"Initializing OPC manager for server {server}" ) server_instance = self.initialize_opc_from_config(