Merge pull request #15 from Aignosi/SIENTIAPDE-1199-revisar-e-testar-observabilidade

Sientiapde 1199 revisar e testar observabilidade

ingestor/ingestor.py

@@ -3,7 +3,7 @@ from copy import deepcopy
 from typing import Dict, Any
 
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
-from sientia_do.temporal.utils.logger import get_logger
+from sientia_do.observability.logger import get_logger
 
 from ingestor.managers.ingestor_manager import IngestorManager
 
@@ -66,14 +66,14 @@ class Ingestor:
             connection_string=self.mongo_connection_string,
             database=self.mongo_database,
             logger=self.logger,
-            project_name="OPC_INGESTOR"
+            project_name="opc_ingestor"
         )
 
         self.metadata = {
             'model_id': '-',
             'model_name': '-',
-            'workflow_name': 'OPC_INGESTOR',
+            'workflow_name': 'opc_ingestor',
-            'schema_name': 'OPC_INGESTOR',
+            'schema_name': 'opc_ingestor',
             'pod_id': self.pod_id,
         }
         self.ingestor_manager = None
@@ -248,18 +248,18 @@ class Ingestor:
 
         for slot, config in new_managed_tags.items():
             if slot not in old_managed_tags:
-                self.logger.debug(f"Subscribing to new slot {slot}")
+                self.logger.info(f"Subscribing to new slot {slot}")
                 self.ingestor_manager.subscribe_to_tags({slot: config})
                 continue
 
             if config != old_managed_tags[slot]:
-                self.logger.debug(f"Resubscribing to slot {slot}")
+                self.logger.info(f"Resubscribing to slot {slot}")
                 self.ingestor_manager.unsubscribe_slot(slot)
                 self.ingestor_manager.subscribe_to_tags({slot: config})
 
         for slot in old_managed_tags.keys():
             if slot not in new_managed_tags:
-                self.logger.debug(f"Unsubscribing from slot {slot}")
+                self.logger.info(f"Unsubscribing from slot {slot}")
                 self.ingestor_manager.unsubscribe_slot(slot)
 
         # Ensure the gauge is updated after any potential changes here
@@ -302,14 +302,14 @@ class Ingestor:
         metrics.ACTIVE_INGESTORS.set(number_of_ingestors)
 
         # Handle no slots
-        self.logger.debug("Managing no slots...")
+        self.logger.info("Managing no slots...")
         self.manage_no_slots(number_of_slots)
 
         available_slots = number_of_slots - number_of_leases
         lacking_ingestors = number_of_slots - number_of_ingestors
         slot_diff = len(self.ingestor_manager.managed_tags) - 1
 
-        self.logger.debug("Managing leases...")
+        self.logger.info("Managing leases...")
         self.manage_leases(available_slots, lacking_ingestors, slot_diff)
 
         # Update managed slots gauge
@@ -329,12 +329,12 @@ class Ingestor:
             self.logger.info("No slots acquired in this loop")
 
         # Update opc servers
-        self.logger.debug("Updating slot config...")
+        self.logger.info("Updating slot config...")
         self.ingestor_manager.update_slot_config()
 
         # Check OPC cycles
-        self.logger.debug("Checking OPC servers integrity...")
+        self.logger.info("Checking OPC servers integrity...")
         self.ingestor_manager.check_opc_servers_integrity()
 
-        self.logger.debug("Updating managed tags...")
+        self.logger.info("Updating managed tags...")
         self.update_ingestor_manager(current_managed_tags)

ingestor/managers/data_manager.py

@@ -7,7 +7,7 @@ from kafka.errors import NoBrokersAvailable
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.notifications.models import NotificationLevel
 from sientia_do.temporal.activities.base import BaseActivity
-from sientia_do.temporal.utils.logger import Logger
+from sientia_do.observability.logger import Logger
 import traceback
 import ingestor.metrics as metrics
 import os

ingestor/managers/ingestor_manager.py

@@ -4,7 +4,7 @@ from copy import deepcopy
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.notifications.models import NotificationLevel
 from sientia_do.temporal.activities.base import BaseActivity
-from sientia_do.temporal.utils.logger import Logger
+from sientia_do.observability.logger import Logger
 from ingestor.managers.data_manager import DataManager
 from ingestor.managers.opc_manager import OpcManager
 from ingestor.managers.resource_manager import ResourceManager
@@ -153,7 +153,7 @@ class IngestorManager(BaseActivity):
                 server_config.pop('tags', None)
                 server_instance = self.opc_managers.get(server, None)
                 if server_instance is None:
-                    self.logger.debug(
+                    self.logger.info(
                         f"Initializing OPC manager for server {server}"
                     )
                     server_instance = self.initialize_opc_from_config(

ingestor/managers/opc_manager.py

@@ -5,7 +5,7 @@ from asyncua.sync import Client
 from sientia_do.notifications.models import NotificationLevel
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.temporal.activities.base import BaseActivity
-from sientia_do.temporal.utils.logger import Logger
+from sientia_do.observability.logger import Logger
 from ingestor.managers.data_manager import DataManager
 import ingestor.metrics as metrics
 

ingestor/managers/resource_manager.py

@@ -5,7 +5,7 @@ from time import time
 import ingestor.metrics as metrics
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.notifications.models import NotificationLevel
-from sientia_do.temporal.utils.logger import Logger
+from sientia_do.observability.logger import Logger
 from sientia_do.temporal.activities.base import BaseActivity
 
 

requirements.txt

@@ -1,5 +1,5 @@
 asyncua==1.1.5
 redis
-git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.8
+git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1
 prometheus_client
 pymongo

tests/unit/test_ingestor.py

@@ -49,8 +49,8 @@ def test___init__(notification_handler, getenv):
     assert ingestor.metadata == {
         "model_id": "-",
         "model_name": "-",
-        "workflow_name": "OPC_INGESTOR",
+        "workflow_name": "opc_ingestor",
-        "schema_name": "OPC_INGESTOR",
+        "schema_name": "opc_ingestor",
         "pod_id": "localhost1"
     }
 
@@ -58,7 +58,7 @@ def test___init__(notification_handler, getenv):
         connection_string="mongodb://sientia:sientia@localhost:27017",
         database="sientia",
         logger=ingestor.logger,
-        project_name="OPC_INGESTOR"
+        project_name="opc_ingestor"
     )
 
 

values.yaml

@@ -11,7 +11,7 @@ image:
   # This sets the pull policy for images.
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.3.2"
+  tag: "0.4.2"
 
 # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets:
@@ -112,20 +112,6 @@ tolerations: []
 affinity: {}
 
 services:
-  api:
-    enabled: false
-    type: ClusterIP
-    port: 4841
-    targetPort: 4841
-    name: api
-
-  opc:
-    enabled: false
-    type: ClusterIP
-    port: 4840
-    targetPort: 4840
-    name: server
-
   metrics:
     enabled: true
     type: ClusterIP
@@ -133,32 +119,27 @@ services:
     targetPort: 9090
     name: metrics
 
-
 # Configuração do ServiceMonitor para o Prometheus Operator
 # ref: https://github.com/prometheus-operator/prometheus-operator
 serviceMonitor:
   # Se true, um recurso ServiceMonitor será criado.
   enabled: true
   # O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m).
-  interval: 30s
+  endpoints:
-  # O path do endpoint de métricas na sua aplicação.
+  - port: metrics
     path: /metrics
-  # Labels adicionais para o recurso ServiceMonitor.
+    interval: 30s
-  # Essencial para que o Prometheus Operator o descubra. Se você usa o helm chart kube-prometheus-stack,
+    relabelings: []
-  # ele procura por ServiceMonitors com o label "release: kube-prometheus-stack".
+
   additionalLabels:
     release: kube-prometheus-stack
-  # Configurações de relabeling adicionais, se necessário.
-  # ref: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
-  relabelings: []
-  port: metrics
   
 env:
   # Entrypoint variables 
   - name: GITHUB_REPO_URL
     value: "git@github.com:Aignosi/sientia-dataops-opc-ingestor.git"
   - name: GITHUB_BRANCH
-    value: "SIENTIAPDE-1174-mapear-e-implementar-metricas-a-serem-criadas"
+    value: "SIENTIAPDE-1199-revisar-e-testar-observabilidade"
   - name: PYTHON_APP
     value: "ingestor.app"
 
@@ -212,7 +193,7 @@ ssh:
 
 # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp
 
-# helm upgrade --install sientia-opc-ingestor sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0
+# helm upgrade --install sientia-opc-ingestor sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.5.0
 
 # kubectl create secret generic git-ssh-key-sientia-opc-ingestor \
 #   --namespace sientia \