Merge pull request #15 from Aignosi/SIENTIAPDE-1169-pensar-e-projetar-testes-de-breakdown-e-performance

Sientiapde 1169 pensar e projetar testes de breakdown e performance

orchestrator/activities/email.py

@@ -9,7 +9,7 @@ with workflow.unsafe.imports_passed_through():
     import smtplib
     from typing import Any
     from sientia_do.temporal.activities.base import BaseActivity
-    from sientia_do.temporal.utils.logger import Logger
+    from sientia_do.observability.logger import Logger
     from sientia_do.notifications.handlers import NotificationHandler
     from orchestrator.utils.email_builder import EmailBuilder
     from email.mime.multipart import MIMEMultipart

orchestrator/utils/email_builder.py

@@ -1,5 +1,5 @@
 import json
-from sientia_do.temporal.utils.logger import Logger
+from sientia_do.observability.logger import Logger
 from sientia_do.notifications.models import NotificationLevel
 from jinja2 import Template
 import re

orchestrator/worker/worker.py

@@ -1,6 +1,6 @@
 from temporalio import workflow, client
 from temporalio.worker import Worker
-
+from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig
 
 with workflow.unsafe.imports_passed_through():
     import os
@@ -21,11 +21,12 @@ with workflow.unsafe.imports_passed_through():
         build_postgres_config
     )
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
-    from sientia_do.temporal.utils.logger import get_logger
+    from sientia_do.observability.logger import get_logger
     from prometheus_client import start_http_server
     from orchestrator import metrics
 
 POD_ID = os.getenv("POD_ID")
+SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091"))
 
 
 async def main():
@@ -55,11 +56,21 @@ async def main():
         project_name=os.getenv('PROJECT_NAME', 'orchestrator'),
     )
 
+    logger.info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...')
+
+    new_runtime = Runtime(
+        telemetry=TelemetryConfig(
+            metrics=PrometheusConfig(
+                bind_address=f"0.0.0.0:{SDK_METRICS_PORT}")
+        )
+    )
+
     logger.info(f'Starting Temporal Client at {host}:{namespace}')
 
     temporal_client = await client.Client.connect(
         target_host=host,
-        namespace=namespace
+        namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'),
+        runtime=new_runtime
     )
 
     logger.info('Starting Activities...')

orchestrator/workflows/alerts.py

@@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through():
     from orchestrator.activities.activities import Activities
     from typing import Any
     from datetime import timedelta
-    from sientia_do.temporal.utils.policies import retry_policy
+    from sientia_do.temporal.policies import retry_policy
 
 
 @workflow.defn(name="alerts")

orchestrator/workflows/orchestrator.py

@@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through():
     from orchestrator.activities.activities import Activities
     from typing import Any
     from datetime import timedelta
-    from sientia_do.temporal.utils.policies import retry_policy
+    from sientia_do.temporal.policies import retry_policy
 
 
 @workflow.defn(name="orchestrator")

orchestrator/workflows/reports.py

@@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through():
     from orchestrator.activities.activities import Activities
     from typing import Any
     from datetime import timedelta
-    from sientia_do.temporal.utils.policies import retry_policy
+    from sientia_do.temporal.policies import retry_policy
 
 
 @workflow.defn(name="reports")

orchestrator/workflows/subworkflows/load_notification_package.py

@@ -3,7 +3,7 @@ from temporalio import workflow
 with workflow.unsafe.imports_passed_through():
     from orchestrator.activities.activities import Activities
     from typing import Any
-    from sientia_do.temporal.utils.policies import retry_policy
+    from sientia_do.temporal.policies import retry_policy
     from datetime import timedelta
 
 

orchestrator/workflows/subworkflows/process_notifications.py

@@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through():
     from orchestrator.activities.activities import Activities
     from typing import Any
     from datetime import timedelta
-    from sientia_do.temporal.utils.policies import retry_policy
+    from sientia_do.temporal.policies import retry_policy
 
 
 @workflow.defn(name="process_notifications")

requirements.txt

@@ -5,5 +5,5 @@ redis
 couchbase
 pymongo
 jinja2
-git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.8
+git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1
 prometheus-client

tests/orchestrator/activities/test_activities.py

@@ -111,3 +111,40 @@ def test___init__(mock_postgres_init,
         logger=logger,
         notification_handler=notification_handler
     )
+
+
+@patch('orchestrator.activities.mongo_db.MongoDB.__init__')
+@patch('orchestrator.activities.temporal_manager.TemporalManager.__init__')
+@patch('orchestrator.activities.slot_manager.SlotManager.__init__')
+@patch('orchestrator.activities.formatters.Formatters.__init__')
+@patch('orchestrator.activities.email.Email.__init__')
+@patch('sientia_do.temporal.activities.postgres.Postgres.__init__')
+@patch('orchestrator.activities.email.Email.shutdown')
+@patch('sientia_do.temporal.activities.postgres.Postgres.close')
+@patch('orchestrator.activities.mongo_db.MongoDB.shutdown')
+def test_shutdown(mock_mongodb_close,
+                  mock_postgres_shutdown,
+                  mock_email_close,
+                  mock_postgres_init,
+                  mock_email_init,
+                  mock_formatters_init,
+                  mock_slot_manager_init,
+                  mock_temporal_manager_init,
+                  mock_mongodb_init,
+                  ):
+
+    activities = Activities(
+        temporal_config=MagicMock(),
+        redis_config=MagicMock(),
+        mongodb_config=MagicMock(),
+        email_config=MagicMock(),
+        postgres_config=MagicMock(),
+        logger=MagicMock(),
+        notification_handler=MagicMock()
+    )
+
+    activities.shutdown()
+
+    mock_mongodb_close.assert_called()
+    mock_postgres_shutdown.assert_called()
+    mock_email_close.assert_called()

tests/orchestrator/activities/test_email.py

@@ -76,6 +76,12 @@ metadata = {
 }
 
 
+def test_shutdown(email):
+    email.shutdown()
+
+    email.server.quit.assert_called_once()
+
+
 @mark.asyncio
 async def test_build_email_html(email):
     email.email_builder.build_email = MagicMock(

values.yaml

@@ -11,7 +11,7 @@ image:
   # This sets the pull policy for images.
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.4.0"
+  tag: "0.4.2"
 
 # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets:
@@ -112,6 +112,13 @@ tolerations: []
 affinity: {}
 
 services:
+  sdk-metrics:
+    enabled: true
+    type: ClusterIP
+    port: 9091
+    targetPort: 9091
+    name: sdk-metrics
+
   metrics:
     enabled: true
     type: ClusterIP
@@ -125,18 +132,18 @@ serviceMonitor:
   # Se true, um recurso ServiceMonitor será criado.
   enabled: true
   # O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m).
-  interval: 30s
+  endpoints:
-  # O path do endpoint de métricas na sua aplicação.
+  - port: metrics
-  path: /metrics
+    path: /metrics
-  # Labels adicionais para o recurso ServiceMonitor.
+    interval: 30s
-  # Essencial para que o Prometheus Operator o descubra. Se você usa o helm chart kube-prometheus-stack,
+    relabelings: []
-  # ele procura por ServiceMonitors com o label "release: kube-prometheus-stack".
+  - port: sdk-metrics
+    path: /metrics
+    interval: 30s
+    relabelings: []
+
   additionalLabels:
     release: kube-prometheus-stack
-  # Configurações de relabeling adicionais, se necessário.
-  # ref: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
-  relabelings: []
-  port: metrics
   
 
 env:
@@ -144,7 +151,7 @@ env:
   - name: GITHUB_REPO_URL
     value: "git@github.com:Aignosi/sientia-dataops-orchestrator_temporal.git"
   - name: GITHUB_BRANCH
-    value: "SIENTIAPDE-1184-investigar-bugs-detectados-no-grafana"
+    value: "SIENTIAPDE-1199-revisar-e-testar-observabilidade"
   - name: PYTHON_APP
     value: "orchestrator.worker.worker"
 
@@ -237,7 +244,7 @@ ssh:
 
 # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp
 
-# helm upgrade --install sientia-orchestrator-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0
+# helm upgrade --install sientia-orchestrator-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.5.0
 
 # kubectl create secret generic git-ssh-key-sientia-orchestrator-worker \
 #   --namespace sientia \