diff --git a/orchestrator/activities/email.py b/orchestrator/activities/email.py index 1401876..35e5951 100644 --- a/orchestrator/activities/email.py +++ b/orchestrator/activities/email.py @@ -9,7 +9,7 @@ with workflow.unsafe.imports_passed_through(): import smtplib from typing import Any from sientia_do.temporal.activities.base import BaseActivity - from sientia_do.temporal.utils.logger import Logger + from sientia_do.observability.logger import Logger from sientia_do.notifications.handlers import NotificationHandler from orchestrator.utils.email_builder import EmailBuilder from email.mime.multipart import MIMEMultipart diff --git a/orchestrator/utils/email_builder.py b/orchestrator/utils/email_builder.py index 64b5963..c78be8b 100644 --- a/orchestrator/utils/email_builder.py +++ b/orchestrator/utils/email_builder.py @@ -1,5 +1,5 @@ import json -from sientia_do.temporal.utils.logger import Logger +from sientia_do.observability.logger import Logger from sientia_do.notifications.models import NotificationLevel from jinja2 import Template import re diff --git a/orchestrator/worker/worker.py b/orchestrator/worker/worker.py index 49fc31e..d5d967e 100644 --- a/orchestrator/worker/worker.py +++ b/orchestrator/worker/worker.py @@ -1,6 +1,6 @@ from temporalio import workflow, client from temporalio.worker import Worker - +from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig with workflow.unsafe.imports_passed_through(): import os @@ -21,11 +21,12 @@ with workflow.unsafe.imports_passed_through(): build_postgres_config ) from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler - from sientia_do.temporal.utils.logger import get_logger + from sientia_do.observability.logger import get_logger from prometheus_client import start_http_server from orchestrator import metrics POD_ID = os.getenv("POD_ID") +SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091")) async def main(): @@ -55,11 +56,21 @@ async def main(): project_name=os.getenv('PROJECT_NAME', 'orchestrator'), ) + logger.info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...') + + new_runtime = Runtime( + telemetry=TelemetryConfig( + metrics=PrometheusConfig( + bind_address=f"0.0.0.0:{SDK_METRICS_PORT}") + ) + ) + logger.info(f'Starting Temporal Client at {host}:{namespace}') temporal_client = await client.Client.connect( target_host=host, - namespace=namespace + namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'), + runtime=new_runtime ) logger.info('Starting Activities...') diff --git a/orchestrator/workflows/alerts.py b/orchestrator/workflows/alerts.py index 1a8509a..64b3606 100644 --- a/orchestrator/workflows/alerts.py +++ b/orchestrator/workflows/alerts.py @@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through(): from orchestrator.activities.activities import Activities from typing import Any from datetime import timedelta - from sientia_do.temporal.utils.policies import retry_policy + from sientia_do.temporal.policies import retry_policy @workflow.defn(name="alerts") diff --git a/orchestrator/workflows/orchestrator.py b/orchestrator/workflows/orchestrator.py index cb81d13..ea292b2 100644 --- a/orchestrator/workflows/orchestrator.py +++ b/orchestrator/workflows/orchestrator.py @@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through(): from orchestrator.activities.activities import Activities from typing import Any from datetime import timedelta - from sientia_do.temporal.utils.policies import retry_policy + from sientia_do.temporal.policies import retry_policy @workflow.defn(name="orchestrator") diff --git a/orchestrator/workflows/reports.py b/orchestrator/workflows/reports.py index eb00231..321578c 100644 --- a/orchestrator/workflows/reports.py +++ b/orchestrator/workflows/reports.py @@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through(): from orchestrator.activities.activities import Activities from typing import Any from datetime import timedelta - from sientia_do.temporal.utils.policies import retry_policy + from sientia_do.temporal.policies import retry_policy @workflow.defn(name="reports") diff --git a/orchestrator/workflows/subworkflows/load_notification_package.py b/orchestrator/workflows/subworkflows/load_notification_package.py index 15d3560..5c83af7 100644 --- a/orchestrator/workflows/subworkflows/load_notification_package.py +++ b/orchestrator/workflows/subworkflows/load_notification_package.py @@ -3,7 +3,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): from orchestrator.activities.activities import Activities from typing import Any - from sientia_do.temporal.utils.policies import retry_policy + from sientia_do.temporal.policies import retry_policy from datetime import timedelta diff --git a/orchestrator/workflows/subworkflows/process_notifications.py b/orchestrator/workflows/subworkflows/process_notifications.py index f2f36c8..b4f6d41 100644 --- a/orchestrator/workflows/subworkflows/process_notifications.py +++ b/orchestrator/workflows/subworkflows/process_notifications.py @@ -4,7 +4,7 @@ with workflow.unsafe.imports_passed_through(): from orchestrator.activities.activities import Activities from typing import Any from datetime import timedelta - from sientia_do.temporal.utils.policies import retry_policy + from sientia_do.temporal.policies import retry_policy @workflow.defn(name="process_notifications") diff --git a/requirements.txt b/requirements.txt index 53ef9e2..50fa9aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,5 @@ redis couchbase pymongo jinja2 -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.8 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1 prometheus-client diff --git a/tests/orchestrator/activities/test_activities.py b/tests/orchestrator/activities/test_activities.py index d2640d1..9a0c1ff 100644 --- a/tests/orchestrator/activities/test_activities.py +++ b/tests/orchestrator/activities/test_activities.py @@ -111,3 +111,40 @@ def test___init__(mock_postgres_init, logger=logger, notification_handler=notification_handler ) + + +@patch('orchestrator.activities.mongo_db.MongoDB.__init__') +@patch('orchestrator.activities.temporal_manager.TemporalManager.__init__') +@patch('orchestrator.activities.slot_manager.SlotManager.__init__') +@patch('orchestrator.activities.formatters.Formatters.__init__') +@patch('orchestrator.activities.email.Email.__init__') +@patch('sientia_do.temporal.activities.postgres.Postgres.__init__') +@patch('orchestrator.activities.email.Email.shutdown') +@patch('sientia_do.temporal.activities.postgres.Postgres.close') +@patch('orchestrator.activities.mongo_db.MongoDB.shutdown') +def test_shutdown(mock_mongodb_close, + mock_postgres_shutdown, + mock_email_close, + mock_postgres_init, + mock_email_init, + mock_formatters_init, + mock_slot_manager_init, + mock_temporal_manager_init, + mock_mongodb_init, + ): + + activities = Activities( + temporal_config=MagicMock(), + redis_config=MagicMock(), + mongodb_config=MagicMock(), + email_config=MagicMock(), + postgres_config=MagicMock(), + logger=MagicMock(), + notification_handler=MagicMock() + ) + + activities.shutdown() + + mock_mongodb_close.assert_called() + mock_postgres_shutdown.assert_called() + mock_email_close.assert_called() diff --git a/tests/orchestrator/activities/test_email.py b/tests/orchestrator/activities/test_email.py index 1c82145..a01167d 100644 --- a/tests/orchestrator/activities/test_email.py +++ b/tests/orchestrator/activities/test_email.py @@ -76,6 +76,12 @@ metadata = { } +def test_shutdown(email): + email.shutdown() + + email.server.quit.assert_called_once() + + @mark.asyncio async def test_build_email_html(email): email.email_builder.build_email = MagicMock( diff --git a/values.yaml b/values.yaml index 7a7aba0..3396de3 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.4.0" + tag: "0.4.2" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: @@ -112,6 +112,13 @@ tolerations: [] affinity: {} services: + sdk-metrics: + enabled: true + type: ClusterIP + port: 9091 + targetPort: 9091 + name: sdk-metrics + metrics: enabled: true type: ClusterIP @@ -125,18 +132,18 @@ serviceMonitor: # Se true, um recurso ServiceMonitor será criado. enabled: true # O intervalo no qual as métricas devem ser coletadas (ex: 30s, 1m). - interval: 30s - # O path do endpoint de métricas na sua aplicação. - path: /metrics - # Labels adicionais para o recurso ServiceMonitor. - # Essencial para que o Prometheus Operator o descubra. Se você usa o helm chart kube-prometheus-stack, - # ele procura por ServiceMonitors com o label "release: kube-prometheus-stack". + endpoints: + - port: metrics + path: /metrics + interval: 30s + relabelings: [] + - port: sdk-metrics + path: /metrics + interval: 30s + relabelings: [] + additionalLabels: release: kube-prometheus-stack - # Configurações de relabeling adicionais, se necessário. - # ref: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config - relabelings: [] - port: metrics env: @@ -144,7 +151,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-orchestrator_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1184-investigar-bugs-detectados-no-grafana" + value: "SIENTIAPDE-1199-revisar-e-testar-observabilidade" - name: PYTHON_APP value: "orchestrator.worker.worker" @@ -237,7 +244,7 @@ ssh: # kubectl create secret docker-registry docker-hub-secret --namespace sientia --docker-server=http://aignosi.azurecr.io --docker-username=aignosi --docker-password=5I5zpQ6sRaHqX1hD3dr+2mo647yO3FRc359/wu6gsP+ACRDRz5mp -# helm upgrade --install sientia-orchestrator-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.4.0 +# helm upgrade --install sientia-orchestrator-worker sientia/sientia-module -n sientia --create-namespace -f ./values.yaml --version 0.5.0 # kubectl create secret generic git-ssh-key-sientia-orchestrator-worker \ # --namespace sientia \