From b014375c3f92d93d3715e9537b355ed46ac19c35 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 15 Jul 2025 12:11:20 -0300 Subject: [PATCH 1/9] SIENTIAPDE-1154 Update GITHUB_BRANCH in values.yaml to SIENTIAPDE-1154-otimizar-conexao-com-banco-de-dados-e-paralelismo for enhanced database connection optimization and parallelism. --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index f20b6a2..0d70087 100644 --- a/values.yaml +++ b/values.yaml @@ -123,7 +123,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1148-separar-scouter-laborious-e-orchestrator-por-namespaces" + value: "SIENTIAPDE-1154-otimizar-conexao-com-banco-de-dados-e-paralelismo" - name: PYTHON_APP value: "scouter.worker.worker" From 95defccbf671c39fc937bd78df63f4e02b0a0e98 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 16 Jul 2025 08:59:21 -0300 Subject: [PATCH 2/9] SIENTIAPDE-1163 Refactor notification handler imports across multiple files to use CoreNotificationHandler instead of NotificationHandler for consistency and clarity. --- scouter/activities/activities.py | 2 +- scouter/activities/faker.py | 2 +- scouter/activities/mongodb.py | 2 +- scouter/activities/redis.py | 2 +- scouter/worker/worker.py | 6 ++++-- tests/activities/test_faker.py | 2 +- tests/activities/test_redis.py | 2 +- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/scouter/activities/activities.py b/scouter/activities/activities.py index 59990b4..775f958 100644 --- a/scouter/activities/activities.py +++ b/scouter/activities/activities.py @@ -2,7 +2,7 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): from sientia_do.temporal.activities.postgres import Postgres - from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.temporal.utils.logger import Logger from scouter.activities.redis import Redis from scouter.activities.gates import Gates diff --git a/scouter/activities/faker.py b/scouter/activities/faker.py index 49311ea..f8e5253 100644 --- a/scouter/activities/faker.py +++ b/scouter/activities/faker.py @@ -5,7 +5,7 @@ import json from kafka import KafkaProducer from temporalio import activity -from sientia_do.notifications.handlers import NotificationHandler +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.temporal.activities.base import BaseActivity from sientia_do.temporal.utils.logger import Logger diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 2191d55..3189235 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through(): from datetime import datetime from pymongo import MongoClient from pandas import DataFrame - from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.base import BaseActivity from sientia_do.temporal.utils.logger import Logger diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index 9c1f3c6..9bad871 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -3,7 +3,7 @@ from temporalio import workflow, activity with workflow.unsafe.imports_passed_through(): from logging import Logger - from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.temporal.activities.redis_base import Redis as RedisBase from sientia_do.temporal.utils.logger import Logger diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index 87e3672..239ecd8 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -4,7 +4,7 @@ from temporalio.worker import Worker with workflow.unsafe.imports_passed_through(): import sys import os - from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.temporal.utils.logger import get_logger from scouter.activities.activities import Activities from scouter.workflow.scouter import Scouter @@ -28,8 +28,10 @@ async def main(): logger.info('Starting Notification Handler...') + mongo_config = build_mongodb_config() notification_handler = NotificationHandler( - servers=os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'http://localhost:9092'), + connection_string=mongo_config['connection_string'], + database=mongo_config['database_name'], logger=logger, project_name=os.getenv('PROJECT_NAME', 'scouter') ) diff --git a/tests/activities/test_faker.py b/tests/activities/test_faker.py index 40f4893..0e3a61c 100644 --- a/tests/activities/test_faker.py +++ b/tests/activities/test_faker.py @@ -1,6 +1,6 @@ from unittest.mock import MagicMock, patch, call import pytest -from sientia_do.notifications.handlers import NotificationHandler +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from scouter.activities.faker import Faker diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index e158794..eff8d40 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -3,7 +3,7 @@ from datetime import datetime import pytest import numpy as np from pandas import DataFrame -from sientia_do.notifications.handlers import NotificationHandler +from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from scouter.activities.redis import Redis From 03f5518cd1a4a911195807a071cb7d383f7275c1 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 16 Jul 2025 09:23:02 -0300 Subject: [PATCH 3/9] SIENTIAPDE-1163 Update sientia-dataops-library dependency version from 1.2.1 to 1.3.0 in requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 496a53b..0131d53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ asyncua redis aiokafka pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.2.1 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.0 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.1 pydruid[pandas] \ No newline at end of file From 9329ae206cb36df6876a5df31c5eae62659807c6 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 17 Jul 2025 11:00:52 -0300 Subject: [PATCH 4/9] SIENTIAPDE-1163 Update sientia-dataops-library dependency version from 1.3.0 to 1.3.3 in requirements.txt and change GITHUB_BRANCH in values.yaml to SIENTIAPDE-1163-alterar-dinamica-de-notificacoes-para-usar-o-mongodb-ao-inves-do-kafka for improved notification handling. --- requirements.txt | 2 +- values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0131d53..7698faf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,6 @@ asyncua redis aiokafka pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.0 +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.3 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.1 pydruid[pandas] \ No newline at end of file diff --git a/values.yaml b/values.yaml index 0d70087..2d3086c 100644 --- a/values.yaml +++ b/values.yaml @@ -123,7 +123,7 @@ env: - name: GITHUB_REPO_URL value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git" - name: GITHUB_BRANCH - value: "SIENTIAPDE-1154-otimizar-conexao-com-banco-de-dados-e-paralelismo" + value: "SIENTIAPDE-1163-alterar-dinamica-de-notificacoes-para-usar-o-mongodb-ao-inves-do-kafka" - name: PYTHON_APP value: "scouter.worker.worker" From 939acb53541b3a06dd4e81524481b5199db48f41 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 17 Jul 2025 12:31:08 -0300 Subject: [PATCH 5/9] SIENTIAPDE-1163 Update image tag in values.yaml from 0.2.3 to 0.2.4 for version increment. --- values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.yaml b/values.yaml index 2d3086c..b04d1ca 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.2.3" + tag: "0.2.4" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: From b0d0478b201ac54da2dc73d3777f3fb186df57a1 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 17 Jul 2025 12:32:38 -0300 Subject: [PATCH 6/9] SIENTIAPDE-1163 Add PR shortcut command to README for easier log generation --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index c16d207..3e90449 100644 --- a/README.md +++ b/README.md @@ -92,3 +92,10 @@ The application can be deployed using the following command: ```bash helm upgrade --install sientia-dataops-opc-ingestor sientia/sientia-module -n sientia-opc --create-namespace -f ./values.yaml ``` + +#PR shortcut +``` +git log origin/main..HEAD --no-merges > git_log +``` +Prompt: +Write a summary of PR changes in markdown. Be objective and direct. Write to file \ No newline at end of file From 839be186447768bf238f9fdbc67619a9de8300e1 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 17 Jul 2025 12:33:27 -0300 Subject: [PATCH 7/9] SIENTIAPDE-1163 Update .gitignore to include .coverage and git_log for better file management --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a446ab3..0445a7f 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,6 @@ __pycache__/ # Ignorar coverage htmlcov/ -.coverage \ No newline at end of file +.coverage + +git_log \ No newline at end of file From 9580b6f8beaf22f77635a7908bf4656fabcfa5f6 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 17 Jul 2025 14:30:34 -0300 Subject: [PATCH 8/9] SIENTIAPDE-1163 SIENTIAPDE-1163 Update test_redis.py to mock existing data in redis_activity.get for improved error handling in group and hold data tests. --- tests/activities/test_redis.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index eff8d40..99eed1f 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -397,7 +397,14 @@ async def test_group_and_hold_data_error_set(redis_activity): 'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records') } - redis_activity.get = MagicMock(return_value=None) + existing_data = { + 'sensor1': 20.0, + 'sensor2': 28.0, + 'timestamp': '2023-01-01 11:00:00' + } + + # Mock get to return existing data + redis_activity.get = MagicMock(return_value=existing_data) redis_activity.set = MagicMock(side_effect=Exception('test')) redis_activity.send_notification = MagicMock() From e6d2bac902c4aa76f1aa1e6bc01960bc07d8e0f2 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 17 Jul 2025 14:59:14 -0300 Subject: [PATCH 9/9] SIENTIAPDE-1163 Remove service configuration from values.yaml to streamline deployment settings. --- values.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/values.yaml b/values.yaml index b04d1ca..540eb7b 100644 --- a/values.yaml +++ b/values.yaml @@ -111,12 +111,7 @@ tolerations: [] affinity: {} -service: - enabled: false - type: ClusterIP - port: 4840 - targetPort: 4840 - +service: {} env: # Entrypoint variables