Merge pull request #11 from Aignosi/SIENTIAPDE-1163-alterar-dinamica-de-notificacoes-para-usar-o-mongodb-ao-inves-do-kafka

Sientiapde 1163 alterar dinamica de notificacoes para usar o mongodb ao inves do kafka

.gitignore

@@ -37,3 +37,5 @@ __pycache__/
 # Ignorar coverage
 htmlcov/
 .coverage
+
+git_log

README.md

@@ -92,3 +92,10 @@ The application can be deployed using the following command:
 ```bash
 helm upgrade --install sientia-dataops-opc-ingestor sientia/sientia-module -n sientia-opc --create-namespace -f ./values.yaml
 ```
+
+#PR shortcut
+```
+git log origin/main..HEAD --no-merges > git_log
+```
+Prompt:
+Write a summary of PR changes in markdown. Be objective and direct. Write to file

requirements.txt

@@ -5,6 +5,6 @@ asyncua
 redis
 aiokafka
 pymongo
-git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.2.1
+git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.3
 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.1
 pydruid[pandas]

scouter/activities/activities.py

@@ -2,7 +2,7 @@ from temporalio import workflow
 
 with workflow.unsafe.imports_passed_through():
     from sientia_do.temporal.activities.postgres import Postgres
-    from sientia_do.notifications.handlers import NotificationHandler
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.temporal.utils.logger import Logger
     from scouter.activities.redis import Redis
     from scouter.activities.gates import Gates

scouter/activities/faker.py

@@ -5,7 +5,7 @@ import json
 from kafka import KafkaProducer
 from temporalio import activity
 
-from sientia_do.notifications.handlers import NotificationHandler
+from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.temporal.activities.base import BaseActivity
 from sientia_do.temporal.utils.logger import Logger
 

scouter/activities/mongodb.py

@@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through():
     from datetime import datetime
     from pymongo import MongoClient
     from pandas import DataFrame
-    from sientia_do.notifications.handlers import NotificationHandler
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.notifications.models import NotificationLevel
     from sientia_do.temporal.activities.base import BaseActivity
     from sientia_do.temporal.utils.logger import Logger

scouter/activities/redis.py

@@ -3,7 +3,7 @@ from temporalio import workflow, activity
 
 with workflow.unsafe.imports_passed_through():
     from logging import Logger
-    from sientia_do.notifications.handlers import NotificationHandler
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.notifications.models import NotificationLevel
     from sientia_do.temporal.activities.redis_base import Redis as RedisBase
     from sientia_do.temporal.utils.logger import Logger

scouter/worker/worker.py

@@ -4,7 +4,7 @@ from temporalio.worker import Worker
 with workflow.unsafe.imports_passed_through():
     import sys
     import os
-    from sientia_do.notifications.handlers import NotificationHandler
+    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.temporal.utils.logger import get_logger
     from scouter.activities.activities import Activities
     from scouter.workflow.scouter import Scouter
@@ -28,8 +28,10 @@ async def main():
 
     logger.info('Starting Notification Handler...')
 
+    mongo_config = build_mongodb_config()
     notification_handler = NotificationHandler(
-        servers=os.getenv('KAFKA_BOOTSTRAP_SERVERS', 'http://localhost:9092'),
+        connection_string=mongo_config['connection_string'],
+        database=mongo_config['database_name'],
         logger=logger,
         project_name=os.getenv('PROJECT_NAME', 'scouter')
     )

tests/activities/test_faker.py

@@ -1,6 +1,6 @@
 from unittest.mock import MagicMock, patch, call
 import pytest
-from sientia_do.notifications.handlers import NotificationHandler
+from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from scouter.activities.faker import Faker
 
 

tests/activities/test_redis.py

@@ -3,7 +3,7 @@ from datetime import datetime
 import pytest
 import numpy as np
 from pandas import DataFrame
-from sientia_do.notifications.handlers import NotificationHandler
+from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.notifications.models import NotificationLevel
 from scouter.activities.redis import Redis
 
@@ -397,7 +397,14 @@ async def test_group_and_hold_data_error_set(redis_activity):
         'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records')
     }
 
-    redis_activity.get = MagicMock(return_value=None)
+    existing_data = {
+        'sensor1': 20.0,
+        'sensor2': 28.0,
+        'timestamp': '2023-01-01 11:00:00'
+    }
+
+    # Mock get to return existing data
+    redis_activity.get = MagicMock(return_value=existing_data)
     redis_activity.set = MagicMock(side_effect=Exception('test'))
     redis_activity.send_notification = MagicMock()
 

values.yaml

@@ -11,7 +11,7 @@ image:
   # This sets the pull policy for images.
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.2.3"
+  tag: "0.2.4"
 
 # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets:
@@ -111,19 +111,14 @@ tolerations: []
 
 affinity: {}
 
-service:
+service: {}
-  enabled: false
-  type: ClusterIP
-  port: 4840
-  targetPort: 4840
-  
 
 env:
   # Entrypoint variables 
   - name: GITHUB_REPO_URL
     value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git"
   - name: GITHUB_BRANCH
-    value: "SIENTIAPDE-1148-separar-scouter-laborious-e-orchestrator-por-namespaces"
+    value: "SIENTIAPDE-1163-alterar-dinamica-de-notificacoes-para-usar-o-mongodb-ao-inves-do-kafka"
   - name: PYTHON_APP
     value: "scouter.worker.worker"