SIENTIAPDE-1325

Update environment configuration and refactor activities to include metrics controller. Remove Kafka settings and adjust Redis and MongoDB initialization. Update tests to reflect changes in initialization and metrics tracking.

.env.example

@@ -6,14 +6,18 @@ POSTGRES_DB=sientia
 POSTGRES_MIN_CONNECTIONS=5
 POSTGRES_MAX_CONNECTIONS=20
 
-KAFKA_BOOTSTRAP_SERVERS=localhost:9092
+MONGODB_USERNAME="root"
-KAFKA_POLLING_TIME=1000
+MONGODB_PASSWORD="password"
+MONGODB_URL="my-release-mongodb.mongodb.svc.cluster.local:27017"
+MONGODB_DATABASE="sientia"
 
 REDIS_HOST=localhost
 REDIS_PORT=6379
+REDIS_USERNAME="user"
+REDIS_PASSWORD="pass"
 
 TEMPORAL_HOST=localhost:7233
-TEMPORAL_NAMESPACE=default
+TEMPORAL_NAMESPACE=scouter
 
 LOG_LEVEL=INFO
 

init_port_forward.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+
+# Usage:
+#   1) Edit the PORT_FORWARDS list below with entries of:
+#        <namespace> <service_name> <local_port> <service_port>
+#   2) Run: ./init_port_forward.sh
+#
+# The script will start all port-forwards in the background and keep running
+# until interrupted (Ctrl+C). On exit, it will clean up started port-forward processes.
+
+set -euo pipefail
+
+# Define your namespace/service/port combinations here
+# Example entries:
+#   "default my-service 8080 80"
+#   "observability grafana 3000 3000"
+PORT_FORWARDS=(
+  "mongodb my-release-mongodb 27017 27017"
+  "paradedb paradedb-rw 5432 5432"
+  "redis redis-master 6379 6379"
+  "temporal temporal-frontend 7233 7233"
+)
+
+if [ ${#PORT_FORWARDS[@]} -eq 0 ]; then
+  echo "No port-forward entries defined. Edit PORT_FORWARDS in $(basename "$0")."
+  exit 1
+fi
+
+PIDS=()
+
+cleanup() {
+  echo "\nStopping port-forward processes..."
+  for pid in "${PIDS[@]}"; do
+    if kill -0 "$pid" >/dev/null 2>&1; then
+      kill "$pid" >/dev/null 2>&1 || true
+    fi
+  done
+}
+
+trap cleanup EXIT INT TERM
+
+timestamp() { date '+%Y-%m-%d %H:%M:%S'; }
+
+# Allow overriding kubectl binary if needed
+KUBECTL=${KUBECTL:-kubectl}
+
+is_port_free() {
+  local port="$1"
+  # Consider port free if nothing is listening locally on it
+  if command -v ss >/dev/null 2>&1; then
+    ! ss -ltn | awk '{print $4}' | grep -E "(^|:|\\])${port}$" >/dev/null 2>&1
+  else
+    if command -v lsof >/dev/null 2>&1; then
+      ! lsof -tiTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1
+    else
+      # Fallback: attempt to open a TCP connection; expect failure when nothing is listening
+      ! (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null
+    fi
+  fi
+}
+
+free_port_if_stuck() {
+  local port="$1"
+  # Try multiple tools to free a stuck listener (often old kubectl PF)
+  if command -v lsof >/dev/null 2>&1; then
+    local pids
+    pids=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null || true)
+    if [ -n "${pids}" ]; then
+      echo "[$(timestamp)] Found listeners on ${port}: ${pids}; terminating"
+      kill ${pids} 2>/dev/null || true
+      sleep 0.5
+    fi
+  fi
+  if ! is_port_free "${port}"; then
+    if command -v fuser >/dev/null 2>&1; then
+      echo "[$(timestamp)] Forcing free of ${port} via fuser"
+      fuser -k "${port}/tcp" 2>/dev/null || true
+      sleep 0.5
+    fi
+  fi
+}
+
+run_port_forward() {
+  local namespace="$1"
+  local service_name="$2"
+  local local_port="$3"
+  local service_port="$4"
+
+  # simple and robust supervisor loop with gentle backoff on failures
+  local delay=2
+  local max_delay=20
+  while true; do
+    free_port_if_stuck "${local_port}"
+    if ! is_port_free "${local_port}"; then
+      echo "[$(timestamp)] ns=${namespace} svc=${service_name} ${local_port}:${service_port} -> local port busy, retrying in 3s"
+      sleep 3
+      continue
+    fi
+
+    echo "[$(timestamp)] Starting port-forward: ns=${namespace} svc=${service_name} ${local_port}:${service_port}"
+    ${KUBECTL} -n "${namespace}" port-forward "svc/${service_name}" "${local_port}:${service_port}" \
+      --address=127.0.0.1 --pod-running-timeout=2m --request-timeout=0
+    rc=$?
+
+    # If kubectl exits (e.g., connection reset by peer), wait a bit and retry
+    echo "[$(timestamp)] Port-forward exited (rc=${rc}): ns=${namespace} svc=${service_name} ${local_port}:${service_port}"
+    sleep "${delay}"
+    # Exponential backoff up to max_delay
+    if [ ${delay} -lt ${max_delay} ]; then
+      delay=$(( delay * 2 ))
+      if [ ${delay} -gt ${max_delay} ]; then
+        delay=${max_delay}
+      fi
+    fi
+  done
+}
+
+ONLY_SERVICE_NAME="${ONLY_SERVICE_NAME:-}"
+
+for entry in "${PORT_FORWARDS[@]}"; do
+  read -r NAMESPACE SERVICE_NAME LOCAL_PORT SERVICE_PORT <<< "$entry"
+  if [ -n "${ONLY_SERVICE_NAME}" ] && [ "${SERVICE_NAME}" != "${ONLY_SERVICE_NAME}" ]; then
+    continue
+  fi
+  run_port_forward "${NAMESPACE}" "${SERVICE_NAME}" "${LOCAL_PORT}" "${SERVICE_PORT}" &
+  PIDS+=("$!")
+done
+
+echo "All port-forwards started: ${#PIDS[@]} process(es). Press Ctrl+C to stop."
+
+# Do not exit the script if one port-forward fails; they self-restart
+set +e
+wait

pyproject.toml

@@ -114,10 +114,6 @@ python_functions = ["test_*"]
 addopts = [
     "-v",
     "--strict-markers",
-    "--cov=model_manager",
-    "--cov-report=term-missing",
-    "--cov-report=html",
-    "--cov-report=xml",
 ]
 markers = [
     "asyncio: marks tests as async",
@@ -126,7 +122,7 @@ markers = [
 ]
 
 [tool.coverage.run]
-source = ["model_manager"]
+source = ["scouter"]
 omit = [
     "*/tests/*",
     "*/venv/*",

scouter/activities/activities.py

@@ -1,3 +1,4 @@
+from sientia_do.observability.metrics_controller import MetricsController
 from temporalio import workflow
 
 with workflow.unsafe.imports_passed_through():
@@ -50,6 +51,10 @@ class Activities(Postgres, Redis, Gates, MongoDB):
             logger (Logger): Logger instance for application logging
             notification_handler (NotificationHandler): Handler for system notifications
         """
+        metrics_controller = MetricsController(
+            logger=logger,
+        )
+
         # Initialize Postgres
         Postgres.__init__(
             self,
@@ -62,6 +67,7 @@ class Activities(Postgres, Redis, Gates, MongoDB):
             max_connections=postgres_config['max_connections'],
             logger=logger,
             notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
         )
 
         # Initialize Redis
@@ -73,10 +79,11 @@ class Activities(Postgres, Redis, Gates, MongoDB):
             notification_handler=notification_handler,
             username=redis_config['username'],
             password=redis_config['password'],
+            metrics_controller=metrics_controller,
         )
 
         # Initialize Gates
-        Gates.__init__(self, logger=logger, notification_handler=notification_handler)
+        Gates.__init__(self, logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller)
 
         # Initialize MongoDB
         MongoDB.__init__(
@@ -85,6 +92,7 @@ class Activities(Postgres, Redis, Gates, MongoDB):
             database_name=mongodb_config['database_name'],
             logger=logger,
             notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
         )
 
         self.pod_id = getenv('HOSTNAME', 'localhost')

scouter/activities/gates.py

@@ -1,5 +1,6 @@
 from collections.abc import Hashable
 
+from sientia_do.observability.metrics_controller import MetricsController
 from temporalio import activity, workflow
 
 with workflow.unsafe.imports_passed_through():
@@ -36,21 +37,22 @@ class Gates(SientiaMonitoring):
     ensure data integrity and enable flexible data processing workflows.
     """
 
-    def __init__(self, logger: Logger, notification_handler: NotificationHandler):
+    def __init__(self, logger: Logger, notification_handler: NotificationHandler, metrics_controller: MetricsController):
         """
         Initialize the Gates class with logging and notification services.
 
         Args:
             logger (Logger): Logger instance for operation logging
             notification_handler (NotificationHandler): Handler for system notifications
+            metrics_controller (MetricsController): Metrics controller instance
         """
-        SientiaMonitoring.__init__(self, logger, notification_handler)
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
 
     def close(self):
         """
         Close the Gates class.
         """
-        SientiaMonitoring.close(self)
+        SientiaMonitoring.shutdown(self)
 
     def apply_aggregation(
         self, values: DataFrame, aggr_function: str, metadata: dict[str, Any]

scouter/activities/mongodb.py

@@ -1,6 +1,6 @@
-from collections.abc import Hashable
 from datetime import UTC
 
+from sientia_do.observability.metrics_controller import MetricsController
 from temporalio import activity, workflow
 
 with workflow.unsafe.imports_passed_through():
@@ -37,6 +37,7 @@ class MongoDB(SientiaMonitoring):
         database_name: str,
         logger: Logger,
         notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
     ):
         """
         Initialize MongoDB connection and services.
@@ -56,9 +57,10 @@ class MongoDB(SientiaMonitoring):
             database_name=database_name,
             logger=logger,
             notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
         )
 
-        SientiaMonitoring.__init__(self, logger=logger, notification_handler=notification_handler)
+        SientiaMonitoring.__init__(self, logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller)
 
     def close(self):
         """
@@ -77,7 +79,7 @@ class MongoDB(SientiaMonitoring):
         self.close()
 
     @activity.defn(name='load_latest_data')
-    async def load_latest_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
+    async def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
         """
         Load the latest data from MongoDB collection since a specified timestamp.
 
@@ -116,7 +118,7 @@ class MongoDB(SientiaMonitoring):
 
             self.debug(f'Data filter: {data_filter}', metadata=metadata)
 
-            data = self.mongodb_repository.find(
+            data = await self.mongodb_repository.find(
                 collection_name=collection_name,
                 filters=data_filter,
                 metadata=metadata,

scouter/activities/redis.py

@@ -1,9 +1,9 @@
-from collections.abc import Hashable
+from sientia_do.observability.metrics_controller import MetricsController
-
 from temporalio import activity, workflow
 
 with workflow.unsafe.imports_passed_through():
     import traceback
+    from collections.abc import Hashable
     from typing import Any
 
     from pandas import DataFrame
@@ -38,6 +38,7 @@ class Redis(SientiaMonitoring):
         password: str,
         logger: Logger,
         notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
     ):
         """
         Initialize Redis connection and services.
@@ -50,7 +51,7 @@ class Redis(SientiaMonitoring):
             logger (Logger): Logger instance for operation logging
             notification_handler (NotificationHandler): Handler for system notifications
         """
-        SientiaMonitoring.__init__(self, logger, notification_handler)
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
         self.redis_repository = RedisRepository(
             host=host,
             port=port,
@@ -58,6 +59,7 @@ class Redis(SientiaMonitoring):
             password=password,
             logger=logger,
             notification_handler=notification_handler,
+            metrics_controller=metrics_controller,
         )
 
     def close(self):
@@ -95,7 +97,7 @@ class Redis(SientiaMonitoring):
         self.info(f'Getting last data timestamp for {key}')
 
         try:
-            data_hold = self.redis_repository.get(key)
+            data_hold = await self.redis_repository.get(key)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,
@@ -153,7 +155,7 @@ class Redis(SientiaMonitoring):
         self.info(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata)
 
         try:
-            self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5)
+            await self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,
@@ -206,7 +208,7 @@ class Redis(SientiaMonitoring):
         self.info(f'Getting held data for {key}')
 
         try:
-            data_hold = self.redis_repository.get(key)
+            data_hold = await self.redis_repository.get(key)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,
@@ -250,7 +252,7 @@ class Redis(SientiaMonitoring):
                 data['timestamp'].max() if not data.empty else data_hold['timestamp']
             )
 
-            self.redis_repository.set(key, data_hold, ttl=retention_time)
+            await self.redis_repository.set(key, data_hold, ttl=retention_time)
 
             data_hold_df = DataFrame(data_hold, index=[0])
             data_hold_melted = data_hold_df.melt(
@@ -296,7 +298,7 @@ class Redis(SientiaMonitoring):
         cache = {'data': data.to_dict(), 'held_data': held_data.to_dict()}
 
         try:
-            self.redis_repository.set(key, cache, ttl=120)
+            await self.redis_repository.set(key, cache, ttl=120)
         except Exception as e:
             self.send_notification(
                 metadata=metadata,

scouter/worker/worker.py

@@ -136,7 +136,7 @@ async def main():
 
     except BaseException as e:  # NOSONAR
         logger.custom_error(
-            'An unhandled exception occurred: %s', e, exc_info=True, metadata=metadata
+            'An unhandled exception occurred: %s', metadata=metadata
         )
     finally:
         if notification_handler:

scouter/workflow/scouter.py

@@ -95,7 +95,7 @@ class Scouter:
             retry_policy=retry_policy,
         )
 
-        if data == {}:
+        if not data:
             return
 
         await workflow.execute_activity_method(

tests/activities/test_activities.py

@@ -12,7 +12,8 @@ from scouter.activities.redis import Redis
 @patch('scouter.activities.activities.Postgres.__init__')
 @patch('scouter.activities.activities.Redis.__init__')
 @patch('scouter.activities.activities.Gates.__init__')
-def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init):
+@patch('scouter.activities.activities.MetricsController')
+def test___init__(mock_metrics_controller, mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init):
     postgres_config = {
         'host': 'localhost',
         'port': 5432,
@@ -38,7 +39,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
         redis_config=redis_config,
         mongodb_config=mongodb_config,
         logger=logger,
-        notification_handler=notification_handler,
+        notification_handler=notification_handler
     )
 
     assert isinstance(activities, Activities)
@@ -58,6 +59,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
         max_connections=postgres_config['max_connections'],
         logger=logger,
         notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
     )
 
     mock_redis_init.assert_called_once_with(
@@ -68,6 +70,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
         password=redis_config['password'],
         logger=logger,
         notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
     )
 
     mock_mongodb_init.assert_called_once_with(
@@ -76,10 +79,11 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
         database_name=mongodb_config['database_name'],
         logger=logger,
         notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
     )
 
     mock_gates_init.assert_called_once_with(
-        ANY, logger=logger, notification_handler=notification_handler
+        ANY, logger=logger, notification_handler=notification_handler, metrics_controller=mock_metrics_controller.return_value
     )
 
 

tests/activities/test_gates.py

@@ -14,10 +14,12 @@ def gates_fixture():
     """Fixture to create a Gates instance with mocked dependencies."""
     logger = Mock()
     notification_handler = MagicMock()
-    gates = Gates(logger=logger, notification_handler=notification_handler)
+    metrics_controller = MagicMock()
+    gates = Gates(logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller)
     gates.send_notification = MagicMock()
     gates.logger = logger
     gates.notification_handler = notification_handler
+    gates.metrics_controller = metrics_controller
     gates.pod_id = 'localhost'
     return gates
 

tests/activities/test_mongo.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from unittest.mock import ANY, MagicMock, patch
+from unittest.mock import ANY, AsyncMock, MagicMock, patch
 
 from pytest import fixture, mark
 from sientia_do.notifications.models import NotificationLevel
@@ -14,12 +14,13 @@ def test_mongodb___init__(mock_mongodb_repository):
 
     logger = MagicMock()
     notification_handler = MagicMock()
-
+    metrics_controller = MagicMock()
     mongo = MongoDB(
         connection_string='mongodb://localhost:27017',
         database_name='test_db',
         logger=logger,
         notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
     )
 
     mock_mongodb_repository.assert_called_once_with(
@@ -27,6 +28,7 @@ def test_mongodb___init__(mock_mongodb_repository):
         database_name='test_db',
         logger=logger,
         notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
     )
 
     assert mongo.mongodb_repository is not None
@@ -42,11 +44,8 @@ def mongodb_activity(mock_mongodb_repository):
         database_name='test_db',
         logger=MagicMock(),
         notification_handler=MagicMock(),
+        metrics_controller=AsyncMock(),
     )
-
-    mongo.logger = MagicMock()
-    mongo.notification_handler = MagicMock()
-    mongo.pod_id = 'localhost'
     return mongo
 
 
@@ -69,7 +68,7 @@ def test_del(mongodb_activity):
 async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
     """Test load_latest_data"""
 
-    mongodb_activity.mongodb_repository.find.return_value = [
+    mongodb_activity.mongodb_repository.find = AsyncMock(return_value = [
         {
             'name': 'test1',
             'value': 1,
@@ -77,7 +76,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
                 '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
             ),
         }
-    ]
+    ])
 
     result = await mongodb_activity.load_latest_data(
         {
@@ -106,7 +105,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
 async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
     """Test load_latest_data"""
 
-    mongodb_activity.mongodb_repository.find.return_value = [
+    mongodb_activity.mongodb_repository.find = AsyncMock(return_value = [
         {
             'name': 'test1',
             'value': 1,
@@ -114,7 +113,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
                 '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
             ),
         }
-    ]
+    ])
 
     result = await mongodb_activity.load_latest_data(
         {

tests/activities/test_redis.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from unittest.mock import ANY, MagicMock, patch
+from unittest.mock import ANY, AsyncMock, MagicMock, patch
 
 import numpy as np
 import pytest
@@ -15,6 +15,7 @@ from scouter.activities.redis import Redis
 def redis_activity(mock_redis_repository):
     logger = MagicMock()
     notification_handler = MagicMock(spec=NotificationHandler)
+    metrics_controller = MagicMock()
     activity = Redis(
         host='localhost',
         port=6379,
@@ -22,6 +23,7 @@ def redis_activity(mock_redis_repository):
         notification_handler=notification_handler,
         username='test',
         password='test',
+        metrics_controller=metrics_controller,
     )
 
     activity.redis_client = MagicMock()
@@ -46,6 +48,7 @@ def test_redis_initialization(mock_redis_repository):
     """Test Redis activity initialization"""
     logger = MagicMock()
     notification_handler = MagicMock(spec=NotificationHandler)
+    metrics_controller = MagicMock()
     activity = Redis(
         host='localhost',
         port=6379,
@@ -53,6 +56,7 @@ def test_redis_initialization(mock_redis_repository):
         notification_handler=notification_handler,
         username='test',
         password='test',
+        metrics_controller=metrics_controller,
     )
     mock_redis_repository.assert_called_once_with(
         host='localhost',
@@ -61,12 +65,9 @@ def test_redis_initialization(mock_redis_repository):
         password='test',
         logger=logger,
         notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
     )
 
-    activity.logger = logger
-    activity.notification_handler = notification_handler
-    activity.pod_id = 'localhost'
-
     assert activity.redis_repository is not None
 
 
@@ -79,7 +80,7 @@ async def test_get_last_data_timestamp_none(redis_activity):
         'schedule_name': 'test_schedule',
     }
 
-    redis_activity.redis_repository.get.return_value = None
+    redis_activity.redis_repository.get = AsyncMock(return_value = None)
 
     result = await redis_activity.get_last_data_timestamp(test_data)
 
@@ -95,7 +96,7 @@ async def test_get_last_data_timestamp_not_none(redis_activity):
     """Test get_last_data_timestamp"""
     test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
 
-    redis_activity.redis_repository.get.return_value = '2023-01-01 12:00:00'
+    redis_activity.redis_repository.get = AsyncMock(return_value = '2023-01-01 12:00:00')
 
     result = await redis_activity.get_last_data_timestamp(test_data)
 
@@ -170,7 +171,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
         'data': data.to_dict('records'),
     }
 
-    redis_activity.redis_repository.set = MagicMock()
+    redis_activity.redis_repository.set = AsyncMock()
 
     result = await redis_activity.put_last_data_timestamp(test_data)
 
@@ -241,8 +242,8 @@ async def test_group_and_hold_data_new_key(redis_activity):
     }
 
     # Mock get to return None for new key
-    redis_activity.redis_repository.get = MagicMock(return_value=None)
+    redis_activity.redis_repository.get = AsyncMock(return_value=None)
-    redis_activity.redis_repository.set = MagicMock()
+    redis_activity.redis_repository.set = AsyncMock()
 
     # Call the method
     result = await redis_activity.group_and_hold_data(test_data)
@@ -257,12 +258,9 @@ async def test_group_and_hold_data_new_key(redis_activity):
     assert result == expected_result
 
     # Verify set was called with correct arguments
-    redis_activity.redis_repository.set.assert_called_once()
+    redis_activity.redis_repository.set.assert_called_once_with(
-    args, kwargs = redis_activity.redis_repository.set.call_args
+        'held_data_test_pipeline_test_schedule', {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600
-    assert args[0] == 'held_data_test_pipeline_test_schedule'
+    )
-    assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}
-    assert kwargs['ttl'] == 3600
-
 
 @pytest.mark.asyncio
 async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
@@ -294,8 +292,8 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
     }
 
     # Mock get to return existing data
-    redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
+    redis_activity.redis_repository.get = AsyncMock(return_value=existing_data)
-    redis_activity.redis_repository.set = MagicMock()
+    redis_activity.redis_repository.set = AsyncMock()
 
     # Call the method
     result = await redis_activity.group_and_hold_data(test_data)
@@ -315,17 +313,10 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
     assert result == expected_result
 
     # Verify set was called with correct arguments
-    redis_activity.redis_repository.set.assert_called_once()
+    redis_activity.redis_repository.set.assert_called_once_with(
-    args, kwargs = redis_activity.redis_repository.set.call_args
+        'held_data_test_workflow_test_schedule', {'sensor1': 25.5, 'sensor2': 28.0, 'sensor3': 42.0, 'sensor4': None, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600
-    assert args[0] == 'held_data_test_workflow_test_schedule'
+    )
-    assert args[1] == {
+
-        'sensor1': 25.5,
-        'sensor2': 28.0,
-        'sensor3': 42.0,
-        'sensor4': None,
-        'timestamp': '2023-01-01 12:00:00',
-    }
-    assert kwargs['ttl'] == 3600
 
 
 @pytest.mark.asyncio
@@ -350,8 +341,8 @@ async def test_group_and_hold_data_with_none_values(redis_activity):
     }
 
     # Mock get to return None for new key
-    redis_activity.redis_repository.get = MagicMock(return_value=None)
+    redis_activity.redis_repository.get = AsyncMock(return_value=None)
-    redis_activity.redis_repository.set = MagicMock()
+    redis_activity.redis_repository.set = AsyncMock()
 
     # Call the method
     result = await redis_activity.group_and_hold_data(test_data)
@@ -375,7 +366,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
         'fill_missing_tags': False,
     }
 
-    redis_activity.redis_repository.get = MagicMock(return_value=None)
+    redis_activity.redis_repository.get = AsyncMock(return_value=None)
 
     # Call the method
     result = await redis_activity.group_and_hold_data(test_data)
@@ -397,7 +388,7 @@ async def test_group_and_hold_data_error_get(redis_activity):
         'fill_missing_tags': False,
     }
 
-    redis_activity.redis_repository.get = MagicMock(side_effect=Exception('test'))
+    redis_activity.redis_repository.get = AsyncMock(side_effect=Exception('test'))
     redis_activity.send_notification = MagicMock()
 
     try:
@@ -436,8 +427,8 @@ async def test_group_and_hold_data_error_set(redis_activity):
     existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
 
     # Mock get to return existing data
-    redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
+    redis_activity.redis_repository.get = AsyncMock(return_value=existing_data)
-    redis_activity.redis_repository.set = MagicMock(side_effect=Exception('test'))
+    redis_activity.redis_repository.set = AsyncMock(side_effect=Exception('test'))
     redis_activity.send_notification = MagicMock()
 
     try:
@@ -450,7 +441,7 @@ async def test_group_and_hold_data_error_set(redis_activity):
 @pytest.mark.asyncio
 async def test_store_data_package(redis_activity):
     """Test store_data_package"""
-    redis_activity.redis_repository.set = MagicMock()
+    redis_activity.redis_repository.set = AsyncMock()
 
     test_data = {
         **metadata,
@@ -483,7 +474,7 @@ async def test_store_data_package(redis_activity):
 @pytest.mark.asyncio
 async def test_store_data_package_error(redis_activity):
     """Test store_data_package error"""
-    redis_activity.redis_repository.set = MagicMock(side_effect=ValueError('test'))
+    redis_activity.redis_repository.set = AsyncMock(side_effect=ValueError('test'))
     redis_activity.send_notification = MagicMock()
 
     test_data = {

tests/conftest.py

@@ -1,12 +0,0 @@
-from unittest.mock import patch
-
-from pytest import fixture
-
-
-@fixture(autouse=True, scope='session')
-def sientia_monitoring_fixture():
-    with patch(
-        'sientia_do.observability.sientia_monitoring.SientiaMonitoring.__init__'
-    ) as mock_sientia_monitoring:
-        mock_sientia_monitoring.return_value = None
-        yield

tests/utils/test_connectors_config.py

@@ -122,6 +122,11 @@ def test_build_redis_config_with_env_vars():
 
 def test_build_mongodb_config_defaults():
     """Test that build_mongodb_config returns default values when no env vars are set"""
+    os.environ['MONGODB_URL'] = 'localhost:27017'
+    os.environ['MONGODB_DATABASE_NAME'] = 'sientia'
+    os.environ['MONGODB_USERNAME'] = 'sientia'
+    os.environ['MONGODB_PASSWORD'] = 'sientia'
+    
     config = build_mongodb_config()
 
     assert config == {

values.yaml

@@ -170,11 +170,6 @@ env:
   - name: POSTGRES_MAX_CONNECTIONS
     value: "40"
 
-  - name: KAFKA_BOOTSTRAP_SERVERS
-    value: "kafka.kafka.svc.cluster.local:9092"
-  - name: KAFKA_POLLING_TIME
-    value: "10000"
-
   - name: REDIS_HOST
     value: "redis-master.redis.svc.cluster.local"
   - name: REDIS_PORT