diff --git a/.env.example b/.env.example index 28055e7..a4ba811 100644 --- a/.env.example +++ b/.env.example @@ -6,14 +6,18 @@ POSTGRES_DB=sientia POSTGRES_MIN_CONNECTIONS=5 POSTGRES_MAX_CONNECTIONS=20 -KAFKA_BOOTSTRAP_SERVERS=localhost:9092 -KAFKA_POLLING_TIME=1000 +MONGODB_USERNAME="root" +MONGODB_PASSWORD="password" +MONGODB_URL="my-release-mongodb.mongodb.svc.cluster.local:27017" +MONGODB_DATABASE="sientia" REDIS_HOST=localhost REDIS_PORT=6379 +REDIS_USERNAME="user" +REDIS_PASSWORD="pass" TEMPORAL_HOST=localhost:7233 -TEMPORAL_NAMESPACE=default +TEMPORAL_NAMESPACE=scouter LOG_LEVEL=INFO diff --git a/init_port_forward.sh b/init_port_forward.sh new file mode 100755 index 0000000..e9c0e3d --- /dev/null +++ b/init_port_forward.sh @@ -0,0 +1,133 @@ +#!/bin/bash + +# Usage: +# 1) Edit the PORT_FORWARDS list below with entries of: +# +# 2) Run: ./init_port_forward.sh +# +# The script will start all port-forwards in the background and keep running +# until interrupted (Ctrl+C). On exit, it will clean up started port-forward processes. + +set -euo pipefail + +# Define your namespace/service/port combinations here +# Example entries: +# "default my-service 8080 80" +# "observability grafana 3000 3000" +PORT_FORWARDS=( + "mongodb my-release-mongodb 27017 27017" + "paradedb paradedb-rw 5432 5432" + "redis redis-master 6379 6379" + "temporal temporal-frontend 7233 7233" +) + +if [ ${#PORT_FORWARDS[@]} -eq 0 ]; then + echo "No port-forward entries defined. Edit PORT_FORWARDS in $(basename "$0")." + exit 1 +fi + +PIDS=() + +cleanup() { + echo "\nStopping port-forward processes..." + for pid in "${PIDS[@]}"; do + if kill -0 "$pid" >/dev/null 2>&1; then + kill "$pid" >/dev/null 2>&1 || true + fi + done +} + +trap cleanup EXIT INT TERM + +timestamp() { date '+%Y-%m-%d %H:%M:%S'; } + +# Allow overriding kubectl binary if needed +KUBECTL=${KUBECTL:-kubectl} + +is_port_free() { + local port="$1" + # Consider port free if nothing is listening locally on it + if command -v ss >/dev/null 2>&1; then + ! ss -ltn | awk '{print $4}' | grep -E "(^|:|\\])${port}$" >/dev/null 2>&1 + else + if command -v lsof >/dev/null 2>&1; then + ! lsof -tiTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1 + else + # Fallback: attempt to open a TCP connection; expect failure when nothing is listening + ! (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null + fi + fi +} + +free_port_if_stuck() { + local port="$1" + # Try multiple tools to free a stuck listener (often old kubectl PF) + if command -v lsof >/dev/null 2>&1; then + local pids + pids=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null || true) + if [ -n "${pids}" ]; then + echo "[$(timestamp)] Found listeners on ${port}: ${pids}; terminating" + kill ${pids} 2>/dev/null || true + sleep 0.5 + fi + fi + if ! is_port_free "${port}"; then + if command -v fuser >/dev/null 2>&1; then + echo "[$(timestamp)] Forcing free of ${port} via fuser" + fuser -k "${port}/tcp" 2>/dev/null || true + sleep 0.5 + fi + fi +} + +run_port_forward() { + local namespace="$1" + local service_name="$2" + local local_port="$3" + local service_port="$4" + + # simple and robust supervisor loop with gentle backoff on failures + local delay=2 + local max_delay=20 + while true; do + free_port_if_stuck "${local_port}" + if ! is_port_free "${local_port}"; then + echo "[$(timestamp)] ns=${namespace} svc=${service_name} ${local_port}:${service_port} -> local port busy, retrying in 3s" + sleep 3 + continue + fi + + echo "[$(timestamp)] Starting port-forward: ns=${namespace} svc=${service_name} ${local_port}:${service_port}" + ${KUBECTL} -n "${namespace}" port-forward "svc/${service_name}" "${local_port}:${service_port}" \ + --address=127.0.0.1 --pod-running-timeout=2m --request-timeout=0 + rc=$? + + # If kubectl exits (e.g., connection reset by peer), wait a bit and retry + echo "[$(timestamp)] Port-forward exited (rc=${rc}): ns=${namespace} svc=${service_name} ${local_port}:${service_port}" + sleep "${delay}" + # Exponential backoff up to max_delay + if [ ${delay} -lt ${max_delay} ]; then + delay=$(( delay * 2 )) + if [ ${delay} -gt ${max_delay} ]; then + delay=${max_delay} + fi + fi + done +} + +ONLY_SERVICE_NAME="${ONLY_SERVICE_NAME:-}" + +for entry in "${PORT_FORWARDS[@]}"; do + read -r NAMESPACE SERVICE_NAME LOCAL_PORT SERVICE_PORT <<< "$entry" + if [ -n "${ONLY_SERVICE_NAME}" ] && [ "${SERVICE_NAME}" != "${ONLY_SERVICE_NAME}" ]; then + continue + fi + run_port_forward "${NAMESPACE}" "${SERVICE_NAME}" "${LOCAL_PORT}" "${SERVICE_PORT}" & + PIDS+=("$!") +done + +echo "All port-forwards started: ${#PIDS[@]} process(es). Press Ctrl+C to stop." + +# Do not exit the script if one port-forward fails; they self-restart +set +e +wait diff --git a/pyproject.toml b/pyproject.toml index 517fead..14d6e25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,10 +114,6 @@ python_functions = ["test_*"] addopts = [ "-v", "--strict-markers", - "--cov=model_manager", - "--cov-report=term-missing", - "--cov-report=html", - "--cov-report=xml", ] markers = [ "asyncio: marks tests as async", @@ -126,7 +122,7 @@ markers = [ ] [tool.coverage.run] -source = ["model_manager"] +source = ["scouter"] omit = [ "*/tests/*", "*/venv/*", diff --git a/scouter/activities/activities.py b/scouter/activities/activities.py index 1dd20d8..19b1ae9 100644 --- a/scouter/activities/activities.py +++ b/scouter/activities/activities.py @@ -1,3 +1,4 @@ +from sientia_do.observability.metrics_controller import MetricsController from temporalio import workflow with workflow.unsafe.imports_passed_through(): @@ -50,6 +51,10 @@ class Activities(Postgres, Redis, Gates, MongoDB): logger (Logger): Logger instance for application logging notification_handler (NotificationHandler): Handler for system notifications """ + metrics_controller = MetricsController( + logger=logger, + ) + # Initialize Postgres Postgres.__init__( self, @@ -62,6 +67,7 @@ class Activities(Postgres, Redis, Gates, MongoDB): max_connections=postgres_config['max_connections'], logger=logger, notification_handler=notification_handler, + metrics_controller=metrics_controller, ) # Initialize Redis @@ -73,10 +79,11 @@ class Activities(Postgres, Redis, Gates, MongoDB): notification_handler=notification_handler, username=redis_config['username'], password=redis_config['password'], + metrics_controller=metrics_controller, ) # Initialize Gates - Gates.__init__(self, logger=logger, notification_handler=notification_handler) + Gates.__init__(self, logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller) # Initialize MongoDB MongoDB.__init__( @@ -85,6 +92,7 @@ class Activities(Postgres, Redis, Gates, MongoDB): database_name=mongodb_config['database_name'], logger=logger, notification_handler=notification_handler, + metrics_controller=metrics_controller, ) self.pod_id = getenv('HOSTNAME', 'localhost') diff --git a/scouter/activities/gates.py b/scouter/activities/gates.py index cb9ebf6..6c7ea32 100644 --- a/scouter/activities/gates.py +++ b/scouter/activities/gates.py @@ -1,5 +1,6 @@ from collections.abc import Hashable +from sientia_do.observability.metrics_controller import MetricsController from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): @@ -36,21 +37,22 @@ class Gates(SientiaMonitoring): ensure data integrity and enable flexible data processing workflows. """ - def __init__(self, logger: Logger, notification_handler: NotificationHandler): + def __init__(self, logger: Logger, notification_handler: NotificationHandler, metrics_controller: MetricsController): """ Initialize the Gates class with logging and notification services. Args: logger (Logger): Logger instance for operation logging notification_handler (NotificationHandler): Handler for system notifications + metrics_controller (MetricsController): Metrics controller instance """ - SientiaMonitoring.__init__(self, logger, notification_handler) + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) def close(self): """ Close the Gates class. """ - SientiaMonitoring.close(self) + SientiaMonitoring.shutdown(self) def apply_aggregation( self, values: DataFrame, aggr_function: str, metadata: dict[str, Any] diff --git a/scouter/activities/mongodb.py b/scouter/activities/mongodb.py index 55450eb..4c73af9 100644 --- a/scouter/activities/mongodb.py +++ b/scouter/activities/mongodb.py @@ -1,6 +1,6 @@ -from collections.abc import Hashable from datetime import UTC +from sientia_do.observability.metrics_controller import MetricsController from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): @@ -37,6 +37,7 @@ class MongoDB(SientiaMonitoring): database_name: str, logger: Logger, notification_handler: NotificationHandler, + metrics_controller: MetricsController, ): """ Initialize MongoDB connection and services. @@ -56,9 +57,10 @@ class MongoDB(SientiaMonitoring): database_name=database_name, logger=logger, notification_handler=notification_handler, + metrics_controller=metrics_controller, ) - SientiaMonitoring.__init__(self, logger=logger, notification_handler=notification_handler) + SientiaMonitoring.__init__(self, logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller) def close(self): """ @@ -77,7 +79,7 @@ class MongoDB(SientiaMonitoring): self.close() @activity.defn(name='load_latest_data') - async def load_latest_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]: + async def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]: """ Load the latest data from MongoDB collection since a specified timestamp. @@ -116,7 +118,7 @@ class MongoDB(SientiaMonitoring): self.debug(f'Data filter: {data_filter}', metadata=metadata) - data = self.mongodb_repository.find( + data = await self.mongodb_repository.find( collection_name=collection_name, filters=data_filter, metadata=metadata, diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py index e4513a2..bcb66f4 100644 --- a/scouter/activities/redis.py +++ b/scouter/activities/redis.py @@ -1,9 +1,9 @@ -from collections.abc import Hashable - +from sientia_do.observability.metrics_controller import MetricsController from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): import traceback + from collections.abc import Hashable from typing import Any from pandas import DataFrame @@ -38,6 +38,7 @@ class Redis(SientiaMonitoring): password: str, logger: Logger, notification_handler: NotificationHandler, + metrics_controller: MetricsController, ): """ Initialize Redis connection and services. @@ -50,7 +51,7 @@ class Redis(SientiaMonitoring): logger (Logger): Logger instance for operation logging notification_handler (NotificationHandler): Handler for system notifications """ - SientiaMonitoring.__init__(self, logger, notification_handler) + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) self.redis_repository = RedisRepository( host=host, port=port, @@ -58,6 +59,7 @@ class Redis(SientiaMonitoring): password=password, logger=logger, notification_handler=notification_handler, + metrics_controller=metrics_controller, ) def close(self): @@ -95,7 +97,7 @@ class Redis(SientiaMonitoring): self.info(f'Getting last data timestamp for {key}') try: - data_hold = self.redis_repository.get(key) + data_hold = await self.redis_repository.get(key) except Exception as e: self.send_notification( metadata=metadata, @@ -153,7 +155,7 @@ class Redis(SientiaMonitoring): self.info(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata) try: - self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5) + await self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5) except Exception as e: self.send_notification( metadata=metadata, @@ -206,7 +208,7 @@ class Redis(SientiaMonitoring): self.info(f'Getting held data for {key}') try: - data_hold = self.redis_repository.get(key) + data_hold = await self.redis_repository.get(key) except Exception as e: self.send_notification( metadata=metadata, @@ -250,7 +252,7 @@ class Redis(SientiaMonitoring): data['timestamp'].max() if not data.empty else data_hold['timestamp'] ) - self.redis_repository.set(key, data_hold, ttl=retention_time) + await self.redis_repository.set(key, data_hold, ttl=retention_time) data_hold_df = DataFrame(data_hold, index=[0]) data_hold_melted = data_hold_df.melt( @@ -296,7 +298,7 @@ class Redis(SientiaMonitoring): cache = {'data': data.to_dict(), 'held_data': held_data.to_dict()} try: - self.redis_repository.set(key, cache, ttl=120) + await self.redis_repository.set(key, cache, ttl=120) except Exception as e: self.send_notification( metadata=metadata, diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index c842761..a89875b 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -136,7 +136,7 @@ async def main(): except BaseException as e: # NOSONAR logger.custom_error( - 'An unhandled exception occurred: %s', e, exc_info=True, metadata=metadata + 'An unhandled exception occurred: %s', metadata=metadata ) finally: if notification_handler: diff --git a/scouter/workflow/scouter.py b/scouter/workflow/scouter.py index 5888627..52393e2 100644 --- a/scouter/workflow/scouter.py +++ b/scouter/workflow/scouter.py @@ -95,7 +95,7 @@ class Scouter: retry_policy=retry_policy, ) - if data == {}: + if not data: return await workflow.execute_activity_method( diff --git a/tests/activities/test_activities.py b/tests/activities/test_activities.py index 8864c5f..7d07ecb 100644 --- a/tests/activities/test_activities.py +++ b/tests/activities/test_activities.py @@ -12,7 +12,8 @@ from scouter.activities.redis import Redis @patch('scouter.activities.activities.Postgres.__init__') @patch('scouter.activities.activities.Redis.__init__') @patch('scouter.activities.activities.Gates.__init__') -def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init): +@patch('scouter.activities.activities.MetricsController') +def test___init__(mock_metrics_controller, mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init): postgres_config = { 'host': 'localhost', 'port': 5432, @@ -38,7 +39,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon redis_config=redis_config, mongodb_config=mongodb_config, logger=logger, - notification_handler=notification_handler, + notification_handler=notification_handler ) assert isinstance(activities, Activities) @@ -58,6 +59,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon max_connections=postgres_config['max_connections'], logger=logger, notification_handler=notification_handler, + metrics_controller=mock_metrics_controller.return_value, ) mock_redis_init.assert_called_once_with( @@ -68,6 +70,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon password=redis_config['password'], logger=logger, notification_handler=notification_handler, + metrics_controller=mock_metrics_controller.return_value, ) mock_mongodb_init.assert_called_once_with( @@ -76,10 +79,11 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon database_name=mongodb_config['database_name'], logger=logger, notification_handler=notification_handler, + metrics_controller=mock_metrics_controller.return_value, ) mock_gates_init.assert_called_once_with( - ANY, logger=logger, notification_handler=notification_handler + ANY, logger=logger, notification_handler=notification_handler, metrics_controller=mock_metrics_controller.return_value ) diff --git a/tests/activities/test_gates.py b/tests/activities/test_gates.py index 726c666..7e8f29e 100644 --- a/tests/activities/test_gates.py +++ b/tests/activities/test_gates.py @@ -14,10 +14,12 @@ def gates_fixture(): """Fixture to create a Gates instance with mocked dependencies.""" logger = Mock() notification_handler = MagicMock() - gates = Gates(logger=logger, notification_handler=notification_handler) + metrics_controller = MagicMock() + gates = Gates(logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller) gates.send_notification = MagicMock() gates.logger = logger gates.notification_handler = notification_handler + gates.metrics_controller = metrics_controller gates.pod_id = 'localhost' return gates diff --git a/tests/activities/test_mongo.py b/tests/activities/test_mongo.py index 268b02c..6b812c2 100644 --- a/tests/activities/test_mongo.py +++ b/tests/activities/test_mongo.py @@ -1,5 +1,5 @@ from datetime import datetime -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import ANY, AsyncMock, MagicMock, patch from pytest import fixture, mark from sientia_do.notifications.models import NotificationLevel @@ -14,12 +14,13 @@ def test_mongodb___init__(mock_mongodb_repository): logger = MagicMock() notification_handler = MagicMock() - + metrics_controller = MagicMock() mongo = MongoDB( connection_string='mongodb://localhost:27017', database_name='test_db', logger=logger, notification_handler=notification_handler, + metrics_controller=metrics_controller, ) mock_mongodb_repository.assert_called_once_with( @@ -27,6 +28,7 @@ def test_mongodb___init__(mock_mongodb_repository): database_name='test_db', logger=logger, notification_handler=notification_handler, + metrics_controller=metrics_controller, ) assert mongo.mongodb_repository is not None @@ -42,11 +44,8 @@ def mongodb_activity(mock_mongodb_repository): database_name='test_db', logger=MagicMock(), notification_handler=MagicMock(), + metrics_controller=AsyncMock(), ) - - mongo.logger = MagicMock() - mongo.notification_handler = MagicMock() - mongo.pod_id = 'localhost' return mongo @@ -69,7 +68,7 @@ def test_del(mongodb_activity): async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): """Test load_latest_data""" - mongodb_activity.mongodb_repository.find.return_value = [ + mongodb_activity.mongodb_repository.find = AsyncMock(return_value = [ { 'name': 'test1', 'value': 1, @@ -77,7 +76,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ ), } - ] + ]) result = await mongodb_activity.load_latest_data( { @@ -106,7 +105,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity): async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): """Test load_latest_data""" - mongodb_activity.mongodb_repository.find.return_value = [ + mongodb_activity.mongodb_repository.find = AsyncMock(return_value = [ { 'name': 'test1', 'value': 1, @@ -114,7 +113,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity): '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ ), } - ] + ]) result = await mongodb_activity.load_latest_data( { diff --git a/tests/activities/test_redis.py b/tests/activities/test_redis.py index 0daf867..3ec93c8 100644 --- a/tests/activities/test_redis.py +++ b/tests/activities/test_redis.py @@ -1,5 +1,5 @@ from datetime import datetime -from unittest.mock import ANY, MagicMock, patch +from unittest.mock import ANY, AsyncMock, MagicMock, patch import numpy as np import pytest @@ -15,6 +15,7 @@ from scouter.activities.redis import Redis def redis_activity(mock_redis_repository): logger = MagicMock() notification_handler = MagicMock(spec=NotificationHandler) + metrics_controller = MagicMock() activity = Redis( host='localhost', port=6379, @@ -22,6 +23,7 @@ def redis_activity(mock_redis_repository): notification_handler=notification_handler, username='test', password='test', + metrics_controller=metrics_controller, ) activity.redis_client = MagicMock() @@ -46,6 +48,7 @@ def test_redis_initialization(mock_redis_repository): """Test Redis activity initialization""" logger = MagicMock() notification_handler = MagicMock(spec=NotificationHandler) + metrics_controller = MagicMock() activity = Redis( host='localhost', port=6379, @@ -53,6 +56,7 @@ def test_redis_initialization(mock_redis_repository): notification_handler=notification_handler, username='test', password='test', + metrics_controller=metrics_controller, ) mock_redis_repository.assert_called_once_with( host='localhost', @@ -61,12 +65,9 @@ def test_redis_initialization(mock_redis_repository): password='test', logger=logger, notification_handler=notification_handler, + metrics_controller=metrics_controller, ) - activity.logger = logger - activity.notification_handler = notification_handler - activity.pod_id = 'localhost' - assert activity.redis_repository is not None @@ -79,7 +80,7 @@ async def test_get_last_data_timestamp_none(redis_activity): 'schedule_name': 'test_schedule', } - redis_activity.redis_repository.get.return_value = None + redis_activity.redis_repository.get = AsyncMock(return_value = None) result = await redis_activity.get_last_data_timestamp(test_data) @@ -95,7 +96,7 @@ async def test_get_last_data_timestamp_not_none(redis_activity): """Test get_last_data_timestamp""" test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'} - redis_activity.redis_repository.get.return_value = '2023-01-01 12:00:00' + redis_activity.redis_repository.get = AsyncMock(return_value = '2023-01-01 12:00:00') result = await redis_activity.get_last_data_timestamp(test_data) @@ -170,7 +171,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity): 'data': data.to_dict('records'), } - redis_activity.redis_repository.set = MagicMock() + redis_activity.redis_repository.set = AsyncMock() result = await redis_activity.put_last_data_timestamp(test_data) @@ -241,8 +242,8 @@ async def test_group_and_hold_data_new_key(redis_activity): } # Mock get to return None for new key - redis_activity.redis_repository.get = MagicMock(return_value=None) - redis_activity.redis_repository.set = MagicMock() + redis_activity.redis_repository.get = AsyncMock(return_value=None) + redis_activity.redis_repository.set = AsyncMock() # Call the method result = await redis_activity.group_and_hold_data(test_data) @@ -257,12 +258,9 @@ async def test_group_and_hold_data_new_key(redis_activity): assert result == expected_result # Verify set was called with correct arguments - redis_activity.redis_repository.set.assert_called_once() - args, kwargs = redis_activity.redis_repository.set.call_args - assert args[0] == 'held_data_test_pipeline_test_schedule' - assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'} - assert kwargs['ttl'] == 3600 - + redis_activity.redis_repository.set.assert_called_once_with( + 'held_data_test_pipeline_test_schedule', {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600 + ) @pytest.mark.asyncio async def test_group_and_hold_data_update_existing_fill_missing(redis_activity): @@ -294,8 +292,8 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity): } # Mock get to return existing data - redis_activity.redis_repository.get = MagicMock(return_value=existing_data) - redis_activity.redis_repository.set = MagicMock() + redis_activity.redis_repository.get = AsyncMock(return_value=existing_data) + redis_activity.redis_repository.set = AsyncMock() # Call the method result = await redis_activity.group_and_hold_data(test_data) @@ -315,17 +313,10 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity): assert result == expected_result # Verify set was called with correct arguments - redis_activity.redis_repository.set.assert_called_once() - args, kwargs = redis_activity.redis_repository.set.call_args - assert args[0] == 'held_data_test_workflow_test_schedule' - assert args[1] == { - 'sensor1': 25.5, - 'sensor2': 28.0, - 'sensor3': 42.0, - 'sensor4': None, - 'timestamp': '2023-01-01 12:00:00', - } - assert kwargs['ttl'] == 3600 + redis_activity.redis_repository.set.assert_called_once_with( + 'held_data_test_workflow_test_schedule', {'sensor1': 25.5, 'sensor2': 28.0, 'sensor3': 42.0, 'sensor4': None, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600 + ) + @pytest.mark.asyncio @@ -350,8 +341,8 @@ async def test_group_and_hold_data_with_none_values(redis_activity): } # Mock get to return None for new key - redis_activity.redis_repository.get = MagicMock(return_value=None) - redis_activity.redis_repository.set = MagicMock() + redis_activity.redis_repository.get = AsyncMock(return_value=None) + redis_activity.redis_repository.set = AsyncMock() # Call the method result = await redis_activity.group_and_hold_data(test_data) @@ -375,7 +366,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity): 'fill_missing_tags': False, } - redis_activity.redis_repository.get = MagicMock(return_value=None) + redis_activity.redis_repository.get = AsyncMock(return_value=None) # Call the method result = await redis_activity.group_and_hold_data(test_data) @@ -397,7 +388,7 @@ async def test_group_and_hold_data_error_get(redis_activity): 'fill_missing_tags': False, } - redis_activity.redis_repository.get = MagicMock(side_effect=Exception('test')) + redis_activity.redis_repository.get = AsyncMock(side_effect=Exception('test')) redis_activity.send_notification = MagicMock() try: @@ -436,8 +427,8 @@ async def test_group_and_hold_data_error_set(redis_activity): existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'} # Mock get to return existing data - redis_activity.redis_repository.get = MagicMock(return_value=existing_data) - redis_activity.redis_repository.set = MagicMock(side_effect=Exception('test')) + redis_activity.redis_repository.get = AsyncMock(return_value=existing_data) + redis_activity.redis_repository.set = AsyncMock(side_effect=Exception('test')) redis_activity.send_notification = MagicMock() try: @@ -450,7 +441,7 @@ async def test_group_and_hold_data_error_set(redis_activity): @pytest.mark.asyncio async def test_store_data_package(redis_activity): """Test store_data_package""" - redis_activity.redis_repository.set = MagicMock() + redis_activity.redis_repository.set = AsyncMock() test_data = { **metadata, @@ -483,7 +474,7 @@ async def test_store_data_package(redis_activity): @pytest.mark.asyncio async def test_store_data_package_error(redis_activity): """Test store_data_package error""" - redis_activity.redis_repository.set = MagicMock(side_effect=ValueError('test')) + redis_activity.redis_repository.set = AsyncMock(side_effect=ValueError('test')) redis_activity.send_notification = MagicMock() test_data = { diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index a5aa843..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,12 +0,0 @@ -from unittest.mock import patch - -from pytest import fixture - - -@fixture(autouse=True, scope='session') -def sientia_monitoring_fixture(): - with patch( - 'sientia_do.observability.sientia_monitoring.SientiaMonitoring.__init__' - ) as mock_sientia_monitoring: - mock_sientia_monitoring.return_value = None - yield diff --git a/tests/utils/test_connectors_config.py b/tests/utils/test_connectors_config.py index 61d2b0a..0eb7012 100644 --- a/tests/utils/test_connectors_config.py +++ b/tests/utils/test_connectors_config.py @@ -122,6 +122,11 @@ def test_build_redis_config_with_env_vars(): def test_build_mongodb_config_defaults(): """Test that build_mongodb_config returns default values when no env vars are set""" + os.environ['MONGODB_URL'] = 'localhost:27017' + os.environ['MONGODB_DATABASE_NAME'] = 'sientia' + os.environ['MONGODB_USERNAME'] = 'sientia' + os.environ['MONGODB_PASSWORD'] = 'sientia' + config = build_mongodb_config() assert config == { diff --git a/values.yaml b/values.yaml index 85e47e5..804b505 100644 --- a/values.yaml +++ b/values.yaml @@ -170,11 +170,6 @@ env: - name: POSTGRES_MAX_CONNECTIONS value: "40" - - name: KAFKA_BOOTSTRAP_SERVERS - value: "kafka.kafka.svc.cluster.local:9092" - - name: KAFKA_POLLING_TIME - value: "10000" - - name: REDIS_HOST value: "redis-master.redis.svc.cluster.local" - name: REDIS_PORT