gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
Update environment configuration and refactor activities to include metrics controller. Remove Kafka settings and adjust Redis and MongoDB initialization. Update tests to reflect changes in initialization and metrics tracking.
This commit is contained in:
vitor-aignosi
2025-10-30 16:49:55 -03:00
parent 0c16075823
commit d4a74ec5eb
16 changed files with 225 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.env.example
View File

@@ -6,14 +6,18 @@ POSTGRES_DB=sientia
POSTGRES_MIN_CONNECTIONS=5
POSTGRES_MAX_CONNECTIONS=20
KAFKA_BOOTSTRAP_SERVERS=localhost:9092
KAFKA_POLLING_TIME=1000
MONGODB_USERNAME="root"
MONGODB_PASSWORD="password"
MONGODB_URL="my-release-mongodb.mongodb.svc.cluster.local:27017"
MONGODB_DATABASE="sientia"
REDIS_HOST=localhost
REDIS_PORT=6379
REDIS_USERNAME="user"
REDIS_PASSWORD="pass"
TEMPORAL_HOST=localhost:7233
TEMPORAL_NAMESPACE=default
TEMPORAL_NAMESPACE=scouter
LOG_LEVEL=INFO

133
init_port_forward.sh Executable file
View File

@@ -0,0 +1,133 @@
#!/bin/bash
# Usage:
# 1) Edit the PORT_FORWARDS list below with entries of:
# <namespace> <service_name> <local_port> <service_port>
# 2) Run: ./init_port_forward.sh
#
# The script will start all port-forwards in the background and keep running
# until interrupted (Ctrl+C). On exit, it will clean up started port-forward processes.
set -euo pipefail
# Define your namespace/service/port combinations here
# Example entries:
# "default my-service 8080 80"
# "observability grafana 3000 3000"
PORT_FORWARDS=(
"mongodb my-release-mongodb 27017 27017"
"paradedb paradedb-rw 5432 5432"
"redis redis-master 6379 6379"
"temporal temporal-frontend 7233 7233"
)
if [ ${#PORT_FORWARDS[@]} -eq 0 ]; then
echo "No port-forward entries defined. Edit PORT_FORWARDS in $(basename "$0")."
exit 1
fi
PIDS=()
cleanup() {
echo "\nStopping port-forward processes..."
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" >/dev/null 2>&1; then
kill "$pid" >/dev/null 2>&1 || true
fi
done
}
trap cleanup EXIT INT TERM
timestamp() { date '+%Y-%m-%d %H:%M:%S'; }
# Allow overriding kubectl binary if needed
KUBECTL=${KUBECTL:-kubectl}
is_port_free() {
local port="$1"
# Consider port free if nothing is listening locally on it
if command -v ss >/dev/null 2>&1; then
! ss -ltn | awk '{print $4}' | grep -E "(^|:|\\])${port}$" >/dev/null 2>&1
else
if command -v lsof >/dev/null 2>&1; then
! lsof -tiTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1
else
# Fallback: attempt to open a TCP connection; expect failure when nothing is listening
! (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null
fi
fi
}
free_port_if_stuck() {
local port="$1"
# Try multiple tools to free a stuck listener (often old kubectl PF)
if command -v lsof >/dev/null 2>&1; then
local pids
pids=$(lsof -tiTCP:"${port}" -sTCP:LISTEN 2>/dev/null || true)
if [ -n "${pids}" ]; then
echo "[$(timestamp)] Found listeners on ${port}: ${pids}; terminating"
kill ${pids} 2>/dev/null || true
sleep 0.5
fi
fi
if ! is_port_free "${port}"; then
if command -v fuser >/dev/null 2>&1; then
echo "[$(timestamp)] Forcing free of ${port} via fuser"
fuser -k "${port}/tcp" 2>/dev/null || true
sleep 0.5
fi
fi
}
run_port_forward() {
local namespace="$1"
local service_name="$2"
local local_port="$3"
local service_port="$4"
# simple and robust supervisor loop with gentle backoff on failures
local delay=2
local max_delay=20
while true; do
free_port_if_stuck "${local_port}"
if ! is_port_free "${local_port}"; then
echo "[$(timestamp)] ns=${namespace} svc=${service_name} ${local_port}:${service_port} -> local port busy, retrying in 3s"
sleep 3
continue
fi
echo "[$(timestamp)] Starting port-forward: ns=${namespace} svc=${service_name} ${local_port}:${service_port}"
${KUBECTL} -n "${namespace}" port-forward "svc/${service_name}" "${local_port}:${service_port}" \
--address=127.0.0.1 --pod-running-timeout=2m --request-timeout=0
rc=$?
# If kubectl exits (e.g., connection reset by peer), wait a bit and retry
echo "[$(timestamp)] Port-forward exited (rc=${rc}): ns=${namespace} svc=${service_name} ${local_port}:${service_port}"
sleep "${delay}"
# Exponential backoff up to max_delay
if [ ${delay} -lt ${max_delay} ]; then
delay=$(( delay * 2 ))
if [ ${delay} -gt ${max_delay} ]; then
delay=${max_delay}
fi
fi
done
}
ONLY_SERVICE_NAME="${ONLY_SERVICE_NAME:-}"
for entry in "${PORT_FORWARDS[@]}"; do
read -r NAMESPACE SERVICE_NAME LOCAL_PORT SERVICE_PORT <<< "$entry"
if [ -n "${ONLY_SERVICE_NAME}" ] && [ "${SERVICE_NAME}" != "${ONLY_SERVICE_NAME}" ]; then
continue
fi
run_port_forward "${NAMESPACE}" "${SERVICE_NAME}" "${LOCAL_PORT}" "${SERVICE_PORT}" &
PIDS+=("$!")
done
echo "All port-forwards started: ${#PIDS[@]} process(es). Press Ctrl+C to stop."
# Do not exit the script if one port-forward fails; they self-restart
set +e
wait

6
pyproject.toml
View File

@@ -114,10 +114,6 @@ python_functions = ["test_*"]
addopts = [
"-v",
"--strict-markers",
"--cov=model_manager",
"--cov-report=term-missing",
"--cov-report=html",
"--cov-report=xml",
]
markers = [
"asyncio: marks tests as async",
@@ -126,7 +122,7 @@ markers = [
]
[tool.coverage.run]
source = ["model_manager"]
source = ["scouter"]
omit = [
"*/tests/*",
"*/venv/*",

10
scouter/activities/activities.py
View File

@@ -1,3 +1,4 @@
from sientia_do.observability.metrics_controller import MetricsController
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
@@ -50,6 +51,10 @@ class Activities(Postgres, Redis, Gates, MongoDB):
logger (Logger): Logger instance for application logging
notification_handler (NotificationHandler): Handler for system notifications
"""
metrics_controller = MetricsController(
logger=logger,
)
# Initialize Postgres
Postgres.__init__(
self,
@@ -62,6 +67,7 @@ class Activities(Postgres, Redis, Gates, MongoDB):
max_connections=postgres_config['max_connections'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
# Initialize Redis
@@ -73,10 +79,11 @@ class Activities(Postgres, Redis, Gates, MongoDB):
notification_handler=notification_handler,
username=redis_config['username'],
password=redis_config['password'],
metrics_controller=metrics_controller,
)
# Initialize Gates
Gates.__init__(self, logger=logger, notification_handler=notification_handler)
Gates.__init__(self, logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller)
# Initialize MongoDB
MongoDB.__init__(
@@ -85,6 +92,7 @@ class Activities(Postgres, Redis, Gates, MongoDB):
database_name=mongodb_config['database_name'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
self.pod_id = getenv('HOSTNAME', 'localhost')

8
scouter/activities/gates.py
View File

@@ -1,5 +1,6 @@
from collections.abc import Hashable
from sientia_do.observability.metrics_controller import MetricsController
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
@@ -36,21 +37,22 @@ class Gates(SientiaMonitoring):
ensure data integrity and enable flexible data processing workflows.
"""
def __init__(self, logger: Logger, notification_handler: NotificationHandler):
def __init__(self, logger: Logger, notification_handler: NotificationHandler, metrics_controller: MetricsController):
"""
Initialize the Gates class with logging and notification services.
Args:
logger (Logger): Logger instance for operation logging
notification_handler (NotificationHandler): Handler for system notifications
metrics_controller (MetricsController): Metrics controller instance
"""
SientiaMonitoring.__init__(self, logger, notification_handler)
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
def close(self):
"""
Close the Gates class.
"""
SientiaMonitoring.close(self)
SientiaMonitoring.shutdown(self)
def apply_aggregation(
self, values: DataFrame, aggr_function: str, metadata: dict[str, Any]

10
scouter/activities/mongodb.py
View File

@@ -1,6 +1,6 @@
from collections.abc import Hashable
from datetime import UTC
from sientia_do.observability.metrics_controller import MetricsController
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
@@ -37,6 +37,7 @@ class MongoDB(SientiaMonitoring):
database_name: str,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
"""
Initialize MongoDB connection and services.
@@ -56,9 +57,10 @@ class MongoDB(SientiaMonitoring):
database_name=database_name,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
SientiaMonitoring.__init__(self, logger=logger, notification_handler=notification_handler)
SientiaMonitoring.__init__(self, logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller)
def close(self):
"""
@@ -77,7 +79,7 @@ class MongoDB(SientiaMonitoring):
self.close()
@activity.defn(name='load_latest_data')
async def load_latest_data(self, input_data: dict[str, Any]) -> dict[Hashable, Any]:
async def load_latest_data(self, input_data: dict[str, Any]) -> list[dict[str, Any]]:
"""
Load the latest data from MongoDB collection since a specified timestamp.
@@ -116,7 +118,7 @@ class MongoDB(SientiaMonitoring):
self.debug(f'Data filter: {data_filter}', metadata=metadata)
data = self.mongodb_repository.find(
data = await self.mongodb_repository.find(
collection_name=collection_name,
filters=data_filter,
metadata=metadata,

18
scouter/activities/redis.py
View File

@@ -1,9 +1,9 @@
from collections.abc import Hashable
from sientia_do.observability.metrics_controller import MetricsController
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import traceback
from collections.abc import Hashable
from typing import Any
from pandas import DataFrame
@@ -38,6 +38,7 @@ class Redis(SientiaMonitoring):
password: str,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
"""
Initialize Redis connection and services.
@@ -50,7 +51,7 @@ class Redis(SientiaMonitoring):
logger (Logger): Logger instance for operation logging
notification_handler (NotificationHandler): Handler for system notifications
"""
SientiaMonitoring.__init__(self, logger, notification_handler)
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
self.redis_repository = RedisRepository(
host=host,
port=port,
@@ -58,6 +59,7 @@ class Redis(SientiaMonitoring):
password=password,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
def close(self):
@@ -95,7 +97,7 @@ class Redis(SientiaMonitoring):
self.info(f'Getting last data timestamp for {key}')
try:
data_hold = self.redis_repository.get(key)
data_hold = await self.redis_repository.get(key)
except Exception as e:
self.send_notification(
metadata=metadata,
@@ -153,7 +155,7 @@ class Redis(SientiaMonitoring):
self.info(f'Last collected timestamp to insert: {last_data_timestamp}', metadata=metadata)
try:
self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5)
await self.redis_repository.set(key, last_data_timestamp, ttl=60 * 60 * 5)
except Exception as e:
self.send_notification(
metadata=metadata,
@@ -206,7 +208,7 @@ class Redis(SientiaMonitoring):
self.info(f'Getting held data for {key}')
try:
data_hold = self.redis_repository.get(key)
data_hold = await self.redis_repository.get(key)
except Exception as e:
self.send_notification(
metadata=metadata,
@@ -250,7 +252,7 @@ class Redis(SientiaMonitoring):
data['timestamp'].max() if not data.empty else data_hold['timestamp']
)
self.redis_repository.set(key, data_hold, ttl=retention_time)
await self.redis_repository.set(key, data_hold, ttl=retention_time)
data_hold_df = DataFrame(data_hold, index=[0])
data_hold_melted = data_hold_df.melt(
@@ -296,7 +298,7 @@ class Redis(SientiaMonitoring):
cache = {'data': data.to_dict(), 'held_data': held_data.to_dict()}
try:
self.redis_repository.set(key, cache, ttl=120)
await self.redis_repository.set(key, cache, ttl=120)
except Exception as e:
self.send_notification(
metadata=metadata,

2
scouter/worker/worker.py
View File

@@ -136,7 +136,7 @@ async def main():
except BaseException as e: # NOSONAR
logger.custom_error(
'An unhandled exception occurred: %s', e, exc_info=True, metadata=metadata
'An unhandled exception occurred: %s', metadata=metadata
)
finally:
if notification_handler:

2
scouter/workflow/scouter.py
View File

@@ -95,7 +95,7 @@ class Scouter:
retry_policy=retry_policy,
)
if data == {}:
if not data:
return
await workflow.execute_activity_method(

10
tests/activities/test_activities.py
View File

@@ -12,7 +12,8 @@ from scouter.activities.redis import Redis
@patch('scouter.activities.activities.Postgres.__init__')
@patch('scouter.activities.activities.Redis.__init__')
@patch('scouter.activities.activities.Gates.__init__')
def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init):
@patch('scouter.activities.activities.MetricsController')
def test___init__(mock_metrics_controller, mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init):
postgres_config = {
'host': 'localhost',
'port': 5432,
@@ -38,7 +39,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
redis_config=redis_config,
mongodb_config=mongodb_config,
logger=logger,
notification_handler=notification_handler,
notification_handler=notification_handler
)
assert isinstance(activities, Activities)
@@ -58,6 +59,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
max_connections=postgres_config['max_connections'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
mock_redis_init.assert_called_once_with(
@@ -68,6 +70,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
password=redis_config['password'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
mock_mongodb_init.assert_called_once_with(
@@ -76,10 +79,11 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
database_name=mongodb_config['database_name'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
mock_gates_init.assert_called_once_with(
ANY, logger=logger, notification_handler=notification_handler
ANY, logger=logger, notification_handler=notification_handler, metrics_controller=mock_metrics_controller.return_value
)

4
tests/activities/test_gates.py
View File

@@ -14,10 +14,12 @@ def gates_fixture():
"""Fixture to create a Gates instance with mocked dependencies."""
logger = Mock()
notification_handler = MagicMock()
gates = Gates(logger=logger, notification_handler=notification_handler)
metrics_controller = MagicMock()
gates = Gates(logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller)
gates.send_notification = MagicMock()
gates.logger = logger
gates.notification_handler = notification_handler
gates.metrics_controller = metrics_controller
gates.pod_id = 'localhost'
return gates

19
tests/activities/test_mongo.py
View File

@@ -1,5 +1,5 @@
from datetime import datetime
from unittest.mock import ANY, MagicMock, patch
from unittest.mock import ANY, AsyncMock, MagicMock, patch
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
@@ -14,12 +14,13 @@ def test_mongodb___init__(mock_mongodb_repository):
logger = MagicMock()
notification_handler = MagicMock()
metrics_controller = MagicMock()
mongo = MongoDB(
connection_string='mongodb://localhost:27017',
database_name='test_db',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
mock_mongodb_repository.assert_called_once_with(
@@ -27,6 +28,7 @@ def test_mongodb___init__(mock_mongodb_repository):
database_name='test_db',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
assert mongo.mongodb_repository is not None
@@ -42,11 +44,8 @@ def mongodb_activity(mock_mongodb_repository):
database_name='test_db',
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
mongo.logger = MagicMock()
mongo.notification_handler = MagicMock()
mongo.pod_id = 'localhost'
return mongo
@@ -69,7 +68,7 @@ def test_del(mongodb_activity):
async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
"""Test load_latest_data"""
mongodb_activity.mongodb_repository.find.return_value = [
mongodb_activity.mongodb_repository.find = AsyncMock(return_value = [
{
'name': 'test1',
'value': 1,
@@ -77,7 +76,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
])
result = await mongodb_activity.load_latest_data(
{
@@ -106,7 +105,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
"""Test load_latest_data"""
mongodb_activity.mongodb_repository.find.return_value = [
mongodb_activity.mongodb_repository.find = AsyncMock(return_value = [
{
'name': 'test1',
'value': 1,
@@ -114,7 +113,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
])
result = await mongodb_activity.load_latest_data(
{

65
tests/activities/test_redis.py
View File

@@ -1,5 +1,5 @@
from datetime import datetime
from unittest.mock import ANY, MagicMock, patch
from unittest.mock import ANY, AsyncMock, MagicMock, patch
import numpy as np
import pytest
@@ -15,6 +15,7 @@ from scouter.activities.redis import Redis
def redis_activity(mock_redis_repository):
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
metrics_controller = MagicMock()
activity = Redis(
host='localhost',
port=6379,
@@ -22,6 +23,7 @@ def redis_activity(mock_redis_repository):
notification_handler=notification_handler,
username='test',
password='test',
metrics_controller=metrics_controller,
)
activity.redis_client = MagicMock()
@@ -46,6 +48,7 @@ def test_redis_initialization(mock_redis_repository):
"""Test Redis activity initialization"""
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
metrics_controller = MagicMock()
activity = Redis(
host='localhost',
port=6379,
@@ -53,6 +56,7 @@ def test_redis_initialization(mock_redis_repository):
notification_handler=notification_handler,
username='test',
password='test',
metrics_controller=metrics_controller,
)
mock_redis_repository.assert_called_once_with(
host='localhost',
@@ -61,12 +65,9 @@ def test_redis_initialization(mock_redis_repository):
password='test',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
activity.logger = logger
activity.notification_handler = notification_handler
activity.pod_id = 'localhost'
assert activity.redis_repository is not None
@@ -79,7 +80,7 @@ async def test_get_last_data_timestamp_none(redis_activity):
'schedule_name': 'test_schedule',
}
redis_activity.redis_repository.get.return_value = None
redis_activity.redis_repository.get = AsyncMock(return_value = None)
result = await redis_activity.get_last_data_timestamp(test_data)
@@ -95,7 +96,7 @@ async def test_get_last_data_timestamp_not_none(redis_activity):
"""Test get_last_data_timestamp"""
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.redis_repository.get.return_value = '2023-01-01 12:00:00'
redis_activity.redis_repository.get = AsyncMock(return_value = '2023-01-01 12:00:00')
result = await redis_activity.get_last_data_timestamp(test_data)
@@ -170,7 +171,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
'data': data.to_dict('records'),
}
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.set = AsyncMock()
result = await redis_activity.put_last_data_timestamp(test_data)
@@ -241,8 +242,8 @@ async def test_group_and_hold_data_new_key(redis_activity):
}
# Mock get to return None for new key
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.get = AsyncMock(return_value=None)
redis_activity.redis_repository.set = AsyncMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -257,12 +258,9 @@ async def test_group_and_hold_data_new_key(redis_activity):
assert result == expected_result
# Verify set was called with correct arguments
redis_activity.redis_repository.set.assert_called_once()
args, kwargs = redis_activity.redis_repository.set.call_args
assert args[0] == 'held_data_test_pipeline_test_schedule'
assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}
assert kwargs['ttl'] == 3600
redis_activity.redis_repository.set.assert_called_once_with(
'held_data_test_pipeline_test_schedule', {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600
)
@pytest.mark.asyncio
async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
@@ -294,8 +292,8 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
}
# Mock get to return existing data
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.get = AsyncMock(return_value=existing_data)
redis_activity.redis_repository.set = AsyncMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -315,17 +313,10 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
assert result == expected_result
# Verify set was called with correct arguments
redis_activity.redis_repository.set.assert_called_once()
args, kwargs = redis_activity.redis_repository.set.call_args
assert args[0] == 'held_data_test_workflow_test_schedule'
assert args[1] == {
'sensor1': 25.5,
'sensor2': 28.0,
'sensor3': 42.0,
'sensor4': None,
'timestamp': '2023-01-01 12:00:00',
}
assert kwargs['ttl'] == 3600
redis_activity.redis_repository.set.assert_called_once_with(
'held_data_test_workflow_test_schedule', {'sensor1': 25.5, 'sensor2': 28.0, 'sensor3': 42.0, 'sensor4': None, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600
)
@pytest.mark.asyncio
@@ -350,8 +341,8 @@ async def test_group_and_hold_data_with_none_values(redis_activity):
}
# Mock get to return None for new key
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.get = AsyncMock(return_value=None)
redis_activity.redis_repository.set = AsyncMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -375,7 +366,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
'fill_missing_tags': False,
}
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.get = AsyncMock(return_value=None)
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -397,7 +388,7 @@ async def test_group_and_hold_data_error_get(redis_activity):
'fill_missing_tags': False,
}
redis_activity.redis_repository.get = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.get = AsyncMock(side_effect=Exception('test'))
redis_activity.send_notification = MagicMock()
try:
@@ -436,8 +427,8 @@ async def test_group_and_hold_data_error_set(redis_activity):
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
# Mock get to return existing data
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
redis_activity.redis_repository.set = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.get = AsyncMock(return_value=existing_data)
redis_activity.redis_repository.set = AsyncMock(side_effect=Exception('test'))
redis_activity.send_notification = MagicMock()
try:
@@ -450,7 +441,7 @@ async def test_group_and_hold_data_error_set(redis_activity):
@pytest.mark.asyncio
async def test_store_data_package(redis_activity):
"""Test store_data_package"""
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.set = AsyncMock()
test_data = {
**metadata,
@@ -483,7 +474,7 @@ async def test_store_data_package(redis_activity):
@pytest.mark.asyncio
async def test_store_data_package_error(redis_activity):
"""Test store_data_package error"""
redis_activity.redis_repository.set = MagicMock(side_effect=ValueError('test'))
redis_activity.redis_repository.set = AsyncMock(side_effect=ValueError('test'))
redis_activity.send_notification = MagicMock()
test_data = {

12
tests/conftest.py
View File

@@ -1,12 +0,0 @@
from unittest.mock import patch
from pytest import fixture
@fixture(autouse=True, scope='session')
def sientia_monitoring_fixture():
with patch(
'sientia_do.observability.sientia_monitoring.SientiaMonitoring.__init__'
) as mock_sientia_monitoring:
mock_sientia_monitoring.return_value = None
yield

5
tests/utils/test_connectors_config.py
View File

@@ -122,6 +122,11 @@ def test_build_redis_config_with_env_vars():
def test_build_mongodb_config_defaults():
"""Test that build_mongodb_config returns default values when no env vars are set"""
os.environ['MONGODB_URL'] = 'localhost:27017'
os.environ['MONGODB_DATABASE_NAME'] = 'sientia'
os.environ['MONGODB_USERNAME'] = 'sientia'
os.environ['MONGODB_PASSWORD'] = 'sientia'
config = build_mongodb_config()
assert config == {

5
values.yaml
View File

@@ -170,11 +170,6 @@ env:
- name: POSTGRES_MAX_CONNECTIONS
value: "40"
- name: KAFKA_BOOTSTRAP_SERVERS
value: "kafka.kafka.svc.cluster.local:9092"
- name: KAFKA_POLLING_TIME
value: "10000"
- name: REDIS_HOST
value: "redis-master.redis.svc.cluster.local"
- name: REDIS_PORT