gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
Update environment configuration and refactor activities to include metrics controller. Remove Kafka settings and adjust Redis and MongoDB initialization. Update tests to reflect changes in initialization and metrics tracking.
This commit is contained in:
vitor-aignosi
2025-10-30 16:49:55 -03:00
parent 0c16075823
commit d4a74ec5eb
16 changed files with 225 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/activities/test_activities.py
View File

@@ -12,7 +12,8 @@ from scouter.activities.redis import Redis
@patch('scouter.activities.activities.Postgres.__init__')
@patch('scouter.activities.activities.Redis.__init__')
@patch('scouter.activities.activities.Gates.__init__')
def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init):
@patch('scouter.activities.activities.MetricsController')
def test___init__(mock_metrics_controller, mock_gates_init, mock_redis_init, mock_postgres_init, mock_mongodb_init):
postgres_config = {
'host': 'localhost',
'port': 5432,
@@ -38,7 +39,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
redis_config=redis_config,
mongodb_config=mongodb_config,
logger=logger,
notification_handler=notification_handler,
notification_handler=notification_handler
)
assert isinstance(activities, Activities)
@@ -58,6 +59,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
max_connections=postgres_config['max_connections'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
mock_redis_init.assert_called_once_with(
@@ -68,6 +70,7 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
password=redis_config['password'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
mock_mongodb_init.assert_called_once_with(
@@ -76,10 +79,11 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
database_name=mongodb_config['database_name'],
logger=logger,
notification_handler=notification_handler,
metrics_controller=mock_metrics_controller.return_value,
)
mock_gates_init.assert_called_once_with(
ANY, logger=logger, notification_handler=notification_handler
ANY, logger=logger, notification_handler=notification_handler, metrics_controller=mock_metrics_controller.return_value
)

4
tests/activities/test_gates.py
View File

@@ -14,10 +14,12 @@ def gates_fixture():
"""Fixture to create a Gates instance with mocked dependencies."""
logger = Mock()
notification_handler = MagicMock()
gates = Gates(logger=logger, notification_handler=notification_handler)
metrics_controller = MagicMock()
gates = Gates(logger=logger, notification_handler=notification_handler, metrics_controller=metrics_controller)
gates.send_notification = MagicMock()
gates.logger = logger
gates.notification_handler = notification_handler
gates.metrics_controller = metrics_controller
gates.pod_id = 'localhost'
return gates

19
tests/activities/test_mongo.py
View File

@@ -1,5 +1,5 @@
from datetime import datetime
from unittest.mock import ANY, MagicMock, patch
from unittest.mock import ANY, AsyncMock, MagicMock, patch
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
@@ -14,12 +14,13 @@ def test_mongodb___init__(mock_mongodb_repository):
logger = MagicMock()
notification_handler = MagicMock()
metrics_controller = MagicMock()
mongo = MongoDB(
connection_string='mongodb://localhost:27017',
database_name='test_db',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
mock_mongodb_repository.assert_called_once_with(
@@ -27,6 +28,7 @@ def test_mongodb___init__(mock_mongodb_repository):
database_name='test_db',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
assert mongo.mongodb_repository is not None
@@ -42,11 +44,8 @@ def mongodb_activity(mock_mongodb_repository):
database_name='test_db',
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
mongo.logger = MagicMock()
mongo.notification_handler = MagicMock()
mongo.pod_id = 'localhost'
return mongo
@@ -69,7 +68,7 @@ def test_del(mongodb_activity):
async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
"""Test load_latest_data"""
mongodb_activity.mongodb_repository.find.return_value = [
mongodb_activity.mongodb_repository.find = AsyncMock(return_value = [
{
'name': 'test1',
'value': 1,
@@ -77,7 +76,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
])
result = await mongodb_activity.load_latest_data(
{
@@ -106,7 +105,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
"""Test load_latest_data"""
mongodb_activity.mongodb_repository.find.return_value = [
mongodb_activity.mongodb_repository.find = AsyncMock(return_value = [
{
'name': 'test1',
'value': 1,
@@ -114,7 +113,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
),
}
]
])
result = await mongodb_activity.load_latest_data(
{

65
tests/activities/test_redis.py
View File

@@ -1,5 +1,5 @@
from datetime import datetime
from unittest.mock import ANY, MagicMock, patch
from unittest.mock import ANY, AsyncMock, MagicMock, patch
import numpy as np
import pytest
@@ -15,6 +15,7 @@ from scouter.activities.redis import Redis
def redis_activity(mock_redis_repository):
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
metrics_controller = MagicMock()
activity = Redis(
host='localhost',
port=6379,
@@ -22,6 +23,7 @@ def redis_activity(mock_redis_repository):
notification_handler=notification_handler,
username='test',
password='test',
metrics_controller=metrics_controller,
)
activity.redis_client = MagicMock()
@@ -46,6 +48,7 @@ def test_redis_initialization(mock_redis_repository):
"""Test Redis activity initialization"""
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
metrics_controller = MagicMock()
activity = Redis(
host='localhost',
port=6379,
@@ -53,6 +56,7 @@ def test_redis_initialization(mock_redis_repository):
notification_handler=notification_handler,
username='test',
password='test',
metrics_controller=metrics_controller,
)
mock_redis_repository.assert_called_once_with(
host='localhost',
@@ -61,12 +65,9 @@ def test_redis_initialization(mock_redis_repository):
password='test',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
activity.logger = logger
activity.notification_handler = notification_handler
activity.pod_id = 'localhost'
assert activity.redis_repository is not None
@@ -79,7 +80,7 @@ async def test_get_last_data_timestamp_none(redis_activity):
'schedule_name': 'test_schedule',
}
redis_activity.redis_repository.get.return_value = None
redis_activity.redis_repository.get = AsyncMock(return_value = None)
result = await redis_activity.get_last_data_timestamp(test_data)
@@ -95,7 +96,7 @@ async def test_get_last_data_timestamp_not_none(redis_activity):
"""Test get_last_data_timestamp"""
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.redis_repository.get.return_value = '2023-01-01 12:00:00'
redis_activity.redis_repository.get = AsyncMock(return_value = '2023-01-01 12:00:00')
result = await redis_activity.get_last_data_timestamp(test_data)
@@ -170,7 +171,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
'data': data.to_dict('records'),
}
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.set = AsyncMock()
result = await redis_activity.put_last_data_timestamp(test_data)
@@ -241,8 +242,8 @@ async def test_group_and_hold_data_new_key(redis_activity):
}
# Mock get to return None for new key
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.get = AsyncMock(return_value=None)
redis_activity.redis_repository.set = AsyncMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -257,12 +258,9 @@ async def test_group_and_hold_data_new_key(redis_activity):
assert result == expected_result
# Verify set was called with correct arguments
redis_activity.redis_repository.set.assert_called_once()
args, kwargs = redis_activity.redis_repository.set.call_args
assert args[0] == 'held_data_test_pipeline_test_schedule'
assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}
assert kwargs['ttl'] == 3600
redis_activity.redis_repository.set.assert_called_once_with(
'held_data_test_pipeline_test_schedule', {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600
)
@pytest.mark.asyncio
async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
@@ -294,8 +292,8 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
}
# Mock get to return existing data
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.get = AsyncMock(return_value=existing_data)
redis_activity.redis_repository.set = AsyncMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -315,17 +313,10 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
assert result == expected_result
# Verify set was called with correct arguments
redis_activity.redis_repository.set.assert_called_once()
args, kwargs = redis_activity.redis_repository.set.call_args
assert args[0] == 'held_data_test_workflow_test_schedule'
assert args[1] == {
'sensor1': 25.5,
'sensor2': 28.0,
'sensor3': 42.0,
'sensor4': None,
'timestamp': '2023-01-01 12:00:00',
}
assert kwargs['ttl'] == 3600
redis_activity.redis_repository.set.assert_called_once_with(
'held_data_test_workflow_test_schedule', {'sensor1': 25.5, 'sensor2': 28.0, 'sensor3': 42.0, 'sensor4': None, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600
)
@pytest.mark.asyncio
@@ -350,8 +341,8 @@ async def test_group_and_hold_data_with_none_values(redis_activity):
}
# Mock get to return None for new key
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.get = AsyncMock(return_value=None)
redis_activity.redis_repository.set = AsyncMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -375,7 +366,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
'fill_missing_tags': False,
}
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.get = AsyncMock(return_value=None)
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -397,7 +388,7 @@ async def test_group_and_hold_data_error_get(redis_activity):
'fill_missing_tags': False,
}
redis_activity.redis_repository.get = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.get = AsyncMock(side_effect=Exception('test'))
redis_activity.send_notification = MagicMock()
try:
@@ -436,8 +427,8 @@ async def test_group_and_hold_data_error_set(redis_activity):
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
# Mock get to return existing data
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
redis_activity.redis_repository.set = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.get = AsyncMock(return_value=existing_data)
redis_activity.redis_repository.set = AsyncMock(side_effect=Exception('test'))
redis_activity.send_notification = MagicMock()
try:
@@ -450,7 +441,7 @@ async def test_group_and_hold_data_error_set(redis_activity):
@pytest.mark.asyncio
async def test_store_data_package(redis_activity):
"""Test store_data_package"""
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.set = AsyncMock()
test_data = {
**metadata,
@@ -483,7 +474,7 @@ async def test_store_data_package(redis_activity):
@pytest.mark.asyncio
async def test_store_data_package_error(redis_activity):
"""Test store_data_package error"""
redis_activity.redis_repository.set = MagicMock(side_effect=ValueError('test'))
redis_activity.redis_repository.set = AsyncMock(side_effect=ValueError('test'))
redis_activity.send_notification = MagicMock()
test_data = {

12
tests/conftest.py
View File

@@ -1,12 +0,0 @@
from unittest.mock import patch
from pytest import fixture
@fixture(autouse=True, scope='session')
def sientia_monitoring_fixture():
with patch(
'sientia_do.observability.sientia_monitoring.SientiaMonitoring.__init__'
) as mock_sientia_monitoring:
mock_sientia_monitoring.return_value = None
yield

5
tests/utils/test_connectors_config.py
View File

@@ -122,6 +122,11 @@ def test_build_redis_config_with_env_vars():
def test_build_mongodb_config_defaults():
"""Test that build_mongodb_config returns default values when no env vars are set"""
os.environ['MONGODB_URL'] = 'localhost:27017'
os.environ['MONGODB_DATABASE_NAME'] = 'sientia'
os.environ['MONGODB_USERNAME'] = 'sientia'
os.environ['MONGODB_PASSWORD'] = 'sientia'
config = build_mongodb_config()
assert config == {