gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
Update environment configuration and refactor activities to include metrics controller. Remove Kafka settings and adjust Redis and MongoDB initialization. Update tests to reflect changes in initialization and metrics tracking.
This commit is contained in:
vitor-aignosi
2025-10-30 16:49:55 -03:00
parent 0c16075823
commit d4a74ec5eb
16 changed files with 225 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
tests/activities/test_redis.py
View File

@@ -1,5 +1,5 @@
from datetime import datetime
from unittest.mock import ANY, MagicMock, patch
from unittest.mock import ANY, AsyncMock, MagicMock, patch
import numpy as np
import pytest
@@ -15,6 +15,7 @@ from scouter.activities.redis import Redis
def redis_activity(mock_redis_repository):
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
metrics_controller = MagicMock()
activity = Redis(
host='localhost',
port=6379,
@@ -22,6 +23,7 @@ def redis_activity(mock_redis_repository):
notification_handler=notification_handler,
username='test',
password='test',
metrics_controller=metrics_controller,
)
activity.redis_client = MagicMock()
@@ -46,6 +48,7 @@ def test_redis_initialization(mock_redis_repository):
"""Test Redis activity initialization"""
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
metrics_controller = MagicMock()
activity = Redis(
host='localhost',
port=6379,
@@ -53,6 +56,7 @@ def test_redis_initialization(mock_redis_repository):
notification_handler=notification_handler,
username='test',
password='test',
metrics_controller=metrics_controller,
)
mock_redis_repository.assert_called_once_with(
host='localhost',
@@ -61,12 +65,9 @@ def test_redis_initialization(mock_redis_repository):
password='test',
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
activity.logger = logger
activity.notification_handler = notification_handler
activity.pod_id = 'localhost'
assert activity.redis_repository is not None
@@ -79,7 +80,7 @@ async def test_get_last_data_timestamp_none(redis_activity):
'schedule_name': 'test_schedule',
}
redis_activity.redis_repository.get.return_value = None
redis_activity.redis_repository.get = AsyncMock(return_value = None)
result = await redis_activity.get_last_data_timestamp(test_data)
@@ -95,7 +96,7 @@ async def test_get_last_data_timestamp_not_none(redis_activity):
"""Test get_last_data_timestamp"""
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.redis_repository.get.return_value = '2023-01-01 12:00:00'
redis_activity.redis_repository.get = AsyncMock(return_value = '2023-01-01 12:00:00')
result = await redis_activity.get_last_data_timestamp(test_data)
@@ -170,7 +171,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
'data': data.to_dict('records'),
}
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.set = AsyncMock()
result = await redis_activity.put_last_data_timestamp(test_data)
@@ -241,8 +242,8 @@ async def test_group_and_hold_data_new_key(redis_activity):
}
# Mock get to return None for new key
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.get = AsyncMock(return_value=None)
redis_activity.redis_repository.set = AsyncMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -257,12 +258,9 @@ async def test_group_and_hold_data_new_key(redis_activity):
assert result == expected_result
# Verify set was called with correct arguments
redis_activity.redis_repository.set.assert_called_once()
args, kwargs = redis_activity.redis_repository.set.call_args
assert args[0] == 'held_data_test_pipeline_test_schedule'
assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}
assert kwargs['ttl'] == 3600
redis_activity.redis_repository.set.assert_called_once_with(
'held_data_test_pipeline_test_schedule', {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600
)
@pytest.mark.asyncio
async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
@@ -294,8 +292,8 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
}
# Mock get to return existing data
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.get = AsyncMock(return_value=existing_data)
redis_activity.redis_repository.set = AsyncMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -315,17 +313,10 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
assert result == expected_result
# Verify set was called with correct arguments
redis_activity.redis_repository.set.assert_called_once()
args, kwargs = redis_activity.redis_repository.set.call_args
assert args[0] == 'held_data_test_workflow_test_schedule'
assert args[1] == {
'sensor1': 25.5,
'sensor2': 28.0,
'sensor3': 42.0,
'sensor4': None,
'timestamp': '2023-01-01 12:00:00',
}
assert kwargs['ttl'] == 3600
redis_activity.redis_repository.set.assert_called_once_with(
'held_data_test_workflow_test_schedule', {'sensor1': 25.5, 'sensor2': 28.0, 'sensor3': 42.0, 'sensor4': None, 'timestamp': '2023-01-01 12:00:00'}, ttl=3600
)
@pytest.mark.asyncio
@@ -350,8 +341,8 @@ async def test_group_and_hold_data_with_none_values(redis_activity):
}
# Mock get to return None for new key
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.get = AsyncMock(return_value=None)
redis_activity.redis_repository.set = AsyncMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -375,7 +366,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
'fill_missing_tags': False,
}
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.get = AsyncMock(return_value=None)
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -397,7 +388,7 @@ async def test_group_and_hold_data_error_get(redis_activity):
'fill_missing_tags': False,
}
redis_activity.redis_repository.get = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.get = AsyncMock(side_effect=Exception('test'))
redis_activity.send_notification = MagicMock()
try:
@@ -436,8 +427,8 @@ async def test_group_and_hold_data_error_set(redis_activity):
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
# Mock get to return existing data
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
redis_activity.redis_repository.set = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.get = AsyncMock(return_value=existing_data)
redis_activity.redis_repository.set = AsyncMock(side_effect=Exception('test'))
redis_activity.send_notification = MagicMock()
try:
@@ -450,7 +441,7 @@ async def test_group_and_hold_data_error_set(redis_activity):
@pytest.mark.asyncio
async def test_store_data_package(redis_activity):
"""Test store_data_package"""
redis_activity.redis_repository.set = MagicMock()
redis_activity.redis_repository.set = AsyncMock()
test_data = {
**metadata,
@@ -483,7 +474,7 @@ async def test_store_data_package(redis_activity):
@pytest.mark.asyncio
async def test_store_data_package_error(redis_activity):
"""Test store_data_package error"""
redis_activity.redis_repository.set = MagicMock(side_effect=ValueError('test'))
redis_activity.redis_repository.set = AsyncMock(side_effect=ValueError('test'))
redis_activity.send_notification = MagicMock()
test_data = {