gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
Update GITHUB_BRANCH in values.yaml to feature/SIENTIAPDE-1325 for specific external operation metrics. Refactor shutdown methods in Activities class to ensure proper closure of MongoDB, Redis, and Gates connections. Adjust tests to reflect these changes and enhance repository pattern implementation for MongoDB and Redis activities.
This commit is contained in:
vitor-aignosi
2025-10-29 16:58:22 -03:00
parent 10b3310677
commit 0c16075823
8 changed files with 152 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
tests/activities/test_redis.py
View File

@@ -11,8 +11,8 @@ from scouter.activities.redis import Redis
@pytest.fixture
@patch('scouter.activities.redis.RedisBase.__init__')
def redis_activity(_mock_redis_init):
@patch('scouter.activities.redis.RedisRepository')
def redis_activity(mock_redis_repository):
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
activity = Redis(
@@ -31,24 +31,6 @@ def redis_activity(_mock_redis_init):
return activity
@patch('scouter.activities.redis.RedisBase.__init__')
def test_redis_initialization(mock_redis_init):
"""Test Redis activity initialization"""
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
Redis(
host='localhost',
port=6379,
logger=logger,
notification_handler=notification_handler,
username='test',
password='test',
)
mock_redis_init.assert_called_once_with(
ANY, 'localhost', 6379, 'test', 'test', logger, notification_handler
)
metadata = {
'metadata': {
'model_id': 'test_model_id',
@@ -59,15 +41,52 @@ metadata = {
}
@patch('scouter.activities.redis.RedisRepository')
def test_redis_initialization(mock_redis_repository):
"""Test Redis activity initialization"""
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
activity = Redis(
host='localhost',
port=6379,
logger=logger,
notification_handler=notification_handler,
username='test',
password='test',
)
mock_redis_repository.assert_called_once_with(
host='localhost',
port=6379,
username='test',
password='test',
logger=logger,
notification_handler=notification_handler,
)
activity.logger = logger
activity.notification_handler = notification_handler
activity.pod_id = 'localhost'
assert activity.redis_repository is not None
@pytest.mark.asyncio
async def test_get_last_data_timestamp_none(redis_activity):
"""Test get_last_data_timestamp"""
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule',
}
redis_activity.get = MagicMock(return_value=None)
redis_activity.redis_repository.get.return_value = None
result = await redis_activity.get_last_data_timestamp(test_data)
redis_activity.redis_repository.get.assert_called_once_with(
'last_data_timestamp:test_pipeline:test_schedule'
)
assert result is None
@@ -76,11 +95,13 @@ async def test_get_last_data_timestamp_not_none(redis_activity):
"""Test get_last_data_timestamp"""
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.get = MagicMock(return_value='2023-01-01 12:00:00')
redis_activity.redis_repository.get.return_value = '2023-01-01 12:00:00'
result = await redis_activity.get_last_data_timestamp(test_data)
redis_activity.get.assert_called_once_with('last_data_timestamp:test_pipeline:test_schedule')
redis_activity.redis_repository.get.assert_called_once_with(
'last_data_timestamp:test_pipeline:test_schedule'
)
assert result == '2023-01-01 12:00:00'
@@ -91,7 +112,7 @@ async def test_get_last_data_timestamp_error(redis_activity):
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.send_notification = MagicMock()
redis_activity.get = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.get.side_effect = Exception('test')
try:
await redis_activity.get_last_data_timestamp(test_data)
@@ -122,13 +143,13 @@ async def test_put_last_data_timestamp_empty_dataframe(redis_activity):
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
}
redis_activity.set = MagicMock()
redis_activity.redis_repository.set = MagicMock()
result = await redis_activity.put_last_data_timestamp(test_data)
assert result is None
redis_activity.set.assert_not_called()
redis_activity.redis_repository.set.assert_not_called()
@pytest.mark.asyncio
@@ -149,13 +170,13 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
'data': data.to_dict('records'),
}
redis_activity.set = MagicMock()
redis_activity.redis_repository.set = MagicMock()
result = await redis_activity.put_last_data_timestamp(test_data)
assert result == '2023-01-01 12:00:01'
redis_activity.set.assert_called_once_with(
redis_activity.redis_repository.set.assert_called_once_with(
'last_data_timestamp:test_pipeline:test_schedule', '2023-01-01 12:00:01', ttl=18000
)
@@ -177,7 +198,7 @@ async def test_put_last_data_timestamp_error(redis_activity):
}
redis_activity.send_notification = MagicMock()
redis_activity.set = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.set = MagicMock(side_effect=Exception('test'))
try:
await redis_activity.put_last_data_timestamp(test_data)
@@ -220,8 +241,8 @@ async def test_group_and_hold_data_new_key(redis_activity):
}
# Mock get to return None for new key
redis_activity.get = MagicMock(return_value=None)
redis_activity.set = MagicMock()
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.set = MagicMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -236,8 +257,8 @@ async def test_group_and_hold_data_new_key(redis_activity):
assert result == expected_result
# Verify set was called with correct arguments
redis_activity.set.assert_called_once()
args, kwargs = redis_activity.set.call_args
redis_activity.redis_repository.set.assert_called_once()
args, kwargs = redis_activity.redis_repository.set.call_args
assert args[0] == 'held_data_test_pipeline_test_schedule'
assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}
assert kwargs['ttl'] == 3600
@@ -273,8 +294,8 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
}
# Mock get to return existing data
redis_activity.get = MagicMock(return_value=existing_data)
redis_activity.set = MagicMock()
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
redis_activity.redis_repository.set = MagicMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -294,8 +315,8 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
assert result == expected_result
# Verify set was called with correct arguments
redis_activity.set.assert_called_once()
args, kwargs = redis_activity.set.call_args
redis_activity.redis_repository.set.assert_called_once()
args, kwargs = redis_activity.redis_repository.set.call_args
assert args[0] == 'held_data_test_workflow_test_schedule'
assert args[1] == {
'sensor1': 25.5,
@@ -329,8 +350,8 @@ async def test_group_and_hold_data_with_none_values(redis_activity):
}
# Mock get to return None for new key
redis_activity.get = MagicMock(return_value=None)
redis_activity.set = MagicMock()
redis_activity.redis_repository.get = MagicMock(return_value=None)
redis_activity.redis_repository.set = MagicMock()
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -354,7 +375,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
'fill_missing_tags': False,
}
redis_activity.get = MagicMock(return_value=None)
redis_activity.redis_repository.get = MagicMock(return_value=None)
# Call the method
result = await redis_activity.group_and_hold_data(test_data)
@@ -376,7 +397,7 @@ async def test_group_and_hold_data_error_get(redis_activity):
'fill_missing_tags': False,
}
redis_activity.get = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.get = MagicMock(side_effect=Exception('test'))
redis_activity.send_notification = MagicMock()
try:
@@ -415,8 +436,8 @@ async def test_group_and_hold_data_error_set(redis_activity):
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
# Mock get to return existing data
redis_activity.get = MagicMock(return_value=existing_data)
redis_activity.set = MagicMock(side_effect=Exception('test'))
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
redis_activity.redis_repository.set = MagicMock(side_effect=Exception('test'))
redis_activity.send_notification = MagicMock()
try:
@@ -429,7 +450,7 @@ async def test_group_and_hold_data_error_set(redis_activity):
@pytest.mark.asyncio
async def test_store_data_package(redis_activity):
"""Test store_data_package"""
redis_activity.set = MagicMock()
redis_activity.redis_repository.set = MagicMock()
test_data = {
**metadata,
@@ -454,7 +475,7 @@ async def test_store_data_package(redis_activity):
await redis_activity.store_data_package(test_data)
redis_activity.set.assert_called_once_with(
redis_activity.redis_repository.set.assert_called_once_with(
ANY, {'data': test_data['data'], 'held_data': test_data['held_data']}, ttl=120
)
@@ -462,7 +483,7 @@ async def test_store_data_package(redis_activity):
@pytest.mark.asyncio
async def test_store_data_package_error(redis_activity):
"""Test store_data_package error"""
redis_activity.set = MagicMock(side_effect=ValueError('test'))
redis_activity.redis_repository.set = MagicMock(side_effect=ValueError('test'))
redis_activity.send_notification = MagicMock()
test_data = {