gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1316

Browse Source 
Update .gitignore and refactor metrics.py, activities.py, and gates.py for improved clarity and consistency. Added coverage.xml and cache directories to .gitignore. Standardized string formatting and parameter handling in metrics and activities classes, enhancing code readability. Removed the deprecated faker.py file and adjusted related tests accordingly.
This commit is contained in:
vitor-aignosi
2025-10-16 13:31:12 -03:00
parent 8bdbf049b8
commit 97eb5bc904
27 changed files with 1101 additions and 1336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

292
tests/activities/test_redis.py
View File

@@ -1,10 +1,12 @@
from unittest.mock import MagicMock, patch, ANY
from datetime import datetime
import pytest
from unittest.mock import ANY, MagicMock, patch
import numpy as np
import pytest
from pandas import DataFrame
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from scouter.activities.redis import Redis
@@ -13,9 +15,14 @@ from scouter.activities.redis import Redis
def redis_activity(_mock_redis_init):
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
activity = Redis(host='localhost', port=6379,
logger=logger, notification_handler=notification_handler,
username='test', password='test')
activity = Redis(
host='localhost',
port=6379,
logger=logger,
notification_handler=notification_handler,
username='test',
password='test',
)
activity.redis_client = MagicMock()
activity.logger = logger
@@ -29,17 +36,16 @@ def test_redis_initialization(mock_redis_init):
"""Test Redis activity initialization"""
logger = MagicMock()
notification_handler = MagicMock(spec=NotificationHandler)
Redis(host='localhost', port=6379,
logger=logger, notification_handler=notification_handler,
username='test', password='test')
Redis(
host='localhost',
port=6379,
logger=logger,
notification_handler=notification_handler,
username='test',
password='test',
)
mock_redis_init.assert_called_once_with(
ANY,
'localhost',
6379,
'test',
'test',
logger,
notification_handler
ANY, 'localhost', 6379, 'test', 'test', logger, notification_handler
)
@@ -48,7 +54,7 @@ metadata = {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
'workflow_name': 'scouter',
}
}
@@ -56,11 +62,7 @@ metadata = {
@pytest.mark.asyncio
async def test_get_last_data_timestamp_none(redis_activity):
"""Test get_last_data_timestamp"""
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule'
}
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.get = MagicMock(return_value=None)
@@ -72,19 +74,13 @@ async def test_get_last_data_timestamp_none(redis_activity):
@pytest.mark.asyncio
async def test_get_last_data_timestamp_not_none(redis_activity):
"""Test get_last_data_timestamp"""
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule'
}
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.get = MagicMock(return_value='2023-01-01 12:00:00')
result = await redis_activity.get_last_data_timestamp(test_data)
redis_activity.get.assert_called_once_with(
'last_data_timestamp:test_pipeline:test_schedule'
)
redis_activity.get.assert_called_once_with('last_data_timestamp:test_pipeline:test_schedule')
assert result == '2023-01-01 12:00:00'
@@ -92,17 +88,12 @@ async def test_get_last_data_timestamp_not_none(redis_activity):
@pytest.mark.asyncio
async def test_get_last_data_timestamp_error(redis_activity):
"""Test get_last_data_timestamp error"""
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule'
}
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
redis_activity.send_notification = MagicMock()
redis_activity.get = MagicMock(side_effect=Exception('test'))
try:
await redis_activity.get_last_data_timestamp(test_data)
except Exception as e:
@@ -110,15 +101,15 @@ async def test_get_last_data_timestamp_error(redis_activity):
redis_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id="REDIS_GET_ERROR",
message="Error getting last data timestamp: test",
block="get_last_data_timestamp",
notification_id='REDIS_GET_ERROR',
message='Error getting last data timestamp: test',
block='get_last_data_timestamp',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)
else:
assert False, "Expected exception"
raise AssertionError('Expected exception')
@pytest.mark.asyncio
@@ -128,7 +119,7 @@ async def test_put_last_data_timestamp_empty_dataframe(redis_activity):
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule',
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records')
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
}
redis_activity.set = MagicMock()
@@ -144,16 +135,18 @@ async def test_put_last_data_timestamp_empty_dataframe(redis_activity):
async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
"""Test put_last_data_timestamp with not empty dataframe"""
data = DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'inserted_at': ['2023-01-01 12:00:00', '2023-01-01 12:00:01']
})
data = DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'inserted_at': ['2023-01-01 12:00:00', '2023-01-01 12:00:01'],
}
)
test_data = {
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule',
'data': data.to_dict('records')
'data': data.to_dict('records'),
}
redis_activity.set = MagicMock()
@@ -163,9 +156,7 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
assert result == '2023-01-01 12:00:01'
redis_activity.set.assert_called_once_with(
'last_data_timestamp:test_pipeline:test_schedule',
'2023-01-01 12:00:01',
ttl=18000
'last_data_timestamp:test_pipeline:test_schedule', '2023-01-01 12:00:01', ttl=18000
)
@@ -176,11 +167,13 @@ async def test_put_last_data_timestamp_error(redis_activity):
**metadata,
'workflow_name': 'test_pipeline',
'schedule_name': 'test_schedule',
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'inserted_at': ['2023-01-01 12:00:00'] * 2
}).to_dict('records')
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'inserted_at': ['2023-01-01 12:00:00'] * 2,
}
).to_dict('records'),
}
redis_activity.send_notification = MagicMock()
@@ -194,15 +187,15 @@ async def test_put_last_data_timestamp_error(redis_activity):
redis_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id="REDIS_SET_ERROR",
message="Error setting last data timestamp: test",
block="put_last_data_timestamp",
notification_id='REDIS_SET_ERROR',
message='Error setting last data timestamp: test',
block='put_last_data_timestamp',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)
else:
assert False, "Expected exception"
raise AssertionError('Expected exception')
@pytest.mark.asyncio
@@ -215,15 +208,14 @@ async def test_group_and_hold_data_new_key(redis_activity):
'schedule_name': 'test_schedule',
'retention_time': 3600,
'model_id': 1,
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict('records'),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
# Mock get to return None for new key
@@ -238,7 +230,7 @@ async def test_group_and_hold_data_new_key(redis_activity):
'timestamp': {0: '2023-01-01 12:00:00', 1: '2023-01-01 12:00:00'},
'variable': {0: 'sensor1', 1: 'sensor2'},
'value': {0: 25.5, 1: 30.0},
'model_id': {0: 1, 1: 1}
'model_id': {0: 1, 1: 1},
}
assert result == expected_result
@@ -246,11 +238,7 @@ async def test_group_and_hold_data_new_key(redis_activity):
redis_activity.set.assert_called_once()
args, kwargs = redis_activity.set.call_args
assert args[0] == 'held_data_test_pipeline_test_schedule'
assert args[1] == {
'sensor1': 25.5,
'sensor2': 30.0,
'timestamp': '2023-01-01 12:00:00'
}
assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}
assert kwargs['ttl'] == 3600
@@ -258,11 +246,7 @@ async def test_group_and_hold_data_new_key(redis_activity):
async def test_group_and_hold_data_update_existing(redis_activity):
"""Test updating existing data with group_and_hold_data"""
# Setup initial data in Redis
existing_data = {
'sensor1': 20.0,
'sensor2': 28.0,
'timestamp': '2023-01-01 11:00:00'
}
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
# New data to update with
test_data = {
@@ -271,16 +255,14 @@ async def test_group_and_hold_data_update_existing(redis_activity):
'schedule_name': 'test_schedule',
'retention_time': 3600,
'model_id': 1,
'data': DataFrame({
'name': ['sensor1', 'sensor3'],
'value': [25.5, 42.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2',
'sensor3': 'sensor3'
}
'data': DataFrame(
{
'name': ['sensor1', 'sensor3'],
'value': [25.5, 42.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict('records'),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2', 'sensor3': 'sensor3'},
}
# Mock get to return existing data
@@ -295,7 +277,7 @@ async def test_group_and_hold_data_update_existing(redis_activity):
'timestamp': {0: '2023-01-01 12:00:00', 1: '2023-01-01 12:00:00', 2: '2023-01-01 12:00:00'},
'variable': {0: 'sensor1', 1: 'sensor2', 2: 'sensor3'},
'value': {0: 25.5, 1: 28.0, 2: 42.0},
'model_id': {0: 1, 1: 1, 2: 1}
'model_id': {0: 1, 1: 1, 2: 1},
}
assert result == expected_result
@@ -307,7 +289,7 @@ async def test_group_and_hold_data_update_existing(redis_activity):
'sensor1': 25.5,
'sensor2': 28.0,
'sensor3': 42.0,
'timestamp': '2023-01-01 12:00:00'
'timestamp': '2023-01-01 12:00:00',
}
assert kwargs['ttl'] == 3600
@@ -322,15 +304,14 @@ async def test_group_and_hold_data_with_none_values(redis_activity):
'schedule_name': 'test_schedule',
'retention_time': 3600,
'model_id': 1,
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [None, 30.0],
'timestamp': [datetime(2023, 1, 1, 12, 0, 0)] * 2
}).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [None, 30.0],
'timestamp': [datetime(2023, 1, 1, 12, 0, 0)] * 2,
}
).to_dict('records'),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
# Mock get to return None for new key
@@ -355,10 +336,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
'schedule_name': 'test_schedule',
'retention_time': 3600,
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
redis_activity.get = MagicMock(return_value=None)
@@ -379,10 +357,7 @@ async def test_group_and_hold_data_error_get(redis_activity):
'retention_time': 3600,
'model_id': 1,
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
redis_activity.get = MagicMock(side_effect=Exception('test'))
@@ -396,15 +371,15 @@ async def test_group_and_hold_data_error_get(redis_activity):
redis_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id="REDIS_GET_ERROR",
message="Error getting held data: test",
block="group_and_hold_data",
notification_id='REDIS_GET_ERROR',
message='Error getting held data: test',
block='group_and_hold_data',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)
else:
assert False, "Expected exception"
raise AssertionError('Expected exception')
@pytest.mark.asyncio
@@ -417,17 +392,10 @@ async def test_group_and_hold_data_error_set(redis_activity):
'retention_time': 3600,
'model_id': 1,
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
existing_data = {
'sensor1': 20.0,
'sensor2': 28.0,
'timestamp': '2023-01-01 11:00:00'
}
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
# Mock get to return existing data
redis_activity.get = MagicMock(return_value=existing_data)
@@ -450,67 +418,65 @@ async def test_store_data_package(redis_activity):
**metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'held_data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict(),
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict(),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'held_data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict(),
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict(),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
await redis_activity.store_data_package(test_data)
redis_activity.set.assert_called_once_with(
ANY,
{
'data': test_data['data'],
'held_data': test_data['held_data']
},
ttl=120)
ANY, {'data': test_data['data'], 'held_data': test_data['held_data']}, ttl=120
)
@pytest.mark.asyncio
async def test_store_data_package_error(redis_activity):
"""Test store_data_package error"""
redis_activity.set = MagicMock(side_effect=Exception('test'))
redis_activity.set = MagicMock(side_effect=ValueError('test'))
redis_activity.send_notification = MagicMock()
test_data = {
**metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'held_data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict(),
'data': DataFrame({
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2
}).to_dict(),
'model_tags': {
'sensor1': 'sensor1',
'sensor2': 'sensor2'
}
'held_data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict(),
'data': DataFrame(
{
'name': ['sensor1', 'sensor2'],
'value': [25.5, 30.0],
'timestamp': ['2023-01-01 12:00:00'] * 2,
}
).to_dict(),
'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
}
with pytest.raises(Exception):
with pytest.raises(ValueError):
await redis_activity.store_data_package(test_data)
redis_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id="REDIS_SET_ERROR",
message="Error setting data package: test",
block="store_data_package",
notification_id='REDIS_SET_ERROR',
message='Error setting data package: test',
block='store_data_package',
level=NotificationLevel.ERROR,
attachment_content=ANY
attachment_content=ANY,
)