SIENTIAPDE-1325
Update GITHUB_BRANCH in values.yaml to feature/SIENTIAPDE-1325 for specific external operation metrics. Refactor shutdown methods in Activities class to ensure proper closure of MongoDB, Redis, and Gates connections. Adjust tests to reflect these changes and enhance repository pattern implementation for MongoDB and Redis activities.
This commit is contained in:
@@ -97,4 +97,6 @@ class Activities(Postgres, Redis, Gates, MongoDB):
|
|||||||
to prevent connection leaks and ensure graceful application termination.
|
to prevent connection leaks and ensure graceful application termination.
|
||||||
"""
|
"""
|
||||||
Postgres.close(self)
|
Postgres.close(self)
|
||||||
MongoDB.shutdown(self)
|
MongoDB.close(self)
|
||||||
|
Redis.close(self)
|
||||||
|
Gates.close(self)
|
||||||
|
|||||||
@@ -8,17 +8,14 @@ with workflow.unsafe.imports_passed_through():
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from pandas import DataFrame
|
|
||||||
from pymongo import MongoClient
|
|
||||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||||
from sientia_do.notifications.models import NotificationLevel
|
from sientia_do.notifications.models import NotificationLevel
|
||||||
from sientia_do.observability.logger import Logger
|
from sientia_do.observability.logger import Logger
|
||||||
from sientia_do.repository.mongodb_repository import MongoDBRepository
|
|
||||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||||
|
from sientia_do.repository.mongodb_repository import MongoDBRepository
|
||||||
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
|
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
class MongoDB(SientiaMonitoring):
|
class MongoDB(SientiaMonitoring):
|
||||||
"""
|
"""
|
||||||
MongoDB operations for data retrieval and storage.
|
MongoDB operations for data retrieval and storage.
|
||||||
@@ -136,7 +133,7 @@ class MongoDB(SientiaMonitoring):
|
|||||||
|
|
||||||
self.debug(f'Loaded data: {data}', metadata=metadata)
|
self.debug(f'Loaded data: {data}', metadata=metadata)
|
||||||
|
|
||||||
return DataFrame(data).to_dict()
|
return data
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
trace = traceback.format_exc()
|
trace = traceback.format_exc()
|
||||||
self.send_notification(
|
self.send_notification(
|
||||||
|
|||||||
@@ -88,8 +88,12 @@ def test___init__(mock_gates_init, mock_redis_init, mock_postgres_init, mock_mon
|
|||||||
@patch('scouter.activities.activities.Gates.__init__')
|
@patch('scouter.activities.activities.Gates.__init__')
|
||||||
@patch('scouter.activities.activities.MongoDB.__init__')
|
@patch('scouter.activities.activities.MongoDB.__init__')
|
||||||
@patch('scouter.activities.activities.Postgres.close')
|
@patch('scouter.activities.activities.Postgres.close')
|
||||||
@patch('scouter.activities.activities.MongoDB.shutdown')
|
@patch('scouter.activities.activities.MongoDB.close')
|
||||||
|
@patch('scouter.activities.activities.Redis.close')
|
||||||
|
@patch('scouter.activities.activities.Gates.close')
|
||||||
def test_shutdown(
|
def test_shutdown(
|
||||||
|
mock_gates_close,
|
||||||
|
mock_redis_close,
|
||||||
mock_mongodb_close,
|
mock_mongodb_close,
|
||||||
mock_postgres_close,
|
mock_postgres_close,
|
||||||
_mock_mongodb_init,
|
_mock_mongodb_init,
|
||||||
@@ -129,3 +133,5 @@ def test_shutdown(
|
|||||||
|
|
||||||
mock_postgres_close.assert_called()
|
mock_postgres_close.assert_called()
|
||||||
mock_mongodb_close.assert_called()
|
mock_mongodb_close.assert_called()
|
||||||
|
mock_redis_close.assert_called()
|
||||||
|
mock_gates_close.assert_called()
|
||||||
|
|||||||
@@ -16,6 +16,9 @@ def gates_fixture():
|
|||||||
notification_handler = MagicMock()
|
notification_handler = MagicMock()
|
||||||
gates = Gates(logger=logger, notification_handler=notification_handler)
|
gates = Gates(logger=logger, notification_handler=notification_handler)
|
||||||
gates.send_notification = MagicMock()
|
gates.send_notification = MagicMock()
|
||||||
|
gates.logger = logger
|
||||||
|
gates.notification_handler = notification_handler
|
||||||
|
gates.pod_id = 'localhost'
|
||||||
return gates
|
return gates
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -5,54 +5,38 @@ from pytest import fixture, mark
|
|||||||
from sientia_do.notifications.models import NotificationLevel
|
from sientia_do.notifications.models import NotificationLevel
|
||||||
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
|
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
|
||||||
|
|
||||||
from scouter.activities.mongodb import MongoDB, clear_mongo_id
|
from scouter.activities.mongodb import MongoDB
|
||||||
|
|
||||||
|
|
||||||
def test_clear_mongo_id():
|
@patch('scouter.activities.mongodb.MongoDBRepository')
|
||||||
"""Test clear_mongo_id"""
|
def test_mongodb___init__(mock_mongodb_repository):
|
||||||
data = [
|
|
||||||
{'_id': '1', 'name': 'test1'},
|
|
||||||
{'_id': '2', 'name': [{'_id': '3', 'name': 'test3'}]},
|
|
||||||
{'_id': '4', 'name': {'_id': '5', 'name': 'test2'}},
|
|
||||||
[{'_id': '6', 'name': 'test2'}],
|
|
||||||
]
|
|
||||||
|
|
||||||
result = clear_mongo_id(data)
|
|
||||||
|
|
||||||
assert result == [
|
|
||||||
{'name': 'test1'},
|
|
||||||
{'name': [{'name': 'test3'}]},
|
|
||||||
{'name': {'name': 'test2'}},
|
|
||||||
[{'name': 'test2'}],
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@patch('scouter.activities.mongodb.MongoClient')
|
|
||||||
def test_mongodb___init__(mock_mongo_client):
|
|
||||||
"""Test MongoDB __init__"""
|
"""Test MongoDB __init__"""
|
||||||
|
|
||||||
|
logger = MagicMock()
|
||||||
|
notification_handler = MagicMock()
|
||||||
|
|
||||||
mongo = MongoDB(
|
mongo = MongoDB(
|
||||||
connection_string='mongodb://localhost:27017',
|
connection_string='mongodb://localhost:27017',
|
||||||
database_name='test_db',
|
database_name='test_db',
|
||||||
logger=MagicMock(),
|
logger=logger,
|
||||||
notification_handler=MagicMock(),
|
notification_handler=notification_handler,
|
||||||
)
|
)
|
||||||
|
|
||||||
mock_mongo_client.assert_called_once_with(
|
mock_mongodb_repository.assert_called_once_with(
|
||||||
'mongodb://localhost:27017', serverSelectionTimeoutMS=5000
|
connection_string='mongodb://localhost:27017',
|
||||||
|
database_name='test_db',
|
||||||
|
logger=logger,
|
||||||
|
notification_handler=notification_handler,
|
||||||
)
|
)
|
||||||
|
|
||||||
mock_mongo_client.return_value.server_info.assert_called_once()
|
assert mongo.mongodb_repository is not None
|
||||||
|
|
||||||
mock_mongo_client.return_value.__getitem__.assert_called_once_with('test_db')
|
|
||||||
|
|
||||||
assert mongo.client is not None
|
|
||||||
assert mongo.database is not None
|
|
||||||
|
|
||||||
|
|
||||||
@fixture
|
@fixture
|
||||||
@patch('scouter.activities.mongodb.MongoClient')
|
@patch('scouter.activities.mongodb.MongoDBRepository')
|
||||||
def mongodb_activity(mock_mongo_client):
|
def mongodb_activity(mock_mongodb_repository):
|
||||||
"""Test MongoDB activity"""
|
"""Test MongoDB activity"""
|
||||||
|
|
||||||
mongo = MongoDB(
|
mongo = MongoDB(
|
||||||
connection_string='mongodb://localhost:27017',
|
connection_string='mongodb://localhost:27017',
|
||||||
database_name='test_db',
|
database_name='test_db',
|
||||||
@@ -60,32 +44,32 @@ def mongodb_activity(mock_mongo_client):
|
|||||||
notification_handler=MagicMock(),
|
notification_handler=MagicMock(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
mongo.logger = MagicMock()
|
||||||
|
mongo.notification_handler = MagicMock()
|
||||||
|
mongo.pod_id = 'localhost'
|
||||||
return mongo
|
return mongo
|
||||||
|
|
||||||
|
|
||||||
def test_shutdown_success(mongodb_activity):
|
def test_close(mongodb_activity):
|
||||||
"""Test shutdown"""
|
"""Test close"""
|
||||||
mongodb_activity.shutdown()
|
mongodb_activity.close()
|
||||||
|
|
||||||
mongodb_activity.client.close.assert_called_once()
|
mongodb_activity.mongodb_repository.close.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
def test_shutdown_error(mongodb_activity):
|
def test_del(mongodb_activity):
|
||||||
"""Test shutdown"""
|
mongodb_activity.close = MagicMock()
|
||||||
mongodb_activity.client.close = MagicMock(side_effect=Exception('test'))
|
|
||||||
|
|
||||||
mongodb_activity.shutdown()
|
mongodb_activity.__del__()
|
||||||
|
|
||||||
mongodb_activity.client.close.assert_called_once()
|
mongodb_activity.close.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
@mark.asyncio
|
@mark.asyncio
|
||||||
async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
|
async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
|
||||||
"""Test load_latest_data"""
|
"""Test load_latest_data"""
|
||||||
collection = MagicMock()
|
|
||||||
mongodb_activity.database.__getitem__.return_value = collection
|
|
||||||
|
|
||||||
collection.find.return_value = [
|
mongodb_activity.mongodb_repository.find.return_value = [
|
||||||
{
|
{
|
||||||
'name': 'test1',
|
'name': 'test1',
|
||||||
'value': 1,
|
'value': 1,
|
||||||
@@ -103,24 +87,26 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
mongodb_activity.database.__getitem__.assert_called_once_with('test_collection')
|
mongodb_activity.mongodb_repository.find.assert_called_once_with(
|
||||||
|
collection_name='test_collection',
|
||||||
|
filters={},
|
||||||
|
metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
|
||||||
|
)
|
||||||
|
|
||||||
collection.find.assert_called_once_with({}, {'_id': 0})
|
assert result == [
|
||||||
|
{
|
||||||
assert result == {
|
'name': 'test1',
|
||||||
'name': {0: 'test1'},
|
'value': 1,
|
||||||
'value': {0: 1},
|
'inserted_at': '2023-01-01 12:00:00.000000+0000',
|
||||||
'inserted_at': {0: '2023-01-01 12:00:00.000000+0000'},
|
}
|
||||||
}
|
]
|
||||||
|
|
||||||
|
|
||||||
@mark.asyncio
|
@mark.asyncio
|
||||||
async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
|
async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
|
||||||
"""Test load_latest_data"""
|
"""Test load_latest_data"""
|
||||||
collection = MagicMock()
|
|
||||||
mongodb_activity.database.__getitem__.return_value = collection
|
|
||||||
|
|
||||||
collection.find.return_value = [
|
mongodb_activity.mongodb_repository.find.return_value = [
|
||||||
{
|
{
|
||||||
'name': 'test1',
|
'name': 'test1',
|
||||||
'value': 1,
|
'value': 1,
|
||||||
@@ -138,34 +124,32 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
mongodb_activity.database.__getitem__.assert_called_once_with('test_collection')
|
mongodb_activity.mongodb_repository.find.assert_called_once_with(
|
||||||
|
collection_name='test_collection',
|
||||||
collection.find.assert_called_once_with(
|
filters={
|
||||||
{
|
|
||||||
'inserted_at': {
|
'inserted_at': {
|
||||||
'$gt': datetime.strptime(
|
'$gt': datetime.strptime(
|
||||||
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
|
'2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{'_id': 0},
|
metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result == {
|
assert result == [
|
||||||
'name': {0: 'test1'},
|
{
|
||||||
'value': {0: 1},
|
'name': 'test1',
|
||||||
'inserted_at': {0: '2023-01-01 12:00:00.000000+0000'},
|
'value': 1,
|
||||||
}
|
'inserted_at': '2023-01-01 12:00:00.000000+0000',
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
@mark.asyncio
|
@mark.asyncio
|
||||||
async def test_load_latest_data_error(mongodb_activity):
|
async def test_load_latest_data_error(mongodb_activity):
|
||||||
"""Test load_latest_data"""
|
"""Test load_latest_data"""
|
||||||
collection = MagicMock()
|
mongodb_activity.mongodb_repository.find.side_effect = Exception('test')
|
||||||
mongodb_activity.send_notification = MagicMock()
|
mongodb_activity.send_notification = MagicMock()
|
||||||
mongodb_activity.database.__getitem__.return_value = collection
|
|
||||||
|
|
||||||
collection.find.side_effect = Exception('test')
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await mongodb_activity.load_latest_data(
|
await mongodb_activity.load_latest_data(
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ from scouter.activities.redis import Redis
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@patch('scouter.activities.redis.RedisBase.__init__')
|
@patch('scouter.activities.redis.RedisRepository')
|
||||||
def redis_activity(_mock_redis_init):
|
def redis_activity(mock_redis_repository):
|
||||||
logger = MagicMock()
|
logger = MagicMock()
|
||||||
notification_handler = MagicMock(spec=NotificationHandler)
|
notification_handler = MagicMock(spec=NotificationHandler)
|
||||||
activity = Redis(
|
activity = Redis(
|
||||||
@@ -31,24 +31,6 @@ def redis_activity(_mock_redis_init):
|
|||||||
return activity
|
return activity
|
||||||
|
|
||||||
|
|
||||||
@patch('scouter.activities.redis.RedisBase.__init__')
|
|
||||||
def test_redis_initialization(mock_redis_init):
|
|
||||||
"""Test Redis activity initialization"""
|
|
||||||
logger = MagicMock()
|
|
||||||
notification_handler = MagicMock(spec=NotificationHandler)
|
|
||||||
Redis(
|
|
||||||
host='localhost',
|
|
||||||
port=6379,
|
|
||||||
logger=logger,
|
|
||||||
notification_handler=notification_handler,
|
|
||||||
username='test',
|
|
||||||
password='test',
|
|
||||||
)
|
|
||||||
mock_redis_init.assert_called_once_with(
|
|
||||||
ANY, 'localhost', 6379, 'test', 'test', logger, notification_handler
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
metadata = {
|
metadata = {
|
||||||
'metadata': {
|
'metadata': {
|
||||||
'model_id': 'test_model_id',
|
'model_id': 'test_model_id',
|
||||||
@@ -59,15 +41,52 @@ metadata = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@patch('scouter.activities.redis.RedisRepository')
|
||||||
|
def test_redis_initialization(mock_redis_repository):
|
||||||
|
"""Test Redis activity initialization"""
|
||||||
|
logger = MagicMock()
|
||||||
|
notification_handler = MagicMock(spec=NotificationHandler)
|
||||||
|
activity = Redis(
|
||||||
|
host='localhost',
|
||||||
|
port=6379,
|
||||||
|
logger=logger,
|
||||||
|
notification_handler=notification_handler,
|
||||||
|
username='test',
|
||||||
|
password='test',
|
||||||
|
)
|
||||||
|
mock_redis_repository.assert_called_once_with(
|
||||||
|
host='localhost',
|
||||||
|
port=6379,
|
||||||
|
username='test',
|
||||||
|
password='test',
|
||||||
|
logger=logger,
|
||||||
|
notification_handler=notification_handler,
|
||||||
|
)
|
||||||
|
|
||||||
|
activity.logger = logger
|
||||||
|
activity.notification_handler = notification_handler
|
||||||
|
activity.pod_id = 'localhost'
|
||||||
|
|
||||||
|
assert activity.redis_repository is not None
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_get_last_data_timestamp_none(redis_activity):
|
async def test_get_last_data_timestamp_none(redis_activity):
|
||||||
"""Test get_last_data_timestamp"""
|
"""Test get_last_data_timestamp"""
|
||||||
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
|
test_data = {
|
||||||
|
**metadata,
|
||||||
|
'workflow_name': 'test_pipeline',
|
||||||
|
'schedule_name': 'test_schedule',
|
||||||
|
}
|
||||||
|
|
||||||
redis_activity.get = MagicMock(return_value=None)
|
redis_activity.redis_repository.get.return_value = None
|
||||||
|
|
||||||
result = await redis_activity.get_last_data_timestamp(test_data)
|
result = await redis_activity.get_last_data_timestamp(test_data)
|
||||||
|
|
||||||
|
redis_activity.redis_repository.get.assert_called_once_with(
|
||||||
|
'last_data_timestamp:test_pipeline:test_schedule'
|
||||||
|
)
|
||||||
|
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
@@ -76,11 +95,13 @@ async def test_get_last_data_timestamp_not_none(redis_activity):
|
|||||||
"""Test get_last_data_timestamp"""
|
"""Test get_last_data_timestamp"""
|
||||||
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
|
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
|
||||||
|
|
||||||
redis_activity.get = MagicMock(return_value='2023-01-01 12:00:00')
|
redis_activity.redis_repository.get.return_value = '2023-01-01 12:00:00'
|
||||||
|
|
||||||
result = await redis_activity.get_last_data_timestamp(test_data)
|
result = await redis_activity.get_last_data_timestamp(test_data)
|
||||||
|
|
||||||
redis_activity.get.assert_called_once_with('last_data_timestamp:test_pipeline:test_schedule')
|
redis_activity.redis_repository.get.assert_called_once_with(
|
||||||
|
'last_data_timestamp:test_pipeline:test_schedule'
|
||||||
|
)
|
||||||
|
|
||||||
assert result == '2023-01-01 12:00:00'
|
assert result == '2023-01-01 12:00:00'
|
||||||
|
|
||||||
@@ -91,7 +112,7 @@ async def test_get_last_data_timestamp_error(redis_activity):
|
|||||||
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
|
test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
|
||||||
|
|
||||||
redis_activity.send_notification = MagicMock()
|
redis_activity.send_notification = MagicMock()
|
||||||
redis_activity.get = MagicMock(side_effect=Exception('test'))
|
redis_activity.redis_repository.get.side_effect = Exception('test')
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await redis_activity.get_last_data_timestamp(test_data)
|
await redis_activity.get_last_data_timestamp(test_data)
|
||||||
@@ -122,13 +143,13 @@ async def test_put_last_data_timestamp_empty_dataframe(redis_activity):
|
|||||||
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
|
'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
|
||||||
}
|
}
|
||||||
|
|
||||||
redis_activity.set = MagicMock()
|
redis_activity.redis_repository.set = MagicMock()
|
||||||
|
|
||||||
result = await redis_activity.put_last_data_timestamp(test_data)
|
result = await redis_activity.put_last_data_timestamp(test_data)
|
||||||
|
|
||||||
assert result is None
|
assert result is None
|
||||||
|
|
||||||
redis_activity.set.assert_not_called()
|
redis_activity.redis_repository.set.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -149,13 +170,13 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
|
|||||||
'data': data.to_dict('records'),
|
'data': data.to_dict('records'),
|
||||||
}
|
}
|
||||||
|
|
||||||
redis_activity.set = MagicMock()
|
redis_activity.redis_repository.set = MagicMock()
|
||||||
|
|
||||||
result = await redis_activity.put_last_data_timestamp(test_data)
|
result = await redis_activity.put_last_data_timestamp(test_data)
|
||||||
|
|
||||||
assert result == '2023-01-01 12:00:01'
|
assert result == '2023-01-01 12:00:01'
|
||||||
|
|
||||||
redis_activity.set.assert_called_once_with(
|
redis_activity.redis_repository.set.assert_called_once_with(
|
||||||
'last_data_timestamp:test_pipeline:test_schedule', '2023-01-01 12:00:01', ttl=18000
|
'last_data_timestamp:test_pipeline:test_schedule', '2023-01-01 12:00:01', ttl=18000
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -177,7 +198,7 @@ async def test_put_last_data_timestamp_error(redis_activity):
|
|||||||
}
|
}
|
||||||
|
|
||||||
redis_activity.send_notification = MagicMock()
|
redis_activity.send_notification = MagicMock()
|
||||||
redis_activity.set = MagicMock(side_effect=Exception('test'))
|
redis_activity.redis_repository.set = MagicMock(side_effect=Exception('test'))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
await redis_activity.put_last_data_timestamp(test_data)
|
await redis_activity.put_last_data_timestamp(test_data)
|
||||||
@@ -220,8 +241,8 @@ async def test_group_and_hold_data_new_key(redis_activity):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Mock get to return None for new key
|
# Mock get to return None for new key
|
||||||
redis_activity.get = MagicMock(return_value=None)
|
redis_activity.redis_repository.get = MagicMock(return_value=None)
|
||||||
redis_activity.set = MagicMock()
|
redis_activity.redis_repository.set = MagicMock()
|
||||||
|
|
||||||
# Call the method
|
# Call the method
|
||||||
result = await redis_activity.group_and_hold_data(test_data)
|
result = await redis_activity.group_and_hold_data(test_data)
|
||||||
@@ -236,8 +257,8 @@ async def test_group_and_hold_data_new_key(redis_activity):
|
|||||||
assert result == expected_result
|
assert result == expected_result
|
||||||
|
|
||||||
# Verify set was called with correct arguments
|
# Verify set was called with correct arguments
|
||||||
redis_activity.set.assert_called_once()
|
redis_activity.redis_repository.set.assert_called_once()
|
||||||
args, kwargs = redis_activity.set.call_args
|
args, kwargs = redis_activity.redis_repository.set.call_args
|
||||||
assert args[0] == 'held_data_test_pipeline_test_schedule'
|
assert args[0] == 'held_data_test_pipeline_test_schedule'
|
||||||
assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}
|
assert args[1] == {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'}
|
||||||
assert kwargs['ttl'] == 3600
|
assert kwargs['ttl'] == 3600
|
||||||
@@ -273,8 +294,8 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Mock get to return existing data
|
# Mock get to return existing data
|
||||||
redis_activity.get = MagicMock(return_value=existing_data)
|
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
|
||||||
redis_activity.set = MagicMock()
|
redis_activity.redis_repository.set = MagicMock()
|
||||||
|
|
||||||
# Call the method
|
# Call the method
|
||||||
result = await redis_activity.group_and_hold_data(test_data)
|
result = await redis_activity.group_and_hold_data(test_data)
|
||||||
@@ -294,8 +315,8 @@ async def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
|
|||||||
assert result == expected_result
|
assert result == expected_result
|
||||||
|
|
||||||
# Verify set was called with correct arguments
|
# Verify set was called with correct arguments
|
||||||
redis_activity.set.assert_called_once()
|
redis_activity.redis_repository.set.assert_called_once()
|
||||||
args, kwargs = redis_activity.set.call_args
|
args, kwargs = redis_activity.redis_repository.set.call_args
|
||||||
assert args[0] == 'held_data_test_workflow_test_schedule'
|
assert args[0] == 'held_data_test_workflow_test_schedule'
|
||||||
assert args[1] == {
|
assert args[1] == {
|
||||||
'sensor1': 25.5,
|
'sensor1': 25.5,
|
||||||
@@ -329,8 +350,8 @@ async def test_group_and_hold_data_with_none_values(redis_activity):
|
|||||||
}
|
}
|
||||||
|
|
||||||
# Mock get to return None for new key
|
# Mock get to return None for new key
|
||||||
redis_activity.get = MagicMock(return_value=None)
|
redis_activity.redis_repository.get = MagicMock(return_value=None)
|
||||||
redis_activity.set = MagicMock()
|
redis_activity.redis_repository.set = MagicMock()
|
||||||
|
|
||||||
# Call the method
|
# Call the method
|
||||||
result = await redis_activity.group_and_hold_data(test_data)
|
result = await redis_activity.group_and_hold_data(test_data)
|
||||||
@@ -354,7 +375,7 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
|
|||||||
'fill_missing_tags': False,
|
'fill_missing_tags': False,
|
||||||
}
|
}
|
||||||
|
|
||||||
redis_activity.get = MagicMock(return_value=None)
|
redis_activity.redis_repository.get = MagicMock(return_value=None)
|
||||||
|
|
||||||
# Call the method
|
# Call the method
|
||||||
result = await redis_activity.group_and_hold_data(test_data)
|
result = await redis_activity.group_and_hold_data(test_data)
|
||||||
@@ -376,7 +397,7 @@ async def test_group_and_hold_data_error_get(redis_activity):
|
|||||||
'fill_missing_tags': False,
|
'fill_missing_tags': False,
|
||||||
}
|
}
|
||||||
|
|
||||||
redis_activity.get = MagicMock(side_effect=Exception('test'))
|
redis_activity.redis_repository.get = MagicMock(side_effect=Exception('test'))
|
||||||
redis_activity.send_notification = MagicMock()
|
redis_activity.send_notification = MagicMock()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -415,8 +436,8 @@ async def test_group_and_hold_data_error_set(redis_activity):
|
|||||||
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
|
existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
|
||||||
|
|
||||||
# Mock get to return existing data
|
# Mock get to return existing data
|
||||||
redis_activity.get = MagicMock(return_value=existing_data)
|
redis_activity.redis_repository.get = MagicMock(return_value=existing_data)
|
||||||
redis_activity.set = MagicMock(side_effect=Exception('test'))
|
redis_activity.redis_repository.set = MagicMock(side_effect=Exception('test'))
|
||||||
redis_activity.send_notification = MagicMock()
|
redis_activity.send_notification = MagicMock()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -429,7 +450,7 @@ async def test_group_and_hold_data_error_set(redis_activity):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_store_data_package(redis_activity):
|
async def test_store_data_package(redis_activity):
|
||||||
"""Test store_data_package"""
|
"""Test store_data_package"""
|
||||||
redis_activity.set = MagicMock()
|
redis_activity.redis_repository.set = MagicMock()
|
||||||
|
|
||||||
test_data = {
|
test_data = {
|
||||||
**metadata,
|
**metadata,
|
||||||
@@ -454,7 +475,7 @@ async def test_store_data_package(redis_activity):
|
|||||||
|
|
||||||
await redis_activity.store_data_package(test_data)
|
await redis_activity.store_data_package(test_data)
|
||||||
|
|
||||||
redis_activity.set.assert_called_once_with(
|
redis_activity.redis_repository.set.assert_called_once_with(
|
||||||
ANY, {'data': test_data['data'], 'held_data': test_data['held_data']}, ttl=120
|
ANY, {'data': test_data['data'], 'held_data': test_data['held_data']}, ttl=120
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -462,7 +483,7 @@ async def test_store_data_package(redis_activity):
|
|||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_store_data_package_error(redis_activity):
|
async def test_store_data_package_error(redis_activity):
|
||||||
"""Test store_data_package error"""
|
"""Test store_data_package error"""
|
||||||
redis_activity.set = MagicMock(side_effect=ValueError('test'))
|
redis_activity.redis_repository.set = MagicMock(side_effect=ValueError('test'))
|
||||||
redis_activity.send_notification = MagicMock()
|
redis_activity.send_notification = MagicMock()
|
||||||
|
|
||||||
test_data = {
|
test_data = {
|
||||||
|
|||||||
12
tests/conftest.py
Normal file
12
tests/conftest.py
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from pytest import fixture
|
||||||
|
|
||||||
|
|
||||||
|
@fixture(autouse=True, scope='session')
|
||||||
|
def sientia_monitoring_fixture():
|
||||||
|
with patch(
|
||||||
|
'sientia_do.observability.sientia_monitoring.SientiaMonitoring.__init__'
|
||||||
|
) as mock_sientia_monitoring:
|
||||||
|
mock_sientia_monitoring.return_value = None
|
||||||
|
yield
|
||||||
@@ -150,7 +150,7 @@ env:
|
|||||||
- name: GITHUB_REPO_URL
|
- name: GITHUB_REPO_URL
|
||||||
value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git"
|
value: "git@github.com:Aignosi/sientia-dataops-scouter_temporal.git"
|
||||||
- name: GITHUB_BRANCH
|
- name: GITHUB_BRANCH
|
||||||
value: "fix/SIENTIAPDE-1314-ajustes-nas-camadas-de-monitoramento-do-sientia"
|
value: "feature/SIENTIAPDE-1325-adicionar-metricas-especificas-de-operacoes-externas"
|
||||||
- name: PYTHON_APP
|
- name: PYTHON_APP
|
||||||
value: "scouter.worker.worker"
|
value: "scouter.worker.worker"
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user