gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
SIENTIAPDE-1325: Refactor Ingestor and Manager Classes for Enhanced Asynchronous Operations

- Introduced asynchronous methods across Ingestor, IngestorManager, DataManager, and OpcManager to improve performance and responsiveness.
- Integrated MetricsController into various classes for better observability and monitoring.
- Updated Redis and MongoDB interactions to support asynchronous operations, enhancing data handling efficiency.
- Removed deprecated Redis metrics and streamlined resource management logic.
- Adjusted unit tests to accommodate the new asynchronous behavior and ensure proper mocking of async methods.
This commit is contained in:
vitor-aignosi
2025-10-31 16:32:45 -03:00
parent 4158a74cac
commit 85a371a38b
7 changed files with 336 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
tests/unit/managers/test_data_manager.py
View File

@@ -1,7 +1,7 @@
from unittest.mock import ANY, MagicMock, patch
from unittest.mock import ANY, AsyncMock, MagicMock, patch
from kafka.errors import NoBrokersAvailable
from pytest import fixture
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
from ingestor.managers.data_manager import DataManager
@@ -19,8 +19,8 @@ metadata = {
@fixture
@patch('ingestor.managers.data_manager.KafkaProducer')
@patch('ingestor.managers.data_manager.MongoClient')
def data_manager(mongo, kafka):
@patch('ingestor.managers.data_manager.MongoDBRepository')
def data_manager(mongodb_repository, kafka):
data_manager = DataManager(
kafka_servers='localhost:9092',
mongo_connection_string='mongodb://localhost:27017',
@@ -29,16 +29,18 @@ def data_manager(mongo, kafka):
logger=MagicMock(),
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=MagicMock(),
)
data_manager.send_notification = MagicMock()
data_manager.send_notification_async = AsyncMock()
data_manager.emit_metric = AsyncMock()
return data_manager
@patch('ingestor.managers.data_manager.KafkaProducer')
@patch('ingestor.managers.data_manager.MongoClient')
def test___init___success(mongo, kafka):
@patch('ingestor.managers.data_manager.MongoDBRepository')
def test___init___success(mongodb_repository, kafka):
logger_mock = MagicMock()
data_manager = DataManager(
@@ -49,6 +51,7 @@ def test___init___success(mongo, kafka):
export_to_kafka=True,
logger=logger_mock,
notification_handler=MagicMock(),
metrics_controller=MagicMock(),
)
kafka.assert_called_once_with(
@@ -64,8 +67,8 @@ def test___init___success(mongo, kafka):
@patch('ingestor.managers.data_manager.KafkaProducer')
@patch('ingestor.managers.data_manager.MongoClient')
def test___init___second_attempt(mongo, kafka):
@patch('ingestor.managers.data_manager.MongoDBRepository')
def test___init___second_attempt(mongodb_repository, kafka):
kafka.side_effect = [NoBrokersAvailable, MagicMock()]
logger_mock = MagicMock()
@@ -77,6 +80,7 @@ def test___init___second_attempt(mongo, kafka):
logger=logger_mock,
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=MagicMock(),
)
kafka.assert_any_call(
@@ -98,8 +102,8 @@ def test___init___second_attempt(mongo, kafka):
@patch('ingestor.managers.data_manager.KafkaProducer')
@patch('ingestor.managers.data_manager.MongoClient')
def test___init___failure_max_attempts(mongo, kafka):
@patch('ingestor.managers.data_manager.MongoDBRepository')
def test___init___failure_max_attempts(mongodb_repository, kafka):
kafka.side_effect = NoBrokersAvailable
logger_mock = MagicMock()
@@ -112,6 +116,7 @@ def test___init___failure_max_attempts(mongo, kafka):
logger=logger_mock,
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=MagicMock(),
)
except NoBrokersAvailable as e:
assert (
@@ -160,15 +165,6 @@ def test_shutdown_no_producer(data_manager):
)
def test_shutdown_no_mongo_client(data_manager):
data_manager.mongo_client = None
data_manager.shutdown()
data_manager.logger.warning.assert_any_call(
'MongoDB client is already closed or not initialized.'
)
def test_shutdown_exception(data_manager):
data_manager.kafka_producer.flush = MagicMock(side_effect=Exception('Test error'))
@@ -179,7 +175,7 @@ def test_shutdown_exception(data_manager):
def test_shutdown_exception_mongo(data_manager):
data_manager.mongo_client.close = MagicMock(side_effect=Exception('Test error'))
data_manager.mongo_repository.close = MagicMock(side_effect=Exception('Test error'))
data_manager.shutdown()
@@ -210,9 +206,10 @@ def test_delivery_error(data_manager):
data_manager.delivery_error(err)
data_manager.logger.error.assert_called_once_with(f'Delivery failed for record : {err}')
def test_publish(data_manager):
@mark.asyncio
async def test_publish(data_manager):
topic = 'test_topic'
data = {'key': 'value'}
@@ -221,7 +218,7 @@ def test_publish(data_manager):
data_manager.kafka_producer.send = send_mock
# Call the publish method
data_manager.publish(topic, data)
await data_manager.publish(topic, data)
# Check if the send method was called with the correct arguments
send_mock.assert_called_once_with(topic=topic, value=data)
@@ -242,7 +239,8 @@ def test_publish_no_kafka(data_manager):
@patch('ingestor.managers.data_manager.traceback')
def test_publish_error(traceback, data_manager):
@mark.asyncio
async def test_publish_error(traceback, data_manager):
topic = 'test_topic'
data = {'key': 'value', 'name': 'test_tag'}
@@ -251,13 +249,13 @@ def test_publish_error(traceback, data_manager):
data_manager.kafka_producer.send = send_mock
# Call the publish method
data_manager.publish(topic, data)
await data_manager.publish(topic, data)
# Check if the send method was called with the correct arguments
send_mock.assert_called_once_with(topic=topic, value=data)
# Check if the error was logged
data_manager.send_notification.assert_called_once_with(
data_manager.send_notification_async.assert_called_once_with(
notification_id=f'KAFKA_PRODUCER_ERROR_{topic}',
message=f'Error publishing message to topic {topic}: Test error',
block='kafka_producer',
@@ -267,15 +265,16 @@ def test_publish_error(traceback, data_manager):
)
def test_publish_error_mongo(data_manager):
@mark.asyncio
async def test_publish_error_mongo(data_manager):
data_manager.export_to_kafka = False
data_manager.mongo_db.__getitem__.return_value.insert_one = MagicMock(
data_manager.mongo_repository.insert = AsyncMock(
side_effect=Exception('Test error')
)
data_manager.publish('test_topic', {'key': 'value'})
await data_manager.publish('test_topic', {'key': 'value'})
data_manager.send_notification.assert_called_once_with(
data_manager.send_notification_async.assert_called_once_with(
notification_id='MONGO_PRODUCER_ERROR_test_topic',
message='Error inserting message to MongoDB: Test error',
block='mongo_producer',