gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1205

Browse Source 
Refactor tests for async compatibility and enhance IngestorManager functionality

- Updated test cases in `test_app.py`, `test_ingestor.py`, and `test_ingestor_manager.py` to use async/await syntax for improved concurrency.
- Refactored methods in IngestorManager and related classes to support asynchronous operations, ensuring non-blocking behavior during execution.
- Enhanced mock setups in tests to accommodate async methods, improving test reliability and performance.
This commit is contained in:
vitor-aignosi
2025-08-27 17:00:06 -03:00
parent c09a96d4a4
commit f55604c7db
6 changed files with 266 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
tests/unit/managers/test_ingestor_manager.py
View File

@@ -1,5 +1,5 @@
from unittest.mock import MagicMock, patch
from pytest import fixture
from unittest.mock import AsyncMock, MagicMock, patch
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
from ingestor.managers.ingestor_manager import IngestorManager
@@ -91,8 +91,9 @@ def test___init__(notification_handler_mock, resource_manager_mock, data_manager
assert ingestor.resource_manager == resource_manager_mock.return_value
@mark.asyncio
@patch('ingestor.managers.ingestor_manager.OpcManager')
def test_initialize_opc_from_config(opc_manager, ingestor_manager):
async def test_initialize_opc_from_config(opc_manager, ingestor_manager):
server_config = {
'name': 'server1',
'url': 'opc.tcp://localhost:4840',
@@ -103,8 +104,8 @@ def test_initialize_opc_from_config(opc_manager, ingestor_manager):
'pod_id': 'test_pod'
}
opc_manager.return_value = MagicMock()
result = ingestor_manager.initialize_opc_from_config(
opc_manager.return_value = MagicMock(connect=AsyncMock())
result = await ingestor_manager.initialize_opc_from_config(
server_config)
opc_manager.assert_called_once_with(
@@ -124,9 +125,10 @@ def test_initialize_opc_from_config(opc_manager, ingestor_manager):
result.connect.assert_called_once()
@mark.asyncio
@patch('ingestor.managers.ingestor_manager.OpcManager')
@patch('ingestor.managers.ingestor_manager.traceback')
def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, ingestor_manager):
async def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, ingestor_manager):
server_config = {
'name': 'server1',
'url': 'opc.tcp://localhost:4840',
@@ -139,7 +141,7 @@ def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, inges
ingestor_manager.logger.error = MagicMock()
opc_manager.side_effect = Exception("Initialization error")
result = ingestor_manager.initialize_opc_from_config(
result = await ingestor_manager.initialize_opc_from_config(
server_config)
assert result is None
@@ -155,17 +157,21 @@ def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, inges
)
@mark.asyncio
@patch('ingestor.managers.ingestor_manager.OpcManager')
@patch('ingestor.managers.ingestor_manager.metrics')
def test_update_opc_servers(metrics, opc_manager, ingestor_manager):
async def test_update_opc_servers(metrics, opc_manager, ingestor_manager):
manager1 = MagicMock(
config={"config": "config1"})
config={"config": "config1"}
)
manager2 = MagicMock(
config={"config": "config2"})
config={"config": "config2"}
)
manager3 = MagicMock(
config={"config": "config3"})
config={"config": "config3"}
)
def mock_initialize_from_config(config):
async def mock_initialize_from_config(config):
if config == {"config": "config1"}:
return manager1
elif config == {"config": "config2"}:
@@ -175,7 +181,7 @@ def test_update_opc_servers(metrics, opc_manager, ingestor_manager):
else:
return None
ingestor_manager.initialize_opc_from_config = MagicMock(
ingestor_manager.initialize_opc_from_config = AsyncMock(
side_effect=mock_initialize_from_config
)
@@ -193,12 +199,12 @@ def test_update_opc_servers(metrics, opc_manager, ingestor_manager):
mock = MagicMock(
config={"config": "old_config2"})
ingestor_manager.opc_managers['server3'] = MagicMock(
ingestor_manager.opc_managers['server3'] = AsyncMock(
config={"config": "config3"})
ingestor_manager.opc_managers['server2'] = mock
ingestor_manager.opc_managers['server4'] = MagicMock()
ingestor_manager.opc_managers['server4'] = AsyncMock()
ingestor_manager.update_opc_servers()
await ingestor_manager.update_opc_servers()
assert len(ingestor_manager.opc_managers) == 3
@@ -340,7 +346,8 @@ def test_get_slot_leases_1_failure(ingestor_manager):
assert result == {}
def test_unsubscribe_slot(ingestor_manager):
@mark.asyncio
async def test_unsubscribe_slot(ingestor_manager):
ingestor_manager.managed_tags = {
"slot1": {
"server1": {"tags": "config1"},
@@ -352,11 +359,11 @@ def test_unsubscribe_slot(ingestor_manager):
}
}
ingestor_manager.opc_managers = {
"server1": MagicMock(),
"server2": MagicMock(),
"server3": MagicMock()
"server1": AsyncMock(),
"server2": AsyncMock(),
"server3": AsyncMock()
}
ingestor_manager.unsubscribe_slot("slot1")
await ingestor_manager.unsubscribe_slot("slot1")
ingestor_manager.opc_managers["server1"].unsubscribe.assert_called_once_with(
"slot1")
@@ -411,7 +418,8 @@ def test_drop_slot_leases(metrics, ingestor_manager):
metrics.SLOTS_RELEASED.labels.return_value.inc.assert_any_call()
def test_manage_server_no_server(ingestor_manager):
@mark.asyncio
async def test_manage_server_no_server(ingestor_manager):
ingestor_manager.opc_managers = {
"server1": MagicMock(),
"server2": MagicMock()
@@ -420,7 +428,7 @@ def test_manage_server_no_server(ingestor_manager):
'tags': 'config1'
}
result = ingestor_manager.manage_server(
result = await ingestor_manager.manage_server(
'slot1', 'server3', server_config, server_config)
assert result == 1
@@ -429,7 +437,8 @@ def test_manage_server_no_server(ingestor_manager):
ingestor_manager.opc_managers["server1"].subscribe.assert_not_called()
def test_manage_server_create_subscription_failure(ingestor_manager):
@mark.asyncio
async def test_manage_server_create_subscription_failure(ingestor_manager):
ingestor_manager.opc_managers = {
"server1": MagicMock(),
"server2": MagicMock()
@@ -444,7 +453,7 @@ def test_manage_server_create_subscription_failure(ingestor_manager):
ingestor_manager.opc_managers["server1"].create_subscription.side_effect = Exception(
"Subscription error")
result = ingestor_manager.manage_server(
result = await ingestor_manager.manage_server(
'slot1', 'server1', server_config, server_config)
assert result == 2
@@ -453,19 +462,20 @@ def test_manage_server_create_subscription_failure(ingestor_manager):
ingestor_manager.opc_managers["server1"].subscribe.assert_not_called()
def test_manage_server(ingestor_manager):
@mark.asyncio
async def test_manage_server(ingestor_manager):
ingestor_manager.opc_managers = {
"server1": MagicMock(),
"server2": MagicMock()
"server1": AsyncMock(),
"server2": AsyncMock()
}
ingestor_manager.subscriptions = {
"server1": MagicMock()
"server1": AsyncMock()
}
server_config = {
'tags': 'config1'
}
result = ingestor_manager.manage_server(
result = await ingestor_manager.manage_server(
'slot1', 'server1', server_config, server_config)
assert result == 0
@@ -476,13 +486,14 @@ def test_manage_server(ingestor_manager):
@patch('ingestor.managers.ingestor_manager.traceback')
def test_manage_server_subscribe_failure(traceback_mock, ingestor_manager):
@mark.asyncio
async def test_manage_server_subscribe_failure(traceback_mock, ingestor_manager):
ingestor_manager.opc_managers = {
"server1": MagicMock(),
"server2": MagicMock()
"server1": AsyncMock(),
"server2": AsyncMock()
}
ingestor_manager.subscriptions = {
"server1": MagicMock()
"server1": AsyncMock()
}
server_config = {
'tags': 'config1'
@@ -491,7 +502,7 @@ def test_manage_server_subscribe_failure(traceback_mock, ingestor_manager):
ingestor_manager.opc_managers["server1"].subscribe.side_effect = Exception(
"Subscription error")
result = ingestor_manager.manage_server(
result = await ingestor_manager.manage_server(
'slot1', 'server1', server_config, server_config)
assert result == 2
@@ -518,8 +529,9 @@ def test_manage_server_subscribe_failure(traceback_mock, ingestor_manager):
)
def test_subscribe_to_tags(ingestor_manager):
ingestor_manager.manage_server = MagicMock(
@mark.asyncio
async def test_subscribe_to_tags(ingestor_manager):
ingestor_manager.manage_server = AsyncMock(
side_effect=[0, 1, 2])
ingestor_manager.managed_tags = {
"slot1": MagicMock(),
@@ -527,11 +539,11 @@ def test_subscribe_to_tags(ingestor_manager):
}
ingestor_manager.opc_managers = {
"server1": MagicMock(),
"server2": MagicMock()
"server1": AsyncMock(),
"server2": AsyncMock()
}
ingestor_manager.subscriptions = {
"server1": MagicMock()
"server1": AsyncMock()
}
tags = {
'slot1': {
@@ -541,7 +553,7 @@ def test_subscribe_to_tags(ingestor_manager):
}
}
ingestor_manager.subscribe_to_tags(tags)
await ingestor_manager.subscribe_to_tags(tags)
ingestor_manager.manage_server.assert_any_call(
'slot1', 'server1', {"tags": "config1"}, tags)

129
tests/unit/managers/test_opc_manager.py
View File

@@ -1,9 +1,7 @@
import json
from datetime import datetime
from unittest.mock import MagicMock, patch
from prometheus_client import Gauge
from pytest import fixture
from unittest.mock import AsyncMock, MagicMock, patch
from pytest import fixture, mark
from asyncua.crypto.security_policies import SecurityPolicyBasic256
import pytest
from ingestor.managers.opc_manager import OpcManager
@@ -59,7 +57,7 @@ def raw_opc_manager(mock_metrics):
@fixture
def opc_manager(raw_opc_manager):
raw_opc_manager.client = MagicMock()
raw_opc_manager.client = AsyncMock()
raw_opc_manager.cert_path = "cert.pem"
raw_opc_manager.private_key_path = "private_key.pem"
raw_opc_manager.server_cert_path = "server_cert.pem"
@@ -70,7 +68,7 @@ def opc_manager(raw_opc_manager):
@fixture
def opc_manager_subscribed(opc_manager):
opc_manager.subscriptions["sub1"] = MagicMock()
opc_manager.subscriptions["sub1"] = AsyncMock()
return opc_manager
@@ -82,10 +80,13 @@ def test___str__(opc_manager):
)
def test_set_security_success(opc_manager):
opc_manager.set_security()
@mark.asyncio
async def test_set_security_success(opc_manager):
await opc_manager.set_security()
assert opc_manager.client.application_uri == opc_manager.server_uri
opc_manager.client.set_application_uri.assert_called_once_with(
opc_manager.server_uri
)
opc_manager.client.set_security.assert_called_once_with(
SecurityPolicyBasic256,
@@ -94,16 +95,18 @@ def test_set_security_success(opc_manager):
server_certificate=opc_manager.server_cert_path,
)
assert opc_manager.client.secure_channel_timeout == 10000000
assert opc_manager.client.session_timeout == 10000000
opc_manager.client.set_secure_channel_timeout.assert_called_once_with(
10000000)
opc_manager.client.set_session_timeout.assert_called_once_with(10000000)
def test_set_security_no_cert(opc_manager):
@mark.asyncio
async def test_set_security_no_cert(opc_manager):
opc_manager.cert_path = None
opc_manager.private_key_path = None
try:
opc_manager.set_security()
await opc_manager.set_security()
except ValueError as e:
assert (
str(e)
@@ -115,12 +118,14 @@ def test_set_security_no_cert(opc_manager):
assert opc_manager.client.set_security.call_count == 0
@mark.asyncio
@patch("ingestor.managers.opc_manager.metrics")
@patch("ingestor.managers.opc_manager.Client")
def test_connect_no_security(client, mock_metrics, raw_opc_manager):
raw_opc_manager.set_security = MagicMock()
async def test_connect_no_security(client, mock_metrics, raw_opc_manager):
raw_opc_manager.set_security = AsyncMock()
client.return_value = AsyncMock()
raw_opc_manager.connect()
await raw_opc_manager.connect()
client.assert_called_once_with(raw_opc_manager.url)
raw_opc_manager.client.connect.assert_called_once()
@@ -140,23 +145,26 @@ def test_connect_no_security(client, mock_metrics, raw_opc_manager):
mock_metrics.OPC_CONNECTIONS_FAILED.labels.assert_not_called()
@mark.asyncio
@patch("ingestor.managers.opc_manager.Client")
def test_connect_with_security(client, raw_opc_manager):
async def test_connect_with_security(client, raw_opc_manager):
raw_opc_manager.cert_path = "cert.pem"
raw_opc_manager.private_key_path = "private_key.pem"
raw_opc_manager.server_cert_path = "server_cert.pem"
raw_opc_manager.set_security = MagicMock()
raw_opc_manager.set_security = AsyncMock()
client.return_value = AsyncMock()
raw_opc_manager.connect()
await raw_opc_manager.connect()
client.assert_called_once_with(raw_opc_manager.url)
raw_opc_manager.client.connect.assert_called_once()
raw_opc_manager.set_security.assert_called_once()
@mark.asyncio
@patch("ingestor.managers.opc_manager.Client")
@patch("ingestor.managers.opc_manager.metrics")
def test_connect_exception_handling_and_metrics(
async def test_connect_exception_handling_and_metrics(
mock_metrics_module, mock_opc_client_class, raw_opc_manager
):
mock_client_instance = mock_opc_client_class.return_value
@@ -168,7 +176,7 @@ def test_connect_exception_handling_and_metrics(
opc_manager_instance.cert_path = None
with pytest.raises(Exception, match=simulated_error_message):
opc_manager_instance.connect()
await opc_manager_instance.connect()
mock_metrics_module.OPC_CONNECTIONS_TOTAL.labels.assert_called_once_with(
pod_id=opc_manager_instance.pod_id, server_name=opc_manager_instance.name
@@ -194,25 +202,28 @@ def test_connect_exception_handling_and_metrics(
)
def test_create_subscription_no_client(raw_opc_manager):
@mark.asyncio
async def test_create_subscription_no_client(raw_opc_manager):
try:
raw_opc_manager.create_subscription("sub1")
await raw_opc_manager.create_subscription("sub1")
except ValueError as e:
assert str(e) == "Client not connected. Call connect first."
else:
assert False, "ValueError not raised"
def test_create_subscription_success_has_period(opc_manager):
opc_manager.create_subscription("sub1", 1000)
@mark.asyncio
async def test_create_subscription_success_has_period(opc_manager):
await opc_manager.create_subscription("sub1", 1000)
opc_manager.client.create_subscription.assert_called_once_with(
1000, opc_manager)
assert opc_manager.subscriptions["sub1"] is not None
def test_create_subscription_success_no_period(opc_manager):
opc_manager.create_subscription("sub1", None)
@mark.asyncio
async def test_create_subscription_success_no_period(opc_manager):
await opc_manager.create_subscription("sub1", None)
opc_manager.client.create_subscription.assert_called_once_with(
500, opc_manager)
@@ -220,8 +231,9 @@ def test_create_subscription_success_no_period(opc_manager):
@patch("ingestor.managers.opc_manager.metrics")
def test_create_subscription_with_metrics(metrics, opc_manager):
opc_manager.create_subscription("sub1", 1000)
@mark.asyncio
async def test_create_subscription_with_metrics(metrics, opc_manager):
await opc_manager.create_subscription("sub1", 1000)
metrics.OPC_SUBSCRIPTIONS_CREATED.labels.assert_called_once_with(
pod_id=opc_manager.pod_id, server_name=opc_manager.name, slot_name="sub1"
@@ -230,11 +242,12 @@ def test_create_subscription_with_metrics(metrics, opc_manager):
@patch("ingestor.managers.opc_manager.metrics")
def test_create_subscription_exception_during_client_call(
@mark.asyncio
async def test_create_subscription_exception_during_client_call(
mock_metrics_module, raw_opc_manager
):
opc_manager_instance = raw_opc_manager
opc_manager_instance.client = MagicMock()
opc_manager_instance.client = AsyncMock()
subscription_name = "test_sub_client_error"
simulated_period = 750
@@ -245,7 +258,7 @@ def test_create_subscription_exception_during_client_call(
)
with pytest.raises(Exception, match=simulated_error_message):
opc_manager_instance.create_subscription(
await opc_manager_instance.create_subscription(
subscription_name, period=simulated_period
)
@@ -261,9 +274,10 @@ def test_create_subscription_exception_during_client_call(
@patch("ingestor.managers.opc_manager.metrics")
def test_subscribe_no_subscription(metrics, opc_manager):
@mark.asyncio
async def test_subscribe_no_subscription(metrics, opc_manager):
try:
opc_manager.subscribe("sub1", tags, 1000)
await opc_manager.subscribe("sub1", tags, 1000)
except ValueError as e:
assert str(
e) == "Subscription not created. Call create_subscription first."
@@ -272,10 +286,12 @@ def test_subscribe_no_subscription(metrics, opc_manager):
metrics.OPC_TAGS_SUBSCRIBED.labels.assert_not_called()
def test_subscribe_success(opc_manager_subscribed):
@mark.asyncio
async def test_subscribe_success(opc_manager_subscribed):
opc_manager_subscribed.client.get_node = MagicMock()
opc_manager_subscribed.nodes = {"ns=3;i=1001": "data"}
opc_manager_subscribed.subscribe("sub1", tags, 1000)
await opc_manager_subscribed.subscribe("sub1", tags, 1000)
assert opc_manager_subscribed.nodes == tags
opc_manager_subscribed.subscriptions["sub1"].subscribe_data_change.assert_called_once_with(
@@ -283,8 +299,9 @@ def test_subscribe_success(opc_manager_subscribed):
)
def test_unsubscribe_no_subscription(opc_manager):
opc_manager.unsubscribe("sub1")
@mark.asyncio
async def test_unsubscribe_no_subscription(opc_manager):
await opc_manager.unsubscribe("sub1")
opc_manager.logger.warning.assert_called_once_with(
"Subscription 'sub1' not found. Cannot unsubscribe."
@@ -292,28 +309,33 @@ def test_unsubscribe_no_subscription(opc_manager):
assert opc_manager.subscriptions.get("sub1") is None
def test_unsubscribe_success(opc_manager_subscribed):
opc_manager_subscribed.unsubscribe("sub1")
@mark.asyncio
async def test_unsubscribe_success(opc_manager_subscribed):
await opc_manager_subscribed.unsubscribe("sub1")
opc_manager_subscribed.subscriptions.get("sub1") is None
def test_disconnect_success(opc_manager_subscribed):
@mark.asyncio
async def test_disconnect_success(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock()
opc_manager_subscribed.disconnect()
await opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once()
assert opc_manager_subscribed.client is None
def test_disconnect_error_unsubscribe(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock()
@mark.asyncio
async def test_disconnect_error_unsubscribe(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock(
disconnect=AsyncMock()
)
opc_manager_subscribed.subscriptions["sub1"] = MagicMock(
delete=MagicMock(side_effect=Exception("Test error"))
delete=AsyncMock(side_effect=Exception("Test error"))
)
opc_manager_subscribed.disconnect()
await opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once()
opc_manager_subscribed.client = None
@@ -322,13 +344,14 @@ def test_disconnect_error_unsubscribe(opc_manager_subscribed):
)
def test_disconnect_error(opc_manager_subscribed):
@mark.asyncio
async def test_disconnect_error(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock()
opc_manager_subscribed.client.disconnect = MagicMock(
side_effect=Exception("Test error")
)
opc_manager_subscribed.disconnect()
await opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions["sub1"].delete.assert_called_once()
opc_manager_subscribed.client = None
@@ -338,7 +361,8 @@ def test_disconnect_error(opc_manager_subscribed):
@patch('ingestor.managers.opc_manager.metrics')
def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_opc_manager):
@mark.asyncio
async def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_opc_manager):
mock_metrics_module.OPC_CONNECTION_STATUS.reset_mock()
mock_metrics_module.OPC_TAGS_SUBSCRIBED.reset_mock()
@@ -347,7 +371,7 @@ def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_opc_mana
mock_sub2 = MagicMock()
raw_opc_manager.subscriptions = {"sub1": mock_sub1, "sub2": mock_sub2}
raw_opc_manager.disconnect()
await raw_opc_manager.disconnect()
mock_metrics_module.OPC_CONNECTION_STATUS.labels.assert_called_once_with(
pod_id=raw_opc_manager.pod_id,
@@ -366,7 +390,8 @@ def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_opc_mana
@patch('ingestor.managers.opc_manager.metrics')
def test_datachange_notification(metrics, opc_manager_subscribed):
@mark.asyncio
async def test_datachange_notification(metrics, opc_manager_subscribed):
data = MagicMock(
monitored_item=MagicMock(
Value=MagicMock(
@@ -387,7 +412,7 @@ def test_datachange_notification(metrics, opc_manager_subscribed):
metrics.OPC_CYCLES_WITHOUT_DATA.reset_mock()
opc_manager_subscribed.datachange_notification("ns=3;i=1001", None, data)
await opc_manager_subscribed.datachange_notification("ns=3;i=1001", None, data)
opc_manager_subscribed.data_manager.publish.assert_any_call(
"topic1",