gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1325

Browse Source 
Refactor Ingestor and Metrics Classes for Enhanced Asynchronous Operations

- Updated IngestorManager, OpcManager, and ResourceManager to utilize asynchronous methods for improved performance.
- Integrated MetricsController into various classes for better observability and monitoring.
- Adjusted unit tests to accommodate the new asynchronous behavior, ensuring proper mocking of async methods.
- Removed deprecated Redis metrics and streamlined resource management logic.
This commit is contained in:
vitor-aignosi
2025-11-03 09:41:49 -03:00
parent 85a371a38b
commit dfd08e883f
11 changed files with 525 additions and 479 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
tests/unit/managers/test_opc_manager.py
View File

@@ -46,17 +46,24 @@ metadata = {
@fixture
@patch('ingestor.managers.opc_manager.metrics')
def raw_opc_manager(mock_metrics):
return OpcManager(
opc_manager = OpcManager(
name='TestConnector',
url='opc.tcp://localhost:4840',
data_manager=MagicMock(),
data_manager=AsyncMock(),
subscription_period_ms=1000,
logger=MagicMock(),
server_uri='opc.tcp://localhost:4840',
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=AsyncMock(),
)
opc_manager.emit_metric = AsyncMock()
opc_manager.send_notification_async = AsyncMock()
opc_manager.send_notification = MagicMock()
return opc_manager
@fixture
def opc_manager(raw_opc_manager):
@@ -64,7 +71,6 @@ def opc_manager(raw_opc_manager):
raw_opc_manager.cert_path = 'cert.pem'
raw_opc_manager.private_key_path = 'private_key.pem'
raw_opc_manager.server_cert_path = 'server_cert.pem'
raw_opc_manager.send_notification = MagicMock()
return raw_opc_manager
@@ -145,17 +151,30 @@ async def test_connect_no_security(client, mock_metrics, raw_opc_manager):
client.assert_called_once_with(raw_opc_manager.url, timeout=10, watchdog_intervall=3600000)
raw_opc_manager.client.connect.assert_called_once()
raw_opc_manager.set_security.assert_not_called()
mock_metrics.OPC_CONNECTIONS_TOTAL.labels.assert_called_once_with(
pod_id=raw_opc_manager.pod_id, server_name=raw_opc_manager.name
raw_opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=mock_metrics.OPC_CONNECTIONS_TOTAL,
method='inc',
value=1,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
},
),
call(
metric_object=mock_metrics.OPC_CONNECTION_STATUS,
method='set',
value=1,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
'server_url': raw_opc_manager.url,
},
),
],
any_order=True,
)
mock_metrics.OPC_CONNECTIONS_TOTAL.labels.return_value.inc.assert_called_once()
mock_metrics.OPC_CONNECTION_STATUS.labels.assert_called_once_with(
pod_id=raw_opc_manager.pod_id,
server_name=raw_opc_manager.name,
server_url=raw_opc_manager.url,
)
mock_metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(1)
mock_metrics.OPC_CONNECTIONS_FAILED.labels.assert_not_called()
@mark.asyncio
@@ -230,10 +249,16 @@ async def test_create_subscription_success_no_period(opc_manager):
async def test_create_subscription_with_metrics(metrics, opc_manager):
await opc_manager.create_subscription('sub1')
metrics.OPC_SUBSCRIPTIONS_CREATED.labels.assert_called_once_with(
pod_id=opc_manager.pod_id, server_name=opc_manager.name, slot_name='sub1'
opc_manager.emit_metric.assert_called_once_with(
metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED,
method='inc',
value=1,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
'slot_name': 'sub1',
},
)
metrics.OPC_SUBSCRIPTIONS_CREATED.labels.return_value.inc.assert_called_once()
@patch('ingestor.managers.opc_manager.metrics')
@@ -390,17 +415,30 @@ async def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_op
await raw_opc_manager.disconnect()
mock_metrics_module.OPC_CONNECTION_STATUS.labels.assert_called_once_with(
pod_id=raw_opc_manager.pod_id,
server_name=raw_opc_manager.name,
server_url=raw_opc_manager.url,
raw_opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=mock_metrics_module.OPC_CONNECTION_STATUS,
method='set',
value=0,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
'server_url': raw_opc_manager.url,
},
),
call(
metric_object=mock_metrics_module.OPC_TAGS_SUBSCRIBED,
method='set',
value=0,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
},
),
],
any_order=True,
)
mock_metrics_module.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0)
mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.assert_called_once_with(
pod_id=raw_opc_manager.pod_id, server_name=raw_opc_manager.name
)
mock_metrics_module.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(0)
@patch('ingestor.managers.opc_manager.metrics')
@@ -422,8 +460,6 @@ async def test_datachange_notification(metrics, opc_manager_subscribed):
}
}
metrics.OPC_CYCLES_WITHOUT_DATA.reset_mock()
await opc_manager_subscribed.datachange_notification('ns=3;i=1001', None, data)
opc_manager_subscribed.data_manager.publish.assert_any_call(
@@ -446,13 +482,19 @@ async def test_datachange_notification(metrics, opc_manager_subscribed):
)
assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0
metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with(
pod_id=opc_manager_subscribed.pod_id, server_name=opc_manager_subscribed.name
opc_manager_subscribed.emit_metric.assert_called_once_with(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=0,
tags={
'pod_id': opc_manager_subscribed.pod_id,
'server_name': opc_manager_subscribed.name,
},
)
metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(0)
def test_check_cycles_no_notification(opc_manager):
@mark.asyncio
async def test_check_cycles_no_notification(opc_manager):
# Setup: node with cycle_count just below threshold
opc_manager.nodes = {
'ns=3;i=1001': {
@@ -461,14 +503,15 @@ def test_check_cycles_no_notification(opc_manager):
}
}
opc_manager.check_cycles()
await opc_manager.check_cycles()
# After one increment, cycle_count = 4.0, still below threshold
assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(4.0)
opc_manager.send_notification.assert_not_called()
opc_manager.send_notification_async.assert_not_called()
def test_check_cycles_triggers_notification(opc_manager):
@mark.asyncio
async def test_check_cycles_triggers_notification(opc_manager):
# Setup: node with cycle_count just below threshold, increment will cross threshold
opc_manager.nodes = {
'ns=3;i=1001': {
@@ -478,11 +521,11 @@ def test_check_cycles_triggers_notification(opc_manager):
}
opc_manager.notification_handler.build_and_send_notification = MagicMock()
opc_manager.check_cycles()
await opc_manager.check_cycles()
# After increment, cycle_count = 5.5, should trigger notification
assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(5.5)
opc_manager.send_notification.assert_called_once_with(
opc_manager.send_notification_async.assert_called_once_with(
notification_id='TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED',
message='5.5 cycles without receive from ns=3;i=1001:Counter',
block='opc_manager',
@@ -492,34 +535,35 @@ def test_check_cycles_triggers_notification(opc_manager):
@patch('ingestor.managers.opc_manager.metrics')
def test_check_opc_listenning_no_notification(metrics, opc_manager):
@mark.asyncio
async def test_check_opc_listenning_no_notification(metrics, opc_manager):
opc_manager.non_receive_count = 3
opc_manager.notification_handler.build_and_send_notification = MagicMock()
metrics.OPC_CYCLES_WITHOUT_DATA.reset_mock()
metrics.OPC_RECONNECTIONS_TOTAL.reset_mock()
result = opc_manager.check_opc_listenning()
result = await opc_manager.check_opc_listenning()
assert opc_manager.non_receive_count == 4
opc_manager.notification_handler.build_and_send_notification.assert_not_called()
opc_manager.send_notification_async.assert_not_called()
assert result is False
metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with(
pod_id=opc_manager.pod_id, server_name=opc_manager.name
opc_manager.emit_metric.assert_called_once_with(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=opc_manager.non_receive_count,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
},
)
metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(
opc_manager.non_receive_count
)
metrics.OPC_RECONNECTIONS_TOTAL.labels.assert_not_called()
def test_check_opc_listenning_warning_notification(opc_manager):
@mark.asyncio
async def test_check_opc_listenning_warning_notification(opc_manager):
opc_manager.non_receive_count = 4
result = opc_manager.check_opc_listenning()
result = await opc_manager.check_opc_listenning()
assert opc_manager.non_receive_count == 5
opc_manager.send_notification.assert_called_once_with(
opc_manager.send_notification_async.assert_called_once_with(
notification_id=f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
message=f'5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}',
block='opc_manager',
@@ -530,18 +574,16 @@ def test_check_opc_listenning_warning_notification(opc_manager):
@patch('ingestor.managers.opc_manager.metrics')
def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager):
@mark.asyncio
async def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager):
opc_manager.non_receive_count = 14
opc_manager.notification_handler.build_and_send_notification = MagicMock()
metrics.OPC_CYCLES_WITHOUT_DATA.reset_mock()
metrics.OPC_RECONNECTIONS_TOTAL.reset_mock()
result = opc_manager.check_opc_listenning()
result = await opc_manager.check_opc_listenning()
assert opc_manager.non_receive_count == 15
# Should be called twice: once for 5, once for 15
assert opc_manager.send_notification.call_count == 2
calls = opc_manager.send_notification.call_args_list
assert opc_manager.send_notification_async.call_count == 2
calls = opc_manager.send_notification_async.call_args_list
# First call: 5 cycles warning
assert calls[0].kwargs == {
'notification_id': f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
@@ -560,42 +602,30 @@ def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager)
}
assert result is True
metrics.OPC_CYCLES_WITHOUT_DATA.labels.assert_called_once_with(
pod_id=opc_manager.pod_id, server_name=opc_manager.name
)
metrics.OPC_CYCLES_WITHOUT_DATA.labels.return_value.set.assert_called_once_with(
opc_manager.non_receive_count
opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=opc_manager.non_receive_count,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
},
),
]
)
metrics.OPC_RECONNECTIONS_TOTAL.labels.assert_called_once_with(
pod_id=opc_manager.pod_id, server_name=opc_manager.name
opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=metrics.OPC_RECONNECTIONS_TOTAL,
method='inc',
value=1,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
},
),
]
)
metrics.OPC_RECONNECTIONS_TOTAL.labels.return_value.inc.assert_called_once()
@patch('ingestor.managers.opc_manager.metrics')
def test_init_metrics_calls_correct_metric_methods(metrics):
opc_manager = OpcManager(
name='TestInitConnector',
url='opc.tcp://init.test:4840',
data_manager=MagicMock(),
subscription_period_ms=1000,
logger=MagicMock(),
server_uri='opc.tcp://init.test:4840/uri',
notification_handler=MagicMock(),
metadata=metadata['metadata'],
)
metrics.OPC_CONNECTION_STATUS.labels.assert_called_with(
pod_id=opc_manager.pod_id,
server_name=opc_manager.name,
server_url=opc_manager.url,
)
metrics.OPC_CONNECTION_STATUS.labels.return_value.set.assert_called_once_with(0)
metrics.OPC_TAGS_SUBSCRIBED.labels.assert_called_once_with(
pod_id=opc_manager.pod_id, server_name=opc_manager.name
)
metrics.OPC_TAGS_SUBSCRIBED.labels.return_value.set.assert_called_once_with(0)