da079c6320
Refactor ingestor lease management and update Docker configurations - Updated CMD in Dockerfile to use the correct application entry point. - Added docker compose command to remove volumes in README. - Removed unnecessary restart policy from ingestor service in docker-compose.yaml. - Refactored manage_leases method in Ingestor class to improve clarity and parameter naming. - Added get_number_of_leases method in IngestorManager to retrieve active leases. - Implemented get_all_leases method in ResourceManager to fetch active leases from Redis. - Enhanced unit tests for lease management in test_ingestor_manager and test_resource_manager.
478 lines
16 KiB
Python
478 lines
16 KiB
Python
from unittest.mock import MagicMock, patch
|
|
from pytest import fixture
|
|
|
|
from ingestor.managers.ingestor_manager import IngestorManager
|
|
|
|
|
|
@fixture
|
|
@patch('ingestor.managers.ingestor_manager.DataManager')
|
|
@patch('ingestor.managers.ingestor_manager.ResourceManager')
|
|
def ingestor_manager(data_manager_mock, resource_manager_mock):
|
|
return IngestorManager(
|
|
kafka_servers="localhost:9092",
|
|
redis_host="localhost",
|
|
redis_port=6379,
|
|
lease_ttl=60,
|
|
heartbeat_ttl=60,
|
|
pod_id="test_pod",
|
|
poll_interval=5,
|
|
logger=MagicMock()
|
|
)
|
|
|
|
|
|
@patch('ingestor.managers.ingestor_manager.OpcManager')
|
|
@patch('ingestor.managers.ingestor_manager.DataManager')
|
|
@patch('ingestor.managers.ingestor_manager.ResourceManager')
|
|
def test___init__(resource_manager_mock, data_manager_mock, opc_manager_mock):
|
|
|
|
ingestor = IngestorManager(
|
|
kafka_servers="localhost:9092",
|
|
redis_host="localhost",
|
|
redis_port=6379,
|
|
lease_ttl=60,
|
|
heartbeat_ttl=60,
|
|
pod_id="test_pod",
|
|
poll_interval=5,
|
|
logger=MagicMock()
|
|
)
|
|
|
|
opc_manager_mock.assert_not_called()
|
|
data_manager_mock.assert_called_once_with(
|
|
"localhost:9092", ingestor.logger)
|
|
resource_manager_mock.assert_called_once_with(
|
|
"localhost", 6379, 60, 60, "test_pod")
|
|
|
|
assert ingestor.poll_interval == 5
|
|
assert ingestor.managed_tags == {}
|
|
assert ingestor.opc_servers == {}
|
|
assert ingestor.opc_managers == {}
|
|
assert ingestor.data_manager == data_manager_mock.return_value
|
|
assert ingestor.resource_manager == resource_manager_mock.return_value
|
|
|
|
|
|
@patch('ingestor.managers.ingestor_manager.OpcManager')
|
|
def test_initialize_opc_from_config(opc_manager, ingestor_manager):
|
|
server_config = {
|
|
'name': 'server1',
|
|
'url': 'opc.tcp://localhost:4840',
|
|
'server_uri': 'http://opcua-server.simulator',
|
|
'cert_path': '/path/to/cert',
|
|
'private_key_path': '/path/to/private_key',
|
|
'server_cert_path': '/path/to/server_cert'
|
|
}
|
|
|
|
opc_manager.return_value = MagicMock()
|
|
result = ingestor_manager.initialize_opc_from_config(
|
|
server_config, ingestor_manager.data_manager, ingestor_manager.logger)
|
|
|
|
opc_manager.assert_called_once_with(
|
|
server_config['name'], server_config['url'], ingestor_manager.data_manager, ingestor_manager.logger,
|
|
server_config['server_uri'], server_config['cert_path'], server_config['private_key_path'],
|
|
server_config['server_cert_path']
|
|
)
|
|
|
|
assert result == opc_manager.return_value
|
|
result.connect.assert_called_once()
|
|
|
|
|
|
@patch('ingestor.managers.ingestor_manager.OpcManager')
|
|
def test_initialize_opc_from_config_exception(opc_manager, ingestor_manager):
|
|
server_config = {
|
|
'name': 'server1',
|
|
'url': 'opc.tcp://localhost:4840',
|
|
'server_uri': 'http://opcua-server.simulator',
|
|
'cert_path': '/path/to/cert',
|
|
'private_key_path': '/path/to/private_key',
|
|
'server_cert_path': '/path/to/server_cert'
|
|
}
|
|
|
|
ingestor_manager.logger.error = MagicMock()
|
|
opc_manager.side_effect = Exception("Initialization error")
|
|
|
|
result = ingestor_manager.initialize_opc_from_config(
|
|
server_config, ingestor_manager.data_manager, ingestor_manager.logger)
|
|
|
|
assert result is None
|
|
ingestor_manager.logger.error.assert_called_once_with(
|
|
"Failed to initialize OpcManager: Initialization error")
|
|
|
|
|
|
@patch('ingestor.managers.ingestor_manager.OpcManager')
|
|
def test_update_opc_servers(opc_manager, ingestor_manager):
|
|
manager1 = MagicMock(
|
|
config={"config": "config1"})
|
|
manager2 = MagicMock(
|
|
config={"config": "config2"})
|
|
manager3 = MagicMock(
|
|
config={"config": "config3"})
|
|
|
|
def mock_initialize_from_config(config, data_manager, logger):
|
|
if config == {"config": "config1"}:
|
|
return manager1
|
|
elif config == {"config": "config2"}:
|
|
return manager2
|
|
elif config == {"config": "config3"}:
|
|
return manager3
|
|
else:
|
|
return None
|
|
|
|
ingestor_manager.initialize_opc_from_config = MagicMock(
|
|
side_effect=mock_initialize_from_config
|
|
)
|
|
|
|
ingestor_manager.managed_tags = {
|
|
"slot1": {
|
|
"server1": {"config": "config1"},
|
|
"server2": {"config": "config2"},
|
|
'server5': {"config": "config5"}
|
|
},
|
|
"slot2": {
|
|
"server3": {"config": "config3"},
|
|
"server1": {"config": "config1"}
|
|
}
|
|
}
|
|
|
|
mock = MagicMock(
|
|
config={"config": "old_config2"})
|
|
ingestor_manager.opc_managers['server3'] = MagicMock(
|
|
config={"config": "config3"})
|
|
ingestor_manager.opc_managers['server2'] = mock
|
|
ingestor_manager.opc_managers['server4'] = MagicMock()
|
|
|
|
ingestor_manager.update_opc_servers()
|
|
|
|
assert len(ingestor_manager.opc_managers) == 3
|
|
|
|
ingestor_manager.initialize_opc_from_config.assert_any_call(
|
|
{"config": "config1"}, ingestor_manager.data_manager, ingestor_manager.logger)
|
|
ingestor_manager.initialize_opc_from_config.assert_any_call(
|
|
{"config": "config2"}, ingestor_manager.data_manager, ingestor_manager.logger)
|
|
ingestor_manager.initialize_opc_from_config.assert_any_call(
|
|
{"config": "config5"}, ingestor_manager.data_manager, ingestor_manager.logger)
|
|
assert ingestor_manager.initialize_opc_from_config.call_count == 3
|
|
|
|
assert ingestor_manager.opc_managers['server1'].config == {
|
|
"config": "config1"}
|
|
assert ingestor_manager.opc_managers['server2'].config == {
|
|
"config": "config2"}
|
|
assert ingestor_manager.opc_managers['server3'].config == {
|
|
"config": "config3"}
|
|
assert 'server4' not in ingestor_manager.opc_managers
|
|
assert 'server5' not in ingestor_manager.opc_managers
|
|
|
|
assert ingestor_manager.opc_managers['server2'] != mock
|
|
|
|
|
|
def test_declare_active(ingestor_manager):
|
|
ingestor_manager.resource_manager.ingestor_heartbeat = MagicMock()
|
|
ingestor_manager.declare_active()
|
|
ingestor_manager.resource_manager.ingestor_heartbeat.assert_called_once()
|
|
|
|
|
|
def test_get_active_ingestors(ingestor_manager):
|
|
ingestor_manager.resource_manager.get_all_ingestors = MagicMock()
|
|
ingestor_manager.get_active_ingestors()
|
|
ingestor_manager.resource_manager.get_all_ingestors.assert_called_once()
|
|
|
|
|
|
def test_get_active_ingestors_empty(ingestor_manager):
|
|
ingestor_manager.resource_manager.get_all_ingestors = MagicMock(
|
|
return_value=None)
|
|
result = ingestor_manager.get_active_ingestors()
|
|
assert result == []
|
|
ingestor_manager.resource_manager.get_all_ingestors.assert_called_once()
|
|
|
|
|
|
def test_get_number_of_leases_success(ingestor_manager):
|
|
ingestor_manager.resource_manager.get_all_leases = MagicMock(
|
|
return_value=["lease1", "lease2"])
|
|
result = ingestor_manager.get_number_of_leases()
|
|
assert result == 2
|
|
ingestor_manager.resource_manager.get_all_leases.assert_called_once()
|
|
|
|
|
|
def test_get_number_of_leases_empty(ingestor_manager):
|
|
ingestor_manager.resource_manager.get_all_leases = MagicMock(
|
|
return_value=None)
|
|
result = ingestor_manager.get_number_of_leases()
|
|
assert result == 0
|
|
ingestor_manager.resource_manager.get_all_leases.assert_called_once()
|
|
|
|
|
|
def test_get_number_of_slots_success(ingestor_manager):
|
|
ingestor_manager.resource_manager.get_all_slots = MagicMock(
|
|
return_value=["slot1", "slot2"])
|
|
result = ingestor_manager.get_number_of_slots()
|
|
assert result == 2
|
|
ingestor_manager.resource_manager.get_all_slots.assert_called_once()
|
|
|
|
|
|
def test_get_number_of_slots_empty(ingestor_manager):
|
|
ingestor_manager.resource_manager.get_all_slots = MagicMock(
|
|
return_value=None)
|
|
result = ingestor_manager.get_number_of_slots()
|
|
assert result == 0
|
|
ingestor_manager.resource_manager.get_all_slots.assert_called_once()
|
|
|
|
|
|
def test_get_slot_leases_1_success(ingestor_manager):
|
|
ingestor_manager.resource_manager.lease_tag = MagicMock(
|
|
return_value=True)
|
|
ingestor_manager.resource_manager.get_tag_slot = MagicMock(
|
|
return_value={"tags": ["tag1"]})
|
|
|
|
ingestor_manager.number_of_slots = 1
|
|
result = ingestor_manager.get_slot_leases()
|
|
|
|
assert result == {
|
|
"1": {"tags": ["tag1"]}
|
|
}
|
|
|
|
|
|
def test_get_slot_leases_2_success(ingestor_manager):
|
|
ingestor_manager.resource_manager.lease_tag = MagicMock(
|
|
side_effect=[True, True])
|
|
ingestor_manager.resource_manager.get_tag_slot = MagicMock(
|
|
side_effect=[{"tags": ["tag1"]}, {"tags": ["tag2"]}])
|
|
|
|
ingestor_manager.number_of_slots = 2
|
|
result = ingestor_manager.get_slot_leases(max_slots=2)
|
|
|
|
assert result == {
|
|
"1": {"tags": ["tag1"]},
|
|
"2": {"tags": ["tag2"]}
|
|
}
|
|
|
|
|
|
def test_get_slot_leases_2_1_none(ingestor_manager):
|
|
ingestor_manager.resource_manager.lease_tag = MagicMock(
|
|
side_effect=[True, True])
|
|
ingestor_manager.resource_manager.get_tag_slot = MagicMock(
|
|
side_effect=[None, {"tags": ["tag1"]}])
|
|
|
|
ingestor_manager.number_of_slots = 1
|
|
result = ingestor_manager.get_slot_leases(max_slots=1)
|
|
|
|
assert result == {}
|
|
|
|
|
|
def test_get_slot_leases_1_failure(ingestor_manager):
|
|
ingestor_manager.resource_manager.lease_tag = MagicMock(
|
|
return_value=False)
|
|
ingestor_manager.resource_manager.get_tag_slot = MagicMock(
|
|
return_value={"tags": ["tag1"]})
|
|
|
|
result = ingestor_manager.get_slot_leases()
|
|
ingestor_manager.resource_manager.get_tag_slot.assert_not_called()
|
|
|
|
assert result == {}
|
|
|
|
|
|
def test_unsubscribe_slot(ingestor_manager):
|
|
ingestor_manager.managed_tags = {
|
|
"slot1": {
|
|
"server1": {"tags": "config1"},
|
|
"server2": {"tags": "config2"}
|
|
},
|
|
"slot2": {
|
|
"server3": {"tags": "config3"},
|
|
"server1": {"tags": "config1"}
|
|
}
|
|
}
|
|
ingestor_manager.opc_managers = {
|
|
"server1": MagicMock(),
|
|
"server2": MagicMock(),
|
|
"server3": MagicMock()
|
|
}
|
|
ingestor_manager.unsubscribe_slot("slot1")
|
|
|
|
ingestor_manager.opc_managers["server1"].unsubscribe.assert_called_once_with(
|
|
"slot1")
|
|
ingestor_manager.opc_managers["server2"].unsubscribe.assert_called_once_with(
|
|
"slot1")
|
|
ingestor_manager.opc_managers["server3"].unsubscribe.assert_not_called()
|
|
|
|
|
|
def test_update_slot_config(ingestor_manager):
|
|
ingestor_manager.managed_tags = {
|
|
"slot1": {"config": "old_config"},
|
|
"slot2": {"config": "new_config"},
|
|
"slot3": {"config": "old_config"}
|
|
}
|
|
|
|
ingestor_manager.resource_manager.get_tag_slot = MagicMock(
|
|
side_effect=[
|
|
{"config": "updated_config"},
|
|
{"config": "new_config"},
|
|
None
|
|
]
|
|
)
|
|
|
|
ingestor_manager.update_opc_servers = MagicMock()
|
|
ingestor_manager.subscribe_to_tags = MagicMock()
|
|
ingestor_manager.unsubscribe_slot = MagicMock()
|
|
|
|
ingestor_manager.update_slot_config()
|
|
|
|
assert ingestor_manager.managed_tags["slot1"] == {
|
|
"config": "updated_config"}
|
|
assert ingestor_manager.managed_tags["slot2"] == {
|
|
"config": "new_config"}
|
|
assert "slot3" not in ingestor_manager.managed_tags
|
|
|
|
assert ingestor_manager.update_opc_servers.call_count == 2
|
|
ingestor_manager.subscribe_to_tags.assert_called_once_with(
|
|
{'slot1': {"config": "updated_config"}}
|
|
)
|
|
ingestor_manager.unsubscribe_slot.assert_any_call("slot3")
|
|
ingestor_manager.unsubscribe_slot.assert_any_call("slot1")
|
|
assert ingestor_manager.unsubscribe_slot.call_count == 2
|
|
|
|
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot1")
|
|
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot3")
|
|
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot2")
|
|
assert ingestor_manager.resource_manager.renew_tag_lease.call_count == 3
|
|
|
|
|
|
def test_drop_slot_leases(ingestor_manager):
|
|
ingestor_manager.resource_manager.drop_tag_lease = MagicMock()
|
|
ingestor_manager.drop_slot_leases(["1", "2"])
|
|
|
|
ingestor_manager.resource_manager.drop_tag_lease.assert_any_call("1")
|
|
ingestor_manager.resource_manager.drop_tag_lease.assert_any_call("2")
|
|
|
|
|
|
def test_manage_server_no_server(ingestor_manager):
|
|
ingestor_manager.opc_managers = {
|
|
"server1": MagicMock(),
|
|
"server2": MagicMock()
|
|
}
|
|
server_config = {
|
|
'tags': 'config1'
|
|
}
|
|
|
|
result = ingestor_manager.manage_server(
|
|
'slot1', 'server3', server_config, server_config)
|
|
|
|
assert result == 1
|
|
ingestor_manager.opc_managers["server1"].create_subscription.assert_not_called(
|
|
)
|
|
ingestor_manager.opc_managers["server1"].subscribe.assert_not_called()
|
|
|
|
|
|
def test_manage_server_create_subscription_failure(ingestor_manager):
|
|
ingestor_manager.opc_managers = {
|
|
"server1": MagicMock(),
|
|
"server2": MagicMock()
|
|
}
|
|
ingestor_manager.subscriptions = {
|
|
"server1": MagicMock()
|
|
}
|
|
server_config = {
|
|
'tags': 'config1'
|
|
}
|
|
|
|
ingestor_manager.opc_managers["server1"].create_subscription.side_effect = Exception(
|
|
"Subscription error")
|
|
|
|
result = ingestor_manager.manage_server(
|
|
'slot1', 'server1', server_config, server_config)
|
|
|
|
assert result == 2
|
|
ingestor_manager.opc_managers["server1"].create_subscription.assert_called_once_with(
|
|
'slot1')
|
|
ingestor_manager.opc_managers["server1"].subscribe.assert_not_called()
|
|
|
|
|
|
def test_manage_server(ingestor_manager):
|
|
ingestor_manager.opc_managers = {
|
|
"server1": MagicMock(),
|
|
"server2": MagicMock()
|
|
}
|
|
ingestor_manager.subscriptions = {
|
|
"server1": MagicMock()
|
|
}
|
|
server_config = {
|
|
'tags': 'config1'
|
|
}
|
|
|
|
result = ingestor_manager.manage_server(
|
|
'slot1', 'server1', server_config, server_config)
|
|
|
|
assert result == 0
|
|
ingestor_manager.opc_managers["server1"].create_subscription.assert_called_once_with(
|
|
'slot1')
|
|
ingestor_manager.opc_managers["server1"].subscribe.assert_called_once_with(
|
|
'slot1', 'config1', ingestor_manager.poll_interval)
|
|
|
|
|
|
def test_manage_server_subscribe_failure(ingestor_manager):
|
|
ingestor_manager.opc_managers = {
|
|
"server1": MagicMock(),
|
|
"server2": MagicMock()
|
|
}
|
|
ingestor_manager.subscriptions = {
|
|
"server1": MagicMock()
|
|
}
|
|
server_config = {
|
|
'tags': 'config1'
|
|
}
|
|
|
|
ingestor_manager.opc_managers["server1"].subscribe.side_effect = Exception(
|
|
"Subscription error")
|
|
|
|
result = ingestor_manager.manage_server(
|
|
'slot1', 'server1', server_config, server_config)
|
|
|
|
assert result == 2
|
|
ingestor_manager.opc_managers["server1"].create_subscription.assert_called_once_with(
|
|
'slot1')
|
|
ingestor_manager.opc_managers["server1"].subscribe.assert_called_once_with(
|
|
'slot1', 'config1', ingestor_manager.poll_interval)
|
|
ingestor_manager.opc_managers["server1"].unsubscribe.assert_called_once_with(
|
|
'slot1')
|
|
ingestor_manager.logger.error.assert_any_call(
|
|
"Failed to subscribe to tags from slot1:server1\n{'tags': 'config1'}: Subscription error"
|
|
)
|
|
ingestor_manager.logger.warning.assert_any_call(
|
|
"Removing subscription from server server1 for slot slot1"
|
|
)
|
|
|
|
|
|
def test_subscribe_to_tags(ingestor_manager):
|
|
ingestor_manager.manage_server = MagicMock(
|
|
side_effect=[0, 1, 2])
|
|
ingestor_manager.managed_tags = {
|
|
"slot1": MagicMock(),
|
|
"slot2": MagicMock()
|
|
}
|
|
|
|
ingestor_manager.opc_managers = {
|
|
"server1": MagicMock(),
|
|
"server2": MagicMock()
|
|
}
|
|
ingestor_manager.subscriptions = {
|
|
"server1": MagicMock()
|
|
}
|
|
tags = {
|
|
'slot1': {
|
|
"server1": {"tags": "config1"},
|
|
"server2": {"tags": "config2"},
|
|
'server3': {"tags": "config3"},
|
|
}
|
|
}
|
|
|
|
ingestor_manager.subscribe_to_tags(tags)
|
|
|
|
ingestor_manager.manage_server.assert_any_call(
|
|
'slot1', 'server1', {"tags": "config1"}, tags)
|
|
ingestor_manager.manage_server.assert_any_call(
|
|
'slot1', 'server2', {"tags": "config2"}, tags)
|
|
ingestor_manager.manage_server.assert_any_call(
|
|
'slot1', 'server3', {"tags": "config3"}, tags)
|
|
|
|
assert ingestor_manager.manage_server.call_count == 3
|
|
|
|
ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with(
|
|
'server3', None)
|