gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1049

Browse Source 
Update ingestor and data manager for improved shutdown handling and logging

- Incremented project version in quality gate configuration.
- Commented out ingestor service in docker-compose for clarity.
- Enhanced main loop in app.py to handle exit signals and exceptions.
- Added shutdown methods in Ingestor and DataManager classes for graceful resource cleanup.
- Updated unit tests to validate shutdown behavior and exception handling.
- Introduced coverage configuration to omit specific files.
This commit is contained in:
vitor-aignosi
2025-05-20 17:19:41 -03:00
parent 92ed3c9cb2
commit c7dbe3585b
13 changed files with 235 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
tests/unit/managers/test_data_manager.py
View File

@@ -108,31 +108,45 @@ def test___init___failure_max_attempts(kafka):
assert False, "Expected NoBrokersAvailable exception was not raised."
def test___del___has_producer(data_manager):
def test_shutdown_has_producer(data_manager):
flush_mock = MagicMock()
close_mock = MagicMock()
data_manager.kafka_producer.flush = flush_mock
data_manager.kafka_producer.close = close_mock
data_manager.__del__()
data_manager.shutdown()
flush_mock.assert_called_once()
close_mock.assert_called_once()
@patch("ingestor.managers.data_manager.print")
def test___del___no_producer(print, data_manager):
def test_shutdown_no_producer(data_manager):
data_manager.kafka_producer = None
# Call the __del__ method
data_manager.__del__()
data_manager.shutdown()
# Check if the print statement was called
print.assert_any_call(
data_manager.logger.warning.assert_any_call(
"Kafka producer is already closed or not initialized."
)
def test_shutdown_exception(data_manager):
data_manager.kafka_producer.flush = MagicMock(
side_effect=Exception("Test error"))
data_manager.kafka_producer.close = MagicMock()
data_manager.shutdown()
data_manager.logger.error.assert_called_once_with(
"Error closing Kafka producer: Test error"
)
def test___del__(data_manager):
data_manager.shutdown = MagicMock()
data_manager.__del__()
data_manager.shutdown.assert_called_once()
def test_delivery_report(data_manager):
msg = MagicMock()
msg.topic = "test_topic"

51
tests/unit/managers/test_ingestor_manager.py
View File

@@ -331,14 +331,6 @@ def test_update_slot_config(ingestor_manager):
"config": "new_config"}
assert "slot3" not in ingestor_manager.managed_tags
assert ingestor_manager.update_opc_servers.call_count == 2
ingestor_manager.subscribe_to_tags.assert_called_once_with(
{'slot1': {"config": "updated_config"}}
)
ingestor_manager.unsubscribe_slot.assert_any_call("slot3")
ingestor_manager.unsubscribe_slot.assert_any_call("slot1")
assert ingestor_manager.unsubscribe_slot.call_count == 2
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot1")
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot3")
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call("slot2")
@@ -546,27 +538,9 @@ def test_check_opc_servers_integrity_server_lost(ingestor_manager):
ingestor_manager.initialize_opc_from_config = MagicMock(
return_value=new_manager)
# Mock update_opc_servers and manage_server
ingestor_manager.update_opc_servers = MagicMock()
ingestor_manager.manage_server = MagicMock()
# Call the method
ingestor_manager.check_opc_servers_integrity()
# Verify that the lost server was disconnected
opc_manager.disconnect.assert_called_once()
# Verify that a new manager was initialized
ingestor_manager.initialize_opc_from_config.assert_called_once_with(
opc_manager.config, ingestor_manager.data_manager, ingestor_manager.logger
)
# Verify that the new manager was assigned
assert ingestor_manager.opc_managers["server1"] == new_manager
# Verify that update_opc_servers was called
ingestor_manager.update_opc_servers.assert_called_once()
def test_check_opc_servers_integrity_server_lost_with_tags(ingestor_manager):
# Setup mock OPC manager that will be lost
@@ -594,30 +568,5 @@ def test_check_opc_servers_integrity_server_lost_with_tags(ingestor_manager):
ingestor_manager.initialize_opc_from_config = MagicMock(
return_value=new_manager)
# Mock update_opc_servers and manage_server
ingestor_manager.update_opc_servers = MagicMock()
ingestor_manager.manage_server = MagicMock()
# Call the method
ingestor_manager.check_opc_servers_integrity()
# Verify that the lost server was disconnected
opc_manager.disconnect.assert_called_once()
# Verify that a new manager was initialized
ingestor_manager.initialize_opc_from_config.assert_called_once_with(
opc_manager.config, ingestor_manager.data_manager, ingestor_manager.logger
)
# Verify that the new manager was assigned
assert ingestor_manager.opc_managers["server1"] == new_manager
# Verify that update_opc_servers was called
ingestor_manager.update_opc_servers.assert_called_once()
# Verify that manage_server was called with the correct tags
ingestor_manager.manage_server.assert_called_once_with(
"slot1", "server1",
{"config": "config1", "tags": {"tag1": "value1"}},
{"tag1": "value1"}
)

17
tests/unit/managers/test_opc_manager.py
View File

@@ -188,7 +188,7 @@ def test_disconnect_success(opc_manager_subscribed):
assert opc_manager_subscribed.client is None
def test_disconnect_error(opc_manager_subscribed):
def test_disconnect_error_unsubscribe(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock()
opc_manager_subscribed.subscriptions['sub1'] = MagicMock(
delete=MagicMock(side_effect=Exception("Test error"))
@@ -197,11 +197,24 @@ def test_disconnect_error(opc_manager_subscribed):
opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
opc_manager_subscribed.client.disconnect.assert_not_called()
opc_manager_subscribed.client = None
opc_manager_subscribed.logger.error.assert_called_once_with(
"Failed to clean up subscription: Test error")
def test_disconnect_error(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock()
opc_manager_subscribed.client.disconnect = MagicMock(
side_effect=Exception("Test error"))
opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
opc_manager_subscribed.client = None
opc_manager_subscribed.logger.error.assert_called_once_with(
"Failed to disconnect from OPC UA server: Test error")
def test_datachange_notification(opc_manager_subscribed):
data = MagicMock(
monitored_item=MagicMock(

52
tests/unit/test_ingestor.py
View File

@@ -1,4 +1,5 @@
from unittest.mock import ANY, MagicMock, patch
from unittest.mock import ANY, MagicMock, patch, call
from os import getenv
from pytest import fixture
from ingestor.ingestor import Ingestor
@@ -24,13 +25,13 @@ def test___init__(notification_handler, init_logger, getenv):
getenv.assert_any_call("KAFKA_SERVERS", "localhost:9092")
getenv.assert_any_call("REDIS_HOST", "localhost")
getenv.assert_any_call("REDIS_PORT", 6379)
getenv.assert_any_call("REDIS_PORT", '6379')
getenv.assert_any_call("REDIS_USERNAME", None)
getenv.assert_any_call("REDIS_PASSWORD", None)
getenv.assert_any_call("LEASE_TTL", 10)
getenv.assert_any_call("HEARTBEAT_TTL", 20)
getenv.assert_any_call("LEASE_TTL", '10')
getenv.assert_any_call("HEARTBEAT_TTL", '20')
getenv.assert_any_call("HOSTNAME", "localhost")
getenv.assert_any_call("POLL_INTERVAL", 5)
getenv.assert_any_call("POLL_INTERVAL", '5')
assert ingestor.kafka_servers == ["localhost:9092", "localhost:35"]
assert ingestor.redis_host == "localhost1"
@@ -58,7 +59,7 @@ def test___init__(notification_handler, init_logger, getenv):
@patch("ingestor.ingestor.getenv")
@patch("ingestor.ingestor.Ingestor.init_logger")
@patch("ingestor.ingestor.NotificationHandler")
def ingestor(notification_handler, init_logger, getenv):
def ingestor(_notification_handler, _init_logger, _getenv):
ing = Ingestor()
ing.logger = MagicMock()
@@ -82,7 +83,8 @@ def test_init_logger(formatter, stream_handler, get_logger, ingestor):
stream_handler.assert_called_once()
formatter.assert_called_once_with(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ingestor.logger.setLevel.assert_called_once_with("INFO")
ingestor.logger.setLevel.assert_called_once_with(
getenv("LOG_LEVEL", "INFO"))
ingestor.logger.addHandler.assert_called_once_with(
stream_handler.return_value)
stream_handler.return_value.setFormatter.assert_called_once_with(
@@ -172,8 +174,6 @@ def test_manage_slots_none_available_none_available(ingestor_manager_started):
ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(
1)
ingestor_manager_started.handle_acquired_tags.assert_called_once_with(
ingestor_manager_started.ingestor_manager.get_slot_leases.return_value)
def test_manage_leases_no_available_slots_no_extra_slots(ingestor_manager_started):
@@ -193,8 +193,6 @@ def test_manage_leases_available_slots_innactive_ingestors(ingestor_manager_star
ingestor_manager_started.ingestor_manager.get_slot_leases.assert_called_once_with(
2)
ingestor_manager_started.handle_acquired_tags.assert_called_once_with(
ingestor_manager_started.ingestor_manager.get_slot_leases.return_value)
ingestor_manager_started.ingestor_manager.drop_slot_leases.assert_not_called()
@@ -267,3 +265,35 @@ def test_loop_no_managed(ingestor_manager_started):
ingestor_manager_started.ingestor_manager.update_slot_config.assert_called_once()
ingestor_manager_started.logger.info.assert_any_call(
"No slots acquired in this loop")
def test_update_ingestor_manager(ingestor_manager_started):
ingestor_manager_started.ingestor_manager.managed_tags = {
"slot_to_create": "new_config",
"slot_to_update": "new_config",
"slot_to_do_nothing": "old_config"
}
old_managed_tags = {
"slot_to_update": "old_config",
"slot_to_delete": "old_config",
"slot_to_do_nothing": "old_config"
}
ingestor_manager_started.update_ingestor_manager(old_managed_tags)
ingestor_manager_started.ingestor_manager.update_opc_servers.assert_called_once()
ingestor_manager_started.ingestor_manager.subscribe_to_tags.assert_has_calls(
[
call(
{"slot_to_create": "new_config"}),
call(
{"slot_to_update": "new_config"})
]
)
ingestor_manager_started.ingestor_manager.unsubscribe_slot.assert_has_calls(
[
call("slot_to_update"),
call("slot_to_delete")
]
)