SIENTIAPDE-988

Implement initial structure for data ingestion with Kafka and OPC UA integration

- Added DataManager for Kafka message handling
- Introduced IngestorManager to manage data ingestion processes
- Created OpcManager for OPC UA client interactions
- Developed ResourceManager for Redis-based resource management
- Established testing framework with unit tests for DataManager and OpcManager
- Configured project settings for Python testing with pytest

tests/managers/test_data_manager.py

@@ -0,0 +1,68 @@
+from unittest.mock import MagicMock, patch
+from pytest import fixture
+
+from ingestor.managers.data_manager import DataManager
+
+
+@fixture
+@patch("ingestor.managers.data_manager.KafkaProducer")
+def data_manager(kafka):
+    return DataManager(
+        kafka_servers="localhost:9092",
+        logger=MagicMock()
+    )
+
+
+def test___del__(data_manager):
+    flush_mock = MagicMock()
+    close_mock = MagicMock()
+
+    data_manager.kafka_producer.flush = flush_mock
+    data_manager.kafka_producer.close = close_mock
+
+    data_manager.__del__()
+    flush_mock.assert_called_once()
+    close_mock.assert_called_once()
+
+
+def test_delivery_report(data_manager):
+    msg = MagicMock()
+    msg.topic = "test_topic"
+    msg.partition = 0
+    msg.offset = 1
+
+    data_manager.delivery_report(msg)
+
+    data_manager.logger.info.assert_called_once_with(
+        f"Record successfully produced to {msg.topic} [{msg.partition}] at offset {msg.offset}"
+    )
+
+
+def test_delivery_error(data_manager):
+    err = "Test error"
+    data_manager.delivery_error(err)
+
+    data_manager.logger.error.assert_called_once_with(
+        f"Delivery failed for record : {err}"
+    )
+
+
+def test_publish(data_manager):
+    topic = "test_topic"
+    data = {"key": "value"}
+
+    # Mock the send method of the Kafka producer
+    send_mock = MagicMock()
+    data_manager.kafka_producer.send = send_mock
+
+    # Call the publish method
+    data_manager.publish(topic, data)
+
+    # Check if the send method was called with the correct arguments
+    send_mock.assert_called_once_with(
+        topic=topic, value=data
+    )
+
+    send_mock.return_value.add_callback.assert_called_once()
+
+    data_manager.kafka_producer.flush.assert_called_once()

tests/managers/test_opc_manager.py

@@ -0,0 +1,316 @@
+import json
+from unittest.mock import MagicMock, patch
+from pytest import fixture
+
+from asyncua.crypto.security_policies import SecurityPolicyBasic256
+from ingestor.managers.opc_manager import OpcManager
+from sientia_do.notifications.models import NotificationLevel
+
+tags = {
+    'ns=3;i=1001': {
+        'aggregation_function': 'LTS',
+        'frequency': 1000,
+        'max_value': 100,
+        'min_value': 0,
+        'tag_name': 'Counter'
+    },
+    'ns=3;i=1003': {
+        'aggregation_function': 'AVG',
+        'frequency': 1000,
+        'max_value': 100,
+        'min_value': 0,
+        'tag_name': 'Random'
+    },
+    'ns=3;i=1004': {
+        'aggregation_function': 'MDN',
+        'frequency': 1000,
+        'max_value': 100,
+        'min_value': 0,
+        'tag_name': 'Sawtooth'
+    }
+}
+
+
+@fixture
+def raw_opc_manager():
+    return OpcManager(
+        'TestConnector', 'opc.tcp://localhost:4840', MagicMock(),
+        MagicMock(), 'opc.tcp://localhost:4840'
+    )
+
+
+@fixture
+def opc_manager(raw_opc_manager):
+    raw_opc_manager.client = MagicMock()
+    raw_opc_manager.cert_path = 'cert.pem'
+    raw_opc_manager.private_key_path = 'private_key.pem'
+    raw_opc_manager.server_cert_path = 'server_cert.pem'
+
+    return raw_opc_manager
+
+
+@fixture
+def opc_manager_subscribed(opc_manager):
+    opc_manager.subscription = MagicMock()
+
+    return opc_manager
+
+
+def test_set_security_success(opc_manager):
+    opc_manager.set_security()
+
+    assert opc_manager.client.application_uri == opc_manager.server_uri
+
+    opc_manager.client.set_security.assert_called_once_with(
+        SecurityPolicyBasic256,
+        certificate=opc_manager.cert_path,
+        private_key=opc_manager.private_key_path,
+        server_certificate=opc_manager.server_cert_path
+    )
+
+    assert opc_manager.client.secure_channel_timeout == 10000000
+    assert opc_manager.client.session_timeout == 10000000
+
+
+def test_set_security_no_cert(opc_manager):
+    opc_manager.cert_path = None
+    opc_manager.private_key_path = None
+
+    try:
+        opc_manager.set_security()
+    except ValueError as e:
+        assert str(
+            e) == "Certificate and private key paths must be provided for secure connection."
+    else:
+        assert False, "ValueError not raised"
+
+    assert opc_manager.client.set_security.call_count == 0
+
+
+@patch('ingestor.managers.opc_manager.Client')
+def test_connect_no_security(client, raw_opc_manager):
+    raw_opc_manager.set_security = MagicMock()
+
+    raw_opc_manager.connect()
+
+    client.assert_called_once_with(raw_opc_manager.url)
+    raw_opc_manager.client.connect.assert_called_once()
+    raw_opc_manager.set_security.assert_not_called()
+
+
+@patch('ingestor.managers.opc_manager.Client')
+def test_connect_with_security(client, raw_opc_manager):
+    raw_opc_manager.cert_path = 'cert.pem'
+    raw_opc_manager.private_key_path = 'private_key.pem'
+    raw_opc_manager.server_cert_path = 'server_cert.pem'
+    raw_opc_manager.set_security = MagicMock()
+
+    raw_opc_manager.connect()
+
+    client.assert_called_once_with(raw_opc_manager.url)
+    raw_opc_manager.client.connect.assert_called_once()
+    raw_opc_manager.set_security.assert_called_once()
+
+
+def test_create_subscription_no_client(raw_opc_manager):
+    try:
+        raw_opc_manager.create_subscription()
+    except ValueError as e:
+        assert str(e) == "Client not connected. Call connect first."
+    else:
+        assert False, "ValueError not raised"
+
+
+def test_create_subscription_success_has_period(opc_manager):
+    opc_manager.create_subscription(1000)
+
+    assert opc_manager.period == 1000
+    opc_manager.client.create_subscription.assert_called_once_with(
+        1000, opc_manager)
+    assert opc_manager.subscription is not None
+
+
+def test_create_subscription_success_no_period(opc_manager):
+    opc_manager.create_subscription(None)
+
+    assert opc_manager.period == 500
+    opc_manager.client.create_subscription.assert_called_once_with(
+        500, opc_manager)
+    assert opc_manager.subscription is not None
+
+
+def test_subscribe_no_subscription(opc_manager):
+    try:
+        opc_manager.subscribe(tags, 1000)
+    except ValueError as e:
+        assert str(
+            e) == "Subscription not created. Call create_subscription first."
+    else:
+        assert False, "ValueError not raised"
+
+
+def test_subscribe_success(opc_manager_subscribed):
+    opc_manager_subscribed.nodes = {
+        'ns=3;i=1001': 'data'
+    }
+    opc_manager_subscribed.subscribe(tags, 1000)
+
+    assert opc_manager_subscribed.nodes == tags
+    assert opc_manager_subscribed.addr_nodes == [
+        opc_manager_subscribed.client.get_node(n) for n in tags if n != 'ns=3;i=1001']
+
+
+def test_disconnect_success(opc_manager_subscribed):
+    opc_manager_subscribed.client = MagicMock()
+    opc_manager_subscribed.subscription = MagicMock()
+
+    opc_manager_subscribed.disconnect()
+
+    opc_manager_subscribed.subscription.delete.assert_called_once()
+    opc_manager_subscribed.client.disconnect.assert_called_once()
+
+
+def test_disconnect_error(opc_manager_subscribed):
+    opc_manager_subscribed.client = MagicMock()
+    opc_manager_subscribed.subscription = MagicMock(
+        delete=MagicMock(side_effect=Exception("Test error"))
+    )
+
+    opc_manager_subscribed.disconnect()
+
+    opc_manager_subscribed.subscription.delete.assert_called_once()
+    opc_manager_subscribed.client.disconnect.assert_not_called()
+    opc_manager_subscribed.logger.error.assert_called_once_with(
+        "Failed to clean up subscription: Test error")
+
+
+def test_init_collector(opc_manager_subscribed):
+    opc_manager_subscribed.connect = MagicMock()
+    opc_manager_subscribed.create_subscription = MagicMock()
+    opc_manager_subscribed.subscribe = MagicMock()
+
+    opc_manager_subscribed.init_collector(tags, 1000, 500)
+
+    opc_manager_subscribed.connect.assert_called_once()
+    opc_manager_subscribed.create_subscription.assert_called_once_with(500)
+    opc_manager_subscribed.subscribe.assert_called_once_with(tags, 1000)
+
+
+def test_datachange_notification(opc_manager_subscribed):
+    data = MagicMock(
+        monitored_item=MagicMock(
+            Value=MagicMock(
+                Value=MagicMock(Value=42),
+                SourceTimestamp='2021-01-01T00:00:00'
+
+            )))
+    opc_manager_subscribed.nodes = {
+        'ns=3;i=1001': {
+            'tag_name': 'Counter',
+            'cycle_rule': {
+                'cycle_increment': 1.0,
+                'cycle_count': 2
+            },
+            'topics': ['topic1', 'topic2']
+        }
+    }
+
+    opc_manager_subscribed.datachange_notification(
+        'ns=3;i=1001', None, data)
+
+    opc_manager_subscribed.data_manager.publish.assert_any_call(
+        'topic1', {
+            'tag': 'ns=3;i=1001',
+            'name': 'Counter',
+            'timestamp': '2021-01-01T00:00:00',
+            'value': 42
+        })
+    opc_manager_subscribed.data_manager.publish.assert_any_call(
+        'topic2', {
+            'tag': 'ns=3;i=1001',
+            'name': 'Counter',
+            'timestamp': '2021-01-01T00:00:00',
+            'value': 42
+        })
+    assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0
+
+
+def test_check_cycles(opc_manager_subscribed):
+    handler = MagicMock()
+    opc_manager_subscribed.nodes = tags
+    opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule'] = {
+        'cycle_increment': 1.0,
+        'cycle_count': 0
+    }
+    opc_manager_subscribed.check_cycles(
+        {}, 'ns=3;i=1001', tags['ns=3;i=1001'], handler)
+
+    assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 1
+
+    opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] = 0
+    opc_manager_subscribed.check_cycles({'ns=3;i=1001': {}},
+                                        'ns=3;i=1001', tags['ns=3;i=1001'], handler)
+
+    assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0
+
+    opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] = 4
+    opc_manager_subscribed.check_cycles(
+        {}, 'ns=3;i=1001', tags['ns=3;i=1001'], handler)
+
+    assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 5
+    handler.assert_called_once_with('5.0 cycles without receive from ns=3;i=1001:Counter',
+                                    'TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED', NotificationLevel.WARNING)
+
+
+def test_check_opc_listenning_5_cycles(opc_manager_subscribed):
+
+    handler = MagicMock()
+    opc_manager_subscribed.nodes = tags
+    opc_manager_subscribed.non_receive_count = 4
+
+    opc_manager_subscribed.check_opc_listenning(handler)
+
+    handler.assert_called_once_with(
+        f'5 cycles without receive from OPC TestConnector. Tags: {json.dumps(tags)}',
+        'OPC_LISTENNING_STOPPED__TestConnector',
+        NotificationLevel.ERROR
+    )
+
+
+def test_check_opc_listenning_no_cycles(opc_manager_subscribed):
+    handler = MagicMock()
+    opc_manager_subscribed.non_receive_count = 0
+
+    opc_manager_subscribed.check_opc_listenning(handler)
+
+    handler.assert_not_called()
+
+
+def test_check_opc_listenning_no_handler(opc_manager_subscribed):
+    opc_manager_subscribed.non_receive_count = 5
+    opc_manager_subscribed.check_opc_listenning(None)
+
+    assert opc_manager_subscribed.non_receive_count == 6
+
+
+def test_check_opc_listenning_15_cycles(opc_manager_subscribed):
+    handler = MagicMock()
+    opc_manager_subscribed.init_collector = MagicMock()
+    opc_manager_subscribed.non_receive_count = 14
+    opc_manager_subscribed.collect_period = 1000
+    opc_manager_subscribed.period = 500
+    opc_manager_subscribed.nodes = tags
+
+    opc_manager_subscribed.check_opc_listenning(handler)
+    handler.assert_any_call(
+        f'15 cycles without receive from OPC TestConnector. Tags: {json.dumps(tags)}',
+        'OPC_LISTENNING_STOPPED__TestConnector',
+        NotificationLevel.ERROR
+    )
+    handler.assert_any_call(
+        'Retrying to connect to server TestConnector',
+        'OPC_CONNECTION_RETRY__TestConnector',
+        NotificationLevel.ERROR
+    )
+    opc_manager_subscribed.init_collector.assert_called_once_with(
+        tags, 1000, 500)

tests/managers/test_resource_manager.py

@@ -0,0 +1,73 @@
+from unittest.mock import MagicMock, patch
+from pytest import fixture
+from ingestor.managers.resource_manager import ResourceManager
+
+
+@fixture
+@patch('ingestor.managers.resource_manager.Redis')
+def resource_manager(redis):
+
+    return ResourceManager(
+        'localhost', 6379, 10, 10, 'pod_id'
+    )
+
+
+def test_get_success(resource_manager):
+    resource_manager.redis.get.return_value = '{"key": "value"}'
+    result = resource_manager.get('key')
+    assert result == {"key": "value"}
+    resource_manager.redis.get.assert_called_once_with('key')
+
+
+def test_get_failure(resource_manager):
+    resource_manager.redis.get.return_value = None
+    result = resource_manager.get('key')
+    assert result is None
+    resource_manager.redis.get.assert_called_once_with('key')
+
+
+def test_get_tag_slot(resource_manager):
+    resource_manager.get = MagicMock(return_value={"tag": "slot"})
+    result = resource_manager.get_tag_slot('id')
+    assert result == {"tag": "slot"}
+    resource_manager.get.assert_called_once_with('slot:opc_tags:id')
+
+
+def test_ingestor_heartbeat(resource_manager):
+    resource_manager.redis.set.return_value = True
+    resource_manager.ingestor_heartbeat()
+    resource_manager.redis.set.assert_called_once_with(
+        'heartbeat:ingestor:pod_id', 1, ex=10
+    )
+
+
+def test_lease_tag(resource_manager):
+    resource_manager.redis.set.return_value = True
+    resource_manager.lease_tag('tag_id')
+    resource_manager.redis.set.assert_called_once_with(
+        'lease:opc_tags:tag_id', 'pod_id', nx=True, ex=10
+    )
+
+
+def test_renew_tag_lease_success(resource_manager):
+    resource_manager.redis.get.return_value = 'pod_id'
+    resource_manager.redis.expire.return_value = True
+    result = resource_manager.renew_tag_lease('tag_id')
+    assert result is True
+    resource_manager.redis.get.assert_called_once_with(
+        'lease:opc_tags:tag_id'
+    )
+    resource_manager.redis.expire.assert_called_once_with(
+        'lease:opc_tags:tag_id', 10
+    )
+
+
+def test_renew_tag_lease_failure(resource_manager):
+    resource_manager.redis.get.return_value = 'other_pod_id'
+    resource_manager.redis.expire.return_value = False
+    result = resource_manager.renew_tag_lease('tag_id')
+    assert result is False
+    resource_manager.redis.get.assert_called_once_with(
+        'lease:opc_tags:tag_id'
+    )
+    resource_manager.redis.expire.assert_not_called()