SIENTIAPDE-994

Refactor and enhance the laborious workflow and utilities

- Removed outdated test file `test_predictions_batch.py` from workflows.
- Added `input_sample.json` for standardized input configuration.
- Introduced `connectors_config.py` to manage database and service configurations.
- Implemented a logging utility in `logger.py` for consistent logging across the application.
- Created `policies.py` to define retry policies for workflows.
- Developed comprehensive tests for `MLFlowRepository` in `test_model_repository.py`.
- Added extensive tests for `OpcRepository` in `test_opc_repository.py`.
- Updated `test_predictions_batch.py` to reflect new workflow structure and testing methodology.

tests/laborious/utils/filters/repository/test_opc_repository.py

@@ -1,106 +0,0 @@
-from unittest.mock import Mock, patch, MagicMock
-from pathlib import Path
-from asyncua.sync import Client
-from asyncua.crypto.security_policies import SecurityPolicyBasic256
-from asyncua.ua import DataValue, Variant, VariantType
-from pytest import fixture
-from laborious.utils.repository.opc_repository import OpcRepository
-
-
-@fixture
-def mock_logger():
-    return Mock()
-
-
-@fixture
-def opc_repository(mock_logger):
-    return OpcRepository(
-        name="test_repo",
-        url="opc.tcp://localhost:4840",
-        logger=mock_logger,
-        server_uri="urn:test:server",
-        cert_path="/path/to/cert.pem",
-        private_key_path="/path/to/key.pem",
-        server_cert_path="/path/to/server_cert.pem"
-    )
-
-
-@fixture
-def mock_client():
-    with patch('laborious.utils.repository.opc_repository.Client') as mock:
-        client_instance = MagicMock()
-        mock.return_value = client_instance
-        yield client_instance
-
-
-def test_init(opc_repository):
-    assert opc_repository.name == "test_repo"
-    assert opc_repository.url == "opc.tcp://localhost:4840"
-    assert opc_repository.server_uri == "urn:test:server"
-    assert opc_repository.cert_path == "/path/to/cert.pem"
-    assert opc_repository.private_key_path == "/path/to/key.pem"
-    assert opc_repository.server_cert_path == "/path/to/server_cert.pem"
-    assert opc_repository.non_receive_count == 0
-    assert opc_repository.client is None
-
-
-def test_set_security(opc_repository, mock_client):
-    opc_repository.client = mock_client
-    opc_repository.set_security()
-
-    mock_client.application_uri = "urn:test:server"
-    mock_client.set_security.assert_called_once_with(
-        SecurityPolicyBasic256,
-        certificate="/path/to/cert.pem",
-        private_key="/path/to/key.pem",
-        server_certificate="/path/to/server_cert.pem"
-    )
-    assert mock_client.secure_channel_timeout == 10000000
-    assert mock_client.session_timeout == 10000000
-
-
-def test_set_security_missing_certificates(opc_repository):
-    opc_repository.cert_path = None
-    opc_repository.private_key_path = None
-
-    try:
-        opc_repository.set_security()
-    except ValueError as e:
-        assert str(
-            e) == "Certificate and private key paths must be provided for secure connection."
-
-
-def test_connect_with_security(opc_repository, mock_client):
-    opc_repository.connect()
-
-    mock_client.connect.assert_called_once()
-    assert opc_repository.client == mock_client
-
-
-def test_connect_without_security(opc_repository, mock_client):
-    opc_repository.cert_path = None
-    opc_repository.connect()
-
-    mock_client.connect.assert_called_once()
-    assert opc_repository.client == mock_client
-
-
-def test_disconnect(opc_repository, mock_client):
-    opc_repository.client = mock_client
-    opc_repository.disconnect()
-
-    mock_client.disconnect.assert_called_once()
-    assert opc_repository.client is None
-
-
-def test_write_data(opc_repository, mock_client, mock_logger):
-    opc_repository.client = mock_client
-    mock_node = MagicMock()
-    mock_client.get_node.return_value = mock_node
-
-    opc_repository.write_data("ns=2;s=TestNode", 42.0, "float", mock_logger)
-
-    mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
-    mock_node.write_value.assert_called_once()
-    mock_logger.info.assert_called_once_with(
-        "Writing 42.0 - <class 'float'> to " + str(mock_node))

tests/laborious/utils/filters/test_conditional_filters.py

@@ -7,18 +7,18 @@ def test_filter_specific_variables_null_values():
     assert filter_specific_variables_null_values(
         DataFrame(
             {'variable': ['variable1', 'variable2'], 'value': [1, 2]}),
-        config={'VARIABLES': ['variable2']}) == True
+        config={'VARIABLES': ['variable2']}) is False
 
 
 def test_filter_specific_variables_null_values_with_null_values():
     assert filter_specific_variables_null_values(
         DataFrame(
             {'variable': ['variable1', 'variable2'], 'value': [1, None]}),
-        config={'VARIABLES': ['variable2']}) == False
+        config={'VARIABLES': ['variable2']}) is True
 
 
 def test_filter_empty_data():
-    assert filter_empty_data(DataFrame(), {}) == True
+    assert filter_empty_data(DataFrame(), {}) is True
 
 
 def test_filter_empty_data_with_data():

tests/laborious/utils/repository/test_opc_repository.py

@@ -0,0 +1,242 @@
+from unittest.mock import Mock, patch, MagicMock, ANY, call
+from asyncua.crypto.security_policies import SecurityPolicyBasic256
+from pytest import fixture
+from laborious.utils.repository.opc_repository import OpcRepository
+from sientia_do.notifications.models import NotificationLevel
+from datetime import datetime
+
+
+@fixture
+def mock_logger():
+    return Mock()
+
+
+@fixture
+def opc_repository(mock_logger):
+    return OpcRepository(
+        name="test_repo",
+        url="opc.tcp://localhost:4840",
+        logger=mock_logger,
+        notification_handler=Mock(),
+        reconnection_interval=60,
+        server_uri="urn:test:server",
+        cert_path="/path/to/cert.pem",
+        private_key_path="/path/to/key.pem",
+        server_cert_path="/path/to/server_cert.pem"
+    )
+
+
+@fixture
+def mock_client():
+    with patch('laborious.utils.repository.opc_repository.Client') as mock:
+        client_instance = MagicMock()
+        mock.return_value = client_instance
+        yield client_instance
+
+
+def test_init(opc_repository):
+    assert opc_repository.name == "test_repo"
+    assert opc_repository.url == "opc.tcp://localhost:4840"
+    assert opc_repository.server_uri == "urn:test:server"
+    assert opc_repository.cert_path == "/path/to/cert.pem"
+    assert opc_repository.private_key_path == "/path/to/key.pem"
+    assert opc_repository.server_cert_path == "/path/to/server_cert.pem"
+    assert opc_repository.reconnection_interval == 60
+    assert opc_repository.client is None
+    assert opc_repository.last_reconnection_time is None
+    assert opc_repository.error_count == 0
+
+
+def test_set_security(opc_repository, mock_client):
+    opc_repository.client = mock_client
+    opc_repository.set_security()
+
+    mock_client.application_uri = "urn:test:server"
+    mock_client.set_security.assert_called_once_with(
+        SecurityPolicyBasic256,
+        certificate="/path/to/cert.pem",
+        private_key="/path/to/key.pem",
+        server_certificate="/path/to/server_cert.pem"
+    )
+    assert mock_client.secure_channel_timeout == 10000000
+    assert mock_client.session_timeout == 10000000
+
+
+def test_set_security_missing_certificates(opc_repository):
+    opc_repository.cert_path = None
+    opc_repository.private_key_path = None
+
+    try:
+        opc_repository.set_security()
+    except ValueError as e:
+        assert str(
+            e) == "Certificate and private key paths must be provided for secure connection."
+
+
+def test_connect_with_security(opc_repository, mock_client):
+    opc_repository.try_connect = MagicMock()
+    opc_repository.connect()
+
+    opc_repository.try_connect.assert_called_once()
+    assert opc_repository.client == mock_client
+
+
+def test_connect_without_security(opc_repository, mock_client):
+    opc_repository.cert_path = None
+    opc_repository.try_connect = MagicMock()
+    opc_repository.set_security = MagicMock()
+    opc_repository.connect()
+
+    opc_repository.try_connect.assert_called_once()
+    opc_repository.set_security.assert_not_called()
+    assert opc_repository.client == mock_client
+
+
+def test_try_connect_sucess(opc_repository):
+    opc_repository.last_reconnection_time = None
+    opc_repository.client = MagicMock()
+    opc_repository.try_connect()
+    opc_repository.client.connect.assert_called_once()
+    assert opc_repository.last_reconnection_time is not None
+
+
+def test_try_connect_fail(opc_repository):
+    opc_repository.last_reconnection_time = None
+    opc_repository.client = MagicMock()
+    opc_repository.client.connect.side_effect = Exception("Test error")
+
+    opc_repository.try_connect()
+
+    opc_repository.client.connect.assert_called_once()
+    assert opc_repository.last_reconnection_time is not None
+    opc_repository.notification_handler.build_and_send_notification.assert_called_once_with(
+        notification_id=f"OPC_CONNECTION_ERROR_{opc_repository.name}",
+        message="Failed to connect to OPC server: Test error",
+        block="opc_repository",
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY
+    )
+
+
+def test_disconnect(opc_repository, mock_client):
+    opc_repository.client = mock_client
+    opc_repository.disconnect()
+
+    mock_client.disconnect.assert_called_once()
+    assert opc_repository.client is None
+
+
+def test_validate_connection_none_client(opc_repository):
+    opc_repository.client = None
+    opc_repository.connect = MagicMock()
+    response = opc_repository.validate_connection()
+    assert response
+    opc_repository.connect.assert_called_once()
+
+
+def test_validate_connection_error_count_disconnect_error(opc_repository):
+    opc_repository.error_count = 6
+    opc_repository.client = MagicMock()
+    opc_repository.disconnect = MagicMock(side_effect=Exception("Test error"))
+    opc_repository.connect = MagicMock()
+
+    response = opc_repository.validate_connection()
+    assert response == opc_repository.connect.return_value
+    opc_repository.disconnect.assert_called_once()
+    opc_repository.connect.assert_called_once()
+    opc_repository.logger.error.assert_has_calls(
+        [
+            call("Failed to disconnect from OPC server: Test error"),
+        ]
+    )
+
+
+@patch('laborious.utils.repository.opc_repository.hasattr', return_value=True)
+@patch('laborious.utils.repository.opc_repository.datetime',
+       MagicMock(now=MagicMock(return_value=datetime(2025, 1, 1, 0, 0, 0))))
+def test_validate_connection_lost_not_time_to_reconect(_mock_datetime, opc_repository):
+    opc_repository.error_count = 0
+    opc_repository.client = MagicMock()
+    opc_repository.client.aio_obj.uaclient.protocol = None
+    opc_repository.last_reconnection_time = datetime(2025, 1, 1, 0, 0, 0)
+    opc_repository.try_connect = MagicMock()
+
+    response = opc_repository.validate_connection()
+    opc_repository.try_connect.assert_not_called()
+    assert response is False
+
+
+@patch('laborious.utils.repository.opc_repository.hasattr', return_value=True)
+@patch('laborious.utils.repository.opc_repository.datetime',
+       MagicMock(now=MagicMock(return_value=datetime(2025, 1, 1, 1, 0, 0))))
+def test_validate_connection_lost_time_to_reconect(_mock_datetime, opc_repository):
+    opc_repository.error_count = 0
+    opc_repository.client = MagicMock()
+    opc_repository.client.aio_obj.uaclient.protocol = None
+    opc_repository.last_reconnection_time = datetime(2025, 1, 1, 0, 0, 0)
+    opc_repository.try_connect = MagicMock()
+
+    response = opc_repository.validate_connection()
+    opc_repository.try_connect.assert_called_once()
+    assert response == opc_repository.try_connect.return_value
+
+
+def test_write_data_validate_connection_failed(opc_repository):
+    opc_repository.validate_connection = MagicMock(return_value=False)
+    opc_repository.client = MagicMock()
+    opc_repository.write_data("ns=2;s=TestNode", 42.0, "float")
+    opc_repository.validate_connection.assert_called_once()
+    opc_repository.client.get_node.assert_not_called()
+
+
+def test_write_data_get_node_failed(opc_repository):
+    opc_repository.validate_connection = MagicMock(return_value=True)
+    opc_repository.client = MagicMock()
+    opc_repository.error_count = 0
+    opc_repository.client.get_node.side_effect = Exception("Test error")
+    opc_repository.write_data("ns=2;s=TestNode", 42.0, "float")
+    opc_repository.validate_connection.assert_called_once()
+    opc_repository.client.get_node.assert_called_once_with("ns=2;s=TestNode")
+    opc_repository.notification_handler.build_and_send_notification.assert_called_once_with(
+        notification_id=f"OPC_WRITE_GET_NODE_ERROR_{opc_repository.name}",
+        message="Failed to get node from OPC server: Test error",
+        block="opc_repository",
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY
+    )
+    assert opc_repository.error_count == 1
+
+
+def test_write_data(opc_repository, mock_client):
+    opc_repository.validate_connection = MagicMock(return_value=True)
+    opc_repository.client = mock_client
+    mock_node = MagicMock()
+    mock_client.get_node.return_value = mock_node
+
+    opc_repository.write_data("ns=2;s=TestNode", 42.0, "float")
+
+    mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
+    mock_node.write_value.assert_called_once()
+    opc_repository.logger.info.assert_called_once_with(
+        "Writing 42.0 - <class 'float'> to " + str(mock_node))
+
+
+def test_write_data_write_value_failed(opc_repository, mock_client):
+    opc_repository.validate_connection = MagicMock(return_value=True)
+    opc_repository.client = mock_client
+    mock_node = MagicMock()
+    opc_repository.error_count = 0
+    mock_client.get_node.return_value = mock_node
+    mock_node.write_value.side_effect = Exception("Test error")
+    opc_repository.write_data("ns=2;s=TestNode", 42.0, "float")
+    opc_repository.validate_connection.assert_called_once()
+    mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
+    mock_node.write_value.assert_called_once()
+    opc_repository.notification_handler.build_and_send_notification.assert_called_once_with(
+        notification_id=f"OPC_WRITE_DATA_ERROR_{opc_repository.name}",
+        message="Failed to write data to OPC server: Test error",
+        block="opc_repository",
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY
+    )
+    assert opc_repository.error_count == 1