SIENTIAPDE-1243: Remove OPC server integration and add code quality tools.
This commit removes the OPC server integration from the Model Manager, including related activities, repositories, metrics, and configuration. It also adds code quality tools such as Ruff (linting/formatting), mypy (type checking), and Bandit (security analysis) along with a validation script and CI/CD integration for automated code validation. The README has been updated to reflect these changes.
This commit is contained in:
Bruno Domingues
@@ -1,388 +0,0 @@
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, Mock, patch, MagicMock, ANY, call
|
||||
from asyncua.crypto.security_policies import SecurityPolicyBasic256
|
||||
from model_manager.utils.repository.opc_repository import OpcRepository
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_logger():
|
||||
return Mock()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def opc_repository(mock_logger):
|
||||
return OpcRepository(
|
||||
id="test_repo",
|
||||
url="opc.tcp://localhost:4840",
|
||||
logger=mock_logger,
|
||||
notification_handler=Mock(),
|
||||
reconnection_interval=60,
|
||||
server_uri="urn:test:server",
|
||||
cert_path="/path/to/cert.pem",
|
||||
private_key_path="/path/to/key.pem",
|
||||
server_cert_path="/path/to/server_cert.pem"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_client():
|
||||
with patch('model_manager.utils.repository.opc_repository.Client') as mock:
|
||||
client_instance = AsyncMock()
|
||||
mock.return_value = client_instance
|
||||
yield client_instance
|
||||
|
||||
|
||||
metadata = {
|
||||
"metadata": {
|
||||
"model_id": "test_model",
|
||||
"model_name": "test_model",
|
||||
"workflow_name": "test_workflow",
|
||||
"schema_name": "test_schedule",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_init(opc_repository):
|
||||
assert opc_repository.id == "test_repo"
|
||||
assert opc_repository.url == "opc.tcp://localhost:4840"
|
||||
assert opc_repository.server_uri == "urn:test:server"
|
||||
assert opc_repository.cert_path == "/path/to/cert.pem"
|
||||
assert opc_repository.private_key_path == "/path/to/key.pem"
|
||||
assert opc_repository.server_cert_path == "/path/to/server_cert.pem"
|
||||
assert opc_repository.reconnection_interval == 60
|
||||
assert opc_repository.client is None
|
||||
assert opc_repository.last_reconnection_time is None
|
||||
assert opc_repository.error_count == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_security(opc_repository, mock_client):
|
||||
opc_repository.client = mock_client
|
||||
await opc_repository.set_security()
|
||||
|
||||
mock_client.application_uri = "urn:test:server"
|
||||
mock_client.set_security.assert_called_once_with(
|
||||
SecurityPolicyBasic256,
|
||||
certificate="/path/to/cert.pem",
|
||||
private_key="/path/to/key.pem",
|
||||
server_certificate="/path/to/server_cert.pem"
|
||||
)
|
||||
assert mock_client.secure_channel_timeout == 10000000
|
||||
assert mock_client.session_timeout == 10000000
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_security_missing_certificates(opc_repository):
|
||||
opc_repository.cert_path = None
|
||||
opc_repository.private_key_path = None
|
||||
|
||||
try:
|
||||
await opc_repository.set_security()
|
||||
except ValueError as e:
|
||||
assert str(
|
||||
e) == "Certificate and private key paths must be provided for secure connection."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_with_security(opc_repository, mock_client):
|
||||
opc_repository.try_connect = AsyncMock(return_value=(True, {}))
|
||||
result = await opc_repository.connect()
|
||||
|
||||
opc_repository.try_connect.assert_called_once()
|
||||
assert opc_repository.client == mock_client
|
||||
assert result == (True, {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_without_security(opc_repository, mock_client):
|
||||
opc_repository.cert_path = None
|
||||
opc_repository.try_connect = AsyncMock(return_value=(True, {}))
|
||||
opc_repository.set_security = AsyncMock()
|
||||
result = await opc_repository.connect()
|
||||
|
||||
opc_repository.try_connect.assert_called_once()
|
||||
opc_repository.set_security.assert_not_called()
|
||||
assert opc_repository.client == mock_client
|
||||
assert result == (True, {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_try_connect_success(opc_repository):
|
||||
opc_repository.last_reconnection_time = None
|
||||
opc_repository.client = AsyncMock()
|
||||
result = await opc_repository.try_connect()
|
||||
|
||||
opc_repository.client.connect.assert_called_once()
|
||||
assert opc_repository.last_reconnection_time is not None
|
||||
assert result == (True, {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_try_connect_fail(opc_repository):
|
||||
opc_repository.last_reconnection_time = None
|
||||
opc_repository.client = MagicMock()
|
||||
opc_repository.client.connect.side_effect = Exception("Test error")
|
||||
|
||||
is_connected, error_data = await opc_repository.try_connect()
|
||||
|
||||
opc_repository.client.connect.assert_called_once()
|
||||
assert is_connected is False
|
||||
assert error_data['notification_id'] == f"OPC_CONNECTION_ERROR_{opc_repository.id}"
|
||||
assert error_data['message'] == "Failed to connect to OPC server: Test error"
|
||||
assert error_data['block'] == "opc_repository"
|
||||
assert error_data['level'] == NotificationLevel.ERROR
|
||||
assert error_data['attachment_content'] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect(opc_repository, mock_client):
|
||||
opc_repository.client = mock_client
|
||||
await opc_repository.disconnect()
|
||||
|
||||
mock_client.disconnect.assert_called_once()
|
||||
assert opc_repository.client is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_no_client(opc_repository):
|
||||
opc_repository.client = None
|
||||
assert await opc_repository.disconnect() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_disconnect_error(opc_repository, mock_client):
|
||||
opc_repository.client = mock_client
|
||||
mock_client.disconnect.side_effect = Exception("Test error")
|
||||
await opc_repository.disconnect()
|
||||
|
||||
opc_repository.logger.custom_error.assert_called_once_with(
|
||||
"Failed to disconnect from OPC server: Test error",
|
||||
ANY
|
||||
)
|
||||
assert opc_repository.client is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_connection_none_client(opc_repository):
|
||||
opc_repository.client = None
|
||||
opc_repository.connect = AsyncMock(return_value=(True, {}))
|
||||
response = await opc_repository.validate_connection()
|
||||
assert response == (True, {})
|
||||
opc_repository.connect.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_connection_error_count_disconnect_error(opc_repository):
|
||||
opc_repository.error_count = 6
|
||||
opc_repository.client = AsyncMock()
|
||||
opc_repository.disconnect = AsyncMock(
|
||||
side_effect=Exception("Test error")
|
||||
)
|
||||
opc_repository.connect = AsyncMock(return_value=(True, {}))
|
||||
|
||||
response = await opc_repository.validate_connection()
|
||||
assert response == opc_repository.connect.return_value
|
||||
opc_repository.disconnect.assert_called_once()
|
||||
opc_repository.connect.assert_called_once()
|
||||
opc_repository.logger.custom_error.assert_has_calls(
|
||||
[
|
||||
call("Failed to disconnect from OPC server: Test error", ANY),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_connection_error_validate_connection_error(opc_repository):
|
||||
opc_repository.client = MagicMock(
|
||||
uaclient=Exception("Test error")
|
||||
)
|
||||
opc_repository.error_count = 0
|
||||
|
||||
response = await opc_repository.validate_connection()
|
||||
|
||||
assert response == (False, {
|
||||
"notification_id": f"OPC_CONNECTION_CHECK_ERROR_{opc_repository.id}",
|
||||
"message": "Failed to validate connection to OPC server: 'Exception' object has no attribute 'protocol'",
|
||||
"block": "opc_repository",
|
||||
"level": NotificationLevel.ERROR,
|
||||
"attachment_content": ANY
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('model_manager.utils.repository.opc_repository.datetime')
|
||||
async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, opc_repository):
|
||||
_mock_datetime.now = MagicMock(
|
||||
return_value=datetime(2025, 1, 1, 0, 0, 0))
|
||||
opc_repository.error_count = 0
|
||||
opc_repository.client = MagicMock()
|
||||
opc_repository.client.uaclient.protocol = None
|
||||
opc_repository.last_reconnection_time = datetime(2025, 1, 1, 0, 0, 0)
|
||||
opc_repository.connect = MagicMock(return_value=(True, {}))
|
||||
|
||||
response = await opc_repository.validate_connection()
|
||||
opc_repository.connect.assert_not_called()
|
||||
assert response == (False, {
|
||||
"notification_id": f"OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{opc_repository.id}",
|
||||
"message": f"OPC server {opc_repository.id} is not connected, waiting for next reconnection window...",
|
||||
"block": "opc_repository",
|
||||
"level": NotificationLevel.WARNING
|
||||
})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('model_manager.utils.repository.opc_repository.datetime')
|
||||
async def test_validate_connection_lost_time_to_reconnect(mock_datetime, opc_repository):
|
||||
mock_datetime.now = MagicMock(
|
||||
return_value=datetime(2025, 1, 1, 1, 0, 0))
|
||||
opc_repository.error_count = 0
|
||||
opc_repository.client = AsyncMock()
|
||||
opc_repository.client.uaclient.protocol = None
|
||||
opc_repository.last_reconnection_time = datetime(2025, 1, 1, 0, 0, 0)
|
||||
opc_repository.connect = AsyncMock(return_value=(True, {}))
|
||||
|
||||
response = await opc_repository.validate_connection()
|
||||
opc_repository.connect.assert_called_once()
|
||||
assert response == opc_repository.connect.return_value
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validate_connection_success(opc_repository):
|
||||
opc_repository.client = MagicMock()
|
||||
opc_repository.error_count = 0
|
||||
opc_repository.client.uaclient.protocol = MagicMock()
|
||||
opc_repository.client.uaclient.protocol.state = "open"
|
||||
|
||||
output = await opc_repository.validate_connection()
|
||||
assert output == (True, {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_data_validate_connection_do_nothing(opc_repository):
|
||||
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
|
||||
opc_repository.client = AsyncMock(
|
||||
get_node=MagicMock()
|
||||
)
|
||||
mock_node = AsyncMock()
|
||||
opc_repository.client.get_node.return_value = mock_node
|
||||
|
||||
result = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
|
||||
"float", opc_repository.logger, metadata['metadata'])
|
||||
|
||||
opc_repository.validate_connection.assert_called_once()
|
||||
opc_repository.client.get_node.assert_called_once_with("ns=2;s=TestNode")
|
||||
assert result == (True, {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_data_validate_connection_failed(opc_repository):
|
||||
opc_repository.validate_connection = AsyncMock(return_value=(False, {}))
|
||||
opc_repository.client = AsyncMock()
|
||||
opc_repository.error_count = 0
|
||||
|
||||
result = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
|
||||
"float", opc_repository.logger, metadata['metadata'])
|
||||
|
||||
opc_repository.validate_connection.assert_called_once()
|
||||
opc_repository.client.get_node.assert_not_called()
|
||||
assert result == (False, {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_data_get_node_failed(opc_repository):
|
||||
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
|
||||
opc_repository.client = AsyncMock()
|
||||
opc_repository.error_count = 0
|
||||
opc_repository.client.get_node = MagicMock(
|
||||
side_effect=Exception("Test error"))
|
||||
|
||||
is_success, error_data = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
|
||||
"float", opc_repository.logger, metadata['metadata'])
|
||||
|
||||
opc_repository.validate_connection.assert_called_once()
|
||||
opc_repository.client.get_node.assert_called_once_with("ns=2;s=TestNode")
|
||||
assert is_success is False
|
||||
assert error_data['notification_id'] == f"OPC_WRITE_GET_NODE_ERROR_{opc_repository.id}"
|
||||
assert error_data['message'] == "Failed to get node from OPC server: Test error | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}"
|
||||
assert error_data['block'] == "opc_repository"
|
||||
assert error_data['level'] == NotificationLevel.ERROR
|
||||
assert error_data['attachment_content'] is not None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_data_invalid_data_type(opc_repository, mock_client):
|
||||
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
|
||||
opc_repository.client = mock_client
|
||||
mock_node = AsyncMock()
|
||||
mock_client.get_node = MagicMock(return_value=mock_node)
|
||||
|
||||
is_success, error_data = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
|
||||
"invalid_type", opc_repository.logger, metadata['metadata'])
|
||||
|
||||
opc_repository.validate_connection.assert_called_once()
|
||||
mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
|
||||
|
||||
assert is_success is False
|
||||
assert error_data['notification_id'] == f"OPC_WRITE_DATA_TYPE_ERROR_{opc_repository.id}"
|
||||
assert error_data['message'] == "Unsupported data type: invalid_type | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}"
|
||||
assert error_data['block'] == "opc_repository"
|
||||
assert error_data['level'] == NotificationLevel.ERROR
|
||||
assert error_data.get('attachment_content') is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('model_manager.utils.repository.opc_repository.metrics')
|
||||
async def test_write_data(mock_metrics, opc_repository, mock_client):
|
||||
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
|
||||
opc_repository.client = mock_client
|
||||
mock_node = AsyncMock()
|
||||
mock_client.get_node = MagicMock(return_value=mock_node)
|
||||
|
||||
result = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
|
||||
"float", opc_repository.logger, metadata['metadata'])
|
||||
|
||||
mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
|
||||
mock_node.write_value.assert_called_once()
|
||||
assert result == (True, {})
|
||||
|
||||
mock_metrics.PREDICTION_OPC_WRITING_COUNT.labels.assert_called_once_with(
|
||||
pod_id=opc_repository.pod_id,
|
||||
model_name=metadata['metadata']['model_name'],
|
||||
pipeline_name=metadata['metadata']['workflow_name'],
|
||||
opc_server_id=opc_repository.id
|
||||
)
|
||||
mock_metrics.PREDICTION_OPC_WRITING_COUNT.labels.return_value.inc.assert_called_once_with()
|
||||
|
||||
mock_metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels.assert_called_once_with(
|
||||
pod_id=opc_repository.pod_id,
|
||||
model_name=metadata['metadata']['model_name'],
|
||||
pipeline_name=metadata['metadata']['workflow_name'],
|
||||
opc_server_id=opc_repository.id
|
||||
)
|
||||
mock_metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels.return_value.observe.assert_called_once_with(
|
||||
ANY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_data_write_value_failed(opc_repository, mock_client):
|
||||
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
|
||||
opc_repository.client = mock_client
|
||||
mock_node = AsyncMock()
|
||||
opc_repository.error_count = 0
|
||||
mock_client.get_node = MagicMock(return_value=mock_node)
|
||||
mock_node.write_value.side_effect = Exception("Test error")
|
||||
|
||||
is_success, error_data = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
|
||||
"float", opc_repository.logger, metadata['metadata'])
|
||||
|
||||
opc_repository.validate_connection.assert_called_once()
|
||||
mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
|
||||
mock_node.write_value.assert_called_once()
|
||||
assert is_success is False
|
||||
assert error_data['notification_id'] == f"OPC_WRITE_DATA_ERROR_{opc_repository.id}"
|
||||
assert error_data['message'] == "Failed to write data to OPC server: Test error | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}"
|
||||
assert error_data['block'] == "opc_repository"
|
||||
assert error_data['level'] == NotificationLevel.ERROR
|
||||
assert error_data['attachment_content'] is not None
|
||||
@@ -1,6 +1,5 @@
|
||||
from os import environ
|
||||
from model_manager.utils.connectors_config import (build_mlflow_config,
|
||||
build_opc_config,
|
||||
build_postgres_config,
|
||||
build_mongodb_config)
|
||||
|
||||
@@ -40,54 +39,6 @@ def test_build_mlflow_config_with_defaults():
|
||||
assert config['password'] == 'aignosi'
|
||||
|
||||
|
||||
def test_build_opc_config_with_env_vars():
|
||||
# Arrange
|
||||
environ['OPC_CONFIG'] = '{"opc": {"name": "test-opc", "url": "opc.tcp://test:4840"}}'
|
||||
|
||||
# Act
|
||||
config = build_opc_config()
|
||||
|
||||
# Assert
|
||||
assert config['opc']['name'] == 'test-opc'
|
||||
assert config['opc']['url'] == 'opc.tcp://test:4840'
|
||||
|
||||
|
||||
def test_build_opc_config_with_individual_env_vars():
|
||||
# Arrange
|
||||
environ.pop('OPC_CONFIG', None)
|
||||
environ['OPC_ID'] = '1'
|
||||
environ['OPC_URL'] = 'opc.tcp://test:4840'
|
||||
environ['OPC_SERVER_URI'] = 'opc.tcp://test:4840'
|
||||
environ['OPC_RECONNECTION_INTERVAL'] = '300'
|
||||
|
||||
# Act
|
||||
config = build_opc_config()
|
||||
|
||||
# Assert
|
||||
assert config['1']['id'] == '1'
|
||||
assert config['1']['url'] == 'opc.tcp://test:4840'
|
||||
assert config['1']['server_uri'] == 'opc.tcp://test:4840'
|
||||
assert config['1']['reconnection_interval'] == 300
|
||||
|
||||
|
||||
def test_build_opc_config_with_defaults():
|
||||
# Arrange
|
||||
environ.pop('OPC_CONFIG', None)
|
||||
environ.pop('OPC_ID', None)
|
||||
environ.pop('OPC_URL', None)
|
||||
environ.pop('OPC_SERVER_URI', None)
|
||||
environ.pop('OPC_RECONNECTION_INTERVAL', None)
|
||||
|
||||
# Act
|
||||
config = build_opc_config()
|
||||
|
||||
# Assert
|
||||
assert config['1']['id'] == '1'
|
||||
assert config['1']['url'] == 'opc.tcp://localhost:4840'
|
||||
assert config['1']['server_uri'] == 'opc.tcp://localhost:4840'
|
||||
assert config['1']['reconnection_interval'] == 120
|
||||
|
||||
|
||||
def test_build_postgres_config_with_env_vars():
|
||||
# Arrange
|
||||
environ['POSTGRES_HOST'] = 'test-host'
|
||||
|
||||
Reference in New Issue
Block a user