gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1243: Remove OPC server integration and add code quality tools.

Browse Source 
This commit removes the OPC server integration from the Model Manager, including related activities, repositories, metrics, and configuration. It also adds code quality tools such as Ruff (linting/formatting), mypy (type checking), and Bandit (security analysis) along with a validation script and CI/CD integration for automated code validation. The README has been updated to reflect these changes.
This commit is contained in:
Bruno Domingues
2025-10-01 16:25:08 -03:00
parent bc4d98f78d
commit b102f79087
24 changed files with 453 additions and 1810 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
tests/laborious/activities/test_activities.py
View File

@@ -4,14 +4,12 @@ from sientia_do.temporal.activities.postgres import Postgres
from model_manager.activities.activities import Activities
from model_manager.activities.mlflow import MLFlow
from model_manager.activities.gates import Gates
from model_manager.activities.opc import OPC
@patch('model_manager.activities.activities.Postgres.__init__')
@patch('model_manager.activities.activities.MLFlow.__init__')
@patch('model_manager.activities.activities.OPC.__init__')
@patch('model_manager.activities.activities.Gates.__init__')
def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgres_init):
def test___init__(mock_gates_init, mock_mlflow_init, mock_postgres_init):
postgres_config = {
'host': 'localhost',
@@ -30,19 +28,12 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre
'password': 'mlflow'
}
opc_config = {
'bootstrap_servers': 'localhost:9092',
'polling_time': 1000,
'group_id': 'test-group'
}
logger = MagicMock()
notification_handler = MagicMock()
activities = Activities(
postgres_config=postgres_config,
mlflow_config=mlflow_config,
opc_config=opc_config,
logger=logger,
notification_handler=notification_handler
)
@@ -50,7 +41,6 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre
assert isinstance(activities, Activities)
assert isinstance(activities, Postgres)
assert isinstance(activities, MLFlow)
assert isinstance(activities, OPC)
assert isinstance(activities, Gates)
mock_postgres_init.assert_called_once_with(
@@ -76,13 +66,6 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre
notification_handler=notification_handler
)
mock_opc_init.assert_called_once_with(
ANY,
opc_servers=opc_config,
logger=logger,
notification_handler=notification_handler
)
mock_gates_init.assert_called_once_with(
ANY,
logger=logger,
@@ -93,9 +76,7 @@ def test___init__(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_postgre
@mark.asyncio
@patch('model_manager.activities.activities.Postgres', return_value=MagicMock())
@patch('model_manager.activities.activities.MLFlow', return_value=MagicMock())
@patch('model_manager.activities.activities.OPC', return_value=MagicMock())
async def test_shutdown(mock_opc_init,
_mock_mlflow_init, mock_postgres_init):
async def test_shutdown(_mock_mlflow_init, mock_postgres_init):
postgres_config = {
'host': 'localhost',
'port': 5432,
@@ -113,23 +94,15 @@ async def test_shutdown(mock_opc_init,
'password': 'mlflow'
}
opc_config = {
'bootstrap_servers': 'localhost:9092',
'polling_time': 1000,
'group_id': 'test-group'
}
logger = MagicMock()
notification_handler = MagicMock()
activities = Activities(
postgres_config=postgres_config,
mlflow_config=mlflow_config,
opc_config=opc_config,
logger=logger,
notification_handler=notification_handler
)
await activities.shutdown()
mock_opc_init.shutdown.assert_called_once()
mock_postgres_init.close.assert_called_once()

369
tests/laborious/activities/test_opc.py
View File

@@ -1,369 +0,0 @@
from unittest.mock import patch, MagicMock, ANY, call, AsyncMock
from pandas import DataFrame
from pytest import fixture, mark
import pytest_asyncio
from sientia_do.notifications.models import NotificationLevel
from model_manager.activities.opc import OPC
metadata = {
"metadata": {
"model_id": "test_model",
"model_name": "test_model",
"workflow_name": "test_workflow",
"schema_name": "test_schedule",
},
}
def test__init__():
servers = {
'server1': 'config'
}
opc = OPC(
opc_servers=servers,
logger=MagicMock(),
notification_handler=MagicMock()
)
assert opc.opc_servers == servers
assert opc.opc_repository == {}
@mark.asyncio
@patch("model_manager.activities.opc.OpcRepository")
@patch("model_manager.activities.opc.OPC.send_notification")
async def test_init_opc(mock_send_notification, mock_opc_repository):
mock_logger = MagicMock()
server1 = MagicMock(
connect=AsyncMock(return_value=(True, {})),
write_data=AsyncMock(return_value=(True, {}))
)
server2 = MagicMock(
connect=AsyncMock(return_value=(True, {})),
write_data=AsyncMock(return_value=(True, {}))
)
server3 = MagicMock(
connect=AsyncMock(return_value=(False, {
'notification_id': 'OPC_CONNECTION_ERROR_server3',
'message': 'Failed to connect to OPC server: Test error',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': 'Test error'
})),
write_data=AsyncMock(return_value=(True, {}))
)
mock_opc_repository.side_effect = [server1, server2, server3]
mock_notification_handler = MagicMock()
servers = {
'server1': {
'id': 'server1',
'url': 'http://localhost:8080',
'server_uri': 'opc.tcp://localhost:4840',
'cert_path': '',
'private_key_path': '',
'server_cert_path': '',
'reconnection_interval': 60,
},
'server2': {
'id': 'server2',
'url': 'http://localhost:8080',
'server_uri': 'opc.tcp://localhost:4840',
'cert_path': '',
'private_key_path': '',
'server_cert_path': '',
'reconnection_interval': 60,
},
'server3': {
'id': 'server3',
'url': 'http://localhost:8080',
'server_uri': 'opc.tcp://localhost:4840',
'cert_path': '',
'private_key_path': '',
'server_cert_path': '',
'reconnection_interval': 60,
}
}
opc = OPC(
opc_servers=servers,
logger=mock_logger,
notification_handler=mock_notification_handler
)
await opc.init_opc()
assert opc.opc_servers == servers
assert opc.logger == mock_logger
assert opc.notification_handler == mock_notification_handler
assert opc.opc_repository['server1'] == server1
assert opc.opc_repository['server2'] == server2
mock_opc_repository.assert_has_calls([
call(
id="server1",
url="http://localhost:8080",
logger=mock_logger,
server_uri="opc.tcp://localhost:4840",
cert_path="",
private_key_path="",
server_cert_path="",
notification_handler=mock_notification_handler,
reconnection_interval=60,
pod_id='localhost'
),
])
mock_opc_repository.assert_has_calls([
call(
id="server2",
url="http://localhost:8080",
logger=mock_logger,
server_uri="opc.tcp://localhost:4840",
cert_path="",
private_key_path="",
server_cert_path="",
notification_handler=mock_notification_handler,
reconnection_interval=60,
pod_id='localhost'
)
])
server1.connect.assert_called_once()
server2.connect.assert_called_once()
mock_send_notification.assert_has_calls([
call(
metadata={
'model_id': '-',
'model_name': '-',
'workflow_name': '-',
'schedule_name': 'INITIALIZATION'
},
notification_id="OPC_CONNECTION_ERROR_server3",
message="Failed to connect to OPC server: Test error",
block="opc_repository",
level=NotificationLevel.ERROR,
attachment_content=ANY
)
])
@pytest_asyncio.fixture
@patch("model_manager.activities.opc.OpcRepository")
async def opc(mock_opc_repository):
servers = {
'server1': {
'id': 'server1',
'url': 'http://localhost:8080',
'server_uri': 'opc.tcp://localhost:4840',
'cert_path': '',
'private_key_path': '',
'server_cert_path': '',
'reconnection_interval': 60,
}
}
mock_opc_repository.return_value.write_data = AsyncMock(
return_value=(True, {})
)
mock_opc_repository.return_value.connect = AsyncMock(
return_value=(True, {})
)
opc = OPC(
opc_servers=servers,
logger=MagicMock(),
notification_handler=MagicMock()
)
await opc.init_opc()
opc.send_notification = MagicMock()
return opc
WRITE_DATA_CASES = [
('tag1', 'int', 50),
('tag2', 'float', 50.5),
('tag3', 'bool', True),
('tag4', 'string', 'test'),
]
@mark.parametrize('tag,data_type,data', WRITE_DATA_CASES)
@mark.asyncio
async def test_write_data_success(opc, tag, data_type, data):
result = await opc.write_data(server_id='server1', tag=tag, data=data,
data_type=data_type, tag_type='prediction', metadata=metadata)
assert result is True
opc.opc_repository['server1'].write_data.assert_called_once_with(
tag, data, data_type, opc.logger, metadata)
@mark.asyncio
async def test_write_data_failed(opc):
opc.opc_repository['server1'].write_data.return_value = (False, {
'notification_id': 'OPC_WRITE_DATA_ERROR_server1',
'message': 'Failed to write data to OPC server: Test error',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': 'Test error'
})
result = await opc.write_data(server_id='server1', tag='tag1', data=50,
data_type='int', tag_type='prediction', metadata=metadata)
assert result is False
opc.send_notification.assert_called_once_with(
metadata=metadata,
notification_id="OPC_WRITE_DATA_ERROR_server1",
message="Failed to write data to OPC server: Test error",
block="opc_repository",
level=NotificationLevel.ERROR,
attachment_content=ANY
)
@mark.asyncio
async def test_write_data_exception(opc):
opc.opc_repository['server1'].write_data.side_effect = Exception(
"Test error")
try:
await opc.write_data(server_id='server1', tag='tag1', data=50,
data_type='int', tag_type='prediction', metadata=metadata)
except Exception:
opc.send_notification.assert_called_once_with(
metadata=metadata,
notification_id="WRITE_OPC_PREDICTION_ERROR",
message="Error writing data to OPC server: Test error",
block="write_opc_data",
level=NotificationLevel.ERROR,
attachment_content=ANY
)
else:
assert False, "Expected an exception to be raised"
@mark.asyncio
async def test_write_opc_data_success(opc):
# Arrange
input_data = {
**metadata,
'data': {
'prediction': [0.75],
'prediction_confidence': [0.95]
},
'opc_output_config': {
'server1': {
'prediction_tags': {
'tag1': {'data_type': 'float'}
},
'confidence_tags': {
'tag2': {'data_type': 'float'}
}
}
}
}
# Act
opc.write_data = AsyncMock(return_value=True)
opc.process_confidence = MagicMock(return_value={'data': 'data'})
output = await opc.write_opc_data(input_data)
# Assert
assert output == {'data': 'data'}
opc.write_data.assert_has_calls([
call(
server_id='server1',
tag='tag1',
data=0.75,
data_type='float',
tag_type='prediction',
metadata=metadata['metadata']
)])
opc.write_data.assert_has_calls([
call(
server_id='server1',
tag='tag2',
data=0.95,
data_type='float',
tag_type='confidence',
metadata=metadata['metadata']
)
])
assert opc.write_data.call_count == 2
@mark.asyncio
async def test_write_opc_data_empty_config(opc):
# Arrange
input_data = {
**metadata,
'data': {
'prediction': [0.75],
'prediction_confidence': [0.95]
},
'opc_servers': ['server1'],
'opc_output_config': {
'server1': {
'prediction_tags': {},
'confidence_tags': {}
}
}
}
# Act
await opc.write_opc_data(input_data)
# Assert
opc.opc_repository['server1'].write_data.assert_not_called()
@mark.asyncio
async def test_write_opc_data_no_validate_server(opc):
opc.validate_server = MagicMock(return_value=False)
input_data = {
**metadata,
'data': {
'prediction': [0.75],
'prediction_confidence': [0.95]
},
'opc_output_config': {
'server1': {
'prediction_tags': {
'tag1': {'data_type': 'float'}
},
'confidence_tags': {
'tag2': {'data_type': 'float'}
}
}
}
}
# Act
await opc.write_opc_data(input_data)
# Assert
opc.opc_repository['server1'].write_data.assert_not_called()
@mark.parametrize('data,success,expected', [
(DataFrame({'prediction_confidence': [0]}), True, 0),
(DataFrame({'prediction_confidence': [0]}), False, 12),
])
def test_process_confidence(opc, data, success, expected):
# Act
result = opc.process_confidence(data, success, metadata)
# Assert
assert result['prediction_confidence'][0] == expected
def test_validate_server(opc):
assert opc.validate_server('server1', metadata) is True
assert opc.validate_server('server2', metadata) is False
@mark.asyncio
async def test_shutdown(opc):
opc.opc_repository['server1'].disconnect = AsyncMock(return_value=True)
await opc.shutdown()
opc.opc_repository['server1'].disconnect.assert_called_once()

388
tests/laborious/utils/repository/test_opc_repository.py
View File

@@ -1,388 +0,0 @@
import pytest
from unittest.mock import AsyncMock, Mock, patch, MagicMock, ANY, call
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from model_manager.utils.repository.opc_repository import OpcRepository
from sientia_do.notifications.models import NotificationLevel
from datetime import datetime
@pytest.fixture
def mock_logger():
return Mock()
@pytest.fixture
def opc_repository(mock_logger):
return OpcRepository(
id="test_repo",
url="opc.tcp://localhost:4840",
logger=mock_logger,
notification_handler=Mock(),
reconnection_interval=60,
server_uri="urn:test:server",
cert_path="/path/to/cert.pem",
private_key_path="/path/to/key.pem",
server_cert_path="/path/to/server_cert.pem"
)
@pytest.fixture
def mock_client():
with patch('model_manager.utils.repository.opc_repository.Client') as mock:
client_instance = AsyncMock()
mock.return_value = client_instance
yield client_instance
metadata = {
"metadata": {
"model_id": "test_model",
"model_name": "test_model",
"workflow_name": "test_workflow",
"schema_name": "test_schedule",
},
}
def test_init(opc_repository):
assert opc_repository.id == "test_repo"
assert opc_repository.url == "opc.tcp://localhost:4840"
assert opc_repository.server_uri == "urn:test:server"
assert opc_repository.cert_path == "/path/to/cert.pem"
assert opc_repository.private_key_path == "/path/to/key.pem"
assert opc_repository.server_cert_path == "/path/to/server_cert.pem"
assert opc_repository.reconnection_interval == 60
assert opc_repository.client is None
assert opc_repository.last_reconnection_time is None
assert opc_repository.error_count == 0
@pytest.mark.asyncio
async def test_set_security(opc_repository, mock_client):
opc_repository.client = mock_client
await opc_repository.set_security()
mock_client.application_uri = "urn:test:server"
mock_client.set_security.assert_called_once_with(
SecurityPolicyBasic256,
certificate="/path/to/cert.pem",
private_key="/path/to/key.pem",
server_certificate="/path/to/server_cert.pem"
)
assert mock_client.secure_channel_timeout == 10000000
assert mock_client.session_timeout == 10000000
@pytest.mark.asyncio
async def test_set_security_missing_certificates(opc_repository):
opc_repository.cert_path = None
opc_repository.private_key_path = None
try:
await opc_repository.set_security()
except ValueError as e:
assert str(
e) == "Certificate and private key paths must be provided for secure connection."
@pytest.mark.asyncio
async def test_connect_with_security(opc_repository, mock_client):
opc_repository.try_connect = AsyncMock(return_value=(True, {}))
result = await opc_repository.connect()
opc_repository.try_connect.assert_called_once()
assert opc_repository.client == mock_client
assert result == (True, {})
@pytest.mark.asyncio
async def test_connect_without_security(opc_repository, mock_client):
opc_repository.cert_path = None
opc_repository.try_connect = AsyncMock(return_value=(True, {}))
opc_repository.set_security = AsyncMock()
result = await opc_repository.connect()
opc_repository.try_connect.assert_called_once()
opc_repository.set_security.assert_not_called()
assert opc_repository.client == mock_client
assert result == (True, {})
@pytest.mark.asyncio
async def test_try_connect_success(opc_repository):
opc_repository.last_reconnection_time = None
opc_repository.client = AsyncMock()
result = await opc_repository.try_connect()
opc_repository.client.connect.assert_called_once()
assert opc_repository.last_reconnection_time is not None
assert result == (True, {})
@pytest.mark.asyncio
async def test_try_connect_fail(opc_repository):
opc_repository.last_reconnection_time = None
opc_repository.client = MagicMock()
opc_repository.client.connect.side_effect = Exception("Test error")
is_connected, error_data = await opc_repository.try_connect()
opc_repository.client.connect.assert_called_once()
assert is_connected is False
assert error_data['notification_id'] == f"OPC_CONNECTION_ERROR_{opc_repository.id}"
assert error_data['message'] == "Failed to connect to OPC server: Test error"
assert error_data['block'] == "opc_repository"
assert error_data['level'] == NotificationLevel.ERROR
assert error_data['attachment_content'] is not None
@pytest.mark.asyncio
async def test_disconnect(opc_repository, mock_client):
opc_repository.client = mock_client
await opc_repository.disconnect()
mock_client.disconnect.assert_called_once()
assert opc_repository.client is None
@pytest.mark.asyncio
async def test_disconnect_no_client(opc_repository):
opc_repository.client = None
assert await opc_repository.disconnect() is None
@pytest.mark.asyncio
async def test_disconnect_error(opc_repository, mock_client):
opc_repository.client = mock_client
mock_client.disconnect.side_effect = Exception("Test error")
await opc_repository.disconnect()
opc_repository.logger.custom_error.assert_called_once_with(
"Failed to disconnect from OPC server: Test error",
ANY
)
assert opc_repository.client is None
@pytest.mark.asyncio
async def test_validate_connection_none_client(opc_repository):
opc_repository.client = None
opc_repository.connect = AsyncMock(return_value=(True, {}))
response = await opc_repository.validate_connection()
assert response == (True, {})
opc_repository.connect.assert_called_once()
@pytest.mark.asyncio
async def test_validate_connection_error_count_disconnect_error(opc_repository):
opc_repository.error_count = 6
opc_repository.client = AsyncMock()
opc_repository.disconnect = AsyncMock(
side_effect=Exception("Test error")
)
opc_repository.connect = AsyncMock(return_value=(True, {}))
response = await opc_repository.validate_connection()
assert response == opc_repository.connect.return_value
opc_repository.disconnect.assert_called_once()
opc_repository.connect.assert_called_once()
opc_repository.logger.custom_error.assert_has_calls(
[
call("Failed to disconnect from OPC server: Test error", ANY),
]
)
@pytest.mark.asyncio
async def test_validate_connection_error_validate_connection_error(opc_repository):
opc_repository.client = MagicMock(
uaclient=Exception("Test error")
)
opc_repository.error_count = 0
response = await opc_repository.validate_connection()
assert response == (False, {
"notification_id": f"OPC_CONNECTION_CHECK_ERROR_{opc_repository.id}",
"message": "Failed to validate connection to OPC server: 'Exception' object has no attribute 'protocol'",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": ANY
})
@pytest.mark.asyncio
@patch('model_manager.utils.repository.opc_repository.datetime')
async def test_validate_connection_lost_not_time_to_reconnect(_mock_datetime, opc_repository):
_mock_datetime.now = MagicMock(
return_value=datetime(2025, 1, 1, 0, 0, 0))
opc_repository.error_count = 0
opc_repository.client = MagicMock()
opc_repository.client.uaclient.protocol = None
opc_repository.last_reconnection_time = datetime(2025, 1, 1, 0, 0, 0)
opc_repository.connect = MagicMock(return_value=(True, {}))
response = await opc_repository.validate_connection()
opc_repository.connect.assert_not_called()
assert response == (False, {
"notification_id": f"OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{opc_repository.id}",
"message": f"OPC server {opc_repository.id} is not connected, waiting for next reconnection window...",
"block": "opc_repository",
"level": NotificationLevel.WARNING
})
@pytest.mark.asyncio
@patch('model_manager.utils.repository.opc_repository.datetime')
async def test_validate_connection_lost_time_to_reconnect(mock_datetime, opc_repository):
mock_datetime.now = MagicMock(
return_value=datetime(2025, 1, 1, 1, 0, 0))
opc_repository.error_count = 0
opc_repository.client = AsyncMock()
opc_repository.client.uaclient.protocol = None
opc_repository.last_reconnection_time = datetime(2025, 1, 1, 0, 0, 0)
opc_repository.connect = AsyncMock(return_value=(True, {}))
response = await opc_repository.validate_connection()
opc_repository.connect.assert_called_once()
assert response == opc_repository.connect.return_value
@pytest.mark.asyncio
async def test_validate_connection_success(opc_repository):
opc_repository.client = MagicMock()
opc_repository.error_count = 0
opc_repository.client.uaclient.protocol = MagicMock()
opc_repository.client.uaclient.protocol.state = "open"
output = await opc_repository.validate_connection()
assert output == (True, {})
@pytest.mark.asyncio
async def test_write_data_validate_connection_do_nothing(opc_repository):
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
opc_repository.client = AsyncMock(
get_node=MagicMock()
)
mock_node = AsyncMock()
opc_repository.client.get_node.return_value = mock_node
result = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
"float", opc_repository.logger, metadata['metadata'])
opc_repository.validate_connection.assert_called_once()
opc_repository.client.get_node.assert_called_once_with("ns=2;s=TestNode")
assert result == (True, {})
@pytest.mark.asyncio
async def test_write_data_validate_connection_failed(opc_repository):
opc_repository.validate_connection = AsyncMock(return_value=(False, {}))
opc_repository.client = AsyncMock()
opc_repository.error_count = 0
result = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
"float", opc_repository.logger, metadata['metadata'])
opc_repository.validate_connection.assert_called_once()
opc_repository.client.get_node.assert_not_called()
assert result == (False, {})
@pytest.mark.asyncio
async def test_write_data_get_node_failed(opc_repository):
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
opc_repository.client = AsyncMock()
opc_repository.error_count = 0
opc_repository.client.get_node = MagicMock(
side_effect=Exception("Test error"))
is_success, error_data = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
"float", opc_repository.logger, metadata['metadata'])
opc_repository.validate_connection.assert_called_once()
opc_repository.client.get_node.assert_called_once_with("ns=2;s=TestNode")
assert is_success is False
assert error_data['notification_id'] == f"OPC_WRITE_GET_NODE_ERROR_{opc_repository.id}"
assert error_data['message'] == "Failed to get node from OPC server: Test error | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}"
assert error_data['block'] == "opc_repository"
assert error_data['level'] == NotificationLevel.ERROR
assert error_data['attachment_content'] is not None
@pytest.mark.asyncio
async def test_write_data_invalid_data_type(opc_repository, mock_client):
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
opc_repository.client = mock_client
mock_node = AsyncMock()
mock_client.get_node = MagicMock(return_value=mock_node)
is_success, error_data = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
"invalid_type", opc_repository.logger, metadata['metadata'])
opc_repository.validate_connection.assert_called_once()
mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
assert is_success is False
assert error_data['notification_id'] == f"OPC_WRITE_DATA_TYPE_ERROR_{opc_repository.id}"
assert error_data['message'] == "Unsupported data type: invalid_type | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}"
assert error_data['block'] == "opc_repository"
assert error_data['level'] == NotificationLevel.ERROR
assert error_data.get('attachment_content') is None
@pytest.mark.asyncio
@patch('model_manager.utils.repository.opc_repository.metrics')
async def test_write_data(mock_metrics, opc_repository, mock_client):
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
opc_repository.client = mock_client
mock_node = AsyncMock()
mock_client.get_node = MagicMock(return_value=mock_node)
result = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
"float", opc_repository.logger, metadata['metadata'])
mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
mock_node.write_value.assert_called_once()
assert result == (True, {})
mock_metrics.PREDICTION_OPC_WRITING_COUNT.labels.assert_called_once_with(
pod_id=opc_repository.pod_id,
model_name=metadata['metadata']['model_name'],
pipeline_name=metadata['metadata']['workflow_name'],
opc_server_id=opc_repository.id
)
mock_metrics.PREDICTION_OPC_WRITING_COUNT.labels.return_value.inc.assert_called_once_with()
mock_metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels.assert_called_once_with(
pod_id=opc_repository.pod_id,
model_name=metadata['metadata']['model_name'],
pipeline_name=metadata['metadata']['workflow_name'],
opc_server_id=opc_repository.id
)
mock_metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels.return_value.observe.assert_called_once_with(
ANY)
@pytest.mark.asyncio
async def test_write_data_write_value_failed(opc_repository, mock_client):
opc_repository.validate_connection = AsyncMock(return_value=(True, {}))
opc_repository.client = mock_client
mock_node = AsyncMock()
opc_repository.error_count = 0
mock_client.get_node = MagicMock(return_value=mock_node)
mock_node.write_value.side_effect = Exception("Test error")
is_success, error_data = await opc_repository.write_data("ns=2;s=TestNode", 42.0,
"float", opc_repository.logger, metadata['metadata'])
opc_repository.validate_connection.assert_called_once()
mock_client.get_node.assert_called_once_with("ns=2;s=TestNode")
mock_node.write_value.assert_called_once()
assert is_success is False
assert error_data['notification_id'] == f"OPC_WRITE_DATA_ERROR_{opc_repository.id}"
assert error_data['message'] == "Failed to write data to OPC server: Test error | metadata: {'model_id': 'test_model', 'model_name': 'test_model', 'workflow_name': 'test_workflow', 'schema_name': 'test_schedule'}"
assert error_data['block'] == "opc_repository"
assert error_data['level'] == NotificationLevel.ERROR
assert error_data['attachment_content'] is not None

49
tests/laborious/utils/test_connectors_config.py
View File

@@ -1,6 +1,5 @@
from os import environ
from model_manager.utils.connectors_config import (build_mlflow_config,
build_opc_config,
build_postgres_config,
build_mongodb_config)
@@ -40,54 +39,6 @@ def test_build_mlflow_config_with_defaults():
assert config['password'] == 'aignosi'
def test_build_opc_config_with_env_vars():
# Arrange
environ['OPC_CONFIG'] = '{"opc": {"name": "test-opc", "url": "opc.tcp://test:4840"}}'
# Act
config = build_opc_config()
# Assert
assert config['opc']['name'] == 'test-opc'
assert config['opc']['url'] == 'opc.tcp://test:4840'
def test_build_opc_config_with_individual_env_vars():
# Arrange
environ.pop('OPC_CONFIG', None)
environ['OPC_ID'] = '1'
environ['OPC_URL'] = 'opc.tcp://test:4840'
environ['OPC_SERVER_URI'] = 'opc.tcp://test:4840'
environ['OPC_RECONNECTION_INTERVAL'] = '300'
# Act
config = build_opc_config()
# Assert
assert config['1']['id'] == '1'
assert config['1']['url'] == 'opc.tcp://test:4840'
assert config['1']['server_uri'] == 'opc.tcp://test:4840'
assert config['1']['reconnection_interval'] == 300
def test_build_opc_config_with_defaults():
# Arrange
environ.pop('OPC_CONFIG', None)
environ.pop('OPC_ID', None)
environ.pop('OPC_URL', None)
environ.pop('OPC_SERVER_URI', None)
environ.pop('OPC_RECONNECTION_INTERVAL', None)
# Act
config = build_opc_config()
# Assert
assert config['1']['id'] == '1'
assert config['1']['url'] == 'opc.tcp://localhost:4840'
assert config['1']['server_uri'] == 'opc.tcp://localhost:4840'
assert config['1']['reconnection_interval'] == 120
def test_build_postgres_config_with_env_vars():
# Arrange
environ['POSTGRES_HOST'] = 'test-host'

32
tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py
View File

@@ -34,8 +34,6 @@ async def test_run_none_path_flag(workflow_mock, format_and_export_prediction):
"prediction_confidence": 0,
"schema": "test_schema",
"table_name": "test_table",
"opc_servers": ["test_server"],
"opc_output_config": {"test": "config"},
"prediction_store_policy": "erl:1"
}
@@ -56,19 +54,6 @@ async def test_run_none_path_flag(workflow_mock, format_and_export_prediction):
start_to_close_timeout=ANY
)])
workflow_mock.execute_activity_method.assert_has_calls([
call(
Activities.write_opc_data,
{
'opc_output_config': input_data['opc_output_config'],
'data': workflow_mock.execute_local_activity_method.return_value,
**metadata
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_activity_method.assert_has_calls([
call(
Activities.export_data_to_postgres,
@@ -86,7 +71,7 @@ async def test_run_none_path_flag(workflow_mock, format_and_export_prediction):
start_to_close_timeout=ANY
)])
assert workflow_mock.execute_activity_method.call_count == 3
assert workflow_mock.execute_activity_method.call_count == 2
assert workflow_mock.execute_local_activity_method.call_count == 1
@@ -103,8 +88,6 @@ async def test_run_default_path_flag(workflow_mock, format_and_export_prediction
"prediction_confidence": 0,
"schema": "test_schema",
"table_name": "test_table",
"opc_servers": ["test_server"],
"opc_output_config": {"test": "config"},
"comment": "test_comment"
}
@@ -125,19 +108,6 @@ async def test_run_default_path_flag(workflow_mock, format_and_export_prediction
)
])
workflow_mock.execute_activity_method.assert_has_calls([
call(
Activities.write_opc_data,
{
'opc_output_config': input_data['opc_output_config'],
'data': workflow_mock.execute_local_activity_method.return_value,
**metadata
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
workflow_mock.execute_activity_method.assert_has_calls([
call(
Activities.export_data_to_postgres,

18
tests/laborious/workflows/subworkflows/test_prediction_process.py
View File

@@ -38,7 +38,7 @@ async def test_run(workflow_mock, prediction_process):
'retention': '30'
},
'path_priority': ['continue', 'repeat', 'stop'],
'opc_output_config': {'test': 'config'},
'prediction_store_policy': 'lts:1'
}
@@ -125,7 +125,7 @@ async def test_run(workflow_mock, prediction_process):
'model_id': 1,
'model_name': 'test_model_name',
'model_config': input_data['model_config'],
'opc_output_config': input_data['opc_output_config'],
'schema': input_data['schema'],
'table_name': input_data['table_name'],
'comment': 'Error',
@@ -153,7 +153,7 @@ async def test_run_stop_at_input_gate(workflow_mock, prediction_process):
'retention': '30'
},
'path_priority': ['continue', 'repeat', 'stop'],
'opc_output_config': {'test': 'config'}
}
# Mock the activity responses
@@ -201,7 +201,7 @@ async def test_run_stop_at_first_mlflow_response_gate(workflow_mock, prediction_
'retention': '30'
},
'path_priority': ['continue', 'repeat', 'stop'],
'opc_output_config': {'test': 'config'}
}
# Mock the activity responses
@@ -272,7 +272,7 @@ async def test_run_stop_at_mlflow_content_gate(workflow_mock, prediction_process
'retention': '30'
},
'path_priority': ['continue', 'repeat', 'stop'],
'opc_output_config': {'test': 'config'}
}
# Mock the activity responses
@@ -352,7 +352,7 @@ async def test_run_stop_at_mlflow_last_response_gate(workflow_mock, prediction_p
'retention': '30'
},
'path_priority': ['continue', 'repeat', 'stop'],
'opc_output_config': {'test': 'config'}
}
# Mock the activity responses
@@ -536,7 +536,7 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process):
'last_timestamp': last_timestamp,
'model_name': model_name,
'model_config': model_config,
'opc_output_config': {'test': 'config'},
'prediction_store_policy': prediction_store_policy
}, confidence, last_timestamp, 'Prediction Process'
)
@@ -558,7 +558,7 @@ async def test_path_flag_handler_continue(workflow_mock, prediction_process):
'schema': schema,
'table_name': table_name,
'comment': 'Prediction Process',
'opc_output_config': {'test': 'config'},
'prediction_store_policy': prediction_store_policy
}
)
@@ -590,7 +590,7 @@ async def test_path_flag_handler_unknown(workflow_mock, prediction_process):
'last_timestamp': last_timestamp,
'model_name': model_name,
'model_config': model_config,
'opc_output_config': {'test': 'config'},
'prediction_store_policy': prediction_store_policy
}, confidence, last_timestamp, ""
)

4
tests/laborious/workflows/test_predictions_batch.py
View File

@@ -32,7 +32,7 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch
'query': 'SELECT * FROM test',
'schema': 'test_schema',
'table_name': 'test_table',
'opc_output_config': 'test_opc_output_config',
'datetime_columns': ['timestamp', 'created_at'],
'prediction_store_policy': 'erl:1',
'model_config': {
@@ -78,7 +78,7 @@ async def test_run(workflow_mock: AsyncMock, predictions_batch: PredictionsBatch
}),
'model_config': input_data.get('model_config', {}),
'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']),
'opc_output_config': input_data.get('opc_output_config', {}),
'prediction_store_policy': input_data.get('prediction_store_policy', 'erl:1')
}