gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Update README, requirements, and E2E tests for improved configuration and functionality

- Enhanced the README with updated model configuration examples, including the addition of an alias for production.
- Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`.
- Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity.
- Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs.
- Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.
This commit is contained in:
Vitor Pimentel dos Santos
2026-05-07 17:02:25 -03:00
parent aaf647efdf
commit e6018af23f
51 changed files with 4408 additions and 2660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
tests/laborious/activities/test_opc.py
View File

@@ -1,6 +1,6 @@
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
from unittest.mock import ANY, MagicMock, call, patch
import pytest_asyncio
import pytest
from pandas import DataFrame
from pytest import mark
from sientia_do.notifications.models import NotificationLevel
@@ -23,39 +23,32 @@ def test__init__():
opc_servers=servers,
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
metrics_controller=MagicMock(),
)
assert opc.opc_servers == servers
assert opc.opc_repository == {}
@mark.asyncio
@patch('laborious.activities.opc.OpcRepository')
@patch('laborious.activities.opc.OPC.send_notification_async')
async def test_init_opc(mock_send_notification, mock_opc_repository):
@patch('laborious.activities.opc.OPC.send_notification')
def test_init_opc(mock_send_notification, mock_opc_repository):
mock_logger = MagicMock()
mock_metrics_controller = AsyncMock()
server1 = MagicMock(
connect=AsyncMock(return_value=(True, {})), write_data=AsyncMock(return_value=(True, {}))
)
server2 = MagicMock(
connect=AsyncMock(return_value=(True, {})), write_data=AsyncMock(return_value=(True, {}))
)
server3 = MagicMock(
connect=AsyncMock(
return_value=(
False,
{
'notification_id': 'OPC_CONNECTION_ERROR_server3',
'message': 'Failed to connect to OPC server: Test error',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': 'Test error',
},
)
),
write_data=AsyncMock(return_value=(True, {})),
mock_metrics_controller = MagicMock()
server1 = MagicMock()
server1.connect.return_value = (True, {})
server2 = MagicMock()
server2.connect.return_value = (True, {})
server3 = MagicMock()
server3.connect.return_value = (
False,
{
'notification_id': 'OPC_CONNECTION_ERROR_server3',
'message': 'Failed to connect to OPC server: Test error',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': 'Test error',
},
)
mock_opc_repository.side_effect = [server1, server2, server3]
mock_notification_handler = MagicMock()
@@ -97,7 +90,7 @@ async def test_init_opc(mock_send_notification, mock_opc_repository):
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
await opc.init_opc()
opc.init_opc()
assert opc.opc_servers == servers
assert opc.logger == mock_logger
@@ -112,13 +105,12 @@ async def test_init_opc(mock_send_notification, mock_opc_repository):
server_name='server1',
url='http://localhost:8080',
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
server_uri='opc.tcp://localhost:4840',
cert_path='',
private_key_path='',
server_cert_path='',
notification_handler=mock_notification_handler,
reconnection_interval=60,
metrics_controller=mock_metrics_controller,
),
]
)
@@ -129,13 +121,12 @@ async def test_init_opc(mock_send_notification, mock_opc_repository):
server_name='server2',
url='http://localhost:8080',
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
server_uri='opc.tcp://localhost:4840',
cert_path='',
private_key_path='',
server_cert_path='',
notification_handler=mock_notification_handler,
reconnection_interval=60,
metrics_controller=mock_metrics_controller,
)
]
)
@@ -162,51 +153,49 @@ async def test_init_opc(mock_send_notification, mock_opc_repository):
)
@pytest_asyncio.fixture
@patch('laborious.activities.opc.OpcRepository')
async def opc(mock_opc_repository):
servers = {
'server1': {
'id': 'server1',
'server_name': 'server1',
'url': 'http://localhost:8080',
'server_uri': 'opc.tcp://localhost:4840',
'cert_path': '',
'private_key_path': '',
'server_cert_path': '',
'reconnection_interval': 60,
@pytest.fixture
def opc():
with patch('laborious.activities.opc.OpcRepository') as mock_opc_repository:
mock_opc_repository.return_value.write_data = MagicMock(return_value=(True, {}))
mock_opc_repository.return_value.connect = MagicMock(return_value=(True, {}))
servers = {
'server1': {
'id': 'server1',
'server_name': 'server1',
'url': 'http://localhost:8080',
'server_uri': 'opc.tcp://localhost:4840',
'cert_path': '',
'private_key_path': '',
'server_cert_path': '',
'reconnection_interval': 60,
}
}
}
mock_opc_repository.return_value.write_data = AsyncMock(return_value=(True, {}))
mock_opc_repository.return_value.connect = AsyncMock(return_value=(True, {}))
opc = OPC(
opc_servers=servers,
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
)
await opc.init_opc()
opc.send_notification = MagicMock()
opc.send_notification_async = AsyncMock()
opc.emit_metric = AsyncMock()
return opc
opc_instance = OPC(
opc_servers=servers,
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=MagicMock(),
)
opc_instance.init_opc()
opc_instance.send_notification = MagicMock()
opc_instance.emit_metric_sync = MagicMock()
yield opc_instance
WRITE_DATA_CASES = [
('tag1', 'int', 50),
('tag2', 'float', 50.5),
('tag3', 'bool', True),
('tag4', 'string', 'test'),
('tag4', 'str', 'test'),
]
@mark.parametrize('tag,data_type,data', WRITE_DATA_CASES)
@mark.asyncio
async def test_write_data_success(opc, tag, data_type, data):
def test_write_data_success(opc, tag, data_type, data):
opc.opc_repository['server1'].write_data.return_value = (True, {'response_time': 0.1})
result = await opc.write_data(
result = opc.write_data(
server_id='server1',
tag=tag,
data=data,
@@ -220,8 +209,7 @@ async def test_write_data_success(opc, tag, data_type, data):
)
@mark.asyncio
async def test_write_data_failed(opc):
def test_write_data_failed(opc):
opc.opc_repository['server1'].write_data.return_value = (
False,
{
@@ -233,7 +221,7 @@ async def test_write_data_failed(opc):
},
)
result = await opc.write_data(
result = opc.write_data(
server_id='server1',
tag='tag1',
data=50,
@@ -243,7 +231,7 @@ async def test_write_data_failed(opc):
)
assert result is None
opc.send_notification_async.assert_called_once_with(
opc.send_notification.assert_called_once_with(
metadata=metadata,
notification_id='OPC_WRITE_DATA_ERROR_server1',
message='Failed to write data to OPC server: Test error',
@@ -253,12 +241,11 @@ async def test_write_data_failed(opc):
)
@mark.asyncio
async def test_write_data_exception(opc):
def test_write_data_exception(opc):
opc.opc_repository['server1'].write_data.side_effect = Exception('Test error')
try:
await opc.write_data(
opc.write_data(
server_id='server1',
tag='tag1',
data=50,
@@ -268,7 +255,7 @@ async def test_write_data_exception(opc):
)
except Exception:
opc.send_notification_async.assert_called_once_with(
opc.send_notification.assert_called_once_with(
metadata=metadata,
notification_id='WRITE_OPC_PREDICTION_ERROR',
message='Error writing data to OPC server: Test error',
@@ -281,9 +268,8 @@ async def test_write_data_exception(opc):
raise AssertionError('Expected an exception to be raised')
@mark.asyncio
async def test_manage_output_tags_success(opc):
opc.write_data = AsyncMock(return_value=0.1)
def test_manage_output_tags_success(opc):
opc.write_data = MagicMock(return_value=0.1)
data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]})
config = {
@@ -291,7 +277,7 @@ async def test_manage_output_tags_success(opc):
'confidence_tags': {'tag2': {'data_type': 'float'}},
}
output_data, opc_metrics = await opc.manage_output_tags(
output_data, opc_metrics = opc.manage_output_tags(
server_id='server1',
config=config,
data=data,
@@ -322,16 +308,15 @@ async def test_manage_output_tags_success(opc):
)
@mark.asyncio
@mark.parametrize('side_effect', [[0.1, None], [None, 0.2]])
async def test_manage_output_tags_failed(opc, side_effect):
opc.write_data = AsyncMock(side_effect=side_effect)
def test_manage_output_tags_failed(opc, side_effect):
opc.write_data = MagicMock(side_effect=side_effect)
data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]})
config = {
'prediction_tags': {'tag1': {'data_type': 'float'}},
'confidence_tags': {'tag2': {'data_type': 'float'}},
}
output_data, opc_metrics = await opc.manage_output_tags(
output_data, opc_metrics = opc.manage_output_tags(
server_id='server1',
config=config,
data=data,
@@ -361,14 +346,13 @@ async def test_manage_output_tags_failed(opc, side_effect):
)
@mark.asyncio
async def test_manage_output_tags_do_nothing(opc):
opc.write_data = AsyncMock(return_value=0.1)
def test_manage_output_tags_do_nothing(opc):
opc.write_data = MagicMock(return_value=0.1)
data = DataFrame({'prediction': [0.75], 'prediction_confidence': [0.95]})
config = {
'_invalid_key': {'tag1': {'data_type': 'float'}},
}
output_data, opc_metrics = await opc.manage_output_tags(
output_data, opc_metrics = opc.manage_output_tags(
server_id='server1',
config=config,
data=data,
@@ -379,11 +363,9 @@ async def test_manage_output_tags_do_nothing(opc):
opc.write_data.assert_not_called()
@mark.asyncio
@patch('laborious.activities.opc.DataFrame')
async def test_write_opc_data_success(mock_dataframe, opc):
# Arrange
input_data = {
def test_write_opc_data_success(mock_dataframe, opc):
input_data: dict[str, object] = {
**metadata,
'data': {'prediction': [0.75], 'prediction_confidence': [0.95]},
'opc_output_config': {
@@ -393,19 +375,19 @@ async def test_write_opc_data_success(mock_dataframe, opc):
}
},
}
opc_output_config = input_data['opc_output_config']
assert isinstance(opc_output_config, dict)
# Act
opc.manage_output_tags = AsyncMock(return_value=(True, {'tag1': 0.1, 'tag2': 0.2}))
opc.manage_output_tags = MagicMock(return_value=(True, {'tag1': 0.1, 'tag2': 0.2}))
opc.process_confidence = MagicMock(return_value={'data': 'data'})
output_data, opc_metrics = await opc.write_opc_data(input_data)
output_data, opc_metrics = opc.write_opc_data(input_data)
# Assert
assert output_data == {'data': 'data'}
assert opc_metrics == {'server1': {'tag1': 0.1, 'tag2': 0.2}}
opc.manage_output_tags.assert_called_once_with(
'server1',
input_data['opc_output_config']['server1'],
opc_output_config['server1'],
mock_dataframe.return_value,
metadata['metadata'],
)
@@ -416,9 +398,7 @@ async def test_write_opc_data_success(mock_dataframe, opc):
)
@mark.asyncio
async def test_write_opc_data_empty_config(opc):
# Arrange
def test_write_opc_data_empty_config(opc):
input_data = {
**metadata,
'data': {'prediction': [0.75], 'prediction_confidence': [0.95]},
@@ -426,16 +406,13 @@ async def test_write_opc_data_empty_config(opc):
'opc_output_config': {'server1': {'prediction_tags': {}, 'confidence_tags': {}}},
}
# Act
await opc.write_opc_data(input_data)
opc.write_opc_data(input_data)
# Assert
opc.opc_repository['server1'].write_data.assert_not_called()
@mark.asyncio
async def test_write_opc_data_no_validate_server(opc):
opc.validate_server = AsyncMock(return_value=False)
def test_write_opc_data_no_validate_server(opc):
opc.validate_server = MagicMock(return_value=False)
input_data = {
**metadata,
'data': {'prediction': [0.75], 'prediction_confidence': [0.95]},
@@ -447,10 +424,8 @@ async def test_write_opc_data_no_validate_server(opc):
},
}
# Act
await opc.write_opc_data(input_data)
opc.write_opc_data(input_data)
# Assert
opc.opc_repository['server1'].write_data.assert_not_called()
@@ -462,21 +437,18 @@ async def test_write_opc_data_no_validate_server(opc):
],
)
def test_process_confidence(opc, data, success, expected):
# Act
result = opc.process_confidence(data, success, metadata)
# Assert
assert result['prediction_confidence'][0] == expected
@mark.asyncio
async def test_validate_server(opc):
assert await opc.validate_server('server1', metadata) is True
assert await opc.validate_server('server2', metadata) is False
def test_validate_server(opc):
assert opc.validate_server('server1', metadata) is True
assert opc.validate_server('server2', metadata) is False
@mark.asyncio
async def test_close(opc):
opc.opc_repository['server1'].disconnect = AsyncMock(return_value=True)
await opc.close()
opc.opc_repository['server1'].disconnect.assert_called_once()
def test_close(opc):
disconnect_mock = MagicMock(return_value=None)
opc.opc_repository['server1'].disconnect = disconnect_mock
opc.close()
disconnect_mock.assert_called_once()