gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-994

Browse Source 
Refactor activity methods and update requirements.txt to enhance functionality and remove deprecated filters. Added detailed docstrings for clarity and improved error handling in data processing workflows.
This commit is contained in:
vitor-aignosi
2025-05-09 16:27:57 -03:00
parent 43f19ed93a
commit d09fb6ac5e
22 changed files with 1435 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
tests/laborious/activities/test_base.py Normal file
View File

@@ -0,0 +1,32 @@
from unittest.mock import MagicMock
from laborious.activities.base import BaseActivity
from pytest import fixture
from sientia_do.notifications.models import Notification
@fixture
def base_activity():
return BaseActivity(
logger=MagicMock(),
notification_handler=MagicMock(),
)
def test_prepare_activity(base_activity):
base_activity.notification_handler.base_notification = Notification(
project="project",
pipeline="pipeline",
trigger="-",
model_name="-",
model_id="-",
)
base_activity.prepare_activity(
schedule_name="test_schedule",
model_name="test_model",
model_id="test_model_id",
)
assert base_activity.notification_handler.base_notification.schedule_name == "test_schedule"
assert base_activity.notification_handler.base_notification.model_name == "test_model"
assert base_activity.notification_handler.base_notification.model_id == "test_model_id"

335
tests/laborious/activities/test_gates.py
View File

@@ -15,9 +15,9 @@ def gates():
@mark.asyncio
@patch('laborious.activities.gates.filter_functions')
@patch('laborious.activities.gates.input_filter_functions')
async def test_input_gate_specific_variables_null_values_with_stop_policy_only(
filter_functions_mock,
input_filter_functions_mock,
gates
):
specific_variables_null_values_mock = MagicMock(return_value=True)
@@ -26,9 +26,15 @@ async def test_input_gate_specific_variables_null_values_with_stop_policy_only(
def functions_side_effect(x):
if x == 'SPECIFIC_VARIABLES_NULL_VALUES':
return specific_variables_null_values_mock
if x == 'path_confidence':
return {
'stop': -1,
'continue': 2,
'repeat': -1
}
return empty_data_mock
filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_data = {
'filters': {
@@ -40,7 +46,8 @@ async def test_input_gate_specific_variables_null_values_with_stop_policy_only(
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
}
},
'path_priority': ['stop', 'continue', 'repeat']
}
result = await gates.input_gate(input_data)
@@ -55,9 +62,9 @@ async def test_input_gate_specific_variables_null_values_with_stop_policy_only(
@mark.asyncio
@patch('laborious.activities.gates.filter_functions')
@patch('laborious.activities.gates.input_filter_functions')
async def test_input_gate_specific_variables_null_values_with_continue_policy_only(
filter_functions_mock,
input_filter_functions_mock,
gates
):
specific_variables_null_values_mock = MagicMock(return_value=True)
@@ -66,9 +73,15 @@ async def test_input_gate_specific_variables_null_values_with_continue_policy_on
def functions_side_effect(x):
if x == 'SPECIFIC_VARIABLES_NULL_VALUES':
return specific_variables_null_values_mock
if x == 'path_confidence':
return {
'stop': -1,
'continue': 2,
'repeat': -1
}
return empty_data_mock
filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_data = {
'filters': {
@@ -80,7 +93,8 @@ async def test_input_gate_specific_variables_null_values_with_continue_policy_on
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
}
},
'path_priority': ['stop', 'continue', 'repeat']
}
result = await gates.input_gate(input_data)
@@ -95,9 +109,9 @@ async def test_input_gate_specific_variables_null_values_with_continue_policy_on
@mark.asyncio
@patch('laborious.activities.gates.filter_functions')
@patch('laborious.activities.gates.input_filter_functions')
async def test_input_gate_specific_variables_null_values_no_filtered(
filter_functions_mock,
input_filter_functions_mock,
gates
):
specific_variables_null_values_mock = MagicMock(return_value=False)
@@ -106,9 +120,15 @@ async def test_input_gate_specific_variables_null_values_no_filtered(
def functions_side_effect(x):
if x == 'SPECIFIC_VARIABLES_NULL_VALUES':
return specific_variables_null_values_mock
if x == 'path_confidence':
return {
'stop': -1,
'continue': 2,
'repeat': -1
}
return empty_data_mock
filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_data = {
'filters': {
@@ -120,7 +140,8 @@ async def test_input_gate_specific_variables_null_values_no_filtered(
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
}
},
'path_priority': ['stop', 'continue', 'repeat']
}
result = await gates.input_gate(input_data)
@@ -137,9 +158,9 @@ async def test_input_gate_specific_variables_null_values_no_filtered(
@mark.asyncio
@patch('laborious.activities.gates.filter_functions')
@patch('laborious.activities.gates.input_filter_functions')
async def test_input_gate_one_stop_policy(
filter_functions_mock,
input_filter_functions_mock,
gates
):
specific_variables_null_values_mock = MagicMock(return_value=True)
@@ -148,9 +169,15 @@ async def test_input_gate_one_stop_policy(
def functions_side_effect(x):
if x == 'SPECIFIC_VARIABLES_NULL_VALUES':
return specific_variables_null_values_mock
if x == 'path_confidence':
return {
'stop': -1,
'continue': 2,
'repeat': -1
}
return empty_data_mock
filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_data = {
'filters': {
@@ -165,7 +192,8 @@ async def test_input_gate_one_stop_policy(
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
}
},
'path_priority': ['stop', 'continue', 'repeat']
}
result = await gates.input_gate(input_data)
@@ -184,9 +212,9 @@ async def test_input_gate_one_stop_policy(
@mark.asyncio
@patch('laborious.activities.gates.filter_functions')
@patch('laborious.activities.gates.input_filter_functions')
async def test_input_gate_one_continue_policy(
filter_functions_mock,
input_filter_functions_mock,
gates
):
specific_variables_null_values_mock = MagicMock(return_value=False)
@@ -195,9 +223,15 @@ async def test_input_gate_one_continue_policy(
def functions_side_effect(x):
if x == 'SPECIFIC_VARIABLES_NULL_VALUES':
return specific_variables_null_values_mock
if x == 'path_confidence':
return {
'stop': -1,
'continue': 2,
'repeat': -1
}
return empty_data_mock
filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_data = {
'filters': {
@@ -212,7 +246,8 @@ async def test_input_gate_one_continue_policy(
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
}
},
'path_priority': ['stop', 'continue', 'repeat']
}
result = await gates.input_gate(input_data)
@@ -231,9 +266,9 @@ async def test_input_gate_one_continue_policy(
@mark.asyncio
@patch('laborious.activities.gates.filter_functions')
@patch('laborious.activities.gates.input_filter_functions')
async def test_input_gate_no_filtered(
filter_functions_mock,
input_filter_functions_mock,
gates
):
specific_variables_null_values_mock = MagicMock(return_value=False)
@@ -242,9 +277,15 @@ async def test_input_gate_no_filtered(
def functions_side_effect(x):
if x == 'SPECIFIC_VARIABLES_NULL_VALUES':
return specific_variables_null_values_mock
if x == 'path_confidence':
return {
'stop': -1,
'continue': 2,
'repeat': -1
}
return empty_data_mock
filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_filter_functions_mock.__getitem__.side_effect = functions_side_effect
input_data = {
'filters': {
@@ -259,7 +300,8 @@ async def test_input_gate_no_filtered(
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
}
},
'path_priority': ['stop', 'continue', 'repeat']
}
result = await gates.input_gate(input_data)
@@ -276,12 +318,12 @@ async def test_input_gate_no_filtered(
@mark.asyncio
@patch('laborious.activities.gates.filter_functions')
@patch('laborious.activities.gates.input_filter_functions')
async def test_input_gate_error(
filter_functions_mock,
input_filter_functions_mock,
gates
):
filter_functions_mock.__getitem__.side_effect = KeyError('test')
input_filter_functions_mock.__getitem__.side_effect = KeyError('test')
input_data = {
'filters': {
@@ -293,7 +335,8 @@ async def test_input_gate_error(
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
}
},
'path_priority': ['stop', 'continue', 'repeat'],
}
result = await gates.input_gate(input_data)
@@ -306,3 +349,239 @@ async def test_input_gate_error(
level=NotificationLevel.ERROR,
attachment_content=ANY
)
transform_filter_path_confidence = {
'stop': -1,
'continue': 255,
'repeat': -1
}
@mark.asyncio
@patch('laborious.activities.gates.mlflow_response_filter_functions')
async def test_mlflow_response_gate_no_filtered(
mlflow_response_filter_functions_mock,
gates
):
api_error_filter_mock = MagicMock(return_value=False)
def transform_filter_functions_side_effect(x: str):
if x == 'API_ERROR':
return api_error_filter_mock
if x == 'path_confidence':
return transform_filter_path_confidence
mlflow_response_filter_functions_mock.__getitem__.side_effect = transform_filter_functions_side_effect
input_data = {
'filters': {
'API_ERROR': {
'POLICY': 'stop',
}
},
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
},
'path_priority': ['stop', 'continue', 'repeat'],
'type': 'predict'
}
result = await gates.mlflow_response_gate(input_data)
assert result == (None, 0)
api_error_filter_mock.assert_called_once_with(
input_data['data'],
input_data['filters']['API_ERROR']
)
gates.notification_handler.build_and_send_notification.assert_not_called()
@mark.asyncio
@patch('laborious.activities.gates.mlflow_response_filter_functions')
async def test_mlflow_response_gate_filtered(
mlflow_response_filter_functions_mock,
gates
):
api_error_filter_mock = MagicMock(return_value=True)
def transform_filter_functions_side_effect(x: str):
if x == 'API_ERROR':
return api_error_filter_mock
if x == 'path_confidence':
return transform_filter_path_confidence
mlflow_response_filter_functions_mock.__getitem__.side_effect = transform_filter_functions_side_effect
input_data = {
'filters': {
'API_ERROR': {
'POLICY': 'continue',
}
},
'data': {
'success': False,
'content': {
'message': 'Error',
'traceback': 'Error'
}
},
'path_priority': ['stop', 'continue', 'repeat'],
'type': 'predict'
}
result = await gates.mlflow_response_gate(input_data)
assert result == ('continue', 255)
api_error_filter_mock.assert_called_once_with(
input_data['data'],
input_data['filters']['API_ERROR']
)
gates.notification_handler.build_and_send_notification.assert_called_once_with(
notification_id='PREDICT_GATE_RESPONSE_FILTER__API_ERROR',
message=input_data['data']['content']['message'],
block='mlflow_gate',
level=NotificationLevel.WARNING,
attachment_content=input_data['data']['content']['traceback']
)
@mark.asyncio
@patch('laborious.activities.gates.mlflow_content_filter_functions')
async def test_mlflow_content_gate_no_filtered(
mlflow_content_filter_functions_mock,
gates
):
nan_values_filter_mock = MagicMock(return_value=False)
def transform_filter_functions_side_effect(x: str):
if x == 'NAN_VALUES':
return nan_values_filter_mock
if x == 'path_confidence':
return transform_filter_path_confidence
mlflow_content_filter_functions_mock.__getitem__.side_effect = transform_filter_functions_side_effect
input_data = {
'filters': {
'NAN_VALUES': {
'POLICY': 'repeat',
}
},
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
},
'path_priority': ['stop', 'continue', 'repeat'],
'type': 'predict'
}
result = await gates.mlflow_content_gate(input_data)
assert result == (None, 0)
nan_values_filter_mock_args = nan_values_filter_mock.call_args
assert nan_values_filter_mock_args[0][0].equals(DataFrame(
{'variable': ['variable1', 'variable2'], 'value': [1, 2]}))
assert nan_values_filter_mock_args[0][1] == input_data['filters']['NAN_VALUES']
gates.notification_handler.build_and_send_notification.assert_not_called()
@mark.asyncio
@patch('laborious.activities.gates.mlflow_content_filter_functions')
async def test_mlflow_content_gate_filtered(
mlflow_content_filter_functions_mock,
gates
):
nan_values_filter_mock = MagicMock(return_value=True)
def transform_filter_functions_side_effect(x: str):
if x == 'NAN_VALUES':
return nan_values_filter_mock
if x == 'path_confidence':
return transform_filter_path_confidence
mlflow_content_filter_functions_mock.__getitem__.side_effect = transform_filter_functions_side_effect
input_data = {
'filters': {
'NAN_VALUES': {
'POLICY': 'repeat',
}
},
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
},
'path_priority': ['stop', 'continue', 'repeat'],
'type': 'predict'
}
result = await gates.mlflow_content_gate(input_data)
assert result == ('repeat', -1)
nan_values_filter_mock_args = nan_values_filter_mock.call_args
assert nan_values_filter_mock_args[0][0].equals(DataFrame(
{'variable': ['variable1', 'variable2'], 'value': [1, 2]}))
assert nan_values_filter_mock_args[0][1] == input_data['filters']['NAN_VALUES']
gates.notification_handler.build_and_send_notification.assert_called_once_with(
notification_id='PREDICT_GATE_CONTENT_FILTER__NAN_VALUES',
message="Data not passed the content filter NAN_VALUES:{'POLICY': 'repeat'}",
block='mlflow_gate',
level=NotificationLevel.WARNING,
attachment_content=DataFrame(input_data['data']).to_string()
)
@mark.asyncio
async def test_format_prediction(
gates
):
input_data = {
'data': {
'variable': ['variable1', 'variable2'],
'value': [1, 2]
},
'timestamp': '2021-01-01',
'model_id': 'model_id',
'prediction_confidence': 0.95
}
expected_output = DataFrame(input_data['data'])
expected_output['timestamp'] = input_data['timestamp']
expected_output['model_id'] = input_data['model_id']
expected_output['prediction_confidence'] = input_data['prediction_confidence']
expected_output['prediction_status'] = 'Good'
expected_output['comment'] = ''
result = await gates.format_prediction(input_data)
assert result == expected_output.to_dict()
@mark.asyncio
async def test_format_default_prediction(
gates
):
input_data = {
'timestamp': '2021-01-01',
'model_id': 'model_id',
'prediction_confidence': 0.95,
'comment': 'Comment'
}
expected_output = DataFrame({
'prediction': [0],
'response_time': [0],
'timestamp': [input_data['timestamp']],
'model_id': [input_data['model_id']],
'prediction_confidence': [input_data['prediction_confidence']],
'prediction_status': ['Bad'],
'comment': [input_data['comment']]
})
result = await gates.format_default_prediction(input_data)
assert result == expected_output.to_dict()

121
tests/laborious/activities/test_mlflow.py Normal file
View File

@@ -0,0 +1,121 @@
from unittest.mock import MagicMock, patch
import numpy as np
from pytest import fixture, mark
from laborious.activities.mlflow import MLFlow
@patch("laborious.activities.mlflow.MLFlowRepository")
def test___init__(mock_mlflow_repository):
mlflow = MLFlow(
mlflow_host="http://localhost",
mlflow_port=5000,
mlflow_username="admin",
mlflow_password="admin",
logger=MagicMock(),
notification_handler=MagicMock()
)
assert mlflow.mlflow_host == "http://localhost"
assert mlflow.mlflow_port == 5000
assert mlflow.mlflow_username == "admin"
assert mlflow.mlflow_password == "admin"
mock_mlflow_repository.assert_called_once_with(
"http://localhost:5000", "admin", "admin"
)
@fixture
@patch("laborious.activities.mlflow.MLFlowRepository")
def mlflow(mock_mlflow_repository):
return MLFlow(
mlflow_host="http://localhost:5000",
mlflow_port=5000,
mlflow_username="admin",
mlflow_password="admin",
logger=MagicMock(),
notification_handler=MagicMock()
)
@mark.asyncio
@patch("laborious.activities.mlflow.DataFrame")
@patch("laborious.activities.mlflow.max")
async def test_request_transform(mock_max, mock_dataframe, mlflow):
mock_max.return_value = '2024-01-02'
# Mock input data
input_data = {
'data': [
{'timestamp': '2024-01-01', 'variable': 'var1', 'value': 1.0},
{'timestamp': '2024-01-01', 'variable': 'var2', 'value': 2.0},
{'timestamp': '2024-01-02', 'variable': 'var1', 'value': 3.0},
{'timestamp': '2024-01-02', 'variable': 'var2', 'value': 4.0}
],
'model_name': 'test_model',
'model_retention': 30
}
# Mock the transform response
expected_response = {'prediction': [0.5, 0.6]}
mlflow.model_monitoring_repository.transform.return_value = expected_response
# Call the method
response_data, timestamp = await mlflow.request_transform(input_data)
# Verify the data was correctly transformed
mock_dataframe.assert_called_once_with(input_data['data'])
mock_dataframe.return_value.pivot.assert_called_once_with(
index='timestamp', columns='variable', values='value'
)
mock_dataframe = mock_dataframe.return_value.pivot.return_value
mock_dataframe.fillna.assert_called_once_with(np.nan, inplace=True)
mock_dataframe.reset_index.assert_called_once()
mock_dataframe.columns.name = None
# Verify the response
assert response_data == expected_response
assert timestamp == '2024-01-02'
# Verify the repository was called with correct arguments
mlflow.model_monitoring_repository.transform.assert_called_once_with(
'test_model', mock_dataframe, 30
)
@mark.asyncio
@patch("laborious.activities.mlflow.DataFrame")
@patch("laborious.activities.mlflow.max")
async def test_request_predict(mock_max, mock_dataframe, mlflow):
mock_max.return_value = '2024-01-02'
# Mock input data
input_data = {
'data': [
{'timestamp': '2024-01-01', 'variable': 'var1', 'value': 1.0},
{'timestamp': '2024-01-01', 'variable': 'var2', 'value': 2.0},
{'timestamp': '2024-01-02', 'variable': 'var1', 'value': 3.0},
{'timestamp': '2024-01-02', 'variable': 'var2', 'value': 4.0}
],
'model_name': 'test_model',
'model_retention': 30
}
# Mock the predict response
expected_response = {'prediction': [0.5, 0.6]}
mlflow.model_monitoring_repository.predict.return_value = expected_response
# Call the method
response_data = await mlflow.request_predict(input_data)
mock_dataframe.assert_called_once_with(input_data['data'])
mock_dataframe.return_value.replace.assert_called_once_with(
np.nan, None, inplace=True
)
# Verify the response
assert response_data == expected_response
# Verify the repository was called with correct arguments
mlflow.model_monitoring_repository.predict.assert_called_once_with(
'test_model', mock_dataframe.return_value, 30
)

178
tests/laborious/activities/test_opc.py Normal file
View File

@@ -0,0 +1,178 @@
from unittest.mock import patch, MagicMock
from pytest import fixture, mark
from laborious.activities.opc import OPC
from sientia_do.notifications.models import NotificationLevel
from unittest.mock import ANY
@patch("laborious.activities.opc.OpcRepository")
def test___init__(mock_opc_repository):
opc = OPC(
name="test",
url="http://localhost:8080",
server_uri="opc.tcp://localhost:4840",
cert_path="",
private_key_path="",
server_cert_path="",
logger=MagicMock(),
notification_handler=MagicMock()
)
assert opc.name == "test"
assert opc.url == "http://localhost:8080"
assert opc.server_uri == "opc.tcp://localhost:4840"
assert opc.cert_path == ""
assert opc.private_key_path == ""
assert opc.server_cert_path == ""
assert opc.opc_repository == mock_opc_repository.return_value
mock_opc_repository.assert_called_once_with(
name="test",
url="http://localhost:8080",
server_uri="opc.tcp://localhost:4840",
cert_path="",
private_key_path="",
server_cert_path="",
logger=opc.logger,
)
opc.opc_repository.connect.assert_called_once()
@fixture
@patch("laborious.activities.opc.OpcRepository")
def opc(mock_opc_repository):
return OPC(
name="test",
url="http://localhost:8080",
server_uri="opc.tcp://localhost:4840",
cert_path="",
private_key_path="",
server_cert_path="",
logger=MagicMock(),
notification_handler=MagicMock()
)
@mark.asyncio
async def test_write_opc_data_success(opc):
# Arrange
input_data = {
'data': {
'prediction': [0.75],
'prediction_confidence': [0.95]
},
'opc_servers': ['server1'],
'opc_output_config': {
'prediction_tags': {
'tag1': {'data_type': 'float'}
},
'confidence_tags': {
'tag2': {'data_type': 'float'}
}
}
}
# Act
await opc.write_opc_data(input_data)
# Assert
opc.opc_repository.write_data.assert_any_call('tag1', 0.75, 'float')
opc.opc_repository.write_data.assert_any_call('tag2', 0.95, 'float')
assert opc.opc_repository.write_data.call_count == 2
@mark.asyncio
async def test_write_opc_data_prediction_error(opc):
# Arrange
input_data = {
'data': {
'prediction': [0.75],
'prediction_confidence': [0.95]
},
'opc_servers': ['server1'],
'opc_output_config': {
'prediction_tags': {
'tag1': {'data_type': 'float'}
}
}
}
opc.opc_repository.write_data.side_effect = Exception("Test error")
# Act
await opc.write_opc_data(input_data)
# Assert
opc.notification_handler.build_and_send_notification.assert_called_with(
notification_id="WRITE_OPC_PREDICTION_ERROR",
message="Error writing data to OPC server: Test error",
block="write_opc_data",
level=NotificationLevel.ERROR,
attachment_content=ANY
)
opc.logger.error.assert_called_once()
@mark.asyncio
async def test_write_opc_data_confidence_error(opc):
# Arrange
input_data = {
'data': {
'prediction': [0.75],
'prediction_confidence': [0.95]
},
'opc_servers': ['server1'],
'opc_output_config': {
'prediction_tags': {
'tag1': {'data_type': 'float'}
},
'confidence_tags': {
'tag2': {'data_type': 'float'}
}
}
}
# Make first call succeed but second fail
def side_effect(*args, **kwargs):
if args[0] == 'tag2':
raise ValueError("Test error")
return None
opc.opc_repository.write_data.side_effect = side_effect
# Act
await opc.write_opc_data(input_data)
# Assert
opc.notification_handler.build_and_send_notification.assert_called_with(
notification_id="WRITE_OPC_CONFIDENCE_ERROR",
message="Error writing data to OPC server: Test error",
block="write_opc_data",
level=NotificationLevel.ERROR,
attachment_content=ANY
)
opc.logger.error.assert_called_once()
@mark.asyncio
async def test_write_opc_data_empty_config(opc):
# Arrange
input_data = {
'data': {
'prediction': [0.75],
'prediction_confidence': [0.95]
},
'opc_servers': ['server1'],
'opc_output_config': {
'prediction_tags': {},
'confidence_tags': {}
}
}
# Act
await opc.write_opc_data(input_data)
# Assert
opc.opc_repository.write_data.assert_not_called()