gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1712

Browse Source 
Update dependencies and refactor input filter handling for consistency

- Updated sientia-dataops-library dependency version from 1.10.3 to 1.10.4 in requirements.txt.
- Refactored input filter handling in the Gates class to read policy and config keys in a case-insensitive manner.
- Updated test cases to ensure consistency in filter key naming conventions across various scenarios.
This commit is contained in:
vitor-aignosi
2026-03-23 14:45:21 -03:00
parent f22cc49b93
commit 503d9aa485
24 changed files with 331 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
e2e/conftest.py
View File

@@ -247,15 +247,17 @@ def mock_minio_repository():
def mock_pi_web_api_repository():
"""Mock PI Web API repository for PI Web API operations."""
mock_repo = MagicMock()
mock_repo.write_value = AsyncMock(
return_value={
'Items': [
{
'WebId': 'web_id_1'
}
]
}
)
async def _write_value(web_ids, value, metadata=None, **kwargs):
"""
Mirror successful PI writes: one response item per requested web_id.
write_pi_web_api_data passes the list into process_pi_web_api_response (not a
wrapped {'Items': ...} envelope).
"""
return [{'WebId': wid, 'Errors': []} for wid in web_ids]
mock_repo.write_value = AsyncMock(side_effect=_write_value)
mock_repo.close = MagicMock()
return mock_repo
@@ -266,7 +268,7 @@ def mock_opc_repository():
mock_repo.write_data = AsyncMock(
return_value=(True, {'response_time': 0.1})
)
mock_repo.disconnect = MagicMock()
mock_repo.disconnect = AsyncMock()
return mock_repo
@pytest_asyncio.fixture
@@ -279,7 +281,8 @@ def patch_create_engine(postgres_engine):
@pytest_asyncio.fixture
def patch_minio_repository(mock_minio_repository):
"""Patch MinioRepository to return mock."""
with patch('sientia_do.repository.minio_repository.MinioRepository', return_value=mock_minio_repository):
# Patch where Activities resolves the symbol (import binds the original class).
with patch('laborious.activities.activities.MinioRepository', return_value=mock_minio_repository):
yield
@pytest_asyncio.fixture
@@ -407,10 +410,13 @@ async def test_activities(
'password': 'test',
},
minio_config={
'endpoint_url': 'http://localhost:9000',
# Host:port only; Minio() prepends http(s):// from the secure flag.
'endpoint_url': 'localhost:9000',
'access_key': 'test',
'secret_key': 'test',
'default_bucket': 'test-bucket',
'retention_hours': 24,
'secure': False,
},
opc_config={},
pi_web_api_config={
@@ -452,7 +458,6 @@ async def temporal_worker(temporal_test_env, test_activities):
test_activities.load_custom_query,
test_activities.load_query_with_minio_offload,
test_activities.cleanup_minio_objects_expired,
test_activities.get_last_timestamp,
test_activities.input_gate,
test_activities.request_transform,
test_activities.mlflow_response_gate,
@@ -465,6 +470,7 @@ async def temporal_worker(temporal_test_env, test_activities):
test_activities.write_pi_web_api_data,
test_activities.write_opc_data,
test_activities.export_data_to_postgres,
test_activities.export_payload_to_postgres,
test_activities.write_metrics,
],
) as worker:

58
e2e/test_predictions_batch_format_export.py
View File

@@ -4,6 +4,7 @@ End-to-end tests for PredictionsBatch workflow - Format and Export scenarios.
import asyncio
from datetime import datetime
from typing import Any, cast
from unittest.mock import ANY, AsyncMock, patch, call
import pandas as pd
@@ -25,13 +26,13 @@ base_input_data = {
'table_name': 'predictions',
'transform_table_name': 'transformed_data',
'input_filters': {
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
'opc_output_config': {},
@@ -191,7 +192,6 @@ async def test_scenario_3_1_1_default_prediction_export(
'Timestamp': '2024-01-01 12:00:00+0000',
'Value': 0.5,
},
endpoint='test_endpoint',
metadata={
'model_id': 311,
'model_name': 'test_model',
@@ -205,7 +205,6 @@ async def test_scenario_3_1_1_default_prediction_export(
'Timestamp': '2024-01-01 12:00:00+0000',
'Value': 0,
},
endpoint='test_endpoint',
metadata={
'model_id': 311,
'model_name': 'test_model',
@@ -217,7 +216,8 @@ async def test_scenario_3_1_1_default_prediction_export(
any_order=True,
)
test_activities.opc_repository['1'].write_data.assert_has_calls(
opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
opc_write_data.assert_has_calls(
[
call('addr_1', 0.5, 'float', ANY,
{
@@ -300,7 +300,8 @@ async def test_scenario_3_1_2_export_with_opc_only(
await start_and_await_workflow(client, input_data, workflow_id)
test_activities.opc_repository['1'].write_data.assert_has_calls(
opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
opc_write_data.assert_has_calls(
[
call('addr_1', 0.5, 'float', ANY,
{
@@ -382,7 +383,6 @@ async def test_scenario_3_1_3_export_with_pi_web_api_only(
'Timestamp': '2024-01-01 12:00:00+0000',
'Value': 0.5,
},
endpoint='test_endpoint',
metadata={
'model_id': 313,
'model_name': 'test_model',
@@ -396,7 +396,6 @@ async def test_scenario_3_1_3_export_with_pi_web_api_only(
'Timestamp': '2024-01-01 12:00:00+0000',
'Value': 0,
},
endpoint='test_endpoint',
metadata={
'model_id': 313,
'model_name': 'test_model',
@@ -408,7 +407,8 @@ async def test_scenario_3_1_3_export_with_pi_web_api_only(
any_order=True,
)
test_activities.opc_repository['1'].write_data.assert_not_called()
opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
opc_write_data.assert_not_called()
assert_prediction(postgres_engine, model_id)
@@ -459,7 +459,8 @@ async def test_scenario_3_1_4_export_without_optional_outputs(
await start_and_await_workflow(client, input_data, workflow_id)
test_activities.pi_web_api_client.write_value.assert_not_called()
test_activities.opc_repository['1'].write_data.assert_not_called()
opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
opc_write_data.assert_not_called()
assert_prediction(postgres_engine, model_id)
@@ -535,7 +536,6 @@ async def test_scenario_3_1_5_export_without_transformed_data(
'Timestamp': '2024-01-01 12:00:00+0000',
'Value': 0.5,
},
endpoint='test_endpoint',
metadata={
'model_id': 315,
'model_name': 'test_model',
@@ -549,7 +549,6 @@ async def test_scenario_3_1_5_export_without_transformed_data(
'Timestamp': '2024-01-01 12:00:00+0000',
'Value': 0,
},
endpoint='test_endpoint',
metadata={
'model_id': 315,
'model_name': 'test_model',
@@ -561,7 +560,8 @@ async def test_scenario_3_1_5_export_without_transformed_data(
any_order=True,
)
test_activities.opc_repository['1'].write_data.assert_has_calls(
opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
opc_write_data.assert_has_calls(
[
call('addr_1', 0.5, 'float', ANY,
{
@@ -695,7 +695,8 @@ async def test_scenario_3_2_2_opc_write_error(
insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
print("[TEST] ✓ Data inserted successfully")
test_activities.opc_repository['1'].write_data.return_value = (False, {
opc_write_data = cast(Any, test_activities.opc_repository['1'].write_data)
opc_write_data.return_value = (False, {
'notification_id': 'OPC_WRITE_DATA_ERROR_1',
'message': 'OPC server unavailable',
'block': 'opc_repository',
@@ -774,25 +775,14 @@ async def test_scenario_3_2_3_pi_web_api_partial_write_error(
insert_sample_data(postgres_engine, model_id, [23.5, 78.2])
print("[TEST] ✓ Data inserted successfully")
test_activities.pi_web_api_client.write_value = AsyncMock(side_effect=[
{
'Items': [
{
'WebId': 'web_id_1',
'Errors': [],
},
]
},
Exception('Tag write failed'),
{
'Items': [
{
'WebId': 'web_id_2',
'Errors': [],
},
]
},
])
test_activities.pi_web_api_client.write_value = AsyncMock(
side_effect=[
# Prediction batch: two web_ids requested, only one acknowledged.
[{'WebId': 'web_id_1', 'Errors': []}],
# Confidence write succeeds.
[{'WebId': 'web_id_2', 'Errors': []}],
]
)
input_data = get_base_input_data(model_id)
input_data['pi_web_api_output_config'] = {

18
e2e/test_predictions_batch_main_workflow.py
View File

@@ -100,13 +100,13 @@ async def test_scenario_1_1_1_happy_path_complete_success(
'table_name': 'predictions',
'transform_table_name': 'transformed_data',
'input_filters': {
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
'opc_output_config': {},
@@ -238,13 +238,13 @@ async def test_scenario_1_2_1_sql_query_execution_error(
'table_name': 'predictions',
'transform_table_name': 'transformed_data',
'input_filters': {
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
'opc_output_config': {},
@@ -407,13 +407,13 @@ async def test_scenario_1_2_3_invalid_datetime_column_specification(
'table_name': 'predictions',
'transform_table_name': 'transformed_data',
'input_filters': {
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
'opc_output_config': {},

33
e2e/test_predictions_batch_prediction_process.py
View File

@@ -5,6 +5,7 @@ End-to-end tests for PredictionsBatch workflow - Prediction Process scenarios.
import asyncio
from datetime import datetime
from decimal import Decimal
from typing import Any
from unittest.mock import MagicMock, patch
import pandas as pd
@@ -27,15 +28,15 @@ base_input_data = {
'transform_table_name': 'transformed_data',
'input_filters': {
'SPECIFIC_VARIABLES_NULL_VALUES': {
'policy': 'CONTINUE', # Continue despite issues, not STOP
'config': {'variables': ['sensor_1']},
'POLICY': 'CONTINUE', # Continue despite issues, not STOP
'CONFIG': {'variables': ['sensor_1']},
},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}},
},
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
'opc_output_config': {},
@@ -59,7 +60,7 @@ def get_base_input_data(model_id):
'query': base_query.format(model_id=model_id),
}
def insert_sample_data(postgres_engine, model_id, values: list[tuple]):
def insert_sample_data(postgres_engine, model_id, values: list[Any]):
with postgres_engine.begin() as conn:
conn.execute(text(f"DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}"))
@@ -111,7 +112,7 @@ async def start_and_await_workflow(client, input_data, workflow_id):
pytest.fail("Workflow execution timed out after 60 seconds")
def assert_continue(
postgres_engine, model_id, prediction_confidence: Decimal = 2,
postgres_engine, model_id, prediction_confidence: Decimal = Decimal(2),
comments: str = 'Input data with bad quality',
):
print("\n[TEST] 4. Verifying prediction was created despite warnings...")
@@ -236,7 +237,7 @@ async def test_scenario_2_1_2_input_gate_triggers_stop(
print("[TEST] ✓ Data inserted successfully")
input_data = get_base_input_data(model_id)
input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['policy'] = 'STOP'
input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'STOP'
print("\n[TEST] 2. Starting workflow that should stop at input gate...")
workflow_id = f'test-input-stop-{datetime.now().timestamp()}'
@@ -285,7 +286,7 @@ async def test_scenario_2_1_3_input_gate_triggers_repeat(
print("[TEST] ✓ Data and previous prediction inserted")
input_data = get_base_input_data(model_id)
input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['policy'] = 'REPEAT'
input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['POLICY'] = 'REPEAT'
print("\n[TEST] 2. Starting workflow that should trigger REPEAT...")
workflow_id = f'test-input-repeat-{datetime.now().timestamp()}'
@@ -345,7 +346,7 @@ async def test_scenario_2_2_1_transform_gate_triggers_continue(
insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
input_data = get_base_input_data(model_id)
input_data['mlflow_transform_filters']['API_ERROR']['policy'] = 'CONTINUE'
input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'CONTINUE'
print("\n[TEST] 2. Starting workflow that should trigger CONTINUE at transform gate...")
workflow_id = f'test-transform-continue-{datetime.now().timestamp()}'
@@ -355,7 +356,7 @@ async def test_scenario_2_2_1_transform_gate_triggers_continue(
postgres_engine=postgres_engine,
model_id=model_id,
prediction_confidence=Decimal(10),
comments='Bad data model',
comments='Unknown MLFlow API error',
)
print("\n[TEST] ✓ All assertions passed!")
@@ -397,7 +398,7 @@ async def test_scenario_2_2_2_transform_gate_triggers_stop(
print("[TEST] ✓ Data inserted successfully")
input_data = get_base_input_data(model_id)
input_data['mlflow_transform_filters']['API_ERROR']['policy'] = 'STOP'
input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'STOP'
print("\n[TEST] 2. Starting workflow that should trigger STOP at transform gate...")
workflow_id = f'test-transform-stop-{datetime.now().timestamp()}'
@@ -449,7 +450,7 @@ async def test_scenario_2_2_3_transform_gate_triggers_repeat(
print("[TEST] ✓ Data inserted successfully")
input_data = get_base_input_data(model_id)
input_data['mlflow_transform_filters']['API_ERROR']['policy'] = 'REPEAT'
input_data['mlflow_transform_filters']['API_ERROR']['POLICY'] = 'REPEAT'
print("\n[TEST] 2. Starting workflow that should trigger REPEAT at transform gate...")
workflow_id = f'test-transform-repeat-{datetime.now().timestamp()}'
@@ -516,7 +517,7 @@ async def test_scenario_2_3_1_predict_gate_triggers_continue(
insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
input_data = get_base_input_data(model_id)
input_data['mlflow_predict_filters']['API_ERROR']['policy'] = 'CONTINUE'
input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'CONTINUE'
print("\n[TEST] 2. Starting workflow that should trigger CONTINUE at predict gate...")
workflow_id = f'test-predict-continue-{datetime.now().timestamp()}'
@@ -526,7 +527,7 @@ async def test_scenario_2_3_1_predict_gate_triggers_continue(
postgres_engine=postgres_engine,
model_id=model_id,
prediction_confidence=Decimal(10),
comments='Bad predict model',
comments='Unknown MLFlow API error',
)
print("\n[TEST] ✓ All assertions passed!")
@@ -567,7 +568,7 @@ async def test_scenario_2_3_2_predict_gate_triggers_stop(
print("[TEST] ✓ Data inserted successfully")
input_data = get_base_input_data(model_id)
input_data['mlflow_predict_filters']['API_ERROR']['policy'] = 'STOP'
input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'STOP'
print("\n[TEST] 2. Starting workflow that should stop at predict gate...")
workflow_id = f'test-predict-stop-{datetime.now().timestamp()}'
@@ -617,7 +618,7 @@ async def test_scenario_2_3_3_predict_gate_triggers_repeat(
print("[TEST] ✓ Data and previous prediction inserted")
input_data = get_base_input_data(model_id)
input_data['mlflow_predict_filters']['API_ERROR']['policy'] = 'REPEAT'
input_data['mlflow_predict_filters']['API_ERROR']['POLICY'] = 'REPEAT'
input_data['path_priority'] = ['REPEAT', 'STOP', 'CONTINUE'] # REPEAT first
print("\n[TEST] 2. Starting workflow that should trigger REPEAT at predict gate...")