gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1478

Browse Source 
SIENTIAPDE-1478
Enhance end-to-end tests for PredictionsBatch workflow

- Added new test scenarios for input and transform gates handling CONTINUE, STOP, and REPEAT policies.
- Implemented sample data insertion functions for testing various prediction outcomes.
- Updated existing tests to verify behavior under different input conditions and response validations.
- Refactored test structure for clarity and maintainability.
This commit is contained in:
vitor-aignosi
2026-01-15 15:46:05 -03:00
parent 85e69a09e0
commit b5b86db10c
2 changed files with 770 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

870
e2e/test_predictions_batch_prediction_process.py
View File

@@ -4,10 +4,12 @@ End-to-end tests for PredictionsBatch workflow - Prediction Process scenarios.
import asyncio
from datetime import datetime
from unittest.mock import patch
from decimal import Decimal
from unittest.mock import MagicMock, patch
import pandas as pd
import pytest
from pytz import timezone
from sqlalchemy import text
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import Worker
@@ -15,58 +17,7 @@ from temporalio.worker import Worker
from laborious.activities.activities import Activities
from laborious.workflows.predictions_batch import PredictionsBatch
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_1_input_gate_triggers_continue(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
):
"""
Scenario 2.1.1: Input Gate Triggers CONTINUE
Description:
Input gate determines data should use previous prediction.
Expected Behavior:
- input_gate returns path_flag='CONTINUE'
- path_flag_handler calls export workflow with input data directly
- MLFlow transform and predict skipped
- Data exported as-is
Assertions:
- input_gate called
- MLFlow operations NOT called
- Export workflow called with original data
- Workflow completes
"""
client = temporal_test_env.client
print("\n[TEST] 1. Inserting test data...")
with postgres_engine.begin() as conn:
conn.execute(text("DELETE FROM predictions_schema.laborious_data WHERE model_id = 201"))
# Insert data with some null values (quality issue)
insert_sql = """
INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
VALUES
(201, 'sensor_1', NULL, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
(201, 'sensor_2', 78.2, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
"""
conn.execute(text(insert_sql))
print("[TEST] ✓ Data inserted successfully")
input_data = {
'metadata': {
'metadata': {
'model_id': 201,
'model_name': 'test_model',
'schedule_name': 'test-schedule',
'workflow_name': 'predictions_batch',
}
},
base_input_data = {
'schedule_name': 'test-schedule',
'model_name': 'test_model',
'model_id': 201,
@@ -99,9 +50,51 @@ async def test_scenario_2_1_1_input_gate_triggers_continue(
'datetime_columns': ['timestamp', 'created_at'],
}
print("\n[TEST] 2. Starting workflow with CONTINUE policy...")
workflow_id = f'test-continue-policy-{datetime.now().timestamp()}'
base_query = "SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = {model_id}"
def get_base_input_data(model_id):
return {
**base_input_data,
'model_id': model_id,
'query': base_query.format(model_id=model_id),
}
def insert_sample_data(postgres_engine, model_id, values: list[tuple]):
with postgres_engine.begin() as conn:
conn.execute(text(f"DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}"))
# Insert data with some null values (quality issue)
values_sql = []
for i, value in enumerate(values):
values_sql.append(f"""
({model_id}, 'sensor_{i+1}', {value}, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
""")
insert_sql = f"""
INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
VALUES
{', '.join(values_sql)}
"""
conn.execute(text(insert_sql))
def insert_sample_prediction(postgres_engine, model_id):
with postgres_engine.begin() as conn:
conn.execute(text(f"DELETE FROM predictions_schema.predictions WHERE model_id = {model_id}"))
# Insert data with some null values (quality issue)
insert_sql = f"""
INSERT INTO predictions_schema.predictions (model_id, timestamp, prediction, prediction_confidence, prediction_status, comments, response_time)
VALUES
({model_id}, '2024-01-01 12:00:00+00:00', 10, 0, 'Good', '', 0.1)
"""
conn.execute(text(insert_sql))
return (model_id, Decimal(10), Decimal(0), 'Good')
async def start_and_await_workflow(client, input_data, workflow_id):
handle = await client.start_workflow(
PredictionsBatch.run,
input_data,
@@ -117,10 +110,11 @@ async def test_scenario_2_1_1_input_gate_triggers_continue(
except asyncio.TimeoutError:
pytest.fail("Workflow execution timed out after 60 seconds")
def assert_continue(postgres_engine, model_id):
print("\n[TEST] 4. Verifying prediction was created despite warnings...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT model_id, prediction, prediction_confidence, prediction_status, comments FROM predictions_schema.predictions WHERE model_id = 201")
text(f"SELECT model_id, prediction, prediction_confidence, prediction_status, comments FROM predictions_schema.predictions WHERE model_id = {model_id}")
)
prediction_rows = result_query.fetchall()
assert len(prediction_rows) == 1, "Expected one prediction record despite warnings"
@@ -132,6 +126,74 @@ async def test_scenario_2_1_1_input_gate_triggers_continue(
assert row[3] == 'Bad', f"Expected prediction_status='Bad', got {row[3]}"
assert row[4] == 'Input data with bad quality', f"Expected comments='Input data with bad quality', got {row[4]}"
def assert_stop(postgres_engine, model_id):
print("\n[TEST] 4. Verifying no predictions were created...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text(f"SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = {model_id}")
)
count = result_query.scalar()
assert count == 0, f"Expected no predictions, but found {count} records"
def assert_repeat(postgres_engine, model_id, last_prediction: list):
print("\n[TEST] 4. Verifying prediction was repeated...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text(f'SELECT model_id, prediction, prediction_confidence, prediction_status FROM predictions_schema.predictions WHERE model_id = {model_id}')
)
prediction_rows = result_query.fetchall()
print(prediction_rows)
assert len(prediction_rows) == 2, "Expected two prediction records"
assert prediction_rows[0] == last_prediction, f"Expected first prediction to be the same as the last prediction, got {prediction_rows[0]}, expected {last_prediction}"
assert prediction_rows[1] == last_prediction, f"Expected second prediction to be the same as the last prediction, got {prediction_rows[1]}, expected {last_prediction}"
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_1_input_gate_triggers_continue(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
):
"""
Scenario 2.1.1: Input Gate Triggers CONTINUE
Description:
Input gate determines data should use previous prediction.
Expected Behavior:
- input_gate returns path_flag='CONTINUE'
- path_flag_handler calls export workflow with input data directly
- MLFlow transform and predict skipped
- Data exported as-is
Assertions:
- input_gate called
- MLFlow operations NOT called
- Export workflow called with original data
- Workflow completes
"""
client = temporal_test_env.client
model_id = 211
print("\n[TEST] 1. Inserting test data...")
insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
print("[TEST] ✓ Data inserted successfully")
input_data = get_base_input_data(model_id)
print("\n[TEST] 2. Starting workflow with CONTINUE policy...")
workflow_id = f'test-continue-policy-{datetime.now().timestamp()}'
await start_and_await_workflow(client, input_data, workflow_id)
assert_continue(postgres_engine, model_id)
print("\n[TEST] ✓ All assertions passed!")
@@ -164,15 +226,133 @@ async def test_scenario_2_1_2_input_gate_triggers_stop(
"""
client = temporal_test_env.client
print("\n[TEST] 1. Ensuring no data exists (empty data will trigger STOP)...")
with postgres_engine.begin() as conn:
conn.execute(text("DELETE FROM predictions_schema.laborious_data WHERE model_id = 202"))
print("[TEST] ✓ Data cleared")
model_id = 212
print("\n[TEST] 1. Inserting test data...")
insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
print("[TEST] ✓ Data inserted successfully")
input_data = get_base_input_data(model_id)
input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['policy'] = 'STOP'
print("\n[TEST] 2. Starting workflow that should stop at input gate...")
workflow_id = f'test-input-stop-{datetime.now().timestamp()}'
await start_and_await_workflow(client, input_data, workflow_id)
assert_stop(postgres_engine, model_id)
print("\n[TEST] ✓ All assertions passed!")
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_1_3_input_gate_triggers_repeat(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
):
"""
Scenario 2.1.3: Input Gate Triggers REPEAT
Description:
Input gate determines data should repeat last prediction.
Expected Behavior:
- input_gate returns path_flag='REPEAT'
- path_flag_handler calls repeat_last_prediction activity
- MLFlow transform and predict skipped
- Last prediction repeated and exported
Assertions:
- input_gate called
- MLFlow operations NOT called
- repeat_last_prediction activity called
- Workflow completes
"""
client = temporal_test_env.client
model_id = 213
print("\n[TEST] 1. Inserting test data and previous prediction...")
insert_sample_data(postgres_engine, model_id, ['NULL', 78.2])
data = insert_sample_prediction(postgres_engine, model_id)
print("[TEST] ✓ Data and previous prediction inserted")
input_data = get_base_input_data(model_id)
input_data['input_filters']['SPECIFIC_VARIABLES_NULL_VALUES']['policy'] = 'REPEAT'
print("\n[TEST] 2. Starting workflow that should trigger REPEAT...")
workflow_id = f'test-input-repeat-{datetime.now().timestamp()}'
await start_and_await_workflow(client, input_data, workflow_id)
assert_repeat(postgres_engine, model_id, data)
print("\n[TEST] ✓ All assertions passed!")
@pytest.fixture
def bad_data_model(patch_mlflow):
model = MagicMock(
predict=MagicMock(
side_effect=Exception("Bad data model")
)
)
patch_mlflow.sklearn.load_model = MagicMock(return_value=model)
return model
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_2_1_transform_gate_triggers_continue(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
bad_data_model,
):
"""
Scenario 2.2.1: Transform Gate Triggers CONTINUE
Description:
Transform response gate determines data should continue despite issues.
Expected Behavior:
- request_transform succeeds
- mlflow_response_gate for transform returns path_flag='CONTINUE'
- path_flag_handler calls export workflow with transform data
- MLFlow predict skipped
- Transform data exported as-is
Assertions:
- Transform completed
- mlflow_response_gate called for transform
- MLFlow predict NOT called
- Export workflow called with transform data
- Workflow completes
"""
client = temporal_test_env.client
print("\n[TEST] 1. Inserting test data...")
with postgres_engine.begin() as conn:
conn.execute(text("DELETE FROM predictions_schema.laborious_data WHERE model_id = 207"))
insert_sql = """
INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
VALUES
(207, 'sensor_1', 23.5, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
(207, 'sensor_2', 78.2, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
"""
conn.execute(text(insert_sql))
print("[TEST] ✓ Data inserted successfully")
input_data = {
'metadata': {
'metadata': {
'model_id': 202,
'model_id': 207,
'model_name': 'test_model',
'schedule_name': 'test-schedule',
'workflow_name': 'predictions_batch',
@@ -180,8 +360,8 @@ async def test_scenario_2_1_2_input_gate_triggers_stop(
},
'schedule_name': 'test-schedule',
'model_name': 'test_model',
'model_id': 202,
'query': 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = 202',
'model_id': 207,
'query': 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = 207',
'schema': 'predictions_schema',
'table_name': 'predictions',
'transform_table_name': 'transformed_data',
@@ -189,12 +369,12 @@ async def test_scenario_2_1_2_input_gate_triggers_stop(
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
'API_ERROR': {'policy': 'CONTINUE', 'config': {}}, # CONTINUE on transform error
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
},
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
'path_priority': ['CONTINUE', 'STOP', 'REPEAT'], # CONTINUE first
'opc_output_config': {},
'pi_web_api_output_config': {},
'save_transform': True,
@@ -207,8 +387,8 @@ async def test_scenario_2_1_2_input_gate_triggers_stop(
'datetime_columns': ['timestamp', 'created_at'],
}
print("\n[TEST] 2. Starting workflow that should stop at input gate...")
workflow_id = f'test-input-stop-{datetime.now().timestamp()}'
print("\n[TEST] 2. Starting workflow that should trigger CONTINUE at transform gate...")
workflow_id = f'test-transform-continue-{datetime.now().timestamp()}'
handle = await client.start_workflow(
PredictionsBatch.run,
@@ -218,24 +398,384 @@ async def test_scenario_2_1_2_input_gate_triggers_stop(
)
print("[TEST] ✓ Workflow started")
print("\n[TEST] 3. Waiting for workflow completion (should exit early)...")
print("\n[TEST] 3. Waiting for workflow completion...")
try:
await asyncio.wait_for(handle.result(), timeout=60.0)
print("[TEST] ✓ Workflow completed (exited early as expected)")
print("[TEST] ✓ Workflow completed successfully")
except asyncio.TimeoutError:
pytest.fail("Workflow execution timed out after 60 seconds")
print("\n[TEST] 4. Verifying no predictions were created...")
print("\n[TEST] 4. Verifying prediction was created (via CONTINUE path)...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 202")
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 207")
)
count = result_query.scalar()
assert count == 0, f"Expected no predictions, but found {count} records"
assert count >= 1, f"Expected at least one prediction, but found {count} records"
print("\n[TEST] ✓ All assertions passed!")
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_2_2_transform_gate_triggers_stop(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
):
"""
Scenario 2.2.2: Transform Gate Triggers STOP
Description:
Transform response validation fails with STOP policy.
Expected Behavior:
- request_transform succeeds but response invalid
- mlflow_response_gate for transform returns path_flag='STOP'
- Workflow exits without calling predict or export
Assertions:
- Transform completed but validation failed
- mlflow_response_gate called for transform
- MLFlow predict NOT called
- Export workflow NOT called
- Workflow completes without error
"""
client = temporal_test_env.client
print("\n[TEST] 1. Inserting test data...")
with postgres_engine.begin() as conn:
conn.execute(text("DELETE FROM predictions_schema.laborious_data WHERE model_id = 208"))
insert_sql = """
INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
VALUES
(208, 'sensor_1', 23.5, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
(208, 'sensor_2', 78.2, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
"""
conn.execute(text(insert_sql))
print("[TEST] ✓ Data inserted successfully")
# Mock mlflow_response_gate for transform to return STOP
original_mlflow_response_gate = test_activities.mlflow_response_gate
async def mock_mlflow_response_gate(input_data):
if input_data.get('type') == 'transform':
return 'STOP', -1, 'Transform response validation failed'
return await original_mlflow_response_gate(input_data)
with patch.object(test_activities, 'mlflow_response_gate', side_effect=mock_mlflow_response_gate):
input_data = {
'metadata': {
'metadata': {
'model_id': 208,
'model_name': 'test_model',
'schedule_name': 'test-schedule',
'workflow_name': 'predictions_batch',
}
},
'schedule_name': 'test-schedule',
'model_name': 'test_model',
'model_id': 208,
'query': 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = 208',
'schema': 'predictions_schema',
'table_name': 'predictions',
'transform_table_name': 'transformed_data',
'input_filters': {
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}}, # STOP on transform error
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
},
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
'opc_output_config': {},
'pi_web_api_output_config': {},
'save_transform': True,
'prediction_store_policy': 'lts:1',
'model_config': {
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
},
'datetime_columns': ['timestamp', 'created_at'],
}
print("\n[TEST] 2. Starting workflow that should stop at transform gate...")
workflow_id = f'test-transform-stop-{datetime.now().timestamp()}'
handle = await client.start_workflow(
PredictionsBatch.run,
input_data,
id=workflow_id,
task_queue='test-queue',
)
print("[TEST] ✓ Workflow started")
print("\n[TEST] 3. Waiting for workflow completion (should exit early)...")
try:
await asyncio.wait_for(handle.result(), timeout=60.0)
print("[TEST] ✓ Workflow completed (exited early as expected)")
except asyncio.TimeoutError:
pytest.fail("Workflow execution timed out after 60 seconds")
print("\n[TEST] 4. Verifying no predictions were created...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 208")
)
count = result_query.scalar()
assert count == 0, f"Expected no predictions, but found {count} records"
print("\n[TEST] ✓ All assertions passed!")
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_2_3_transform_gate_triggers_repeat(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
):
"""
Scenario 2.2.3: Transform Gate Triggers REPEAT
Description:
Transform response gate determines data should repeat last prediction.
Expected Behavior:
- request_transform succeeds but response has issues
- mlflow_response_gate for transform returns path_flag='REPEAT'
- path_flag_handler calls repeat_last_prediction activity
- MLFlow predict skipped
- Last prediction repeated and exported
Assertions:
- Transform completed but validation triggered REPEAT
- mlflow_response_gate called for transform
- MLFlow predict NOT called
- repeat_last_prediction activity called
- Workflow completes
"""
client = temporal_test_env.client
print("\n[TEST] 1. Inserting test data and previous prediction...")
with postgres_engine.begin() as conn:
conn.execute(text("DELETE FROM predictions_schema.laborious_data WHERE model_id = 209"))
conn.execute(text("DELETE FROM predictions_schema.predictions WHERE model_id = 209"))
insert_sql = """
INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
VALUES
(209, 'sensor_1', 23.5, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
(209, 'sensor_2', 78.2, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
"""
conn.execute(text(insert_sql))
# Insert a previous prediction to repeat
insert_prediction = """
INSERT INTO predictions_schema.predictions (model_id, timestamp, prediction, prediction_confidence, prediction_status, comments, response_time)
VALUES (209, '2024-01-01 11:00:00+00:00', 45.0, 10, 'Good', 'Previous prediction', 0.1)
"""
conn.execute(text(insert_prediction))
print("[TEST] ✓ Data and previous prediction inserted")
# Mock request_transform to return an error response
def mock_request_transform(*args, **kwargs):
return {
'success': False, # This will trigger API_ERROR filter
'content': [],
}
with patch.object(test_activities, 'request_transform', side_effect=mock_request_transform):
input_data = {
'metadata': {
'metadata': {
'model_id': 209,
'model_name': 'test_model',
'schedule_name': 'test-schedule',
'workflow_name': 'predictions_batch',
}
},
'schedule_name': 'test-schedule',
'model_name': 'test_model',
'model_id': 209,
'query': 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = 209',
'schema': 'predictions_schema',
'table_name': 'predictions',
'transform_table_name': 'transformed_data',
'input_filters': {
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'REPEAT', 'config': {}}, # REPEAT on transform error
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
},
'path_priority': ['REPEAT', 'STOP', 'CONTINUE'], # REPEAT first
'opc_output_config': {},
'pi_web_api_output_config': {},
'save_transform': True,
'prediction_store_policy': 'lts:1',
'model_config': {
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
},
'datetime_columns': ['timestamp', 'created_at'],
}
print("\n[TEST] 2. Starting workflow that should trigger REPEAT at transform gate...")
workflow_id = f'test-transform-repeat-{datetime.now().timestamp()}'
handle = await client.start_workflow(
PredictionsBatch.run,
input_data,
id=workflow_id,
task_queue='test-queue',
)
print("[TEST] ✓ Workflow started")
print("\n[TEST] 3. Waiting for workflow completion...")
try:
await asyncio.wait_for(handle.result(), timeout=60.0)
print("[TEST] ✓ Workflow completed successfully")
except asyncio.TimeoutError:
pytest.fail("Workflow execution timed out after 60 seconds")
print("\n[TEST] 4. Verifying prediction was repeated...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 209")
)
count = result_query.scalar()
assert count >= 2, f"Expected at least 2 predictions (original + repeated), but found {count} records"
print("\n[TEST] ✓ All assertions passed!")
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_3_1_predict_gate_triggers_continue(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
):
"""
Scenario 2.3.1: Predict Gate Triggers CONTINUE
Description:
Predict response gate determines data should continue despite issues.
Expected Behavior:
- request_predict succeeds
- mlflow_response_gate for predict returns path_flag='CONTINUE'
- path_flag_handler calls export workflow with predict data
- Prediction exported despite quality issues
Assertions:
- Transform and predict completed
- mlflow_response_gate called for predict
- Export workflow called with predict data
- Workflow completes
"""
client = temporal_test_env.client
print("\n[TEST] 1. Inserting test data...")
with postgres_engine.begin() as conn:
conn.execute(text("DELETE FROM predictions_schema.laborious_data WHERE model_id = 210"))
insert_sql = """
INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
VALUES
(210, 'sensor_1', 23.5, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
(210, 'sensor_2', 78.2, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
"""
conn.execute(text(insert_sql))
print("[TEST] ✓ Data inserted successfully")
# Mock mlflow_response_gate for predict to return CONTINUE
original_mlflow_response_gate = test_activities.mlflow_response_gate
async def mock_mlflow_response_gate(input_data):
if input_data.get('type') == 'predict':
return 'CONTINUE', 10, 'Predict response has issues but continuing'
return await original_mlflow_response_gate(input_data)
with patch.object(test_activities, 'mlflow_response_gate', side_effect=mock_mlflow_response_gate):
input_data = {
'metadata': {
'metadata': {
'model_id': 210,
'model_name': 'test_model',
'schedule_name': 'test-schedule',
'workflow_name': 'predictions_batch',
}
},
'schedule_name': 'test-schedule',
'model_name': 'test_model',
'model_id': 210,
'query': 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = 210',
'schema': 'predictions_schema',
'table_name': 'predictions',
'transform_table_name': 'transformed_data',
'input_filters': {
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'CONTINUE', 'config': {}}, # CONTINUE on predict error
},
'path_priority': ['CONTINUE', 'STOP', 'REPEAT'], # CONTINUE first
'opc_output_config': {},
'pi_web_api_output_config': {},
'save_transform': True,
'prediction_store_policy': 'lts:1',
'model_config': {
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
},
'datetime_columns': ['timestamp', 'created_at'],
}
print("\n[TEST] 2. Starting workflow that should trigger CONTINUE at predict gate...")
workflow_id = f'test-predict-continue-{datetime.now().timestamp()}'
handle = await client.start_workflow(
PredictionsBatch.run,
input_data,
id=workflow_id,
task_queue='test-queue',
)
print("[TEST] ✓ Workflow started")
print("\n[TEST] 3. Waiting for workflow completion...")
try:
await asyncio.wait_for(handle.result(), timeout=60.0)
print("[TEST] ✓ Workflow completed successfully")
except asyncio.TimeoutError:
pytest.fail("Workflow execution timed out after 60 seconds")
print("\n[TEST] 4. Verifying prediction was created (via CONTINUE path)...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 210")
)
count = result_query.scalar()
assert count >= 1, f"Expected at least one prediction, but found {count} records"
print("\n[TEST] ✓ All assertions passed!")
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_3_2_predict_gate_triggers_stop(
@@ -276,14 +816,15 @@ async def test_scenario_2_3_2_predict_gate_triggers_stop(
conn.execute(text(insert_sql))
print("[TEST] ✓ Data inserted successfully")
# Mock request_predict to return an error response
def mock_request_predict(*args, **kwargs):
return {
'success': False, # This will trigger API_ERROR filter
'content': [],
}
# Mock mlflow_response_gate for predict to return STOP
original_mlflow_response_gate = test_activities.mlflow_response_gate
with patch.object(test_activities, 'request_predict', side_effect=mock_request_predict):
async def mock_mlflow_response_gate(input_data):
if input_data.get('type') == 'predict':
return 'STOP', -1, 'Predict response validation failed'
return await original_mlflow_response_gate(input_data)
with patch.object(test_activities, 'mlflow_response_gate', side_effect=mock_mlflow_response_gate):
input_data = {
'metadata': {
'metadata': {
@@ -353,7 +894,132 @@ async def test_scenario_2_3_2_predict_gate_triggers_stop(
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_2_1_mlflow_transform_api_error(
async def test_scenario_2_3_3_predict_gate_triggers_repeat(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
):
"""
Scenario 2.3.3: Predict Gate Triggers REPEAT
Description:
Predict response gate determines data should repeat last prediction.
Expected Behavior:
- request_predict succeeds but response has issues
- mlflow_response_gate for predict returns path_flag='REPEAT'
- path_flag_handler calls repeat_last_prediction activity
- Last prediction repeated and exported
Assertions:
- Transform and predict completed but validation triggered REPEAT
- mlflow_response_gate called for predict
- repeat_last_prediction activity called
- Export workflow NOT called with current prediction
- Workflow completes
"""
client = temporal_test_env.client
print("\n[TEST] 1. Inserting test data and previous prediction...")
with postgres_engine.begin() as conn:
conn.execute(text("DELETE FROM predictions_schema.laborious_data WHERE model_id = 211"))
conn.execute(text("DELETE FROM predictions_schema.predictions WHERE model_id = 211"))
insert_sql = """
INSERT INTO predictions_schema.laborious_data (model_id, variable, value, timestamp, created_at)
VALUES
(211, 'sensor_1', 23.5, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00'),
(211, 'sensor_2', 78.2, '2024-01-01 12:00:00+00:00', '2024-01-01 12:00:00+00:00')
"""
conn.execute(text(insert_sql))
# Insert a previous prediction to repeat
insert_prediction = """
INSERT INTO predictions_schema.predictions (model_id, timestamp, prediction, prediction_confidence, prediction_status, comments, response_time)
VALUES (211, '2024-01-01 11:00:00+00:00', 47.5, 10, 'Good', 'Previous prediction', 0.1)
"""
conn.execute(text(insert_prediction))
print("[TEST] ✓ Data and previous prediction inserted")
# Mock request_predict to return an error response
def mock_request_predict(*args, **kwargs):
return {
'success': False, # This will trigger API_ERROR filter
'content': [],
}
with patch.object(test_activities, 'request_predict', side_effect=mock_request_predict):
input_data = {
'metadata': {
'metadata': {
'model_id': 211,
'model_name': 'test_model',
'schedule_name': 'test-schedule',
'workflow_name': 'predictions_batch',
}
},
'schedule_name': 'test-schedule',
'model_name': 'test_model',
'model_id': 211,
'query': 'SELECT timestamp, variable, value, created_at FROM predictions_schema.laborious_data WHERE model_id = 211',
'schema': 'predictions_schema',
'table_name': 'predictions',
'transform_table_name': 'transformed_data',
'input_filters': {
'EMPTY_DATA': {'policy': 'STOP', 'config': {}},
},
'mlflow_transform_filters': {
'API_ERROR': {'policy': 'STOP', 'config': {}},
},
'mlflow_predict_filters': {
'API_ERROR': {'policy': 'REPEAT', 'config': {}}, # REPEAT on predict error
},
'path_priority': ['REPEAT', 'STOP', 'CONTINUE'], # REPEAT first
'opc_output_config': {},
'pi_web_api_output_config': {},
'save_transform': True,
'prediction_store_policy': 'lts:1',
'model_config': {
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
},
'datetime_columns': ['timestamp', 'created_at'],
}
print("\n[TEST] 2. Starting workflow that should trigger REPEAT at predict gate...")
workflow_id = f'test-predict-repeat-{datetime.now().timestamp()}'
handle = await client.start_workflow(
PredictionsBatch.run,
input_data,
id=workflow_id,
task_queue='test-queue',
)
print("[TEST] ✓ Workflow started")
print("\n[TEST] 3. Waiting for workflow completion...")
try:
await asyncio.wait_for(handle.result(), timeout=60.0)
print("[TEST] ✓ Workflow completed successfully")
except asyncio.TimeoutError:
pytest.fail("Workflow execution timed out after 60 seconds")
print("\n[TEST] 4. Verifying prediction was repeated...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 211")
)
count = result_query.scalar()
assert count >= 2, f"Expected at least 2 predictions (original + repeated), but found {count} records"
print("\n[TEST] ✓ All assertions passed!")
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_4_1_mlflow_transform_api_error(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
@@ -445,26 +1111,28 @@ async def test_scenario_2_2_1_mlflow_transform_api_error(
)
print("[TEST] ✓ Workflow started")
print("\n[TEST] 3. Waiting for workflow to fail...")
print("\n[TEST] 3. Waiting for workflow completion...")
try:
await asyncio.wait_for(handle.result(), timeout=60.0)
pytest.fail("Expected workflow to fail, but it completed successfully")
print("[TEST] ✓ Workflow completed (may have failed after retries or completed with error handling)")
except Exception as e:
print(f"[TEST] ✓ Workflow failed as expected: {type(e).__name__}")
# Verify no predictions were created
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 207")
)
count = result_query.scalar()
assert count == 0, f"Expected no predictions, but found {count} records"
# Verify no predictions were created (regardless of whether workflow failed or completed)
print("\n[TEST] 4. Verifying no predictions were created...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 207")
)
count = result_query.scalar()
assert count == 0, f"Expected no predictions, but found {count} records"
print("\n[TEST] ✓ All assertions passed!")
@pytest.mark.asyncio
@pytest.mark.integration
async def test_scenario_2_2_2_mlflow_predict_api_error(
async def test_scenario_2_4_2_mlflow_predict_api_error(
temporal_test_env: WorkflowEnvironment,
temporal_worker: Worker,
test_activities: Activities,
@@ -556,18 +1224,20 @@ async def test_scenario_2_2_2_mlflow_predict_api_error(
)
print("[TEST] ✓ Workflow started")
print("\n[TEST] 3. Waiting for workflow to fail...")
print("\n[TEST] 3. Waiting for workflow completion...")
try:
await asyncio.wait_for(handle.result(), timeout=60.0)
pytest.fail("Expected workflow to fail, but it completed successfully")
print("[TEST] ✓ Workflow completed (may have failed after retries or completed with error handling)")
except Exception as e:
print(f"[TEST] ✓ Workflow failed as expected: {type(e).__name__}")
# Verify no predictions were created
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 208")
)
count = result_query.scalar()
assert count == 0, f"Expected no predictions, but found {count} records"
# Verify no predictions were created (regardless of whether workflow failed or completed)
print("\n[TEST] 4. Verifying no predictions were created...")
with postgres_engine.connect() as conn:
result_query = conn.execute(
text("SELECT COUNT(*) FROM predictions_schema.predictions WHERE model_id = 208")
)
count = result_query.scalar()
assert count == 0, f"Expected no predictions, but found {count} records"
print("\n[TEST] ✓ All assertions passed!")