gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1773

Browse Source 
Enhance environment configuration and update dependencies

- Added new environment variables for PluginStore and MLflow configuration in `.env.example`, including `RUNTIME`, `STORE_BASE_URL`, `STORE_OWNER`, `STORE_REPO`, `STORE_BRANCH`, `STORE_USERNAME`, `STORE_PASSWORD`, `STORE_CACHE_TTL_SECONDS`, `PYPI_SERVER`, `PYPI_USERNAME`, and `PYPI_PASSWORD`.
- Updated `git-requirements-mapping.txt` to reflect changes in repository names.
- Modified `requirements-light.txt` and `requirements.txt` to upgrade `sientia-dataops-library` to version 1.12.0 and `sientia-mlops-library` to version 0.8.1.
- Updated `values.yaml` to include new environment variables for worker runtime and PluginStore configuration.
- Refactored E2E tests to utilize new MLflow repository stubs and PluginStore mocks for improved testing accuracy.
This commit is contained in:
vitor-aignosi
2026-05-05 16:52:51 -03:00
parent 473bd0b03f
commit 1ce8b9d3a7
23 changed files with 1280 additions and 3970 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
e2e/conftest.py
View File

@@ -313,88 +313,55 @@ def patch_minio_repository(mock_minio_repository):
@pytest_asyncio.fixture
def patch_pi_web_api_repository(mock_pi_web_api_repository):
"""Patch MLflowRepository to return mock."""
"""Patch PI Web API client to return mock."""
with patch('laborious.activities.api.PIWebAPIClient', return_value=mock_pi_web_api_repository):
yield
@pytest_asyncio.fixture
def mock_mlflow_models():
"""Create mock models for MLflow load_model methods."""
# Mock transform model - returns DataFrame with same index as input
mock_transform_model = MagicMock()
def mock_transform_predict(data):
num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1
print(data.to_csv())
print(data.index)
result = pd.DataFrame({
'feature_1': [0.234] * num_rows,
'feature_2': [0.783] * num_rows,
})
def plugin_store_stub():
"""
PluginStore stub for Activities construction.
Runtime installation happens in the worker process; activities only hold a reference.
"""
return MagicMock()
@pytest_asyncio.fixture
def mlflow_repository_stub():
"""
SientiaMLflowRepository stub that returns a SientiaModel-like wrapper for E2E tests.
Transform/predict mirror the legacy sklearn/pyfunc mock behavior using pandas outputs.
"""
repo = MagicMock()
def _transform_side_effect(data: pd.DataFrame):
result = pd.DataFrame(
{
'feature_1': [0.234] * len(data),
'feature_2': [0.783] * len(data),
}
)
result.index = data.index
return result
mock_transform_model.predict = MagicMock(side_effect=mock_transform_predict)
# Mock predict model - returns array/list of predictions
mock_predict_model = MagicMock()
def mock_predict_predict(data):
num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1
return [0.5] * num_rows
mock_predict_model.predict = MagicMock(side_effect=mock_predict_predict)
# Mock PyFuncModel for compressed models
mock_pyfunc_model = MagicMock()
mock_pyfunc_model._model_impl = MagicMock()
mock_pyfunc_model._model_impl.python_model = mock_transform_model
return {
'transform_model': mock_transform_model,
'predict_model': mock_predict_model,
'pyfunc_model': mock_pyfunc_model,
}
return result, {}
def _predict_side_effect(_params: dict, data: pd.DataFrame):
pred = pd.DataFrame([0.5] * len(data), columns=['placeholder'])
pred.index = data.index
return pred, {}
@pytest_asyncio.fixture
def patch_mlflow(mock_mlflow_models):
"""Patch mlflow module in repository with load_model mocks."""
mock_mlflow = MagicMock()
# Mock sklearn.load_model
def mock_sklearn_load_model(model_uri):
if 'data_model' in model_uri or 'transform' in model_uri.lower():
return mock_mlflow_models['transform_model']
return mock_mlflow_models['predict_model']
mock_mlflow.sklearn = MagicMock()
mock_mlflow.sklearn.load_model = MagicMock(side_effect=mock_sklearn_load_model)
# Mock pyfunc.load_model
def mock_pyfunc_load_model(model_uri):
if 'artifacts' in model_uri or 'tmp' in model_uri:
return mock_mlflow_models['pyfunc_model']
if 'data_model' in model_uri or 'transform' in model_uri.lower():
return mock_mlflow_models['transform_model']
return mock_mlflow_models['predict_model']
mock_mlflow.pyfunc = MagicMock()
mock_mlflow.pyfunc.load_model = MagicMock(side_effect=mock_pyfunc_load_model)
# Mock pytorch.load_model
mock_mlflow.pytorch = MagicMock()
mock_mlflow.pytorch.load_model = MagicMock(return_value=mock_mlflow_models['predict_model'])
# Mock other mlflow methods that might be called
mock_mlflow.set_tracking_uri = MagicMock()
mock_mlflow.get_run = MagicMock(return_value=MagicMock(info=MagicMock(artifact_uri='mlflow-artifacts:/test_run_id')))
mock_mlflow.tracking = MagicMock()
mock_mlflow.tracking.MlflowClient = MagicMock(return_value=MagicMock(
search_registered_models=MagicMock(return_value=[MagicMock(name='test_model')]),
search_model_versions=MagicMock(return_value=[MagicMock(
current_stage='Production',
version='1',
source='runs:/artifacts/test_run_id'
)])
))
with patch('laborious.utils.repository.model_repository.mlflow', new=mock_mlflow):
yield mock_mlflow
wrapper = MagicMock()
wrapper.transform.side_effect = _transform_side_effect
wrapper.predict.side_effect = _predict_side_effect
repo.get_cached_model = MagicMock(return_value=wrapper)
repo.stub_wrapper = wrapper
repo._client = MagicMock()
return repo
@pytest_asyncio.fixture(scope='function')
@@ -407,7 +374,8 @@ async def test_activities(
mock_minio_repository,
patch_create_engine,
patch_minio_repository,
patch_mlflow,
mlflow_repository_stub,
plugin_store_stub,
patch_pi_web_api_repository,
mock_opc_repository
):
@@ -429,12 +397,7 @@ async def test_activities(
'min_connections': 1,
'max_connections': 5,
},
mlflow_config={
'host': 'http://localhost',
'port': '5000',
'username': 'test',
'password': 'test',
},
plugin_store=plugin_store_stub,
minio_config={
# Host:port only; Minio() prepends http(s):// from the secure flag.
'endpoint_url': 'localhost:9000',
@@ -452,6 +415,8 @@ async def test_activities(
},
logger=mock_logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
mlflow_repository=mlflow_repository_stub,
)
activities.opc_repository = {
@@ -474,7 +439,8 @@ async def test_activities_real_minio(
notification_handler,
metrics_controller,
patch_create_engine,
patch_mlflow,
mlflow_repository_stub,
plugin_store_stub,
patch_pi_web_api_repository,
mock_opc_repository,
):
@@ -495,12 +461,7 @@ async def test_activities_real_minio(
'min_connections': 1,
'max_connections': 5,
},
mlflow_config={
'host': 'http://localhost',
'port': '5000',
'username': 'test',
'password': 'test',
},
plugin_store=plugin_store_stub,
minio_config={
'endpoint_url': f'localhost:{minio_port}',
'access_key': 'minioadmin',
@@ -517,6 +478,8 @@ async def test_activities_real_minio(
},
logger=mock_logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
mlflow_repository=mlflow_repository_stub,
)
activities.opc_repository = {'1': mock_opc_repository}
try:

3
e2e/test_minio_offload.py
View File

@@ -103,8 +103,7 @@ async def test_predictions_batch_with_minio_offload_path(
'prediction_store_policy': 'lts:1',
'model_config': {
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
'target': 'sensor_1',
},
'datetime_columns': ['timestamp', 'created_at'],
}

3
e2e/test_predictions_batch_format_export.py
View File

@@ -40,8 +40,7 @@ base_input_data = {
'prediction_store_policy': 'lts:1',
'model_config': {
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
'target': 'sensor_1',
},
'datetime_columns': ['timestamp', 'created_at'],
}

9
e2e/test_predictions_batch_main_workflow.py
View File

@@ -68,9 +68,8 @@ async def test_scenario_1_1_1_happy_path_complete_success(
'save_transform': True,
'prediction_store_policy': 'lts:1',
'model_config': {
'target': 'sensor_1',
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
},
'datetime_columns': ['timestamp', 'created_at'],
}
@@ -157,9 +156,8 @@ async def test_scenario_1_2_1_sql_query_execution_error(
'save_transform': True,
'prediction_store_policy': 'lts:1',
'model_config': {
'target': 'sensor_1',
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
},
}
@@ -277,9 +275,8 @@ async def test_scenario_1_2_3_invalid_datetime_column_specification(
'save_transform': True,
'prediction_store_policy': 'lts:1',
'model_config': {
'target': 'sensor_1',
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
},
'datetime_columns': ['nonexistent_column'],
}

66
e2e/test_predictions_batch_prediction_process.py
View File

@@ -50,8 +50,7 @@ base_input_data = {
'prediction_store_policy': 'lts:1',
'model_config': {
'retention_minutes': 0,
'transform_flavor': 'sklearn',
'predict_flavor': 'sklearn',
'target': 'sensor_1',
},
'datetime_columns': ['timestamp', 'created_at'],
}
@@ -80,24 +79,30 @@ def insert_sample_prediction(postgres_engine, model_id):
@pytest.fixture
def bad_data_model(patch_mlflow):
model = MagicMock(predict=MagicMock(side_effect=Exception('Bad data model')))
patch_mlflow.sklearn.load_model = MagicMock(return_value=model)
return model
def bad_data_model(mlflow_repository_stub):
mlflow_repository_stub.stub_wrapper.transform = MagicMock(
side_effect=Exception('Bad data model')
)
return mlflow_repository_stub.stub_wrapper
@pytest.fixture
def bad_predict_model(patch_mlflow, mock_mlflow_models):
model = MagicMock(predict=MagicMock(side_effect=Exception('Bad predict model')))
def bad_predict_model(mlflow_repository_stub):
wrapper = mlflow_repository_stub.stub_wrapper
def mock_sklearn_load_model(model_uri):
if 'data_model' in model_uri or 'transform' in model_uri.lower():
return mock_mlflow_models['transform_model']
return model
def _good_transform(data):
result = pd.DataFrame(
{
'feature_1': [0.234] * len(data),
'feature_2': [0.783] * len(data),
}
)
result.index = data.index
return result, {}
patch_mlflow.sklearn = MagicMock()
patch_mlflow.sklearn.load_model = MagicMock(side_effect=mock_sklearn_load_model)
return model
wrapper.transform.side_effect = _good_transform
wrapper.predict = MagicMock(side_effect=Exception('Bad predict model'))
return wrapper
@pytest.mark.asyncio
@@ -107,7 +112,7 @@ async def test_scenario_2_1_1_input_gate_triggers_continue(
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
mock_mlflow_models,
mlflow_repository_stub,
):
"""Input gate CONTINUE: export default prediction; MLflow transform/predict not used."""
client = temporal_test_env.client
@@ -118,8 +123,8 @@ async def test_scenario_2_1_1_input_gate_triggers_continue(
client, PredictionsBatch.run, input_data, make_workflow_id('test-continue-policy')
)
assert_continue(postgres_engine, model_id)
mock_mlflow_models['transform_model'].predict.assert_not_called()
mock_mlflow_models['predict_model'].predict.assert_not_called()
mlflow_repository_stub.stub_wrapper.transform.assert_not_called()
mlflow_repository_stub.stub_wrapper.predict.assert_not_called()
@pytest.mark.asyncio
@@ -129,7 +134,7 @@ async def test_scenario_2_1_2_input_gate_triggers_stop(
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
mock_mlflow_models,
mlflow_repository_stub,
):
"""Input gate STOP: no export, no MLflow."""
client = temporal_test_env.client
@@ -141,7 +146,7 @@ async def test_scenario_2_1_2_input_gate_triggers_stop(
client, PredictionsBatch.run, input_data, make_workflow_id('test-input-stop')
)
assert_stop(postgres_engine, model_id)
mock_mlflow_models['transform_model'].predict.assert_not_called()
mlflow_repository_stub.stub_wrapper.transform.assert_not_called()
@pytest.mark.asyncio
@@ -151,7 +156,7 @@ async def test_scenario_2_1_3_input_gate_triggers_repeat(
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
mock_mlflow_models,
mlflow_repository_stub,
):
"""Input gate REPEAT with existing history."""
client = temporal_test_env.client
@@ -164,7 +169,7 @@ async def test_scenario_2_1_3_input_gate_triggers_repeat(
client, PredictionsBatch.run, input_data, make_workflow_id('test-input-repeat')
)
assert_repeat(postgres_engine, model_id, data)
mock_mlflow_models['transform_model'].predict.assert_not_called()
mlflow_repository_stub.stub_wrapper.transform.assert_not_called()
@pytest.mark.asyncio
@@ -222,7 +227,7 @@ async def test_scenario_2_2_2_transform_gate_triggers_stop(
test_activities: Activities,
postgres_engine,
bad_data_model,
mock_mlflow_models,
mlflow_repository_stub,
):
client = temporal_test_env.client
model_id = 222
@@ -233,7 +238,7 @@ async def test_scenario_2_2_2_transform_gate_triggers_stop(
client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-stop')
)
assert_stop(postgres_engine, model_id)
mock_mlflow_models['predict_model'].predict.assert_not_called()
mlflow_repository_stub.stub_wrapper.predict.assert_not_called()
@pytest.mark.asyncio
@@ -264,19 +269,20 @@ async def test_scenario_2_2_4_transform_content_gate_nan_values_stop(
temporal_worker: Worker,
test_activities: Activities,
postgres_engine,
mock_mlflow_models,
mlflow_repository_stub,
):
"""mlflow_content_gate triggers STOP when transform output is all NaN (NAN_VALUES filter)."""
client = temporal_test_env.client
model_id = 224
def all_nan_transform(data):
num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1
result = pd.DataFrame({'feature_1': [np.nan] * num_rows, 'feature_2': [np.nan] * num_rows})
result = pd.DataFrame(
{'feature_1': [np.nan] * len(data), 'feature_2': [np.nan] * len(data)}
)
result.index = data.index
return result
return result, {}
mock_mlflow_models['transform_model'].predict = MagicMock(side_effect=all_nan_transform)
mlflow_repository_stub.stub_wrapper.transform = MagicMock(side_effect=all_nan_transform)
insert_sample_data(postgres_engine, model_id, [60.0, 78.2])
input_data = get_base_input_data(model_id)
@@ -288,7 +294,7 @@ async def test_scenario_2_2_4_transform_content_gate_nan_values_stop(
client, PredictionsBatch.run, input_data, make_workflow_id('test-transform-content-stop')
)
assert_stop(postgres_engine, model_id)
mock_mlflow_models['predict_model'].predict.assert_not_called()
mlflow_repository_stub.stub_wrapper.predict.assert_not_called()
@pytest.mark.asyncio