SIENTIAPDE-1773
Enhance environment configuration and update dependencies - Added new environment variables for PluginStore and MLflow configuration in `.env.example`, including `RUNTIME`, `STORE_BASE_URL`, `STORE_OWNER`, `STORE_REPO`, `STORE_BRANCH`, `STORE_USERNAME`, `STORE_PASSWORD`, `STORE_CACHE_TTL_SECONDS`, `PYPI_SERVER`, `PYPI_USERNAME`, and `PYPI_PASSWORD`. - Updated `git-requirements-mapping.txt` to reflect changes in repository names. - Modified `requirements-light.txt` and `requirements.txt` to upgrade `sientia-dataops-library` to version 1.12.0 and `sientia-mlops-library` to version 0.8.1. - Updated `values.yaml` to include new environment variables for worker runtime and PluginStore configuration. - Refactored E2E tests to utilize new MLflow repository stubs and PluginStore mocks for improved testing accuracy.
This commit is contained in:
vitor-aignosi
141
e2e/conftest.py
141
e2e/conftest.py
@@ -313,88 +313,55 @@ def patch_minio_repository(mock_minio_repository):
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def patch_pi_web_api_repository(mock_pi_web_api_repository):
|
||||
"""Patch MLflowRepository to return mock."""
|
||||
"""Patch PI Web API client to return mock."""
|
||||
with patch('laborious.activities.api.PIWebAPIClient', return_value=mock_pi_web_api_repository):
|
||||
yield
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def mock_mlflow_models():
|
||||
"""Create mock models for MLflow load_model methods."""
|
||||
# Mock transform model - returns DataFrame with same index as input
|
||||
mock_transform_model = MagicMock()
|
||||
def mock_transform_predict(data):
|
||||
num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1
|
||||
print(data.to_csv())
|
||||
print(data.index)
|
||||
result = pd.DataFrame({
|
||||
'feature_1': [0.234] * num_rows,
|
||||
'feature_2': [0.783] * num_rows,
|
||||
})
|
||||
def plugin_store_stub():
|
||||
"""
|
||||
PluginStore stub for Activities construction.
|
||||
|
||||
Runtime installation happens in the worker process; activities only hold a reference.
|
||||
"""
|
||||
|
||||
return MagicMock()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def mlflow_repository_stub():
|
||||
"""
|
||||
SientiaMLflowRepository stub that returns a SientiaModel-like wrapper for E2E tests.
|
||||
|
||||
Transform/predict mirror the legacy sklearn/pyfunc mock behavior using pandas outputs.
|
||||
"""
|
||||
|
||||
repo = MagicMock()
|
||||
|
||||
def _transform_side_effect(data: pd.DataFrame):
|
||||
result = pd.DataFrame(
|
||||
{
|
||||
'feature_1': [0.234] * len(data),
|
||||
'feature_2': [0.783] * len(data),
|
||||
}
|
||||
)
|
||||
result.index = data.index
|
||||
return result
|
||||
mock_transform_model.predict = MagicMock(side_effect=mock_transform_predict)
|
||||
|
||||
# Mock predict model - returns array/list of predictions
|
||||
mock_predict_model = MagicMock()
|
||||
def mock_predict_predict(data):
|
||||
num_rows = max(len(data), 1) if hasattr(data, '__len__') else 1
|
||||
return [0.5] * num_rows
|
||||
mock_predict_model.predict = MagicMock(side_effect=mock_predict_predict)
|
||||
|
||||
# Mock PyFuncModel for compressed models
|
||||
mock_pyfunc_model = MagicMock()
|
||||
mock_pyfunc_model._model_impl = MagicMock()
|
||||
mock_pyfunc_model._model_impl.python_model = mock_transform_model
|
||||
|
||||
return {
|
||||
'transform_model': mock_transform_model,
|
||||
'predict_model': mock_predict_model,
|
||||
'pyfunc_model': mock_pyfunc_model,
|
||||
}
|
||||
return result, {}
|
||||
|
||||
def _predict_side_effect(_params: dict, data: pd.DataFrame):
|
||||
pred = pd.DataFrame([0.5] * len(data), columns=['placeholder'])
|
||||
pred.index = data.index
|
||||
return pred, {}
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
def patch_mlflow(mock_mlflow_models):
|
||||
"""Patch mlflow module in repository with load_model mocks."""
|
||||
mock_mlflow = MagicMock()
|
||||
|
||||
# Mock sklearn.load_model
|
||||
def mock_sklearn_load_model(model_uri):
|
||||
if 'data_model' in model_uri or 'transform' in model_uri.lower():
|
||||
return mock_mlflow_models['transform_model']
|
||||
return mock_mlflow_models['predict_model']
|
||||
mock_mlflow.sklearn = MagicMock()
|
||||
mock_mlflow.sklearn.load_model = MagicMock(side_effect=mock_sklearn_load_model)
|
||||
|
||||
# Mock pyfunc.load_model
|
||||
def mock_pyfunc_load_model(model_uri):
|
||||
if 'artifacts' in model_uri or 'tmp' in model_uri:
|
||||
return mock_mlflow_models['pyfunc_model']
|
||||
if 'data_model' in model_uri or 'transform' in model_uri.lower():
|
||||
return mock_mlflow_models['transform_model']
|
||||
return mock_mlflow_models['predict_model']
|
||||
mock_mlflow.pyfunc = MagicMock()
|
||||
mock_mlflow.pyfunc.load_model = MagicMock(side_effect=mock_pyfunc_load_model)
|
||||
|
||||
# Mock pytorch.load_model
|
||||
mock_mlflow.pytorch = MagicMock()
|
||||
mock_mlflow.pytorch.load_model = MagicMock(return_value=mock_mlflow_models['predict_model'])
|
||||
|
||||
# Mock other mlflow methods that might be called
|
||||
mock_mlflow.set_tracking_uri = MagicMock()
|
||||
mock_mlflow.get_run = MagicMock(return_value=MagicMock(info=MagicMock(artifact_uri='mlflow-artifacts:/test_run_id')))
|
||||
mock_mlflow.tracking = MagicMock()
|
||||
mock_mlflow.tracking.MlflowClient = MagicMock(return_value=MagicMock(
|
||||
search_registered_models=MagicMock(return_value=[MagicMock(name='test_model')]),
|
||||
search_model_versions=MagicMock(return_value=[MagicMock(
|
||||
current_stage='Production',
|
||||
version='1',
|
||||
source='runs:/artifacts/test_run_id'
|
||||
)])
|
||||
))
|
||||
|
||||
with patch('laborious.utils.repository.model_repository.mlflow', new=mock_mlflow):
|
||||
yield mock_mlflow
|
||||
wrapper = MagicMock()
|
||||
wrapper.transform.side_effect = _transform_side_effect
|
||||
wrapper.predict.side_effect = _predict_side_effect
|
||||
|
||||
repo.get_cached_model = MagicMock(return_value=wrapper)
|
||||
repo.stub_wrapper = wrapper
|
||||
repo._client = MagicMock()
|
||||
return repo
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='function')
|
||||
@@ -407,7 +374,8 @@ async def test_activities(
|
||||
mock_minio_repository,
|
||||
patch_create_engine,
|
||||
patch_minio_repository,
|
||||
patch_mlflow,
|
||||
mlflow_repository_stub,
|
||||
plugin_store_stub,
|
||||
patch_pi_web_api_repository,
|
||||
mock_opc_repository
|
||||
):
|
||||
@@ -429,12 +397,7 @@ async def test_activities(
|
||||
'min_connections': 1,
|
||||
'max_connections': 5,
|
||||
},
|
||||
mlflow_config={
|
||||
'host': 'http://localhost',
|
||||
'port': '5000',
|
||||
'username': 'test',
|
||||
'password': 'test',
|
||||
},
|
||||
plugin_store=plugin_store_stub,
|
||||
minio_config={
|
||||
# Host:port only; Minio() prepends http(s):// from the secure flag.
|
||||
'endpoint_url': 'localhost:9000',
|
||||
@@ -452,6 +415,8 @@ async def test_activities(
|
||||
},
|
||||
logger=mock_logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
mlflow_repository=mlflow_repository_stub,
|
||||
)
|
||||
|
||||
activities.opc_repository = {
|
||||
@@ -474,7 +439,8 @@ async def test_activities_real_minio(
|
||||
notification_handler,
|
||||
metrics_controller,
|
||||
patch_create_engine,
|
||||
patch_mlflow,
|
||||
mlflow_repository_stub,
|
||||
plugin_store_stub,
|
||||
patch_pi_web_api_repository,
|
||||
mock_opc_repository,
|
||||
):
|
||||
@@ -495,12 +461,7 @@ async def test_activities_real_minio(
|
||||
'min_connections': 1,
|
||||
'max_connections': 5,
|
||||
},
|
||||
mlflow_config={
|
||||
'host': 'http://localhost',
|
||||
'port': '5000',
|
||||
'username': 'test',
|
||||
'password': 'test',
|
||||
},
|
||||
plugin_store=plugin_store_stub,
|
||||
minio_config={
|
||||
'endpoint_url': f'localhost:{minio_port}',
|
||||
'access_key': 'minioadmin',
|
||||
@@ -517,6 +478,8 @@ async def test_activities_real_minio(
|
||||
},
|
||||
logger=mock_logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
mlflow_repository=mlflow_repository_stub,
|
||||
)
|
||||
activities.opc_repository = {'1': mock_opc_repository}
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user