feat: enhance E2E testing setup and model reporting
- Added a new fixture to manage runtime report artifacts in a writable temp directory during E2E tests, addressing permission issues in local CI/dev environments. - Updated `conftest.py` to include a requirements.txt file in the model packaging path for training activities. - Refactored existing fixtures to use `pytest.fixture` instead of `pytest_asyncio.fixture` for better compatibility. - Enhanced the `Reports` class to include a target alias for report metrics, ensuring compatibility with Evidently's reporting requirements. - Introduced new test scenarios to validate the handling of missing and whitespace-only `date_column` inputs in the training workflow. These changes improve the robustness of the E2E testing framework and enhance the clarity of model reporting metrics.
This commit is contained in:
vitor-aignosi
@@ -17,6 +17,7 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
import base64
|
||||
import csv
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
@@ -27,10 +28,10 @@ import pytest_asyncio
|
||||
import requests
|
||||
from minio import Minio
|
||||
from sqlalchemy import create_engine, text
|
||||
from testcontainers.core.container import DockerContainer
|
||||
from testcontainers.minio import MinioContainer
|
||||
from testcontainers.mongodb import MongoDbContainer
|
||||
from testcontainers.postgres import PostgresContainer
|
||||
from testcontainers.core.container import DockerContainer # type: ignore[import-untyped]
|
||||
from testcontainers.minio import MinioContainer # type: ignore[import-untyped]
|
||||
from testcontainers.mongodb import MongoDbContainer # type: ignore[import-untyped]
|
||||
from testcontainers.postgres import PostgresContainer # type: ignore[import-untyped]
|
||||
from temporalio.testing import WorkflowEnvironment
|
||||
from temporalio.worker import Worker
|
||||
|
||||
@@ -288,6 +289,12 @@ class DummyTransformer:
|
||||
pass
|
||||
"""
|
||||
|
||||
# requirements.txt required by current model packaging path in training activity.
|
||||
requirements_txt = """
|
||||
pandas
|
||||
numpy
|
||||
"""
|
||||
|
||||
def push_file(path: str, content: str):
|
||||
encoded = base64.b64encode(content.encode()).decode()
|
||||
_gitea_api(
|
||||
@@ -307,6 +314,7 @@ class DummyTransformer:
|
||||
push_file(f'{prefix}/schemas.yaml', schemas_yaml)
|
||||
push_file(f'{prefix}/wrapper.py', wrapper_py)
|
||||
push_file(f'{prefix}/model_logic.py', model_logic_py)
|
||||
push_file(f'{prefix}/requirements.txt', requirements_txt.strip() + '\n')
|
||||
push_file(f'{prefix}/__init__.py', "")
|
||||
|
||||
# Push runtime
|
||||
@@ -317,7 +325,7 @@ class DummyTransformer:
|
||||
# Session-scoped containers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
@pytest.fixture(scope='session')
|
||||
def postgres_container():
|
||||
"""PostgreSQL 15 container for experiment_run table."""
|
||||
container = PostgresContainer('postgres:15')
|
||||
@@ -326,7 +334,7 @@ def postgres_container():
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
@pytest.fixture(scope='session')
|
||||
def minio_container():
|
||||
"""MinIO container for training CSV storage."""
|
||||
container = MinioContainer()
|
||||
@@ -335,7 +343,7 @@ def minio_container():
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
@pytest.fixture(scope='session')
|
||||
def mongodb_container():
|
||||
"""MongoDB container for CoreNotificationHandler."""
|
||||
container = MongoDbContainer('mongo:7')
|
||||
@@ -344,7 +352,7 @@ def mongodb_container():
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
@pytest.fixture(scope='session')
|
||||
def gitea_container():
|
||||
"""
|
||||
Gitea container with a ``model-store`` repo seeded via REST API
|
||||
@@ -397,7 +405,7 @@ def gitea_container():
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
@pytest.fixture(scope='session')
|
||||
def mlflow_tracking_dir():
|
||||
"""Local MLflow filesystem tracking directory (no network needed)."""
|
||||
tmpdir = tempfile.mkdtemp(prefix='mlflow-e2e-')
|
||||
@@ -406,11 +414,51 @@ def mlflow_tracking_dir():
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope='session', autouse=True)
|
||||
def e2e_runtime_reports_dir():
|
||||
"""
|
||||
Route runtime report artifacts to a writable temp directory during E2E.
|
||||
|
||||
Production defaults point to /var/lib/model-manager; in local CI/dev runs this
|
||||
path may be unavailable. This fixture keeps the same code paths while avoiding
|
||||
host permission issues.
|
||||
"""
|
||||
import model_manager.runtime_paths as runtime_paths
|
||||
import model_manager.utils.repository.data_manager_repository as data_repo_module
|
||||
|
||||
base_dir = tempfile.mkdtemp(prefix='model-manager-e2e-runtime-')
|
||||
reports_root = f'{base_dir}/reports'
|
||||
reports_temp_dir = f'{reports_root}/temp'
|
||||
project_base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'model_manager'))
|
||||
|
||||
old_runtime_reports_root = runtime_paths.REPORTS_ROOT
|
||||
old_runtime_reports_temp = runtime_paths.REPORTS_TEMP_DIR
|
||||
old_runtime_project_base = runtime_paths.PROJECT_BASE_PATH
|
||||
old_repo_reports_root = data_repo_module.REPORTS_ROOT
|
||||
old_repo_project_base = data_repo_module.PROJECT_BASE_PATH
|
||||
|
||||
runtime_paths.REPORTS_ROOT = reports_root
|
||||
runtime_paths.REPORTS_TEMP_DIR = reports_temp_dir
|
||||
runtime_paths.PROJECT_BASE_PATH = project_base_path
|
||||
data_repo_module.REPORTS_ROOT = reports_root
|
||||
data_repo_module.PROJECT_BASE_PATH = project_base_path
|
||||
|
||||
try:
|
||||
yield reports_root
|
||||
finally:
|
||||
runtime_paths.REPORTS_ROOT = old_runtime_reports_root
|
||||
runtime_paths.REPORTS_TEMP_DIR = old_runtime_reports_temp
|
||||
runtime_paths.PROJECT_BASE_PATH = old_runtime_project_base
|
||||
data_repo_module.REPORTS_ROOT = old_repo_reports_root
|
||||
data_repo_module.PROJECT_BASE_PATH = old_repo_project_base
|
||||
shutil.rmtree(base_dir, ignore_errors=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session-scoped: seed MinIO with training CSV
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest_asyncio.fixture(scope='session', autouse=True)
|
||||
@pytest.fixture(scope='session', autouse=True)
|
||||
def upload_training_csv(minio_container, mlflow_tracking_dir): # noqa: ARG001
|
||||
"""
|
||||
Upload training CSV files to the MinIO container before any test runs.
|
||||
@@ -479,7 +527,7 @@ def upload_training_csv(minio_container, mlflow_tracking_dir): # noqa: ARG001
|
||||
# Function-scoped: database engine + schema setup
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@pytest.fixture
|
||||
def postgres_engine(postgres_container):
|
||||
"""SQLAlchemy engine connected to the test PostgreSQL container."""
|
||||
engine = create_engine(postgres_container.get_connection_url())
|
||||
@@ -487,7 +535,7 @@ def postgres_engine(postgres_container):
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_experiment_run_table(postgres_engine):
|
||||
"""
|
||||
Create the experiment_run table before each test and drop it afterwards
|
||||
@@ -517,13 +565,13 @@ def setup_experiment_run_table(postgres_engine):
|
||||
# Observability (real sientia_do implementations)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest_asyncio.fixture(scope='session')
|
||||
@pytest.fixture(scope='session')
|
||||
def e2e_logger():
|
||||
"""Shared production-style Logger for the whole E2E session."""
|
||||
return get_logger('model-manager-e2e')
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@pytest.fixture
|
||||
def metrics_controller(e2e_logger):
|
||||
"""MetricsController bound to the E2E logger (fresh instance per test)."""
|
||||
return MetricsController(logger=e2e_logger)
|
||||
@@ -533,7 +581,7 @@ def metrics_controller(e2e_logger):
|
||||
# Application fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@pytest.fixture
|
||||
def notification_handler(mongodb_container, e2e_logger):
|
||||
"""
|
||||
Real CoreNotificationHandler connected to the MongoDB testcontainer.
|
||||
@@ -549,7 +597,7 @@ def notification_handler(mongodb_container, e2e_logger):
|
||||
handler.shutdown()
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@pytest.fixture
|
||||
def plugin_store(gitea_container, e2e_logger, metrics_controller, notification_handler):
|
||||
"""
|
||||
Real PluginStore pointed at the Gitea testcontainer.
|
||||
@@ -569,7 +617,7 @@ def plugin_store(gitea_container, e2e_logger, metrics_controller, notification_h
|
||||
yield store
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
@pytest.fixture
|
||||
def test_activities(
|
||||
postgres_container,
|
||||
minio_container,
|
||||
|
||||
Reference in New Issue
Block a user