gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Refactor ModelMetrics to utilize DriftAnalysis for drift detection

- Replaced ModelAnalysis with DriftAnalysis in the ModelMetrics class to enhance drift detection capabilities.
- Updated method signatures and documentation to reflect the changes in target_name and return values.
- Adjusted data handling to ensure compatibility with the new analysis methods and improved clarity in the drift metrics dataframe preparation.
This commit is contained in:
vitor-aignosi
2026-05-08 16:39:12 -03:00
parent e6018af23f
commit 10c7e292b9
29 changed files with 843 additions and 1266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
e2e/conftest.py
View File

@@ -1,109 +1,20 @@
"""Pytest configuration and fixtures for E2E tests."""
from concurrent.futures import ThreadPoolExecutor
from typing import Any
from pathlib import Path
from unittest.mock import MagicMock
import pandas as pd
import pytest
import pytest_asyncio
from sqlalchemy import create_engine, text
from sqlalchemy import create_engine
from testcontainers.core.container import DockerContainer
from testcontainers.minio import MinioContainer
from testcontainers.postgres import PostgresContainer
from temporalio.testing import WorkflowEnvironment
from temporalio.worker import Worker
class _FakeModelAnalysis:
"""
Configurable double for ``sientia_model.analytics.model_analysis.ModelAnalysis``
used by E2E drift tests.
Replaces the real analyzer in ``sientia_model.analytics.model_analysis``
BEFORE the production import chain runs so
``laborious.activities.model_metrics`` resolves ``ModelAnalysis`` to this
class at import time. Keeps drift assertions stable across runs (the real
analyzer is data-dependent).
Tests configure responses through class-level attributes which are reset
between tests by ``reset_model_analysis_stub``:
- ``_drift_response_factory``: callable ``(univariate, multivariate) -> DataFrame``
controlling the consolidated drift dataframe seen by ``calculate_drift``.
- ``_univariate_side_effect`` / ``_multivariate_side_effect``: optional
side effects (Exception or callable) for the raw detection methods.
- ``_drift_metrics_dataframe_exception``: when set, raised by
``get_drift_metrics_dataframe`` to simulate analyzer failures.
"""
_drift_response_factory: Any = None
_univariate_side_effect: Any = None
_multivariate_side_effect: Any = None
_drift_metrics_dataframe_exception: Exception | None = None
last_instance: Any = None
def __init__(self, config):
self.config = config
type(self).last_instance = self
self.detect_univariate_drift_calls = []
self.detect_multivariate_drift_calls = []
self.get_drift_metrics_dataframe_calls = []
def detect_univariate_drift(self, **kwargs):
"""Record arguments and return ``{}`` unless ``_univariate_side_effect`` overrides it."""
self.detect_univariate_drift_calls.append(kwargs)
side_effect = type(self)._univariate_side_effect
if isinstance(side_effect, Exception):
raise side_effect
if callable(side_effect):
return side_effect(**kwargs)
return {}
def detect_multivariate_drift(self, **kwargs):
"""Record arguments and return ``{}`` unless ``_multivariate_side_effect`` overrides it."""
self.detect_multivariate_drift_calls.append(kwargs)
side_effect = type(self)._multivariate_side_effect
if isinstance(side_effect, Exception):
raise side_effect
if callable(side_effect):
return side_effect(**kwargs)
return {}
def get_drift_metrics_dataframe(self, univariate_drift, multivariate_drift):
"""Return the configured drift dataframe (copy) or raise the configured exception."""
self.get_drift_metrics_dataframe_calls.append(
{'univariate_drift': univariate_drift, 'multivariate_drift': multivariate_drift}
)
if type(self)._drift_metrics_dataframe_exception is not None:
raise type(self)._drift_metrics_dataframe_exception
factory = type(self)._drift_response_factory
if factory is None:
return pd.DataFrame()
result = factory(univariate_drift, multivariate_drift)
return result.copy() if isinstance(result, pd.DataFrame) else result
@classmethod
def reset(cls):
"""Clear all configured side effects and the last constructed instance."""
cls._drift_response_factory = None
cls._univariate_side_effect = None
cls._multivariate_side_effect = None
cls._drift_metrics_dataframe_exception = None
cls.last_instance = None
# Patch ``sientia_model.analytics.model_analysis.ModelAnalysis`` BEFORE the
# production import chain runs so drift E2E tests can drive deterministic
# analyzer outputs (the real implementation is data-dependent and would yield
# values that drift across runs). The patch is applied to the real installed
# module so that ``from laborious.activities.activities import Activities``
# resolves ``ModelAnalysis`` to ``_FakeModelAnalysis`` at import time.
import sientia_model.analytics.model_analysis as _sientia_model_analysis_module # noqa: E402
_sientia_model_analysis_module.ModelAnalysis = _FakeModelAnalysis
from laborious.activities.activities import Activities # noqa: E402
from laborious.activities.activities import Activities
from laborious.workflows.drift import Drift
from laborious.workflows.minimal_retrain import MinimalRetrain
from laborious.workflows.predictions_batch import PredictionsBatch
@@ -116,6 +27,11 @@ from sientia_do.notifications.handlers import CoreNotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
# Single source of truth for the test database schema. Mirrors the production
# DDL for ``sientia_data`` so any production change can be pasted directly into
# this file (see ``e2e/db_schema.sql``) without touching Python.
DB_SCHEMA_SQL_PATH = Path(__file__).parent / 'db_schema.sql'
@pytest_asyncio.fixture(scope='session')
def postgres_container():
@@ -153,113 +69,16 @@ def postgres_engine(postgres_container):
def _create_schema_and_tables(engine):
"""Create all schemas/tables required by workflow and activity paths."""
"""
Create all schemas/tables required by workflow and activity paths.
Loads the DDL from ``e2e/db_schema.sql`` (single source of truth that
mirrors the production schema). The SQL file is executed via the raw
DBAPI cursor so multi-statement DDL is supported.
"""
sql_text = DB_SCHEMA_SQL_PATH.read_text(encoding='utf-8')
with engine.begin() as conn:
conn.execute(text('CREATE SCHEMA IF NOT EXISTS predictions_schema'))
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS predictions_schema.laborious_data (
id SERIAL NOT NULL,
model_id int4 NOT NULL,
variable text NOT NULL,
value numeric NULL,
"timestamp" timestamptz NOT NULL,
created_at timestamptz NOT NULL,
PRIMARY KEY (id)
);
"""
)
)
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS predictions_schema.predictions (
id SERIAL NOT NULL,
model_id int4 NOT NULL,
prediction numeric NULL,
prediction_confidence numeric NOT NULL,
response_time numeric NOT NULL,
prediction_status text NOT NULL,
"timestamp" timestamptz NOT NULL,
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
comments text NULL,
PRIMARY KEY (id, created_at)
);
"""
)
)
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS predictions_schema.transformed_data (
id SERIAL NOT NULL,
model_id int4 NOT NULL,
variable text NOT NULL,
value numeric NULL,
"timestamp" timestamptz NOT NULL,
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
PRIMARY KEY (id)
);
"""
)
)
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS predictions_schema.drift (
id SERIAL NOT NULL,
model_id int4 NOT NULL,
feature text NOT NULL,
method text NOT NULL,
value numeric NULL,
drift bool NOT NULL,
chunk int4 NOT NULL,
"timestamp" timestamptz NOT NULL,
timestamp_end text NULL,
accurate bool NOT NULL,
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
updated_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
PRIMARY KEY (id)
);
"""
)
)
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS predictions_schema.simple_metrics_data (
id SERIAL NOT NULL,
model_id int4 NOT NULL,
metric text NOT NULL,
value numeric NOT NULL,
"timestamp" timestamptz NOT NULL,
data_size int4 NOT NULL,
interval_minutes int4 NOT NULL,
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
PRIMARY KEY (id)
);
"""
)
)
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS predictions_schema.retrain_reports (
id SERIAL NOT NULL,
model_id int4 NOT NULL,
model_name text NOT NULL,
"timestamp" text NOT NULL,
status text NOT NULL,
version text NULL,
mlflow_run_id text NULL,
mlflow_experiment_id text NULL,
created_at timestamptz DEFAULT CURRENT_TIMESTAMP NOT NULL,
PRIMARY KEY (id)
);
"""
)
)
conn.exec_driver_sql(sql_text)
@pytest_asyncio.fixture(autouse=True)
@@ -269,14 +88,6 @@ def setup_postgres_schema_and_tables(postgres_engine):
yield
@pytest.fixture(autouse=True)
def reset_model_analysis_stub():
"""Reset class-level state on the ModelAnalysis stub between tests."""
_FakeModelAnalysis.reset()
yield
_FakeModelAnalysis.reset()
@pytest_asyncio.fixture
def mock_logger():
"""Logger double with readable console output for E2E runs."""
@@ -650,36 +461,3 @@ async def temporal_worker_minimal_retrain(temporal_test_env, test_activities):
yield worker
@pytest.fixture
def model_analysis_stub():
"""
Expose the ``_FakeModelAnalysis`` class so drift tests can configure responses.
Use class-level attributes to control the analyzer outputs:
- ``stub.set_drift_dataframe(factory)`` to provide rows for ``calculate_drift``.
- ``stub.raise_on_get_drift_metrics_dataframe(exception)`` to simulate failures.
"""
class _Helper:
"""Thin convenience wrapper around _FakeModelAnalysis class state."""
cls = _FakeModelAnalysis
def set_drift_dataframe(self, factory):
self.cls._drift_response_factory = factory
def raise_on_get_drift_metrics_dataframe(self, exc: Exception):
self.cls._drift_metrics_dataframe_exception = exc
def set_univariate_side_effect(self, side_effect):
self.cls._univariate_side_effect = side_effect
def set_multivariate_side_effect(self, side_effect):
self.cls._multivariate_side_effect = side_effect
@property
def last_instance(self):
return self.cls.last_instance
return _Helper()