gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Refactor ModelMetrics to utilize DriftAnalysis for drift detection

- Replaced ModelAnalysis with DriftAnalysis in the ModelMetrics class to enhance drift detection capabilities.
- Updated method signatures and documentation to reflect the changes in target_name and return values.
- Adjusted data handling to ensure compatibility with the new analysis methods and improved clarity in the drift metrics dataframe preparation.
This commit is contained in:
vitor-aignosi
2026-05-08 16:39:12 -03:00
parent e6018af23f
commit 10c7e292b9
29 changed files with 843 additions and 1266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

369
tests/laborious/activities/test_model_metrics.py
View File

@@ -1,8 +1,9 @@
from unittest.mock import ANY, MagicMock, patch
from pandas import DataFrame
from pandas import DataFrame, Timestamp
from pytest import fixture
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from laborious.activities.model_metrics import ModelMetrics
@@ -72,39 +73,28 @@ def test_calculate_drift_invalid_chunk_period(model_metrics_activity):
raise AssertionError('Expected ValueError')
@patch('laborious.activities.model_metrics.DataFrame')
@patch('laborious.activities.model_metrics.to_datetime')
def test_calculate_drift_with_reference_data(
mock_to_datetime, mock_dataframe, model_metrics_activity
):
# Arrange
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
mock_to_datetime.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
'2023-05-26 11:12:27+00:00'
)
mock_drift_df = MagicMock()
mock_drift_df.empty = False
mock_drift_df.drop.return_value = mock_drift_df
mock_drift_df.__getitem__.return_value.isin.return_value = [True]
mock_drift_df.__getitem__.return_value = mock_drift_df
mock_drift_df.__getitem__.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
'2023-05-26 11:12:27+00:00'
)
mock_drift_df.rename.return_value = mock_drift_df
mock_drift_df.drop_duplicates.return_value = mock_drift_df
mock_drift_df.to_dict.return_value = [
def _sample_drift_metrics_df(ts: Timestamp) -> DataFrame:
"""Minimal analyzer-shaped dataframe (univariate row + columns the activity expects)."""
return DataFrame(
{
'method': 'ks_test',
'value': 0.5,
'feature': 'feature1',
'timestamp': '2023-05-26 11:12:27+00:00',
'model_id': 'test_model_id',
'accurate': True,
'timestamp': [ts],
'feature': ['feature1'],
'method': ['ks_test'],
'value': [0.5],
'alert': [False],
'chunk_index': [0],
'chunk_start_date': [ts],
'chunk_end_date': [ts],
'threshold': [0.1],
'drift_type': ['univariate'],
}
]
)
model_metrics_activity.get_drift_metrics = MagicMock(return_value=mock_drift_df)
def test_calculate_drift_with_reference_data(model_metrics_activity):
ts = Timestamp('2023-05-26 11:12:27')
drift_df = _sample_drift_metrics_df(ts)
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
reference_data = DataFrame(
{
@@ -114,20 +104,11 @@ def test_calculate_drift_with_reference_data(
}
)
mock_target_df = MagicMock()
mock_target_df.pivot.return_value = mock_target_df
mock_target_df.index = ['2023-05-26 11:12:27']
mock_target_df.reset_index.return_value = mock_target_df
mock_target_df.dropna.return_value = mock_target_df
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
mock_target_df.drop.return_value.columns = ['feature1']
mock_dataframe.return_value = mock_target_df
input_data = {
**metadata,
'model_name': 'test_model',
'model_id': 'test_model_id',
'reference_data': reference_data.to_dict(),
'reference_data': reference_data.to_dict('list'),
'target_data': {
'timestamp': ['2023-05-26 11:12:27'],
'variable': ['feature1'],
@@ -138,92 +119,40 @@ def test_calculate_drift_with_reference_data(
'chunk_period': 'min',
}
# Act
result = model_metrics_activity.calculate_drift(input_data)
# Assert
assert isinstance(result, list)
assert result == mock_drift_df.to_dict.return_value # type: ignore[comparison-overlap]
model_metrics_activity.info.assert_called()
model_metrics_activity.get_drift_metrics.assert_called_once()
# Verify transformations were called
mock_drift_df.drop.assert_called_once_with(
columns=['p_value', 'chunk_start_date'], inplace=True, errors='ignore'
)
mock_drift_df.__getitem__.assert_called()
mock_drift_df.rename.assert_called_once_with(
columns={
'metric': 'method',
'statistic': 'value',
'alert': 'drift',
'chunk_index': 'chunk',
'chunk_end_date': 'timestamp_end',
}
)
mock_drift_df.drop_duplicates.assert_called_once_with(
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
)
mock_drift_df.to_dict.assert_called_once_with(orient='records')
@patch('laborious.activities.model_metrics.DataFrame')
@patch('laborious.activities.model_metrics.to_datetime')
def test_calculate_drift_without_reference_data(
mock_to_datetime, mock_dataframe, model_metrics_activity
):
# Arrange
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
mock_to_datetime.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
'2023-05-26 11:12:27+00:00'
)
mock_drift_df = MagicMock()
mock_drift_df.empty = False
mock_drift_df.drop.return_value = mock_drift_df
mock_drift_df.__getitem__.return_value.isin.return_value = [True]
mock_drift_df.__getitem__.return_value = mock_drift_df
mock_drift_df.__getitem__.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
'2023-05-26 11:12:27+00:00'
)
mock_drift_df.rename.return_value = mock_drift_df
mock_drift_df.drop_duplicates.return_value = mock_drift_df
mock_drift_df.to_dict.return_value = [
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
assert result == [
{
'timestamp': expected_timestamp,
'feature': 'feature1',
'method': 'ks_test',
'value': 0.5,
'feature': 'feature1',
'timestamp': '2023-05-26 11:12:27+00:00',
'alert': False,
'chunk_index': 0,
'chunk_start_date': ts.isoformat(),
'chunk_end_date': ts.isoformat(),
'model_id': 'test_model_id',
'accurate': False,
'accurate': True,
}
]
model_metrics_activity.info.assert_called()
model_metrics_activity.get_drift_metrics.assert_called_once()
model_metrics_activity.get_drift_metrics = MagicMock(return_value=mock_drift_df)
def test_calculate_drift_without_reference_data(model_metrics_activity):
# Ten rows so int(len * 0.3) >= 1 for the built-in reference slice.
ts_last = Timestamp('2023-05-26 11:12:36')
drift_df = _sample_drift_metrics_df(ts_last)
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
timestamps = [f'2023-05-26 11:12:{27 + i:02d}' for i in range(10)]
target_data_dict = {
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
'variable': ['feature1', 'feature1', 'feature1'],
'value': [1.0, 2.0, 3.0],
'timestamp': timestamps,
'variable': ['feature1'] * 10,
'value': [float(i) for i in range(10)],
}
mock_target_df = MagicMock()
mock_target_df.pivot.return_value = mock_target_df
mock_target_df.index = ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29']
mock_target_df.reset_index.return_value = mock_target_df
mock_target_df.dropna.return_value = mock_target_df
mock_target_df.sort_values.return_value = mock_target_df
mock_target_df.head.return_value = DataFrame(
{'timestamp': ['2023-05-26 11:12:27'], 'feature1': [1.0]}
)
mock_target_df.__getitem__.return_value.apply.return_value = [
'2023-05-26 11:12:27',
'2023-05-26 11:12:28',
'2023-05-26 11:12:29',
]
mock_target_df.drop.return_value.columns = ['feature1']
mock_dataframe.return_value = mock_target_df
mock_dataframe.side_effect = lambda x=None: mock_target_df if x is not None else mock_target_df
input_data = {
**metadata,
'model_name': 'test_model',
@@ -235,12 +164,23 @@ def test_calculate_drift_without_reference_data(
'chunk_period': 's',
}
# Act
result = model_metrics_activity.calculate_drift(input_data)
# Assert
assert isinstance(result, list)
assert result == mock_drift_df.to_dict.return_value
expected_timestamp = ts_last.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
assert result == [
{
'timestamp': expected_timestamp,
'feature': 'feature1',
'method': 'ks_test',
'value': 0.5,
'alert': False,
'chunk_index': 0,
'chunk_start_date': ts_last.isoformat(),
'chunk_end_date': ts_last.isoformat(),
'model_id': 'test_model_id',
'accurate': False,
}
]
model_metrics_activity.warning.assert_called()
model_metrics_activity.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
@@ -250,24 +190,6 @@ def test_calculate_drift_without_reference_data(
level=NotificationLevel.WARNING,
attachment_content=ANY,
)
# Verify transformations were called
mock_drift_df.drop.assert_called_once_with(
columns=['p_value', 'chunk_start_date'], inplace=True, errors='ignore'
)
mock_drift_df.__getitem__.assert_called()
mock_drift_df.rename.assert_called_once_with(
columns={
'metric': 'method',
'statistic': 'value',
'alert': 'drift',
'chunk_index': 'chunk',
'chunk_end_date': 'timestamp_end',
}
)
mock_drift_df.drop_duplicates.assert_called_once_with(
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
)
mock_drift_df.to_dict.assert_called_once_with(orient='records')
@patch('laborious.activities.model_metrics.DataFrame')
@@ -391,44 +313,18 @@ def test_calculate_drift_empty_after_timestamp_filter(
'No drift metrics found after dropping rows where timestamp is not in target data',
metadata['metadata'],
)
# When the timestamp filter empties the dataframe, the rename/drop pipeline
# is short-circuited, so neither ``drop`` nor ``rename`` should run.
# When the timestamp filter empties the dataframe, the post-filter
# pipeline is short-circuited, so neither ``drop`` nor ``rename`` runs
# (they wouldn't run anyway, as the activity preserves the lib's schema).
mock_drift_df.drop.assert_not_called()
mock_drift_df.rename.assert_not_called()
mock_drift_df.__getitem__.assert_called()
@patch('laborious.activities.model_metrics.DataFrame')
@patch('laborious.activities.model_metrics.to_datetime')
def test_calculate_drift_success_min(mock_to_datetime, mock_dataframe, model_metrics_activity):
# Arrange
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
mock_to_datetime.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
'2023-05-26 11:12:27+00:00'
)
mock_drift_df = MagicMock()
mock_drift_df.empty = False
mock_drift_df.drop.return_value = mock_drift_df
mock_drift_df.__getitem__.return_value.isin.return_value = [True]
mock_drift_df.__getitem__.return_value = mock_drift_df
mock_drift_df.__getitem__.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
'2023-05-26 11:12:27+00:00'
)
mock_drift_df.rename.return_value = mock_drift_df
mock_drift_df.drop_duplicates.return_value = mock_drift_df
mock_drift_df.to_dict.return_value = [
{
'method': 'ks_test',
'value': 0.5,
'feature': 'feature1',
'timestamp': '2023-05-26 11:12:27+00:00',
'model_id': 'test_model_id',
'accurate': True,
}
]
model_metrics_activity.get_drift_metrics = MagicMock(return_value=mock_drift_df)
def test_calculate_drift_success_min(model_metrics_activity):
ts = Timestamp('2023-05-26 11:12:27')
drift_df = _sample_drift_metrics_df(ts)
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
reference_data = DataFrame(
{
@@ -438,20 +334,11 @@ def test_calculate_drift_success_min(mock_to_datetime, mock_dataframe, model_met
}
)
mock_target_df = MagicMock()
mock_target_df.pivot.return_value = mock_target_df
mock_target_df.index = ['2023-05-26 11:12:27']
mock_target_df.reset_index.return_value = mock_target_df
mock_target_df.dropna.return_value = mock_target_df
mock_target_df.__getitem__.return_value.isin.return_value = [True]
mock_target_df.drop.return_value.columns = ['feature1']
mock_dataframe.return_value = mock_target_df
input_data = {
**metadata,
'model_name': 'test_model',
'model_id': 'test_model_id',
'reference_data': reference_data.to_dict(),
'reference_data': reference_data.to_dict('list'),
'target_data': {
'timestamp': ['2023-05-26 11:12:27'],
'variable': ['feature1'],
@@ -462,65 +349,31 @@ def test_calculate_drift_success_min(mock_to_datetime, mock_dataframe, model_met
'chunk_period': 'min',
}
# Act
result = model_metrics_activity.calculate_drift(input_data)
# Assert
assert isinstance(result, list)
assert result == mock_drift_df.to_dict.return_value # type: ignore[comparison-overlap]
model_metrics_activity.info.assert_called()
model_metrics_activity.get_drift_metrics.assert_called_once()
# Verify transformations were called
mock_drift_df.drop.assert_called_once_with(
columns=['p_value', 'chunk_start_date'], inplace=True, errors='ignore'
)
mock_drift_df.__getitem__.assert_called()
mock_drift_df.rename.assert_called_once_with(
columns={
'metric': 'method',
'statistic': 'value',
'alert': 'drift',
'chunk_index': 'chunk',
'chunk_end_date': 'timestamp_end',
}
)
mock_drift_df.drop_duplicates.assert_called_once_with(
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
)
mock_drift_df.to_dict.assert_called_once_with(orient='records')
@patch('laborious.activities.model_metrics.DataFrame')
@patch('laborious.activities.model_metrics.to_datetime')
def test_calculate_drift_success_s(mock_to_datetime, mock_dataframe, model_metrics_activity):
# Arrange
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
mock_to_datetime.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
'2023-05-26 11:12:27+00:00'
)
mock_drift_df = MagicMock()
mock_drift_df.empty = False
mock_drift_df.drop.return_value = mock_drift_df
mock_drift_df.__getitem__.return_value.isin.return_value = [True]
mock_drift_df.__getitem__.return_value = mock_drift_df
mock_drift_df.__getitem__.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
'2023-05-26 11:12:27+00:00'
)
mock_drift_df.rename.return_value = mock_drift_df
mock_drift_df.drop_duplicates.return_value = mock_drift_df
mock_drift_df.to_dict.return_value = [
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
assert result == [
{
'timestamp': expected_timestamp,
'feature': 'feature1',
'method': 'ks_test',
'value': 0.5,
'feature': 'feature1',
'timestamp': '2023-05-26 11:12:27+00:00',
'alert': False,
'chunk_index': 0,
'chunk_start_date': ts.isoformat(),
'chunk_end_date': ts.isoformat(),
'model_id': 'test_model_id',
'accurate': True,
}
]
model_metrics_activity.info.assert_called()
model_metrics_activity.get_drift_metrics.assert_called_once()
model_metrics_activity.get_drift_metrics = MagicMock(return_value=mock_drift_df)
def test_calculate_drift_success_s(model_metrics_activity):
ts = Timestamp('2023-05-26 11:12:27')
drift_df = _sample_drift_metrics_df(ts)
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
reference_data = DataFrame(
{
@@ -530,20 +383,11 @@ def test_calculate_drift_success_s(mock_to_datetime, mock_dataframe, model_metri
}
)
mock_target_df = MagicMock()
mock_target_df.pivot.return_value = mock_target_df
mock_target_df.index = ['2023-05-26 11:12:27']
mock_target_df.reset_index.return_value = mock_target_df
mock_target_df.dropna.return_value = mock_target_df
mock_target_df.__getitem__.return_value.isin.return_value = [True]
mock_target_df.drop.return_value.columns = ['feature1']
mock_dataframe.return_value = mock_target_df
input_data = {
**metadata,
'model_name': 'test_model',
'model_id': 'test_model_id',
'reference_data': reference_data.to_dict(),
'reference_data': reference_data.to_dict('list'),
'target_data': {
'timestamp': ['2023-05-26 11:12:27'],
'variable': ['feature1'],
@@ -554,32 +398,25 @@ def test_calculate_drift_success_s(mock_to_datetime, mock_dataframe, model_metri
'chunk_period': 's',
}
# Act
result = model_metrics_activity.calculate_drift(input_data)
# Assert
assert isinstance(result, list)
assert result == mock_drift_df.to_dict.return_value # type: ignore[comparison-overlap]
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
assert result == [
{
'timestamp': expected_timestamp,
'feature': 'feature1',
'method': 'ks_test',
'value': 0.5,
'alert': False,
'chunk_index': 0,
'chunk_start_date': ts.isoformat(),
'chunk_end_date': ts.isoformat(),
'model_id': 'test_model_id',
'accurate': True,
}
]
model_metrics_activity.info.assert_called()
model_metrics_activity.get_drift_metrics.assert_called_once()
# Verify transformations were called
mock_drift_df.drop.assert_called_once_with(
columns=['p_value', 'chunk_start_date'], inplace=True, errors='ignore'
)
mock_drift_df.__getitem__.assert_called()
mock_drift_df.rename.assert_called_once_with(
columns={
'metric': 'method',
'statistic': 'value',
'alert': 'drift',
'chunk_index': 'chunk',
'chunk_end_date': 'timestamp_end',
}
)
mock_drift_df.drop_duplicates.assert_called_once_with(
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
)
mock_drift_df.to_dict.assert_called_once_with(orient='records')
@patch('laborious.activities.model_metrics.DataFrame')
@@ -646,7 +483,7 @@ def test_calculate_drift_get_drift_metrics_error(
@patch('laborious.activities.model_metrics.to_datetime')
@patch('laborious.activities.model_metrics.time.time')
@patch('laborious.activities.model_metrics.ModelAnalysis')
@patch('laborious.activities.model_metrics.DriftAnalysis')
@patch('laborious.activities.model_metrics.metrics')
def test_get_drift_metrics_success(
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
@@ -657,8 +494,8 @@ def test_get_drift_metrics_success(
mock_drift_df = DataFrame(
{
'timestamp': ['2023-05-26 11:12:27'],
'metric': ['ks_test'],
'statistic': [0.5],
'method': ['ks_test'],
'value': [0.5],
'feature': ['feature1'],
}
)
@@ -707,7 +544,7 @@ def test_get_drift_metrics_success(
@patch('laborious.activities.model_metrics.to_datetime')
@patch('laborious.activities.model_metrics.time.time')
@patch('laborious.activities.model_metrics.ModelAnalysis')
@patch('laborious.activities.model_metrics.DriftAnalysis')
@patch('laborious.activities.model_metrics.metrics')
def test_get_drift_metrics_univariate_error(
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
@@ -764,7 +601,7 @@ def test_get_drift_metrics_univariate_error(
@patch('laborious.activities.model_metrics.to_datetime')
@patch('laborious.activities.model_metrics.time.time')
@patch('laborious.activities.model_metrics.ModelAnalysis')
@patch('laborious.activities.model_metrics.DriftAnalysis')
@patch('laborious.activities.model_metrics.metrics')
def test_get_drift_metrics_multivariate_error(
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
@@ -810,7 +647,7 @@ def test_get_drift_metrics_multivariate_error(
@patch('laborious.activities.model_metrics.to_datetime')
@patch('laborious.activities.model_metrics.time.time')
@patch('laborious.activities.model_metrics.ModelAnalysis')
@patch('laborious.activities.model_metrics.DriftAnalysis')
@patch('laborious.activities.model_metrics.metrics')
def test_get_drift_metrics_dataframe_error(
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity

4
tests/laborious/utils/test_connectors_config.py
View File

@@ -125,6 +125,8 @@ def test_build_minio_config_with_env_vars():
environ['MINIO_SECRET_KEY'] = 'test-secret'
environ['MINIO_REGION_NAME'] = 'test-region'
environ['MINIO_DEFAULT_BUCKET'] = 'test-bucket'
# Isolate from IDE/CI env (e.g. VS Code may export MINIO_SECURE=true).
environ['MINIO_SECURE'] = 'false'
assert build_minio_config() == {
'endpoint_url': 'http://test-host',
'access_key': 'test-key',
@@ -141,6 +143,8 @@ def test_build_minio_config_with_defaults():
environ.pop('MINIO_SECRET_KEY', None)
environ.pop('MINIO_REGION_NAME', None)
environ.pop('MINIO_DEFAULT_BUCKET', None)
environ.pop('MINIO_SECURE', None)
environ.pop('MINIO_RETENTION_HOURS', None)
assert build_minio_config() == {
'endpoint_url': 'http://localhost:9000',
'access_key': 'minioadmin',