SIENTIAPDE-1646
Update E2E test report and enhance drift analysis handling - Updated the E2E test report metrics to reflect the latest test results, showing 47 collected tests with all passing. - Removed outdated sections related to failed tests and their causes, streamlining the report. - Implemented a regression fix in the drift analysis to handle empty merged frames, ensuring workflows skip export when no drift metrics are available. - Enhanced the `insert_sample_data` and `insert_sample_prediction` functions to allow customizable timestamps for better test accuracy. - Refactored E2E tests to improve clarity and maintainability, particularly in handling repeat scenarios with distinct timestamps.
This commit is contained in:
vitor-aignosi
@@ -1,9 +1,10 @@
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
|
||||
from pandas import DataFrame, Timestamp
|
||||
from pytest import fixture
|
||||
from pytest import fixture, raises
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_model.analytics.drift_analysis import DriftInsufficientDataError
|
||||
|
||||
from laborious.activities.model_metrics import ModelMetrics
|
||||
|
||||
@@ -192,13 +193,11 @@ def test_calculate_drift_without_reference_data(model_metrics_activity):
|
||||
)
|
||||
|
||||
|
||||
@patch('laborious.activities.model_metrics.DataFrame')
|
||||
@patch('laborious.activities.model_metrics.to_datetime')
|
||||
def test_calculate_drift_empty_drift_df(mock_to_datetime, mock_dataframe, model_metrics_activity):
|
||||
# Arrange
|
||||
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
||||
|
||||
model_metrics_activity.get_drift_metrics = MagicMock(return_value=DataFrame())
|
||||
def test_calculate_drift_empty_drift_df(model_metrics_activity):
|
||||
"""Empty analyzer merge yields no rows and no insufficient-data alert (lib owns that failure mode)."""
|
||||
ts = Timestamp('2023-05-26 11:12:27')
|
||||
empty_df = _sample_drift_metrics_df(ts).iloc[0:0]
|
||||
model_metrics_activity.get_drift_metrics = MagicMock(return_value=empty_df)
|
||||
|
||||
reference_data = DataFrame(
|
||||
{
|
||||
@@ -208,15 +207,6 @@ def test_calculate_drift_empty_drift_df(mock_to_datetime, mock_dataframe, model_
|
||||
}
|
||||
)
|
||||
|
||||
mock_target_df = MagicMock()
|
||||
mock_target_df.pivot.return_value = mock_target_df
|
||||
mock_target_df.index = ['2023-05-26 11:12:27']
|
||||
mock_target_df.reset_index.return_value = mock_target_df
|
||||
mock_target_df.dropna.return_value = mock_target_df
|
||||
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
||||
mock_target_df.drop.return_value.columns = ['feature1']
|
||||
mock_dataframe.return_value = mock_target_df
|
||||
|
||||
input_data = {
|
||||
**metadata,
|
||||
'model_name': 'test_model',
|
||||
@@ -232,45 +222,60 @@ def test_calculate_drift_empty_drift_df(mock_to_datetime, mock_dataframe, model_
|
||||
'chunk_period': 'min',
|
||||
}
|
||||
|
||||
# Act
|
||||
result = model_metrics_activity.calculate_drift(input_data)
|
||||
|
||||
# Assert
|
||||
assert result == []
|
||||
model_metrics_activity.warning.assert_called_with(
|
||||
'No drift metrics found', metadata['metadata']
|
||||
model_metrics_activity.send_notification.assert_not_called()
|
||||
|
||||
|
||||
def test_calculate_drift_empty_after_timestamp_filter(model_metrics_activity):
|
||||
"""Rows dropped by target-window alignment yield an empty export list, not an insufficient-data error."""
|
||||
ts_target = Timestamp('2023-05-26 11:12:27')
|
||||
drift_df = _sample_drift_metrics_df(Timestamp('2020-01-01'))
|
||||
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
||||
|
||||
reference_data = DataFrame(
|
||||
{
|
||||
'timestamp': ['2023-05-26 11:12:27'],
|
||||
'target': [1.0],
|
||||
'feature1': [1.0],
|
||||
}
|
||||
)
|
||||
|
||||
input_data = {
|
||||
**metadata,
|
||||
'model_name': 'test_model',
|
||||
'model_id': 'test_model_id',
|
||||
'reference_data': reference_data.to_dict(),
|
||||
'target_data': {
|
||||
'timestamp': ['2023-05-26 11:12:27'],
|
||||
'variable': ['feature1'],
|
||||
'value': [1.0],
|
||||
},
|
||||
'target_name': 'target',
|
||||
'drift_metrics': ['ks_test'],
|
||||
'chunk_period': 'min',
|
||||
}
|
||||
|
||||
result = model_metrics_activity.calculate_drift(input_data)
|
||||
assert result == []
|
||||
model_metrics_activity.send_notification.assert_not_called()
|
||||
|
||||
|
||||
@patch('laborious.activities.model_metrics.DataFrame')
|
||||
@patch('laborious.activities.model_metrics.to_datetime')
|
||||
def test_calculate_drift_empty_after_timestamp_filter(
|
||||
def test_calculate_drift_drift_insufficient_data_error_from_lib(
|
||||
mock_to_datetime, mock_dataframe, model_metrics_activity
|
||||
):
|
||||
# Arrange
|
||||
"""``DriftInsufficientDataError`` maps to MODEL_METRICS_DRIFT_INSUFFICIENT_DATA, not GET error."""
|
||||
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
||||
|
||||
mock_drift_df = MagicMock()
|
||||
mock_drift_df.empty = False
|
||||
mock_drift_df.drop.return_value = mock_drift_df
|
||||
|
||||
# Set up __getitem__ to handle filtering - timestamp access returns series with isin=False
|
||||
# and filtering returns empty DataFrame
|
||||
mock_timestamp_series = MagicMock()
|
||||
mock_timestamp_series.isin.return_value = [False]
|
||||
mock_empty_df = MagicMock()
|
||||
mock_empty_df.empty = True
|
||||
|
||||
def getitem_side_effect(key):
|
||||
if key == 'timestamp':
|
||||
return mock_timestamp_series
|
||||
else:
|
||||
# This is the filtering operation - return empty DataFrame
|
||||
return mock_empty_df
|
||||
|
||||
mock_drift_df.__getitem__.side_effect = getitem_side_effect
|
||||
|
||||
model_metrics_activity.get_drift_metrics = MagicMock(return_value=mock_drift_df)
|
||||
lib_msg = (
|
||||
'[MODEL_METRICS_DRIFT_INSUFFICIENT_DATA] Drift analysis produced no time chunks '
|
||||
'(chunk_period=\'min\', analysis_rows=1).'
|
||||
)
|
||||
model_metrics_activity.get_drift_metrics = MagicMock(
|
||||
side_effect=DriftInsufficientDataError(lib_msg, analysis_rows=1)
|
||||
)
|
||||
|
||||
reference_data = DataFrame(
|
||||
{
|
||||
@@ -304,21 +309,18 @@ def test_calculate_drift_empty_after_timestamp_filter(
|
||||
'chunk_period': 'min',
|
||||
}
|
||||
|
||||
# Act
|
||||
result = model_metrics_activity.calculate_drift(input_data)
|
||||
with raises(DriftInsufficientDataError, match='MODEL_METRICS_DRIFT_INSUFFICIENT_DATA'):
|
||||
model_metrics_activity.calculate_drift(input_data)
|
||||
|
||||
# Assert
|
||||
assert result == []
|
||||
model_metrics_activity.warning.assert_called_with(
|
||||
'No drift metrics found after dropping rows where timestamp is not in target data',
|
||||
metadata['metadata'],
|
||||
model_metrics_activity.error.assert_called_once_with(lib_msg, metadata['metadata'])
|
||||
model_metrics_activity.send_notification.assert_called_once_with(
|
||||
metadata=metadata['metadata'],
|
||||
notification_id='MODEL_METRICS_DRIFT_INSUFFICIENT_DATA',
|
||||
message=lib_msg,
|
||||
block='model_metrics',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=ANY,
|
||||
)
|
||||
# When the timestamp filter empties the dataframe, the post-filter
|
||||
# pipeline is short-circuited, so neither ``drop`` nor ``rename`` runs
|
||||
# (they wouldn't run anyway, as the activity preserves the lib's schema).
|
||||
mock_drift_df.drop.assert_not_called()
|
||||
mock_drift_df.rename.assert_not_called()
|
||||
mock_drift_df.__getitem__.assert_called()
|
||||
|
||||
|
||||
def test_calculate_drift_success_min(model_metrics_activity):
|
||||
@@ -463,11 +465,10 @@ def test_calculate_drift_get_drift_metrics_error(
|
||||
'chunk_period': 'min',
|
||||
}
|
||||
|
||||
# Act
|
||||
result = model_metrics_activity.calculate_drift(input_data)
|
||||
# Act / Assert
|
||||
with raises(Exception, match='Get drift metrics error'):
|
||||
model_metrics_activity.calculate_drift(input_data)
|
||||
|
||||
# Assert
|
||||
assert result == []
|
||||
model_metrics_activity.error.assert_called_once_with(
|
||||
'Error getting drift metrics: Get drift metrics error', metadata['metadata']
|
||||
)
|
||||
@@ -683,7 +684,7 @@ def test_get_drift_metrics_dataframe_error(
|
||||
except Exception as e:
|
||||
assert str(e) == 'Dataframe error'
|
||||
model_metrics_activity.error.assert_called_once_with(
|
||||
'Error getting drift metrics: Dataframe error', metadata['metadata']
|
||||
'Error building drift metrics dataframe: Dataframe error', metadata['metadata']
|
||||
)
|
||||
model_metrics_activity.emit_metric_sync.assert_called_with(
|
||||
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
||||
|
||||
Reference in New Issue
Block a user