vitor-aignosi
8d34228d7b
Update E2E test report and enhance drift analysis handling - Updated the E2E test report metrics to reflect the latest test results, showing 47 collected tests with all passing. - Removed outdated sections related to failed tests and their causes, streamlining the report. - Implemented a regression fix in the drift analysis to handle empty merged frames, ensuring workflows skip export when no drift metrics are available. - Enhanced the `insert_sample_data` and `insert_sample_prediction` functions to allow customizable timestamps for better test accuracy. - Refactored E2E tests to improve clarity and maintainability, particularly in handling repeat scenarios with distinct timestamps.
956 lines
31 KiB
Python
956 lines
31 KiB
Python
from unittest.mock import ANY, MagicMock, patch
|
|
|
|
from pandas import DataFrame, Timestamp
|
|
from pytest import fixture, raises
|
|
from sientia_do.notifications.models import NotificationLevel
|
|
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
|
from sientia_model.analytics.drift_analysis import DriftInsufficientDataError
|
|
|
|
from laborious.activities.model_metrics import ModelMetrics
|
|
|
|
|
|
@fixture
|
|
def model_metrics_activity():
|
|
model_metrics = ModelMetrics(
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=MagicMock(),
|
|
)
|
|
model_metrics.error = MagicMock()
|
|
model_metrics.debug = MagicMock()
|
|
model_metrics.info = MagicMock()
|
|
model_metrics.warning = MagicMock()
|
|
model_metrics.critical = MagicMock()
|
|
model_metrics.send_notification = MagicMock()
|
|
model_metrics.emit_metric_sync = MagicMock()
|
|
model_metrics.get_core_labels = MagicMock(
|
|
return_value={
|
|
'pod_id': 'test_pod',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
}
|
|
)
|
|
model_metrics.observe_lag_sync = MagicMock()
|
|
model_metrics.pod_id = 'test_pod'
|
|
return model_metrics
|
|
|
|
|
|
metadata = {
|
|
'metadata': {
|
|
'model_id': 'test_model',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
'schema_name': 'test_schedule',
|
|
},
|
|
}
|
|
|
|
|
|
def test_calculate_drift_invalid_chunk_period(model_metrics_activity):
|
|
# Arrange
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'invalid',
|
|
}
|
|
|
|
# Act & Assert
|
|
try:
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
except ValueError as e:
|
|
assert str(e) == 'Invalid chunk period: invalid, must be "min" or "s"'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Invalid chunk period: invalid', metadata['metadata']
|
|
)
|
|
else:
|
|
raise AssertionError('Expected ValueError')
|
|
|
|
|
|
def _sample_drift_metrics_df(ts: Timestamp) -> DataFrame:
|
|
"""Minimal analyzer-shaped dataframe (univariate row + columns the activity expects)."""
|
|
return DataFrame(
|
|
{
|
|
'timestamp': [ts],
|
|
'feature': ['feature1'],
|
|
'method': ['ks_test'],
|
|
'value': [0.5],
|
|
'alert': [False],
|
|
'chunk_index': [0],
|
|
'chunk_start_date': [ts],
|
|
'chunk_end_date': [ts],
|
|
'threshold': [0.1],
|
|
'drift_type': ['univariate'],
|
|
}
|
|
)
|
|
|
|
|
|
def test_calculate_drift_with_reference_data(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
def test_calculate_drift_without_reference_data(model_metrics_activity):
|
|
# Ten rows so int(len * 0.3) >= 1 for the built-in reference slice.
|
|
ts_last = Timestamp('2023-05-26 11:12:36')
|
|
drift_df = _sample_drift_metrics_df(ts_last)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
timestamps = [f'2023-05-26 11:12:{27 + i:02d}' for i in range(10)]
|
|
target_data_dict = {
|
|
'timestamp': timestamps,
|
|
'variable': ['feature1'] * 10,
|
|
'value': [float(i) for i in range(10)],
|
|
}
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': target_data_dict,
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts_last.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts_last.isoformat(),
|
|
'chunk_end_date': ts_last.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': False,
|
|
}
|
|
]
|
|
model_metrics_activity.warning.assert_called()
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING',
|
|
message='Using 30% first rows of target data as reference data',
|
|
block='model_metrics',
|
|
level=NotificationLevel.WARNING,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
def test_calculate_drift_empty_drift_df(model_metrics_activity):
|
|
"""Empty analyzer merge yields no rows and no insufficient-data alert (lib owns that failure mode)."""
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
empty_df = _sample_drift_metrics_df(ts).iloc[0:0]
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=empty_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
assert result == []
|
|
model_metrics_activity.send_notification.assert_not_called()
|
|
|
|
|
|
def test_calculate_drift_empty_after_timestamp_filter(model_metrics_activity):
|
|
"""Rows dropped by target-window alignment yield an empty export list, not an insufficient-data error."""
|
|
ts_target = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(Timestamp('2020-01-01'))
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
assert result == []
|
|
model_metrics_activity.send_notification.assert_not_called()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
def test_calculate_drift_drift_insufficient_data_error_from_lib(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
"""``DriftInsufficientDataError`` maps to MODEL_METRICS_DRIFT_INSUFFICIENT_DATA, not GET error."""
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
lib_msg = (
|
|
'[MODEL_METRICS_DRIFT_INSUFFICIENT_DATA] Drift analysis produced no time chunks '
|
|
'(chunk_period=\'min\', analysis_rows=1).'
|
|
)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(
|
|
side_effect=DriftInsufficientDataError(lib_msg, analysis_rows=1)
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
with raises(DriftInsufficientDataError, match='MODEL_METRICS_DRIFT_INSUFFICIENT_DATA'):
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
|
|
model_metrics_activity.error.assert_called_once_with(lib_msg, metadata['metadata'])
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_DRIFT_INSUFFICIENT_DATA',
|
|
message=lib_msg,
|
|
block='model_metrics',
|
|
level=NotificationLevel.ERROR,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
def test_calculate_drift_success_min(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
def test_calculate_drift_success_s(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
def test_calculate_drift_get_drift_metrics_error(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
model_metrics_activity.get_drift_metrics = MagicMock(
|
|
side_effect=Exception('Get drift metrics error')
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act / Assert
|
|
with raises(Exception, match='Get drift metrics error'):
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error getting drift metrics: Get drift metrics error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_GET_DRIFT_METRICS_ERROR',
|
|
message='Error getting drift metrics: Get drift metrics error',
|
|
block='model_metrics',
|
|
level=NotificationLevel.ERROR,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_success(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_drift_df = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'method': ['ks_test'],
|
|
'value': [0.5],
|
|
'feature': ['feature1'],
|
|
}
|
|
)
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.return_value = mock_drift_df
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act
|
|
result = model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
|
|
# Assert
|
|
assert isinstance(result, DataFrame)
|
|
model_metrics_activity.debug.assert_called()
|
|
model_metrics_activity.observe_lag_sync.assert_called()
|
|
model_metrics_activity.emit_metric_sync.assert_called()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_univariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.side_effect = Exception(
|
|
'Univariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act & Assert
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Univariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting univariate drift: Univariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_multivariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.side_effect = Exception(
|
|
'Multivariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Multivariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting multivariate drift: Multivariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_dataframe_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.side_effect = Exception(
|
|
'Dataframe error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Dataframe error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error building drift metrics dataframe: Dataframe error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
def test_calculate_simple_metrics_success_all_metrics(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mse', 'mae', 'r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 4
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert 'r2' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mse', 'mae', 'r2']",
|
|
metadata['metadata'],
|
|
)
|
|
model_metrics_activity.debug.assert_called_once()
|
|
|
|
|
|
def test_calculate_simple_metrics_success_rmse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_mse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_mae_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mae'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_r2_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_r2_zero_ss_tot(model_metrics_activity):
|
|
# Arrange
|
|
# All target values are the same, so ss_tot will be 0
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 1.0],
|
|
'prediction': [1.1, 1.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['value'].values[0] == 0.0 # Should return 0.0 when ss_tot == 0
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_multiple_metrics_subset(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 2
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_unknown_metric_ignored(model_metrics_activity):
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['unknown_metric', 'rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|