vitor-aignosi
10c7e292b9
Refactor ModelMetrics to utilize DriftAnalysis for drift detection - Replaced ModelAnalysis with DriftAnalysis in the ModelMetrics class to enhance drift detection capabilities. - Updated method signatures and documentation to reflect the changes in target_name and return values. - Adjusted data handling to ensure compatibility with the new analysis methods and improved clarity in the drift metrics dataframe preparation.
955 lines
31 KiB
Python
955 lines
31 KiB
Python
from unittest.mock import ANY, MagicMock, patch
|
|
|
|
from pandas import DataFrame, Timestamp
|
|
from pytest import fixture
|
|
from sientia_do.notifications.models import NotificationLevel
|
|
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
|
|
|
from laborious.activities.model_metrics import ModelMetrics
|
|
|
|
|
|
@fixture
|
|
def model_metrics_activity():
|
|
model_metrics = ModelMetrics(
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=MagicMock(),
|
|
)
|
|
model_metrics.error = MagicMock()
|
|
model_metrics.debug = MagicMock()
|
|
model_metrics.info = MagicMock()
|
|
model_metrics.warning = MagicMock()
|
|
model_metrics.critical = MagicMock()
|
|
model_metrics.send_notification = MagicMock()
|
|
model_metrics.emit_metric_sync = MagicMock()
|
|
model_metrics.get_core_labels = MagicMock(
|
|
return_value={
|
|
'pod_id': 'test_pod',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
}
|
|
)
|
|
model_metrics.observe_lag_sync = MagicMock()
|
|
model_metrics.pod_id = 'test_pod'
|
|
return model_metrics
|
|
|
|
|
|
metadata = {
|
|
'metadata': {
|
|
'model_id': 'test_model',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
'schema_name': 'test_schedule',
|
|
},
|
|
}
|
|
|
|
|
|
def test_calculate_drift_invalid_chunk_period(model_metrics_activity):
|
|
# Arrange
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'invalid',
|
|
}
|
|
|
|
# Act & Assert
|
|
try:
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
except ValueError as e:
|
|
assert str(e) == 'Invalid chunk period: invalid, must be "min" or "s"'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Invalid chunk period: invalid', metadata['metadata']
|
|
)
|
|
else:
|
|
raise AssertionError('Expected ValueError')
|
|
|
|
|
|
def _sample_drift_metrics_df(ts: Timestamp) -> DataFrame:
|
|
"""Minimal analyzer-shaped dataframe (univariate row + columns the activity expects)."""
|
|
return DataFrame(
|
|
{
|
|
'timestamp': [ts],
|
|
'feature': ['feature1'],
|
|
'method': ['ks_test'],
|
|
'value': [0.5],
|
|
'alert': [False],
|
|
'chunk_index': [0],
|
|
'chunk_start_date': [ts],
|
|
'chunk_end_date': [ts],
|
|
'threshold': [0.1],
|
|
'drift_type': ['univariate'],
|
|
}
|
|
)
|
|
|
|
|
|
def test_calculate_drift_with_reference_data(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
def test_calculate_drift_without_reference_data(model_metrics_activity):
|
|
# Ten rows so int(len * 0.3) >= 1 for the built-in reference slice.
|
|
ts_last = Timestamp('2023-05-26 11:12:36')
|
|
drift_df = _sample_drift_metrics_df(ts_last)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
timestamps = [f'2023-05-26 11:12:{27 + i:02d}' for i in range(10)]
|
|
target_data_dict = {
|
|
'timestamp': timestamps,
|
|
'variable': ['feature1'] * 10,
|
|
'value': [float(i) for i in range(10)],
|
|
}
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': target_data_dict,
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts_last.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts_last.isoformat(),
|
|
'chunk_end_date': ts_last.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': False,
|
|
}
|
|
]
|
|
model_metrics_activity.warning.assert_called()
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING',
|
|
message='Using 30% first rows of target data as reference data',
|
|
block='model_metrics',
|
|
level=NotificationLevel.WARNING,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
def test_calculate_drift_empty_drift_df(mock_to_datetime, mock_dataframe, model_metrics_activity):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=DataFrame())
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert result == []
|
|
model_metrics_activity.warning.assert_called_with(
|
|
'No drift metrics found', metadata['metadata']
|
|
)
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
def test_calculate_drift_empty_after_timestamp_filter(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
mock_drift_df = MagicMock()
|
|
mock_drift_df.empty = False
|
|
mock_drift_df.drop.return_value = mock_drift_df
|
|
|
|
# Set up __getitem__ to handle filtering - timestamp access returns series with isin=False
|
|
# and filtering returns empty DataFrame
|
|
mock_timestamp_series = MagicMock()
|
|
mock_timestamp_series.isin.return_value = [False]
|
|
mock_empty_df = MagicMock()
|
|
mock_empty_df.empty = True
|
|
|
|
def getitem_side_effect(key):
|
|
if key == 'timestamp':
|
|
return mock_timestamp_series
|
|
else:
|
|
# This is the filtering operation - return empty DataFrame
|
|
return mock_empty_df
|
|
|
|
mock_drift_df.__getitem__.side_effect = getitem_side_effect
|
|
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=mock_drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert result == []
|
|
model_metrics_activity.warning.assert_called_with(
|
|
'No drift metrics found after dropping rows where timestamp is not in target data',
|
|
metadata['metadata'],
|
|
)
|
|
# When the timestamp filter empties the dataframe, the post-filter
|
|
# pipeline is short-circuited, so neither ``drop`` nor ``rename`` runs
|
|
# (they wouldn't run anyway, as the activity preserves the lib's schema).
|
|
mock_drift_df.drop.assert_not_called()
|
|
mock_drift_df.rename.assert_not_called()
|
|
mock_drift_df.__getitem__.assert_called()
|
|
|
|
|
|
def test_calculate_drift_success_min(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
def test_calculate_drift_success_s(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
def test_calculate_drift_get_drift_metrics_error(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
model_metrics_activity.get_drift_metrics = MagicMock(
|
|
side_effect=Exception('Get drift metrics error')
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert result == []
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error getting drift metrics: Get drift metrics error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_GET_DRIFT_METRICS_ERROR',
|
|
message='Error getting drift metrics: Get drift metrics error',
|
|
block='model_metrics',
|
|
level=NotificationLevel.ERROR,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_success(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_drift_df = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'method': ['ks_test'],
|
|
'value': [0.5],
|
|
'feature': ['feature1'],
|
|
}
|
|
)
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.return_value = mock_drift_df
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act
|
|
result = model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
|
|
# Assert
|
|
assert isinstance(result, DataFrame)
|
|
model_metrics_activity.debug.assert_called()
|
|
model_metrics_activity.observe_lag_sync.assert_called()
|
|
model_metrics_activity.emit_metric_sync.assert_called()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_univariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.side_effect = Exception(
|
|
'Univariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act & Assert
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Univariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting univariate drift: Univariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_multivariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.side_effect = Exception(
|
|
'Multivariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Multivariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting multivariate drift: Multivariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_dataframe_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.side_effect = Exception(
|
|
'Dataframe error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Dataframe error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error getting drift metrics: Dataframe error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
def test_calculate_simple_metrics_success_all_metrics(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mse', 'mae', 'r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 4
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert 'r2' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mse', 'mae', 'r2']",
|
|
metadata['metadata'],
|
|
)
|
|
model_metrics_activity.debug.assert_called_once()
|
|
|
|
|
|
def test_calculate_simple_metrics_success_rmse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_mse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_mae_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mae'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_r2_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_r2_zero_ss_tot(model_metrics_activity):
|
|
# Arrange
|
|
# All target values are the same, so ss_tot will be 0
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 1.0],
|
|
'prediction': [1.1, 1.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['value'].values[0] == 0.0 # Should return 0.0 when ss_tot == 0
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_multiple_metrics_subset(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 2
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_unknown_metric_ignored(model_metrics_activity):
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['unknown_metric', 'rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|