vitor-aignosi
00b6382a88
Simplify bandit nosec comments and mock metrics_controller with real spe - drift.py/simple_metrics.py: move nosec rationale to a code comment above the query instead of an inline suppression comment, keeping bandit happy with a plain `# nosec` - tests: replace bare `MagicMock()` with `MagicMock(spec=MetricsController)` for metrics_controller across activities/repository tests so mocks fail fast on attribute typos instead of silently accepting any call
956 lines
31 KiB
Python
956 lines
31 KiB
Python
from unittest.mock import ANY, MagicMock, patch
|
|
|
|
from pandas import DataFrame, Timestamp
|
|
from pytest import fixture, raises
|
|
from sientia_do.notifications.models import NotificationLevel
|
|
from sientia_do.observability.metrics_controller import MetricsController
|
|
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
|
from sientia_model.analytics.drift_analysis import DriftInsufficientDataError
|
|
|
|
from laborious.activities.model_metrics import ModelMetrics
|
|
|
|
|
|
@fixture
|
|
def model_metrics_activity():
|
|
model_metrics = ModelMetrics(
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=MagicMock(spec=MetricsController),
|
|
)
|
|
model_metrics.error = MagicMock()
|
|
model_metrics.debug = MagicMock()
|
|
model_metrics.info = MagicMock()
|
|
model_metrics.warning = MagicMock()
|
|
model_metrics.critical = MagicMock()
|
|
model_metrics.send_notification = MagicMock()
|
|
model_metrics.emit_metric_sync = MagicMock()
|
|
model_metrics.get_core_labels = MagicMock(
|
|
return_value={
|
|
'pod_id': 'test_pod',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
}
|
|
)
|
|
model_metrics.observe_lag_sync = MagicMock()
|
|
model_metrics.pod_id = 'test_pod'
|
|
return model_metrics
|
|
|
|
|
|
metadata = {
|
|
'metadata': {
|
|
'model_id': 'test_model',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
'schema_name': 'test_schedule',
|
|
},
|
|
}
|
|
|
|
|
|
def test_calculate_drift_invalid_chunk_period(model_metrics_activity):
|
|
# Arrange
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'invalid',
|
|
}
|
|
|
|
# Act & Assert
|
|
try:
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
except ValueError as e:
|
|
assert str(e) == 'Invalid chunk period: invalid, must be "min" or "s"'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Invalid chunk period: invalid', metadata['metadata']
|
|
)
|
|
else:
|
|
raise AssertionError('Expected ValueError')
|
|
|
|
|
|
def _sample_drift_metrics_df(ts: Timestamp) -> DataFrame:
|
|
"""Minimal analyzer-shaped dataframe (univariate row + columns the activity expects)."""
|
|
return DataFrame(
|
|
{
|
|
'timestamp': [ts],
|
|
'feature': ['feature1'],
|
|
'method': ['ks_test'],
|
|
'value': [0.5],
|
|
'alert': [False],
|
|
'chunk_index': [0],
|
|
'chunk_start_date': [ts],
|
|
'chunk_end_date': [ts],
|
|
'threshold': [0.1],
|
|
'drift_type': ['univariate'],
|
|
}
|
|
)
|
|
|
|
|
|
def test_calculate_drift_with_reference_data(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
def test_calculate_drift_without_reference_data(model_metrics_activity):
|
|
# Ten rows so int(len * 0.3) >= 1 for the built-in reference slice.
|
|
ts_last = Timestamp('2023-05-26 11:12:36')
|
|
drift_df = _sample_drift_metrics_df(ts_last)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
timestamps = [f'2023-05-26 11:12:{27 + i:02d}' for i in range(10)]
|
|
target_data_dict = {
|
|
'timestamp': timestamps,
|
|
'variable': ['feature1'] * 10,
|
|
'value': [float(i) for i in range(10)],
|
|
}
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': target_data_dict,
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts_last.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts_last.isoformat(),
|
|
'chunk_end_date': ts_last.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': False,
|
|
}
|
|
]
|
|
model_metrics_activity.warning.assert_called()
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING',
|
|
message='Using 30% first rows of target data as reference data',
|
|
block='model_metrics',
|
|
level=NotificationLevel.WARNING,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
def test_calculate_drift_empty_drift_df(model_metrics_activity):
|
|
"""Empty analyzer merge yields no rows and no insufficient-data alert (lib owns that failure mode)."""
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
empty_df = _sample_drift_metrics_df(ts).iloc[0:0]
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=empty_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
assert result == []
|
|
model_metrics_activity.send_notification.assert_not_called()
|
|
|
|
|
|
def test_calculate_drift_empty_after_timestamp_filter(model_metrics_activity):
|
|
"""Rows dropped by target-window alignment yield an empty export list, not an insufficient-data error."""
|
|
drift_df = _sample_drift_metrics_df(Timestamp('2020-01-01'))
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
assert result == []
|
|
model_metrics_activity.send_notification.assert_not_called()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
def test_calculate_drift_drift_insufficient_data_error_from_lib(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
"""``DriftInsufficientDataError`` maps to MODEL_METRICS_DRIFT_INSUFFICIENT_DATA, not GET error."""
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
lib_msg = (
|
|
'[MODEL_METRICS_DRIFT_INSUFFICIENT_DATA] Drift analysis produced no time chunks '
|
|
"(chunk_period='min', analysis_rows=1)."
|
|
)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(
|
|
side_effect=DriftInsufficientDataError(lib_msg, analysis_rows=1)
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
with raises(DriftInsufficientDataError, match='MODEL_METRICS_DRIFT_INSUFFICIENT_DATA'):
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
|
|
model_metrics_activity.error.assert_called_once_with(lib_msg, metadata['metadata'])
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_DRIFT_INSUFFICIENT_DATA',
|
|
message=lib_msg,
|
|
block='model_metrics',
|
|
level=NotificationLevel.ERROR,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
def test_calculate_drift_success_min(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
def test_calculate_drift_success_s(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
def test_calculate_drift_get_drift_metrics_error(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
model_metrics_activity.get_drift_metrics = MagicMock(
|
|
side_effect=Exception('Get drift metrics error')
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act / Assert
|
|
with raises(Exception, match='Get drift metrics error'):
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error getting drift metrics: Get drift metrics error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_GET_DRIFT_METRICS_ERROR',
|
|
message='Error getting drift metrics: Get drift metrics error',
|
|
block='model_metrics',
|
|
level=NotificationLevel.ERROR,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_success(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_drift_df = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'method': ['ks_test'],
|
|
'value': [0.5],
|
|
'feature': ['feature1'],
|
|
}
|
|
)
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.return_value = mock_drift_df
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act
|
|
result = model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
|
|
# Assert
|
|
assert isinstance(result, DataFrame)
|
|
model_metrics_activity.debug.assert_called()
|
|
model_metrics_activity.observe_lag_sync.assert_called()
|
|
model_metrics_activity.emit_metric_sync.assert_called()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_univariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.side_effect = Exception(
|
|
'Univariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act & Assert
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Univariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting univariate drift: Univariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_multivariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.side_effect = Exception(
|
|
'Multivariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Multivariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting multivariate drift: Multivariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_dataframe_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.side_effect = Exception(
|
|
'Dataframe error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Dataframe error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error building drift metrics dataframe: Dataframe error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
def test_calculate_simple_metrics_success_all_metrics(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mse', 'mae', 'r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 4
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert 'r2' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mse', 'mae', 'r2']",
|
|
metadata['metadata'],
|
|
)
|
|
model_metrics_activity.debug.assert_called_once()
|
|
|
|
|
|
def test_calculate_simple_metrics_success_rmse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_mse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_mae_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mae'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_r2_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_r2_zero_ss_tot(model_metrics_activity):
|
|
# Arrange
|
|
# All target values are the same, so ss_tot will be 0
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 1.0],
|
|
'prediction': [1.1, 1.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['value'].values[0] == 0.0 # Should return 0.0 when ss_tot == 0
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_multiple_metrics_subset(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 2
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_unknown_metric_ignored(model_metrics_activity):
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['unknown_metric', 'rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|