vitor-aignosi
9920cd61c2
Add tests for retrain, drift, storage, OPC repo, worker paths - covers csv lookup fallback, drift insufficient-data passthrough, timestamp tz normalization, storage close without engine, OPC watchdog/write_value fallbacks and reconnect reporting, and legacy runtime install in worker main
1003 lines
33 KiB
Python
1003 lines
33 KiB
Python
from unittest.mock import ANY, MagicMock, patch
|
|
|
|
from pandas import DataFrame, Series, Timestamp
|
|
from pytest import fixture, mark, raises
|
|
from sientia_do.notifications.models import NotificationLevel
|
|
from sientia_do.observability.metrics_controller import MetricsController
|
|
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
|
from sientia_model.analytics.drift_analysis import DriftInsufficientDataError
|
|
|
|
from laborious.activities.model_metrics import ModelMetrics
|
|
|
|
|
|
@fixture
|
|
def model_metrics_activity():
|
|
model_metrics = ModelMetrics(
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=MagicMock(spec=MetricsController),
|
|
)
|
|
model_metrics.error = MagicMock()
|
|
model_metrics.debug = MagicMock()
|
|
model_metrics.info = MagicMock()
|
|
model_metrics.warning = MagicMock()
|
|
model_metrics.critical = MagicMock()
|
|
model_metrics.send_notification = MagicMock()
|
|
model_metrics.emit_metric_sync = MagicMock()
|
|
model_metrics.get_core_labels = MagicMock(
|
|
return_value={
|
|
'pod_id': 'test_pod',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
}
|
|
)
|
|
model_metrics.observe_lag_sync = MagicMock()
|
|
model_metrics.pod_id = 'test_pod'
|
|
return model_metrics
|
|
|
|
|
|
metadata = {
|
|
'metadata': {
|
|
'model_id': 'test_model',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
'schema_name': 'test_schedule',
|
|
},
|
|
}
|
|
|
|
|
|
def test_calculate_drift_invalid_chunk_period(model_metrics_activity):
|
|
# Arrange
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'invalid',
|
|
}
|
|
|
|
# Act & Assert
|
|
try:
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
except ValueError as e:
|
|
assert str(e) == 'Invalid chunk period: invalid, must be "min" or "s"'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Invalid chunk period: invalid', metadata['metadata']
|
|
)
|
|
else:
|
|
raise AssertionError('Expected ValueError')
|
|
|
|
|
|
def _sample_drift_metrics_df(ts: Timestamp) -> DataFrame:
|
|
"""Minimal analyzer-shaped dataframe (univariate row + columns the activity expects)."""
|
|
return DataFrame(
|
|
{
|
|
'timestamp': [ts],
|
|
'feature': ['feature1'],
|
|
'method': ['ks_test'],
|
|
'value': [0.5],
|
|
'alert': [False],
|
|
'chunk_index': [0],
|
|
'chunk_start_date': [ts],
|
|
'chunk_end_date': [ts],
|
|
'threshold': [0.1],
|
|
'drift_type': ['univariate'],
|
|
}
|
|
)
|
|
|
|
|
|
def test_calculate_drift_with_reference_data(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
def test_calculate_drift_without_reference_data(model_metrics_activity):
|
|
# Ten rows so int(len * 0.3) >= 1 for the built-in reference slice.
|
|
ts_last = Timestamp('2023-05-26 11:12:36')
|
|
drift_df = _sample_drift_metrics_df(ts_last)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
timestamps = [f'2023-05-26 11:12:{27 + i:02d}' for i in range(10)]
|
|
target_data_dict = {
|
|
'timestamp': timestamps,
|
|
'variable': ['feature1'] * 10,
|
|
'value': [float(i) for i in range(10)],
|
|
}
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': target_data_dict,
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts_last.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts_last.isoformat(),
|
|
'chunk_end_date': ts_last.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': False,
|
|
}
|
|
]
|
|
model_metrics_activity.warning.assert_called()
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING',
|
|
message='Using 30% first rows of target data as reference data',
|
|
block='model_metrics',
|
|
level=NotificationLevel.WARNING,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
def test_calculate_drift_empty_drift_df(model_metrics_activity):
|
|
"""Empty analyzer merge yields no rows and no insufficient-data alert (lib owns that failure mode)."""
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
empty_df = _sample_drift_metrics_df(ts).iloc[0:0]
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=empty_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
assert result == []
|
|
model_metrics_activity.send_notification.assert_not_called()
|
|
|
|
|
|
def test_calculate_drift_empty_after_timestamp_filter(model_metrics_activity):
|
|
"""Rows dropped by target-window alignment yield an empty export list, not an insufficient-data error."""
|
|
drift_df = _sample_drift_metrics_df(Timestamp('2020-01-01'))
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
assert result == []
|
|
model_metrics_activity.send_notification.assert_not_called()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
def test_calculate_drift_drift_insufficient_data_error_from_lib(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
"""``DriftInsufficientDataError`` maps to MODEL_METRICS_DRIFT_INSUFFICIENT_DATA, not GET error."""
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
lib_msg = (
|
|
'[MODEL_METRICS_DRIFT_INSUFFICIENT_DATA] Drift analysis produced no time chunks '
|
|
"(chunk_period='min', analysis_rows=1)."
|
|
)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(
|
|
side_effect=DriftInsufficientDataError(lib_msg, analysis_rows=1)
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
with raises(DriftInsufficientDataError, match='MODEL_METRICS_DRIFT_INSUFFICIENT_DATA'):
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
|
|
model_metrics_activity.error.assert_called_once_with(lib_msg, metadata['metadata'])
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_DRIFT_INSUFFICIENT_DATA',
|
|
message=lib_msg,
|
|
block='model_metrics',
|
|
level=NotificationLevel.ERROR,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
def test_calculate_drift_success_min(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
def test_calculate_drift_success_s(model_metrics_activity):
|
|
ts = Timestamp('2023-05-26 11:12:27')
|
|
drift_df = _sample_drift_metrics_df(ts)
|
|
model_metrics_activity.get_drift_metrics = MagicMock(return_value=drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict('list'),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
result = model_metrics_activity.calculate_drift(input_data)
|
|
|
|
expected_timestamp = ts.tz_localize('UTC').strftime(DATETIME_FORMAT_WITH_TZ)
|
|
assert result == [
|
|
{
|
|
'timestamp': expected_timestamp,
|
|
'feature': 'feature1',
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'alert': False,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': ts.isoformat(),
|
|
'chunk_end_date': ts.isoformat(),
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
def test_calculate_drift_get_drift_metrics_error(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
model_metrics_activity.get_drift_metrics = MagicMock(
|
|
side_effect=Exception('Get drift metrics error')
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act / Assert
|
|
with raises(Exception, match='Get drift metrics error'):
|
|
model_metrics_activity.calculate_drift(input_data)
|
|
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error getting drift metrics: Get drift metrics error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.send_notification.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_GET_DRIFT_METRICS_ERROR',
|
|
message='Error getting drift metrics: Get drift metrics error',
|
|
block='model_metrics',
|
|
level=NotificationLevel.ERROR,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_success(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_drift_df = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'method': ['ks_test'],
|
|
'value': [0.5],
|
|
'feature': ['feature1'],
|
|
}
|
|
)
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.return_value = mock_drift_df
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act
|
|
result = model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
|
|
# Assert
|
|
assert isinstance(result, DataFrame)
|
|
model_metrics_activity.debug.assert_called()
|
|
model_metrics_activity.observe_lag_sync.assert_called()
|
|
model_metrics_activity.emit_metric_sync.assert_called()
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_univariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.side_effect = Exception(
|
|
'Univariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act & Assert
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Univariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting univariate drift: Univariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_multivariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.side_effect = Exception(
|
|
'Multivariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Multivariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting multivariate drift: Multivariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
def test_get_drift_metrics_dataframe_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.side_effect = Exception(
|
|
'Dataframe error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Dataframe error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error building drift metrics dataframe: Dataframe error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric_sync.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
def test_calculate_simple_metrics_success_all_metrics(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mse', 'mae', 'r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 4
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert 'r2' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mse', 'mae', 'r2']",
|
|
metadata['metadata'],
|
|
)
|
|
model_metrics_activity.debug.assert_called_once()
|
|
|
|
|
|
def test_calculate_simple_metrics_success_rmse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_mse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_mae_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mae'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_r2_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_r2_zero_ss_tot(model_metrics_activity):
|
|
# Arrange
|
|
# All target values are the same, so ss_tot will be 0
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 1.0],
|
|
'prediction': [1.1, 1.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['value'].values[0] == 0.0 # Should return 0.0 when ss_tot == 0
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_success_multiple_metrics_subset(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 2
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
def test_calculate_simple_metrics_unknown_metric_ignored(model_metrics_activity):
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['unknown_metric', 'rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
result = DataFrame(model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|
|
|
|
|
|
@mark.parametrize(
|
|
'failing_stage',
|
|
['detect_univariate_drift', 'detect_multivariate_drift', 'get_drift_metrics_dataframe'],
|
|
)
|
|
@patch('laborious.activities.model_metrics.DriftAnalysis')
|
|
def test_get_drift_metrics_reraises_insufficient_data_untouched(
|
|
mock_drift_analysis, failing_stage, model_metrics_activity
|
|
):
|
|
"""Insufficient data is not a stage failure: it propagates without the stage error report."""
|
|
error = DriftInsufficientDataError('not enough rows', analysis_rows=1)
|
|
getattr(mock_drift_analysis.return_value, failing_stage).side_effect = error
|
|
data = DataFrame({'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]})
|
|
model_metrics_activity._drift_analyze_stage_error = MagicMock()
|
|
|
|
with raises(DriftInsufficientDataError) as exc_info:
|
|
model_metrics_activity.get_drift_metrics(
|
|
reference_data=data,
|
|
target_data=data,
|
|
target_name='target',
|
|
reference_columns=data.drop(columns=['target', 'timestamp']).columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
|
|
assert exc_info.value is error
|
|
model_metrics_activity._drift_analyze_stage_error.assert_not_called()
|
|
|
|
|
|
def test_to_naive_utc_converts_aware_timestamps_to_naive_utc():
|
|
series = Series(['2024-01-01T03:00:00-03:00', '2024-01-01T09:00:00-03:00'])
|
|
|
|
result = ModelMetrics._to_naive_utc(series)
|
|
|
|
assert result.dt.tz is None
|
|
assert result.tolist() == [Timestamp('2024-01-01 06:00:00'), Timestamp('2024-01-01 12:00:00')]
|
|
|
|
|
|
def test_to_naive_utc_keeps_naive_timestamps():
|
|
series = Series(['2024-01-01 06:00:00'])
|
|
|
|
result = ModelMetrics._to_naive_utc(series)
|
|
|
|
assert result.dt.tz is None
|
|
assert result.tolist() == [Timestamp('2024-01-01 06:00:00')]
|