vitor-aignosi
5d0d049082
Remove code validation script and refactor imports in activities and workflows - Deleted the `validate.sh` script, which was responsible for running code quality checks. - Cleaned up import statements in `activities.py`, `gates.py`, `mlflow.py`, and `storage.py` by removing unused imports and organizing them. - Refactored initialization methods in `MinioManager` and `MLFlow` classes for improved readability. - Updated various workflows to ensure compatibility with the new structure and removed unnecessary comments. - Enhanced test cases to accommodate changes in the activities and workflows, ensuring proper mocking of dependencies.
1109 lines
38 KiB
Python
1109 lines
38 KiB
Python
from unittest.mock import ANY, AsyncMock, MagicMock, patch
|
|
|
|
from pandas import DataFrame
|
|
from pytest import fixture, mark
|
|
from sientia_do.notifications.models import NotificationLevel
|
|
|
|
from laborious.activities.model_metrics import ModelMetrics
|
|
|
|
|
|
@fixture
|
|
def model_metrics_activity():
|
|
model_metrics = ModelMetrics(
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=AsyncMock(),
|
|
)
|
|
model_metrics.error = MagicMock()
|
|
model_metrics.debug = MagicMock()
|
|
model_metrics.info = MagicMock()
|
|
model_metrics.warning = MagicMock()
|
|
model_metrics.critical = MagicMock()
|
|
model_metrics.send_notification = MagicMock()
|
|
model_metrics.send_notification_async = AsyncMock()
|
|
model_metrics.emit_metric = AsyncMock()
|
|
model_metrics.get_core_labels = MagicMock(
|
|
return_value={
|
|
'pod_id': 'test_pod',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
}
|
|
)
|
|
model_metrics.observe_lag = AsyncMock()
|
|
model_metrics.pod_id = 'test_pod'
|
|
return model_metrics
|
|
|
|
|
|
metadata = {
|
|
'metadata': {
|
|
'model_id': 'test_model',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
'schema_name': 'test_schedule',
|
|
},
|
|
}
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_calculate_drift_invalid_chunk_period(model_metrics_activity):
|
|
# Arrange
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'invalid',
|
|
}
|
|
|
|
# Act & Assert
|
|
try:
|
|
await model_metrics_activity.calculate_drift(input_data)
|
|
except ValueError as e:
|
|
assert str(e) == 'Invalid chunk period: invalid, must be "min" or "s"'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Invalid chunk period: invalid', metadata['metadata']
|
|
)
|
|
else:
|
|
raise AssertionError('Expected ValueError')
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
async def test_calculate_drift_with_reference_data(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
mock_to_datetime.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
|
|
'2023-05-26 11:12:27+00:00'
|
|
)
|
|
|
|
mock_drift_df = MagicMock()
|
|
mock_drift_df.empty = False
|
|
mock_drift_df.drop.return_value = mock_drift_df
|
|
mock_drift_df.__getitem__.return_value.isin.return_value = [True]
|
|
mock_drift_df.__getitem__.return_value = mock_drift_df
|
|
mock_drift_df.__getitem__.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
|
|
'2023-05-26 11:12:27+00:00'
|
|
)
|
|
mock_drift_df.rename.return_value = mock_drift_df
|
|
mock_drift_df.drop_duplicates.return_value = mock_drift_df
|
|
mock_drift_df.to_dict.return_value = [
|
|
{
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'feature': 'feature1',
|
|
'timestamp': '2023-05-26 11:12:27+00:00',
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
|
|
model_metrics_activity.get_drift_metrics = AsyncMock(return_value=mock_drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act
|
|
result = await model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert isinstance(result, list)
|
|
assert result == mock_drift_df.to_dict.return_value # type: ignore[comparison-overlap]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
# Verify transformations were called
|
|
mock_drift_df.drop.assert_called_once_with(columns=['p_value'], inplace=True)
|
|
mock_drift_df.__getitem__.assert_called()
|
|
mock_drift_df.rename.assert_called_once_with(
|
|
columns={'metric': 'method', 'statistic': 'value'}, inplace=True
|
|
)
|
|
mock_drift_df.drop_duplicates.assert_called_once_with(
|
|
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
|
|
)
|
|
mock_drift_df.to_dict.assert_called_once_with(orient='records')
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
async def test_calculate_drift_without_reference_data(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
mock_to_datetime.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
|
|
'2023-05-26 11:12:27+00:00'
|
|
)
|
|
|
|
mock_drift_df = MagicMock()
|
|
mock_drift_df.empty = False
|
|
mock_drift_df.drop.return_value = mock_drift_df
|
|
mock_drift_df.__getitem__.return_value.isin.return_value = [True]
|
|
mock_drift_df.__getitem__.return_value = mock_drift_df
|
|
mock_drift_df.__getitem__.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
|
|
'2023-05-26 11:12:27+00:00'
|
|
)
|
|
mock_drift_df.rename.return_value = mock_drift_df
|
|
mock_drift_df.drop_duplicates.return_value = mock_drift_df
|
|
mock_drift_df.to_dict.return_value = [
|
|
{
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'feature': 'feature1',
|
|
'timestamp': '2023-05-26 11:12:27+00:00',
|
|
'model_id': 'test_model_id',
|
|
'accurate': False,
|
|
}
|
|
]
|
|
|
|
model_metrics_activity.get_drift_metrics = AsyncMock(return_value=mock_drift_df)
|
|
|
|
target_data_dict = {
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'variable': ['feature1', 'feature1', 'feature1'],
|
|
'value': [1.0, 2.0, 3.0],
|
|
}
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.sort_values.return_value = mock_target_df
|
|
mock_target_df.head.return_value = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'feature1': [1.0]}
|
|
)
|
|
mock_target_df.__getitem__.return_value.apply.return_value = [
|
|
'2023-05-26 11:12:27',
|
|
'2023-05-26 11:12:28',
|
|
'2023-05-26 11:12:29',
|
|
]
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
mock_dataframe.side_effect = lambda x=None: mock_target_df if x is not None else mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': None,
|
|
'target_data': target_data_dict,
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
# Act
|
|
result = await model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert isinstance(result, list)
|
|
assert result == mock_drift_df.to_dict.return_value
|
|
model_metrics_activity.warning.assert_called()
|
|
model_metrics_activity.send_notification_async.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING',
|
|
message='Using 30% first rows of target data as reference data',
|
|
block='model_metrics',
|
|
level=NotificationLevel.WARNING,
|
|
attachment_content=ANY,
|
|
)
|
|
# Verify transformations were called
|
|
mock_drift_df.drop.assert_called_once_with(columns=['p_value'], inplace=True)
|
|
mock_drift_df.__getitem__.assert_called()
|
|
mock_drift_df.rename.assert_called_once_with(
|
|
columns={'metric': 'method', 'statistic': 'value'}, inplace=True
|
|
)
|
|
mock_drift_df.drop_duplicates.assert_called_once_with(
|
|
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
|
|
)
|
|
mock_drift_df.to_dict.assert_called_once_with(orient='records')
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
async def test_calculate_drift_empty_drift_df(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
model_metrics_activity.get_drift_metrics = AsyncMock(return_value=DataFrame())
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act
|
|
result = await model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert result == []
|
|
model_metrics_activity.warning.assert_called_with(
|
|
'No drift metrics found', metadata['metadata']
|
|
)
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
async def test_calculate_drift_empty_after_timestamp_filter(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
mock_drift_df = MagicMock()
|
|
mock_drift_df.empty = False
|
|
mock_drift_df.drop.return_value = mock_drift_df
|
|
|
|
# Set up __getitem__ to handle filtering - timestamp access returns series with isin=False
|
|
# and filtering returns empty DataFrame
|
|
mock_timestamp_series = MagicMock()
|
|
mock_timestamp_series.isin.return_value = [False]
|
|
mock_empty_df = MagicMock()
|
|
mock_empty_df.empty = True
|
|
|
|
def getitem_side_effect(key):
|
|
if key == 'timestamp':
|
|
return mock_timestamp_series
|
|
else:
|
|
# This is the filtering operation - return empty DataFrame
|
|
return mock_empty_df
|
|
|
|
mock_drift_df.__getitem__.side_effect = getitem_side_effect
|
|
|
|
model_metrics_activity.get_drift_metrics = AsyncMock(return_value=mock_drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act
|
|
result = await model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert result == []
|
|
model_metrics_activity.warning.assert_called_with(
|
|
'No drift metrics found after dropping rows where timestamp is not in target data',
|
|
metadata['metadata'],
|
|
)
|
|
# Verify transformations were called
|
|
mock_drift_df.drop.assert_called_once_with(columns=['p_value'], inplace=True)
|
|
mock_drift_df.__getitem__.assert_called()
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
async def test_calculate_drift_success_min(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
mock_to_datetime.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
|
|
'2023-05-26 11:12:27+00:00'
|
|
)
|
|
|
|
mock_drift_df = MagicMock()
|
|
mock_drift_df.empty = False
|
|
mock_drift_df.drop.return_value = mock_drift_df
|
|
mock_drift_df.__getitem__.return_value.isin.return_value = [True]
|
|
mock_drift_df.__getitem__.return_value = mock_drift_df
|
|
mock_drift_df.__getitem__.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
|
|
'2023-05-26 11:12:27+00:00'
|
|
)
|
|
mock_drift_df.rename.return_value = mock_drift_df
|
|
mock_drift_df.drop_duplicates.return_value = mock_drift_df
|
|
mock_drift_df.to_dict.return_value = [
|
|
{
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'feature': 'feature1',
|
|
'timestamp': '2023-05-26 11:12:27+00:00',
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
|
|
model_metrics_activity.get_drift_metrics = AsyncMock(return_value=mock_drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.isin.return_value = [True]
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act
|
|
result = await model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert isinstance(result, list)
|
|
assert result == mock_drift_df.to_dict.return_value # type: ignore[comparison-overlap]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
# Verify transformations were called
|
|
mock_drift_df.drop.assert_called_once_with(columns=['p_value'], inplace=True)
|
|
mock_drift_df.__getitem__.assert_called()
|
|
mock_drift_df.rename.assert_called_once_with(
|
|
columns={'metric': 'method', 'statistic': 'value'}, inplace=True
|
|
)
|
|
mock_drift_df.drop_duplicates.assert_called_once_with(
|
|
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
|
|
)
|
|
mock_drift_df.to_dict.assert_called_once_with(orient='records')
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
async def test_calculate_drift_success_s(mock_to_datetime, mock_dataframe, model_metrics_activity):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
mock_to_datetime.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
|
|
'2023-05-26 11:12:27+00:00'
|
|
)
|
|
|
|
mock_drift_df = MagicMock()
|
|
mock_drift_df.empty = False
|
|
mock_drift_df.drop.return_value = mock_drift_df
|
|
mock_drift_df.__getitem__.return_value.isin.return_value = [True]
|
|
mock_drift_df.__getitem__.return_value = mock_drift_df
|
|
mock_drift_df.__getitem__.return_value.dt.tz_localize.return_value.dt.strftime.return_value = (
|
|
'2023-05-26 11:12:27+00:00'
|
|
)
|
|
mock_drift_df.rename.return_value = mock_drift_df
|
|
mock_drift_df.drop_duplicates.return_value = mock_drift_df
|
|
mock_drift_df.to_dict.return_value = [
|
|
{
|
|
'method': 'ks_test',
|
|
'value': 0.5,
|
|
'feature': 'feature1',
|
|
'timestamp': '2023-05-26 11:12:27+00:00',
|
|
'model_id': 'test_model_id',
|
|
'accurate': True,
|
|
}
|
|
]
|
|
|
|
model_metrics_activity.get_drift_metrics = AsyncMock(return_value=mock_drift_df)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.isin.return_value = [True]
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 's',
|
|
}
|
|
|
|
# Act
|
|
result = await model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert isinstance(result, list)
|
|
assert result == mock_drift_df.to_dict.return_value # type: ignore[comparison-overlap]
|
|
model_metrics_activity.info.assert_called()
|
|
model_metrics_activity.get_drift_metrics.assert_called_once()
|
|
# Verify transformations were called
|
|
mock_drift_df.drop.assert_called_once_with(columns=['p_value'], inplace=True)
|
|
mock_drift_df.__getitem__.assert_called()
|
|
mock_drift_df.rename.assert_called_once_with(
|
|
columns={'metric': 'method', 'statistic': 'value'}, inplace=True
|
|
)
|
|
mock_drift_df.drop_duplicates.assert_called_once_with(
|
|
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
|
|
)
|
|
mock_drift_df.to_dict.assert_called_once_with(orient='records')
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.DataFrame')
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
async def test_calculate_drift_get_drift_metrics_error(
|
|
mock_to_datetime, mock_dataframe, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_to_datetime.return_value.dt.strftime.return_value = '2023-05-26 11:12:27'
|
|
|
|
model_metrics_activity.get_drift_metrics = AsyncMock(
|
|
side_effect=Exception('Get drift metrics error')
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
mock_target_df = MagicMock()
|
|
mock_target_df.pivot.return_value = mock_target_df
|
|
mock_target_df.index = ['2023-05-26 11:12:27']
|
|
mock_target_df.reset_index.return_value = mock_target_df
|
|
mock_target_df.dropna.return_value = mock_target_df
|
|
mock_target_df.__getitem__.return_value.apply.return_value = ['2023-05-26 11:12:27']
|
|
mock_target_df.drop.return_value.columns = ['feature1']
|
|
mock_dataframe.return_value = mock_target_df
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_name': 'test_model',
|
|
'model_id': 'test_model_id',
|
|
'reference_data': reference_data.to_dict(),
|
|
'target_data': {
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'variable': ['feature1'],
|
|
'value': [1.0],
|
|
},
|
|
'target_name': 'target',
|
|
'drift_metrics': ['ks_test'],
|
|
'chunk_period': 'min',
|
|
}
|
|
|
|
# Act
|
|
result = await model_metrics_activity.calculate_drift(input_data)
|
|
|
|
# Assert
|
|
assert result == []
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error getting drift metrics: Get drift metrics error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.send_notification_async.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='MODEL_METRICS_GET_DRIFT_METRICS_ERROR',
|
|
message='Error getting drift metrics: Get drift metrics error',
|
|
block='model_metrics',
|
|
level=NotificationLevel.ERROR,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.ModelAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
async def test_get_drift_metrics_success(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_drift_df = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'metric': ['ks_test'],
|
|
'statistic': [0.5],
|
|
'feature': ['feature1'],
|
|
}
|
|
)
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.return_value = mock_drift_df
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act
|
|
result = await model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
|
|
# Assert
|
|
assert isinstance(result, DataFrame)
|
|
model_metrics_activity.debug.assert_called()
|
|
model_metrics_activity.observe_lag.assert_called()
|
|
model_metrics_activity.emit_metric.assert_called()
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.ModelAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
async def test_get_drift_metrics_univariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
# Arrange
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.side_effect = Exception(
|
|
'Univariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27'],
|
|
'target': [1.0],
|
|
'feature1': [1.0],
|
|
}
|
|
)
|
|
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
# Act & Assert
|
|
try:
|
|
await model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Univariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting univariate drift: Univariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.ModelAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
async def test_get_drift_metrics_multivariate_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.side_effect = Exception(
|
|
'Multivariate drift error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
await model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Multivariate drift error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error detecting multivariate drift: Multivariate drift error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.model_metrics.to_datetime')
|
|
@patch('laborious.activities.model_metrics.time.time')
|
|
@patch('laborious.activities.model_metrics.ModelAnalysis')
|
|
@patch('laborious.activities.model_metrics.metrics')
|
|
async def test_get_drift_metrics_dataframe_error(
|
|
mock_metrics, mock_model_analysis, mock_time, mock_to_datetime, model_metrics_activity
|
|
):
|
|
mock_time.return_value = 1000.0
|
|
|
|
mock_model_analysis.return_value.detect_univariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.detect_multivariate_drift.return_value = MagicMock()
|
|
mock_model_analysis.return_value.get_drift_metrics_dataframe.side_effect = Exception(
|
|
'Dataframe error'
|
|
)
|
|
|
|
reference_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
target_data = DataFrame(
|
|
{'timestamp': ['2023-05-26 11:12:27'], 'target': [1.0], 'feature1': [1.0]}
|
|
)
|
|
reference_columns = reference_data.drop(
|
|
columns=['target', 'timestamp'], errors='ignore'
|
|
).columns
|
|
|
|
try:
|
|
await model_metrics_activity.get_drift_metrics(
|
|
reference_data=reference_data,
|
|
target_data=target_data,
|
|
target_name='target',
|
|
reference_columns=reference_columns,
|
|
drift_metrics=['ks_test'],
|
|
chunk_period='min',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
except Exception as e:
|
|
assert str(e) == 'Dataframe error'
|
|
model_metrics_activity.error.assert_called_once_with(
|
|
'Error getting drift metrics: Dataframe error', metadata['metadata']
|
|
)
|
|
model_metrics_activity.emit_metric.assert_called_with(
|
|
metric_object=mock_metrics.MODEL_ANALYZE_ERROR_COUNT, tags=ANY
|
|
)
|
|
else:
|
|
raise AssertionError('Expected Exception')
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_calculate_simple_metrics_success_all_metrics(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mse', 'mae', 'r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(await model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 4
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert 'r2' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mse', 'mae', 'r2']",
|
|
metadata['metadata'],
|
|
)
|
|
model_metrics_activity.debug.assert_called_once()
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_calculate_simple_metrics_success_rmse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(await model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_calculate_simple_metrics_success_mse_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(await model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mse'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mse']", metadata['metadata']
|
|
)
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_calculate_simple_metrics_success_mae_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(await model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'mae'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_calculate_simple_metrics_success_r2_only(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(await model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_calculate_simple_metrics_r2_zero_ss_tot(model_metrics_activity):
|
|
# Arrange
|
|
# All target values are the same, so ss_tot will be 0
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 1.0],
|
|
'prediction': [1.1, 1.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['r2'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(await model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'r2'
|
|
assert result['value'].values[0] == 0.0 # Should return 0.0 when ss_tot == 0
|
|
assert result['model_id'].values[0] == 'test_model_id'
|
|
assert result['timestamp'].values[0] == '2023-05-26 11:12:28'
|
|
assert result['data_size'].values[0] == 2
|
|
assert result['interval_minutes'].values[0] == 5
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['r2']", metadata['metadata']
|
|
)
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_calculate_simple_metrics_success_multiple_metrics_subset(model_metrics_activity):
|
|
# Arrange
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28', '2023-05-26 11:12:29'],
|
|
'target': [1.0, 2.0, 3.0],
|
|
'prediction': [1.1, 2.1, 2.9],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['rmse', 'mae'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
# Act
|
|
result = DataFrame(await model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
# Assert
|
|
assert len(result['metric']) == 2
|
|
assert 'rmse' in result['metric'].values
|
|
assert 'mae' in result['metric'].values
|
|
assert all(model_id == 'test_model_id' for model_id in result['model_id'].values)
|
|
assert all(timestamp == '2023-05-26 11:12:29' for timestamp in result['timestamp'].values)
|
|
assert all(data_size == 3 for data_size in result['data_size'].values)
|
|
assert all(interval_minutes == 5 for interval_minutes in result['interval_minutes'].values)
|
|
model_metrics_activity.info.assert_called_once_with(
|
|
"Calculating simple metrics for model test_model_id: ['rmse', 'mae']", metadata['metadata']
|
|
)
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_calculate_simple_metrics_unknown_metric_ignored(model_metrics_activity):
|
|
target_data = DataFrame(
|
|
{
|
|
'timestamp': ['2023-05-26 11:12:27', '2023-05-26 11:12:28'],
|
|
'target': [1.0, 2.0],
|
|
'prediction': [1.1, 2.1],
|
|
}
|
|
)
|
|
|
|
input_data = {
|
|
**metadata,
|
|
'model_id': 'test_model_id',
|
|
'target_data': target_data.to_dict(),
|
|
'metrics': ['unknown_metric', 'rmse'],
|
|
'interval_minutes': 5,
|
|
}
|
|
|
|
result = DataFrame(await model_metrics_activity.calculate_simple_metrics(input_data))
|
|
|
|
assert len(result['metric']) == 1
|
|
assert result['metric'].values[0] == 'rmse'
|