d1f9394879
- Added a new fixture to manage runtime report artifacts in a writable temp directory during E2E tests, addressing permission issues in local CI/dev environments. - Updated `conftest.py` to include a requirements.txt file in the model packaging path for training activities. - Refactored existing fixtures to use `pytest.fixture` instead of `pytest_asyncio.fixture` for better compatibility. - Enhanced the `Reports` class to include a target alias for report metrics, ensuring compatibility with Evidently's reporting requirements. - Introduced new test scenarios to validate the handling of missing and whitespace-only `date_column` inputs in the training workflow. These changes improve the robustness of the E2E testing framework and enhance the clarity of model reporting metrics.
644 lines
20 KiB
Python
644 lines
20 KiB
Python
"""Unit tests for ExperimentTracking class with 100% coverage."""
|
|
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_logger():
|
|
"""Create a mock logger."""
|
|
return MagicMock()
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_notification_handler():
|
|
"""Create a mock notification handler."""
|
|
return MagicMock()
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_metrics_controller():
|
|
"""Create a mock metrics controller."""
|
|
controller = MagicMock()
|
|
controller.shutdown = AsyncMock()
|
|
controller.emit = AsyncMock()
|
|
return controller
|
|
|
|
|
|
@pytest.fixture
|
|
def db_config():
|
|
"""Create a valid database configuration."""
|
|
return {
|
|
'host': 'localhost',
|
|
'port': 5432,
|
|
'user': 'testuser',
|
|
'password': 'testpass',
|
|
'dbname': 'testdb',
|
|
'min_connections': 1,
|
|
'max_connections': 10,
|
|
}
|
|
|
|
|
|
def test_experiment_tracking_init(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test ExperimentTracking initialization."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking
|
|
|
|
ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
|
|
def test_experiment_tracking_del_without_engine(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test __del__ when engine attribute does not exist."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
if hasattr(et, 'engine'):
|
|
delattr(et, 'engine')
|
|
|
|
et.__del__()
|
|
|
|
|
|
def test_experiment_tracking_del_with_engine(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test __del__ when engine exists."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
et.engine = MagicMock()
|
|
|
|
class MockSuper:
|
|
def __del__(self):
|
|
pass
|
|
|
|
with patch('builtins.super', return_value=MockSuper()):
|
|
et.__del__()
|
|
|
|
|
|
def test_experiment_tracking_del_with_engine_exception(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test __del__ catches exceptions."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
et.engine = MagicMock()
|
|
|
|
class MockSuperWithError:
|
|
_should_raise: bool
|
|
|
|
def __init__(self) -> None:
|
|
self._should_raise = False
|
|
|
|
def __del__(self):
|
|
# Only raise error if not being cleaned up by garbage collector
|
|
# This prevents the PytestUnraisableExceptionWarning
|
|
if hasattr(self, '_should_raise') and self._should_raise:
|
|
raise RuntimeError('Test error')
|
|
|
|
# Suppress the PytestUnraisableExceptionWarning for this specific test
|
|
import warnings
|
|
|
|
warnings.filterwarnings('ignore', category=pytest.PytestUnraisableExceptionWarning)
|
|
|
|
mock_super = MockSuperWithError()
|
|
mock_super._should_raise = True
|
|
try:
|
|
with patch('builtins.super', return_value=mock_super):
|
|
et.__del__()
|
|
finally:
|
|
# Prevent the exception from being raised during garbage collection
|
|
mock_super._should_raise = False
|
|
|
|
|
|
def test_execute_update_success(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test _execute_update executes query successfully."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
mock_connection = MagicMock()
|
|
mock_result = MagicMock()
|
|
mock_result.rowcount = 1
|
|
mock_connection.execute.return_value = mock_result
|
|
mock_engine = MagicMock()
|
|
mock_engine.begin.return_value.__enter__.return_value = mock_connection
|
|
et.engine = mock_engine
|
|
|
|
result = et._execute_update('UPDATE test SET x = :x', {'x': 1})
|
|
|
|
assert result == {'rowcount': 1}
|
|
mock_connection.execute.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_status_success(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run with STATUS update type."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
mock_execute = MagicMock()
|
|
|
|
def mock_execute_update(*args, **kwargs):
|
|
mock_execute(*args, **kwargs)
|
|
return {'rowcount': 1}
|
|
|
|
et._execute_update = mock_execute_update # type: ignore[method-assign]
|
|
et.info = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': UpdateType.STATUS,
|
|
'status': 'running',
|
|
}
|
|
|
|
et.update_experiment_run(input_data)
|
|
|
|
mock_execute.assert_called_once()
|
|
call_args = mock_execute.call_args
|
|
assert 'status' in call_args[0][1]
|
|
assert call_args[0][1]['status'] == 'running'
|
|
assert call_args[0][1]['experiment_run_id'] == 1
|
|
et.info.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_status_missing_status(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run with STATUS but missing status parameter."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
et.send_notification = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': UpdateType.STATUS,
|
|
}
|
|
|
|
with pytest.raises(RuntimeError):
|
|
et.update_experiment_run(input_data)
|
|
|
|
et.send_notification.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_status_with_error_success(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run with STATUS_WITH_ERROR update type."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
mock_execute = MagicMock()
|
|
|
|
def mock_execute_update(*args, **kwargs):
|
|
mock_execute(*args, **kwargs)
|
|
return {'rowcount': 1}
|
|
|
|
et._execute_update = mock_execute_update # type: ignore[method-assign]
|
|
et.info = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': UpdateType.STATUS_WITH_ERROR,
|
|
'status': 'failed',
|
|
'error_message': 'Test error',
|
|
}
|
|
|
|
et.update_experiment_run(input_data)
|
|
|
|
mock_execute.assert_called_once()
|
|
call_args = mock_execute.call_args
|
|
assert 'status' in call_args[0][1]
|
|
assert call_args[0][1]['status'] == 'failed'
|
|
assert call_args[0][1]['error_message'] == 'Test error'
|
|
et.info.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_status_with_error_truncate_message(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run truncates error message if too long."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
mock_execute = MagicMock()
|
|
|
|
def mock_execute_update(*args, **kwargs):
|
|
mock_execute(*args, **kwargs)
|
|
return {'rowcount': 1}
|
|
|
|
et._execute_update = mock_execute_update # type: ignore[method-assign]
|
|
et.info = MagicMock()
|
|
|
|
long_error = 'x' * 2000
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': UpdateType.STATUS_WITH_ERROR,
|
|
'status': 'failed',
|
|
'error_message': long_error,
|
|
}
|
|
|
|
et.update_experiment_run(input_data)
|
|
|
|
call_args = mock_execute.call_args
|
|
assert len(call_args[0][1]['error_message']) == 1024
|
|
|
|
|
|
def test_update_experiment_run_status_with_error_missing_error_message(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run with STATUS_WITH_ERROR but missing error_message."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
et.send_notification = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': UpdateType.STATUS_WITH_ERROR,
|
|
'status': 'failed',
|
|
}
|
|
|
|
with pytest.raises(RuntimeError):
|
|
et.update_experiment_run(input_data)
|
|
|
|
et.send_notification.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_model_saved_success(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run with MODEL_SAVED update type."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
mock_execute = MagicMock()
|
|
|
|
def mock_execute_update(*args, **kwargs):
|
|
mock_execute(*args, **kwargs)
|
|
return {'rowcount': 1}
|
|
|
|
et._execute_update = mock_execute_update # type: ignore[method-assign]
|
|
et.info = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': UpdateType.MODEL_SAVED,
|
|
'status': 'completed',
|
|
'run_name': 'run_001',
|
|
}
|
|
|
|
et.update_experiment_run(input_data)
|
|
|
|
mock_execute.assert_called_once()
|
|
call_args = mock_execute.call_args
|
|
assert 'run_name' in call_args[0][1]
|
|
assert call_args[0][1]['run_name'] == 'run_001'
|
|
assert call_args[0][1]['status'] == 'completed'
|
|
et.info.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_model_saved_missing_run_name(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run with MODEL_SAVED but missing run_name."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
et.send_notification = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': UpdateType.MODEL_SAVED,
|
|
'status': 'completed',
|
|
}
|
|
|
|
with pytest.raises(RuntimeError):
|
|
et.update_experiment_run(input_data)
|
|
|
|
et.send_notification.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_invalid_update_type(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run with invalid update_type."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
et.send_notification = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': 'invalid_type',
|
|
}
|
|
|
|
with pytest.raises(RuntimeError):
|
|
et.update_experiment_run(input_data)
|
|
|
|
et.send_notification.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_no_rows_updated(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run raises error when no rows are updated."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
def mock_execute_update(*args, **kwargs):
|
|
return {'rowcount': 0}
|
|
|
|
et._execute_update = mock_execute_update # type: ignore[method-assign]
|
|
et.send_notification = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 999,
|
|
'update_type': UpdateType.STATUS,
|
|
'status': 'running',
|
|
}
|
|
|
|
with pytest.raises(RuntimeError):
|
|
et.update_experiment_run(input_data)
|
|
|
|
et.send_notification.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_status_with_error_missing_status(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run with STATUS_WITH_ERROR but missing status - covers line 179."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
et.send_notification = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': UpdateType.STATUS_WITH_ERROR,
|
|
'error_message': 'Some error',
|
|
}
|
|
|
|
with pytest.raises(RuntimeError):
|
|
et.update_experiment_run(input_data)
|
|
|
|
et.send_notification.assert_called_once()
|
|
|
|
|
|
def test_update_experiment_run_model_saved_missing_status(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test update_experiment_run with MODEL_SAVED but missing status - covers line 204."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking, UpdateType
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
et.send_notification = MagicMock()
|
|
|
|
input_data = {
|
|
'metadata': {'workflow_id': 'test-123'},
|
|
'experiment_run_id': 1,
|
|
'update_type': UpdateType.MODEL_SAVED,
|
|
'run_name': 'run_001',
|
|
}
|
|
|
|
with pytest.raises(RuntimeError):
|
|
et.update_experiment_run(input_data)
|
|
|
|
et.send_notification.assert_called_once()
|
|
|
|
|
|
def test_experiment_tracking_del_with_engine_no_super_del(
|
|
db_config, mock_logger, mock_notification_handler, mock_metrics_controller
|
|
):
|
|
"""Test __del__ when engine exists but super has no __del__ - covers line 103."""
|
|
from model_manager.activities.experiment_tracking import ExperimentTracking
|
|
|
|
et = ExperimentTracking(
|
|
host=db_config['host'],
|
|
port=db_config['port'],
|
|
user=db_config['user'],
|
|
password=db_config['password'],
|
|
dbname=db_config['dbname'],
|
|
min_connections=db_config['min_connections'],
|
|
max_connections=db_config['max_connections'],
|
|
logger=mock_logger,
|
|
notification_handler=mock_notification_handler,
|
|
metrics_controller=mock_metrics_controller,
|
|
)
|
|
|
|
et.engine = MagicMock()
|
|
|
|
class MockSuperNoDel:
|
|
pass
|
|
|
|
with patch('builtins.super', return_value=MockSuperNoDel()):
|
|
et.__del__()
|