vitor-aignosi
a3da800cab
Refactor monitoring and metrics integration across various components - Removed coverage options from `pyproject.toml`. - Updated prediction metrics in `README.md` to replace `pipeline_name` with `workflow_name`. - Upgraded `sientia-dataops-library` dependency version in `requirements-light.txt` and `requirements.txt`. - Enhanced metrics handling in `laborious` activities, including `Activities`, `Gates`, `MLFlow`, and `OPC`, to utilize a new `MetricsController`. - Refactored metric emission methods to improve clarity and consistency across the codebase. - Updated tests to reflect changes in metrics handling and ensure proper functionality.
157 lines
4.8 KiB
Python
157 lines
4.8 KiB
Python
from unittest.mock import ANY, AsyncMock, MagicMock, patch
|
|
|
|
from pytest import mark
|
|
|
|
from laborious.activities.activities import Activities
|
|
from laborious.activities.gates import Gates
|
|
from laborious.activities.mlflow import MLFlow
|
|
from laborious.activities.opc import OPC
|
|
from laborious.activities.storage import Storage
|
|
|
|
|
|
@patch('laborious.activities.activities.Storage.__init__')
|
|
@patch('laborious.activities.activities.MLFlow.__init__')
|
|
@patch('laborious.activities.activities.OPC.__init__')
|
|
@patch('laborious.activities.activities.Gates.__init__')
|
|
@patch('laborious.activities.activities.MetricsController')
|
|
def test___init__(
|
|
mock_metrics_controller, mock_gates_init, mock_opc_init, mock_mlflow_init, mock_storage_init
|
|
):
|
|
postgres_config = {
|
|
'host': 'localhost',
|
|
'port': 5432,
|
|
'user': 'postgres',
|
|
'password': 'postgres',
|
|
'dbname': 'postgres',
|
|
'min_connections': 1,
|
|
'max_connections': 10,
|
|
}
|
|
|
|
minio_config = {
|
|
'endpoint_url': 'localhost:9000',
|
|
'access_key': 'minio',
|
|
'secret_key': 'minio123',
|
|
'region_name': 'us-east-1',
|
|
'default_bucket': 'test',
|
|
}
|
|
|
|
mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'}
|
|
|
|
opc_config = {
|
|
'bootstrap_servers': 'localhost:9092',
|
|
'polling_time': 1000,
|
|
'group_id': 'test-group',
|
|
}
|
|
|
|
logger = MagicMock()
|
|
notification_handler = MagicMock()
|
|
|
|
activities = Activities(
|
|
postgres_config=postgres_config,
|
|
mlflow_config=mlflow_config,
|
|
minio_config=minio_config,
|
|
opc_config=opc_config,
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
)
|
|
|
|
assert isinstance(activities, Activities)
|
|
assert isinstance(activities, Storage)
|
|
assert isinstance(activities, MLFlow)
|
|
assert isinstance(activities, OPC)
|
|
assert isinstance(activities, Gates)
|
|
|
|
mock_storage_init.assert_called_once_with(
|
|
ANY,
|
|
host=postgres_config['host'],
|
|
port=postgres_config['port'],
|
|
user=postgres_config['user'],
|
|
password=postgres_config['password'],
|
|
dbname=postgres_config['dbname'],
|
|
min_connections=postgres_config['min_connections'],
|
|
max_connections=postgres_config['max_connections'],
|
|
minio_config=minio_config,
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
metrics_controller=mock_metrics_controller.return_value,
|
|
)
|
|
|
|
mock_mlflow_init.assert_called_once_with(
|
|
ANY,
|
|
mlflow_host=mlflow_config['host'],
|
|
mlflow_port=mlflow_config['port'],
|
|
mlflow_username=mlflow_config['username'],
|
|
mlflow_password=mlflow_config['password'],
|
|
minio_config=minio_config,
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
metrics_controller=mock_metrics_controller.return_value,
|
|
)
|
|
|
|
mock_opc_init.assert_called_once_with(
|
|
ANY,
|
|
opc_servers=opc_config,
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
metrics_controller=mock_metrics_controller.return_value,
|
|
)
|
|
|
|
mock_gates_init.assert_called_once_with(
|
|
ANY,
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
metrics_controller=mock_metrics_controller.return_value,
|
|
)
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.activities.Storage')
|
|
@patch('laborious.activities.activities.MLFlow')
|
|
@patch('laborious.activities.activities.OPC')
|
|
@patch('laborious.activities.activities.Gates')
|
|
async def test_shutdown(mock_gates_init, mock_opc_init, mock_mlflow_init, mock_storage_init):
|
|
mock_opc_init.close = AsyncMock()
|
|
postgres_config = {
|
|
'host': 'localhost',
|
|
'port': 5432,
|
|
'user': 'postgres',
|
|
'password': 'postgres',
|
|
'dbname': 'postgres',
|
|
'min_connections': 1,
|
|
'max_connections': 10,
|
|
}
|
|
|
|
minio_config = {
|
|
'endpoint_url': 'localhost:9000',
|
|
'access_key': 'minio',
|
|
'secret_key': 'minio123',
|
|
'region_name': 'us-east-1',
|
|
'default_bucket': 'test',
|
|
}
|
|
|
|
mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'}
|
|
|
|
opc_config = {
|
|
'bootstrap_servers': 'localhost:9092',
|
|
'polling_time': 1000,
|
|
'group_id': 'test-group',
|
|
}
|
|
|
|
logger = MagicMock()
|
|
notification_handler = MagicMock()
|
|
|
|
activities = Activities(
|
|
postgres_config=postgres_config,
|
|
mlflow_config=mlflow_config,
|
|
minio_config=minio_config,
|
|
opc_config=opc_config,
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
)
|
|
|
|
await activities.shutdown()
|
|
mock_opc_init.close.assert_called_once()
|
|
mock_storage_init.close.assert_called_once()
|
|
mock_mlflow_init.close.assert_called_once()
|
|
mock_gates_init.close.assert_called_once()
|