vitor-aignosi
a3da800cab
Refactor monitoring and metrics integration across various components - Removed coverage options from `pyproject.toml`. - Updated prediction metrics in `README.md` to replace `pipeline_name` with `workflow_name`. - Upgraded `sientia-dataops-library` dependency version in `requirements-light.txt` and `requirements.txt`. - Enhanced metrics handling in `laborious` activities, including `Activities`, `Gates`, `MLFlow`, and `OPC`, to utilize a new `MetricsController`. - Refactored metric emission methods to improve clarity and consistency across the codebase. - Updated tests to reflect changes in metrics handling and ensure proper functionality.
225 lines
6.8 KiB
Python
225 lines
6.8 KiB
Python
import datetime
|
|
from unittest.mock import ANY, AsyncMock, MagicMock, patch
|
|
|
|
from pytest import fixture, mark, raises
|
|
from sientia_do.notifications.models import NotificationLevel
|
|
from sientia_do.temporal.activities.postgres import Postgres
|
|
|
|
from laborious.activities.storage import Storage
|
|
|
|
metadata = {
|
|
'metadata': {
|
|
'model_id': 'test_model_id',
|
|
'model_name': 'test_model',
|
|
'workflow_name': 'test_workflow',
|
|
'schedule_name': 'test_schedule',
|
|
}
|
|
}
|
|
|
|
|
|
@fixture
|
|
@patch('laborious.activities.storage.MinioRepository')
|
|
def storage(mock_minio_repository):
|
|
return Storage(
|
|
host='localhost',
|
|
port=5432,
|
|
user='postgres',
|
|
password='postgres',
|
|
dbname='postgres',
|
|
min_connections=1,
|
|
max_connections=10,
|
|
minio_config={
|
|
'endpoint_url': 'localhost:9000',
|
|
'access_key': 'minio',
|
|
'secret_key': 'minio123',
|
|
'region_name': 'us-east-1',
|
|
'default_bucket': 'test',
|
|
},
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=AsyncMock(),
|
|
)
|
|
|
|
|
|
@patch('laborious.activities.storage.MinioRepository')
|
|
def test___init___not_hasattr(mock_minio_repository):
|
|
logger = MagicMock()
|
|
notification_handler = MagicMock()
|
|
metrics_controller = AsyncMock()
|
|
storage = Storage(
|
|
host='localhost',
|
|
port=5432,
|
|
user='postgres',
|
|
password='postgres',
|
|
dbname='postgres',
|
|
min_connections=1,
|
|
max_connections=10,
|
|
minio_config={
|
|
'endpoint_url': 'localhost:9000',
|
|
'access_key': 'minio',
|
|
'secret_key': 'minio123',
|
|
'region_name': 'us-east-1',
|
|
'default_bucket': 'test',
|
|
},
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
metrics_controller=metrics_controller,
|
|
)
|
|
assert isinstance(storage, Postgres)
|
|
|
|
mock_minio_repository.assert_called_once_with(
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
minio_endpoint_url='localhost:9000',
|
|
minio_access_key='minio',
|
|
minio_secret_key='minio123',
|
|
minio_region_name='us-east-1',
|
|
minio_default_bucket='test',
|
|
metrics_controller=metrics_controller,
|
|
)
|
|
|
|
|
|
@patch('laborious.activities.storage.MinioRepository')
|
|
def test___init___none_minio_repository(mock_minio_repository, storage):
|
|
storage.minio_repository = None
|
|
logger = MagicMock()
|
|
notification_handler = MagicMock()
|
|
metrics_controller = AsyncMock()
|
|
storage.__init__(
|
|
host='localhost',
|
|
port=5432,
|
|
user='postgres',
|
|
password='postgres',
|
|
dbname='postgres',
|
|
min_connections=1,
|
|
max_connections=10,
|
|
minio_config={
|
|
'endpoint_url': 'localhost:9000',
|
|
'access_key': 'minio',
|
|
'secret_key': 'minio123',
|
|
'region_name': 'us-east-1',
|
|
'default_bucket': 'test',
|
|
},
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
metrics_controller=metrics_controller,
|
|
)
|
|
|
|
mock_minio_repository.assert_called_once_with(
|
|
logger=logger,
|
|
notification_handler=notification_handler,
|
|
minio_endpoint_url='localhost:9000',
|
|
minio_access_key='minio',
|
|
minio_secret_key='minio123',
|
|
minio_region_name='us-east-1',
|
|
minio_default_bucket='test',
|
|
metrics_controller=metrics_controller,
|
|
)
|
|
|
|
|
|
@patch('laborious.activities.storage.MinioRepository')
|
|
def test___init___done_repository(mock_minio_repository, storage):
|
|
storage.__init__(
|
|
host='localhost',
|
|
port=5432,
|
|
user='postgres',
|
|
password='postgres',
|
|
dbname='postgres',
|
|
min_connections=1,
|
|
max_connections=10,
|
|
minio_config={
|
|
'endpoint_url': 'localhost:9000',
|
|
'access_key': 'minio',
|
|
'secret_key': 'minio123',
|
|
'region_name': 'us-east-1',
|
|
'default_bucket': 'test',
|
|
},
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=AsyncMock(),
|
|
)
|
|
mock_minio_repository.assert_not_called()
|
|
assert storage.minio_repository is not None
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_query_to_minio_minio_repository_not_initialized(storage):
|
|
storage.minio_repository = None
|
|
|
|
with raises(ValueError) as e:
|
|
await storage.query_to_minio({})
|
|
|
|
assert str(e.value) == 'Minio repository not initialized'
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_query_to_minio_not_data(storage):
|
|
storage.load_custom_query = AsyncMock(return_value=None)
|
|
result = await storage.query_to_minio({})
|
|
|
|
storage.load_custom_query.assert_called_once_with({})
|
|
assert result['success'] is False
|
|
assert result['message'] == 'No data returned from query'
|
|
|
|
|
|
@mark.asyncio
|
|
@patch('laborious.activities.storage.pd.DataFrame')
|
|
@patch('laborious.activities.storage.now')
|
|
async def test_query_to_minio_success(now, dataframe, storage):
|
|
data = [{'a': 1}, {'a': 2}, {'a': 3}]
|
|
storage.load_custom_query = AsyncMock(return_value=data)
|
|
now.return_value = datetime.datetime(2024, 1, 1, 0, 0, 0)
|
|
storage.minio_repository.store_dataframe_as_parquet = AsyncMock()
|
|
storage.minio_repository.minio_bucket = 'test'
|
|
|
|
result = await storage.query_to_minio({'object_prefix': 'test', **metadata})
|
|
|
|
dataframe.assert_called_once_with(data)
|
|
|
|
storage.minio_repository.store_dataframe_as_parquet.assert_called_once_with(
|
|
dataframe=dataframe.return_value,
|
|
uri='s3://test/test_2024-01-01_00-00-00.parquet',
|
|
object_name='test_2024-01-01_00-00-00.parquet',
|
|
metadata=metadata['metadata'],
|
|
)
|
|
|
|
assert result['success'] is True
|
|
assert result['object_key'] == 'test_2024-01-01_00-00-00.parquet'
|
|
assert result['uri'] == 's3://test/test_2024-01-01_00-00-00.parquet'
|
|
|
|
|
|
@mark.asyncio
|
|
async def test_query_to_minio_error(storage):
|
|
storage.send_notification = MagicMock()
|
|
storage.send_notification_async = AsyncMock()
|
|
storage.minio_repository.store_dataframe_as_parquet = AsyncMock()
|
|
|
|
storage.load_custom_query = AsyncMock(side_effect=Exception('test'))
|
|
result = await storage.query_to_minio({**metadata, 'object_prefix': 'test'})
|
|
assert result['success'] is False
|
|
assert result['message'] == 'test'
|
|
storage.send_notification_async.assert_called_once_with(
|
|
metadata=metadata['metadata'],
|
|
notification_id='ERROR_STORING_QUERY_TO_MINIO',
|
|
message='Error storing query to MinIO: test',
|
|
block='query_to_minio',
|
|
level=NotificationLevel.ERROR,
|
|
attachment_content=ANY,
|
|
)
|
|
|
|
|
|
def test_close(storage):
|
|
storage.minio_repository = MagicMock()
|
|
|
|
storage.close()
|
|
|
|
assert storage.minio_repository is None
|
|
|
|
|
|
def test___del__(storage):
|
|
storage.close = MagicMock()
|
|
|
|
storage.__del__()
|
|
|
|
storage.close.assert_called_once()
|