SIENTIAPDE-1248: Integrate MinIO for object storage and add related configurations
This commit introduces MinIO integration for object storage within the Model Manager system. It includes: - Added MinIO activity class for file operations (fetch, delete). - Updated Activities orchestrator to include MinIO activities. - Added MinIO configuration builder to utils/connectors_config.py. - Added environment variables for MinIO configuration in .env.example. - Added boto3 and botocore dependencies to requirements.txt. - Added unit tests for MinIO activities.
This commit is contained in:
Bruno Domingues
@@ -10,8 +10,9 @@ from model_manager.activities.mlflow import MLFlow
|
||||
|
||||
@patch('model_manager.activities.activities.Postgres.__init__')
|
||||
@patch('model_manager.activities.activities.MLFlow.__init__')
|
||||
@patch('model_manager.activities.activities.MinIO.__init__')
|
||||
@patch('model_manager.activities.activities.Gates.__init__')
|
||||
def test___init__(mock_gates_init, mock_mlflow_init, mock_postgres_init):
|
||||
def test___init__(mock_gates_init, mock_minio_init, mock_mlflow_init, mock_postgres_init):
|
||||
postgres_config = {
|
||||
'host': 'localhost',
|
||||
'port': 5432,
|
||||
@@ -24,12 +25,25 @@ def test___init__(mock_gates_init, mock_mlflow_init, mock_postgres_init):
|
||||
|
||||
mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'}
|
||||
|
||||
minio_config = {
|
||||
'endpoint_url': 'http://localhost:9000',
|
||||
'access_key': 'minioadmin',
|
||||
'secret_key': 'minioadmin',
|
||||
'region': 'us-east-1',
|
||||
'use_ssl': False,
|
||||
'max_retry_attempts': 3,
|
||||
'retry_mode': 'adaptive',
|
||||
'connect_timeout': 10,
|
||||
'read_timeout': 60,
|
||||
}
|
||||
|
||||
logger = MagicMock()
|
||||
notification_handler = MagicMock()
|
||||
|
||||
activities = Activities(
|
||||
postgres_config=postgres_config,
|
||||
mlflow_config=mlflow_config,
|
||||
minio_config=minio_config,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
@@ -62,6 +76,21 @@ def test___init__(mock_gates_init, mock_mlflow_init, mock_postgres_init):
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
mock_minio_init.assert_called_once_with(
|
||||
ANY,
|
||||
endpoint_url=minio_config['endpoint_url'],
|
||||
access_key=minio_config['access_key'],
|
||||
secret_key=minio_config['secret_key'],
|
||||
region=minio_config['region'],
|
||||
use_ssl=minio_config['use_ssl'],
|
||||
max_retry_attempts=minio_config['max_retry_attempts'],
|
||||
retry_mode=minio_config['retry_mode'],
|
||||
connect_timeout=minio_config['connect_timeout'],
|
||||
read_timeout=minio_config['read_timeout'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
mock_gates_init.assert_called_once_with(
|
||||
ANY, logger=logger, notification_handler=notification_handler
|
||||
)
|
||||
@@ -83,12 +112,25 @@ async def test_shutdown(_mock_mlflow_init, mock_postgres_init):
|
||||
|
||||
mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'}
|
||||
|
||||
minio_config = {
|
||||
'endpoint_url': 'http://localhost:9000',
|
||||
'access_key': 'minioadmin',
|
||||
'secret_key': 'minioadmin',
|
||||
'region': 'us-east-1',
|
||||
'use_ssl': False,
|
||||
'max_retry_attempts': 3,
|
||||
'retry_mode': 'adaptive',
|
||||
'connect_timeout': 10,
|
||||
'read_timeout': 60,
|
||||
}
|
||||
|
||||
logger = MagicMock()
|
||||
notification_handler = MagicMock()
|
||||
|
||||
activities = Activities(
|
||||
postgres_config=postgres_config,
|
||||
mlflow_config=mlflow_config,
|
||||
minio_config=minio_config,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
@@ -117,6 +117,25 @@ async def test_input_gate_with_filter(gates_activity):
|
||||
gates_activity.debug.assert_called()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_input_gate_filter_returns_false(gates_activity):
|
||||
"""Test to cover line 129 branch when filter returns False (filter passes)."""
|
||||
# Arrange - Use data that will NOT trigger EMPTY_DATA filter (has data)
|
||||
input_data = {
|
||||
**metadata,
|
||||
'filters': {'EMPTY_DATA': {'policy': 'STOP', 'config': {}}},
|
||||
'data': {'value': [1, 2, 3, 4, 5]}, # Has data, filter returns False
|
||||
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
|
||||
}
|
||||
|
||||
# Act
|
||||
result = await gates_activity.input_gate(input_data)
|
||||
|
||||
# Assert - Filter returns False, so no policy is added to filter_output
|
||||
assert result == (None, 0, '') # No filter triggered
|
||||
gates_activity.debug.assert_called()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_mlflow_response_gate_invalid_filter(gates_activity):
|
||||
# Arrange
|
||||
@@ -210,6 +229,29 @@ async def test_mlflow_response_gate_with_filter(gates_activity):
|
||||
gates_activity.send_notification.assert_called()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_mlflow_response_gate_filter_returns_false(gates_activity):
|
||||
"""Test to cover line 208 branch when filter returns False (no API error)."""
|
||||
# Arrange - Use data that will NOT trigger API_ERROR filter (success=True)
|
||||
input_data = {
|
||||
**metadata,
|
||||
'filters': {'API_ERROR': {'policy': 'STOP'}},
|
||||
'data': {
|
||||
'success': True, # Success=True, filter returns False
|
||||
'content': {'message': 'Operation successful', 'result': 'data'},
|
||||
},
|
||||
'type': 'transform',
|
||||
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
|
||||
}
|
||||
|
||||
# Act
|
||||
result = await gates_activity.mlflow_response_gate(input_data)
|
||||
|
||||
# Assert - Filter returns False, so no policy is added to filter_output
|
||||
assert result == (None, 0, '') # No filter triggered
|
||||
gates_activity.debug.assert_called()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_mlflow_content_gate_invalid_filter(gates_activity):
|
||||
# Arrange
|
||||
@@ -304,6 +346,26 @@ async def test_mlflow_content_gate_with_filter(gates_activity):
|
||||
gates_activity.send_notification.assert_called()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_mlflow_content_gate_filter_returns_false(gates_activity):
|
||||
"""Test to cover line 293 branch when filter returns False (no NaN values)."""
|
||||
# Arrange - Use data that will NOT trigger NAN_VALUES filter (no NaN)
|
||||
input_data = {
|
||||
**metadata,
|
||||
'filters': {'NAN_VALUES': {'policy': 'STOP', 'config': {}}},
|
||||
'data': {'value': [1.0, 2.0, 3.0, 4.0, 5.0]}, # All valid numbers, no NaN
|
||||
'type': 'predict',
|
||||
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
|
||||
}
|
||||
|
||||
# Act
|
||||
result = await gates_activity.mlflow_content_gate(input_data)
|
||||
|
||||
# Assert - Filter returns False, so no policy is added to filter_output
|
||||
assert result == (None, 0, '') # No filter triggered
|
||||
gates_activity.debug.assert_called()
|
||||
|
||||
|
||||
def test_get_prediction_store_policy_invalid_policy(gates_activity):
|
||||
# Arrange
|
||||
prediction_store_policy = 'INVALID_POLICY'
|
||||
|
||||
328
tests/laborious/activities/test_minio.py
Normal file
328
tests/laborious/activities/test_minio.py
Normal file
@@ -0,0 +1,328 @@
|
||||
from io import BytesIO
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from pytest import fixture, mark, raises
|
||||
|
||||
from model_manager.activities.minio import MinIO
|
||||
|
||||
|
||||
@patch('model_manager.activities.minio.boto3.client')
|
||||
def test___init__(mock_boto3_client):
|
||||
"""Test MinIO initialization with correct configuration."""
|
||||
mock_client = MagicMock()
|
||||
mock_boto3_client.return_value = mock_client
|
||||
|
||||
logger = MagicMock()
|
||||
notification_handler = MagicMock()
|
||||
|
||||
minio = MinIO(
|
||||
endpoint_url='http://localhost:9000',
|
||||
access_key='minioadmin',
|
||||
secret_key='minioadmin',
|
||||
region='us-east-1',
|
||||
use_ssl=False,
|
||||
max_retry_attempts=3,
|
||||
retry_mode='adaptive',
|
||||
connect_timeout=10,
|
||||
read_timeout=60,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
assert minio.endpoint_url == 'http://localhost:9000'
|
||||
assert minio.access_key == 'minioadmin'
|
||||
assert minio.secret_key == 'minioadmin'
|
||||
assert minio.region == 'us-east-1'
|
||||
assert minio.use_ssl is False
|
||||
assert minio.max_retry_attempts == 3
|
||||
assert minio.retry_mode == 'adaptive'
|
||||
assert minio.connect_timeout == 10
|
||||
assert minio.read_timeout == 60
|
||||
|
||||
# Verify boto3 client was created with correct parameters
|
||||
mock_boto3_client.assert_called_once()
|
||||
call_kwargs = mock_boto3_client.call_args[1]
|
||||
assert call_kwargs['endpoint_url'] == 'http://localhost:9000'
|
||||
assert call_kwargs['aws_access_key_id'] == 'minioadmin'
|
||||
assert call_kwargs['aws_secret_access_key'] == 'minioadmin'
|
||||
assert call_kwargs['use_ssl'] is False
|
||||
|
||||
|
||||
@patch('model_manager.activities.minio.boto3.client')
|
||||
def test___init___failure(mock_boto3_client):
|
||||
"""Test MinIO initialization failure handling."""
|
||||
mock_boto3_client.side_effect = Exception('Connection failed')
|
||||
|
||||
logger = MagicMock()
|
||||
notification_handler = MagicMock()
|
||||
|
||||
with raises(Exception, match='Failed to initialize MinIO client'):
|
||||
MinIO(
|
||||
endpoint_url='http://localhost:9000',
|
||||
access_key='minioadmin',
|
||||
secret_key='minioadmin',
|
||||
region='us-east-1',
|
||||
use_ssl=False,
|
||||
max_retry_attempts=3,
|
||||
retry_mode='adaptive',
|
||||
connect_timeout=10,
|
||||
read_timeout=60,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
|
||||
@fixture
|
||||
@patch('model_manager.activities.minio.boto3.client')
|
||||
def minio(mock_boto3_client):
|
||||
"""Fixture to create a MinIO instance for testing."""
|
||||
mock_client = MagicMock()
|
||||
mock_boto3_client.return_value = mock_client
|
||||
|
||||
logger = MagicMock()
|
||||
notification_handler = MagicMock()
|
||||
|
||||
minio_instance = MinIO(
|
||||
endpoint_url='http://localhost:9000',
|
||||
access_key='minioadmin',
|
||||
secret_key='minioadmin',
|
||||
region='us-east-1',
|
||||
use_ssl=False,
|
||||
max_retry_attempts=3,
|
||||
retry_mode='adaptive',
|
||||
connect_timeout=10,
|
||||
read_timeout=60,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
minio_instance.send_notification = MagicMock()
|
||||
minio_instance.minio_client = mock_client
|
||||
|
||||
return minio_instance
|
||||
|
||||
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'workflow_name': 'test_workflow',
|
||||
'model_name': 'test_model',
|
||||
'model_id': 'test_model_id',
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_fetch_file_from_minio_success(minio):
|
||||
"""Test successful file fetch from MinIO."""
|
||||
# Arrange
|
||||
test_content = b'test file content'
|
||||
mock_response = {'Body': MagicMock()}
|
||||
mock_response['Body'].__enter__ = MagicMock(
|
||||
return_value=MagicMock(read=MagicMock(return_value=test_content))
|
||||
)
|
||||
mock_response['Body'].__exit__ = MagicMock(return_value=None)
|
||||
|
||||
minio.minio_client.get_object.return_value = mock_response
|
||||
|
||||
input_data = {
|
||||
'metadata': metadata['metadata'],
|
||||
'bucket_name': 'test-bucket',
|
||||
'file_name': 'test-file.txt',
|
||||
}
|
||||
|
||||
# Act
|
||||
result = await minio.fetch_file_from_minio(input_data)
|
||||
|
||||
# Assert
|
||||
assert isinstance(result, BytesIO)
|
||||
result.seek(0)
|
||||
assert result.read() == test_content
|
||||
|
||||
minio.minio_client.get_object.assert_called_once_with(Bucket='test-bucket', Key='test-file.txt')
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_fetch_file_from_minio_file_not_found(minio):
|
||||
"""Test file fetch when file doesn't exist."""
|
||||
# Arrange
|
||||
minio.minio_client.get_object.side_effect = Exception(
|
||||
'NoSuchKey: The specified key does not exist'
|
||||
)
|
||||
|
||||
input_data = {
|
||||
'metadata': metadata['metadata'],
|
||||
'bucket_name': 'test-bucket',
|
||||
'file_name': 'nonexistent.txt',
|
||||
}
|
||||
|
||||
# Act & Assert
|
||||
with raises(Exception, match='Error fetching file from MinIO'):
|
||||
await minio.fetch_file_from_minio(input_data)
|
||||
|
||||
# Verify notification was sent
|
||||
minio.send_notification.assert_called_once()
|
||||
call_kwargs = minio.send_notification.call_args[1]
|
||||
assert call_kwargs['notification_id'] == 'FETCH_FILE_FROM_MINIO_ERROR'
|
||||
assert call_kwargs['block'] == 'fetch_file_from_minio'
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_fetch_file_from_minio_network_error(minio):
|
||||
"""Test file fetch with network error."""
|
||||
# Arrange
|
||||
minio.minio_client.get_object.side_effect = Exception('Network timeout')
|
||||
|
||||
input_data = {
|
||||
'metadata': metadata['metadata'],
|
||||
'bucket_name': 'test-bucket',
|
||||
'file_name': 'test-file.txt',
|
||||
}
|
||||
|
||||
# Act & Assert
|
||||
with raises(Exception, match='Error fetching file from MinIO'):
|
||||
await minio.fetch_file_from_minio(input_data)
|
||||
|
||||
minio.send_notification.assert_called_once()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_delete_file_from_minio_success(minio):
|
||||
"""Test successful file deletion from MinIO."""
|
||||
# Arrange
|
||||
minio.minio_client.delete_object.return_value = None
|
||||
|
||||
input_data = {
|
||||
'metadata': metadata['metadata'],
|
||||
'bucket_name': 'test-bucket',
|
||||
'file_name': 'test-file.txt',
|
||||
}
|
||||
|
||||
# Act
|
||||
result = await minio.delete_file_from_minio(input_data)
|
||||
|
||||
# Assert
|
||||
assert result is None
|
||||
|
||||
minio.minio_client.delete_object.assert_called_once_with(
|
||||
Bucket='test-bucket', Key='test-file.txt'
|
||||
)
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_delete_file_from_minio_idempotent(minio):
|
||||
"""Test that delete is idempotent (no error if file doesn't exist)."""
|
||||
# Arrange
|
||||
# MinIO delete_object is idempotent - no error if file doesn't exist
|
||||
minio.minio_client.delete_object.return_value = None
|
||||
|
||||
input_data = {
|
||||
'metadata': metadata['metadata'],
|
||||
'bucket_name': 'test-bucket',
|
||||
'file_name': 'nonexistent.txt',
|
||||
}
|
||||
|
||||
# Act
|
||||
result = await minio.delete_file_from_minio(input_data)
|
||||
|
||||
# Assert
|
||||
assert result is None
|
||||
minio.minio_client.delete_object.assert_called_once()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_delete_file_from_minio_access_denied(minio):
|
||||
"""Test file deletion with access denied error."""
|
||||
# Arrange
|
||||
minio.minio_client.delete_object.side_effect = Exception('AccessDenied: Access Denied')
|
||||
|
||||
input_data = {
|
||||
'metadata': metadata['metadata'],
|
||||
'bucket_name': 'test-bucket',
|
||||
'file_name': 'test-file.txt',
|
||||
}
|
||||
|
||||
# Act & Assert
|
||||
with raises(Exception, match='Error deleting file from MinIO'):
|
||||
await minio.delete_file_from_minio(input_data)
|
||||
|
||||
# Verify notification was sent
|
||||
minio.send_notification.assert_called_once()
|
||||
call_kwargs = minio.send_notification.call_args[1]
|
||||
assert call_kwargs['notification_id'] == 'DELETE_FILE_FROM_MINIO_ERROR'
|
||||
assert call_kwargs['block'] == 'delete_file_from_minio'
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_delete_file_from_minio_network_error(minio):
|
||||
"""Test file deletion with network error."""
|
||||
# Arrange
|
||||
minio.minio_client.delete_object.side_effect = Exception('Connection timeout')
|
||||
|
||||
input_data = {
|
||||
'metadata': metadata['metadata'],
|
||||
'bucket_name': 'test-bucket',
|
||||
'file_name': 'test-file.txt',
|
||||
}
|
||||
|
||||
# Act & Assert
|
||||
with raises(Exception, match='Error deleting file from MinIO'):
|
||||
await minio.delete_file_from_minio(input_data)
|
||||
|
||||
minio.send_notification.assert_called_once()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_fetch_file_from_minio_large_file(minio):
|
||||
"""Test fetching a large file from MinIO."""
|
||||
# Arrange
|
||||
# Simulate a 10MB file
|
||||
large_content = b'x' * (10 * 1024 * 1024)
|
||||
mock_response = {'Body': MagicMock()}
|
||||
mock_response['Body'].__enter__ = MagicMock(
|
||||
return_value=MagicMock(read=MagicMock(return_value=large_content))
|
||||
)
|
||||
mock_response['Body'].__exit__ = MagicMock(return_value=None)
|
||||
|
||||
minio.minio_client.get_object.return_value = mock_response
|
||||
|
||||
input_data = {
|
||||
'metadata': metadata['metadata'],
|
||||
'bucket_name': 'test-bucket',
|
||||
'file_name': 'large-file.bin',
|
||||
}
|
||||
|
||||
# Act
|
||||
result = await minio.fetch_file_from_minio(input_data)
|
||||
|
||||
# Assert
|
||||
assert isinstance(result, BytesIO)
|
||||
result.seek(0)
|
||||
assert len(result.read()) == 10 * 1024 * 1024
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_fetch_file_from_minio_empty_file(minio):
|
||||
"""Test fetching an empty file from MinIO."""
|
||||
# Arrange
|
||||
empty_content = b''
|
||||
mock_response = {'Body': MagicMock()}
|
||||
mock_response['Body'].__enter__ = MagicMock(
|
||||
return_value=MagicMock(read=MagicMock(return_value=empty_content))
|
||||
)
|
||||
mock_response['Body'].__exit__ = MagicMock(return_value=None)
|
||||
|
||||
minio.minio_client.get_object.return_value = mock_response
|
||||
|
||||
input_data = {
|
||||
'metadata': metadata['metadata'],
|
||||
'bucket_name': 'test-bucket',
|
||||
'file_name': 'empty-file.txt',
|
||||
}
|
||||
|
||||
# Act
|
||||
result = await minio.fetch_file_from_minio(input_data)
|
||||
|
||||
# Assert
|
||||
assert isinstance(result, BytesIO)
|
||||
result.seek(0)
|
||||
assert result.read() == b''
|
||||
Reference in New Issue
Block a user