gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1248: Integrate MinIO for object storage and add related configurations

Browse Source 
This commit introduces MinIO integration for object storage within the Model Manager system. It includes:

- Added MinIO activity class for file operations (fetch, delete).
- Updated Activities orchestrator to include MinIO activities.
- Added MinIO configuration builder to utils/connectors_config.py.
- Added environment variables for MinIO configuration in .env.example.
- Added boto3 and botocore dependencies to requirements.txt.
- Added unit tests for MinIO activities.
This commit is contained in:
Bruno Domingues
2025-10-03 21:04:27 -03:00
parent 11c25d126c
commit 9afe711075
12 changed files with 850 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
tests/laborious/activities/test_activities.py
View File

@@ -10,8 +10,9 @@ from model_manager.activities.mlflow import MLFlow
@patch('model_manager.activities.activities.Postgres.__init__')
@patch('model_manager.activities.activities.MLFlow.__init__')
@patch('model_manager.activities.activities.MinIO.__init__')
@patch('model_manager.activities.activities.Gates.__init__')
def test___init__(mock_gates_init, mock_mlflow_init, mock_postgres_init):
def test___init__(mock_gates_init, mock_minio_init, mock_mlflow_init, mock_postgres_init):
postgres_config = {
'host': 'localhost',
'port': 5432,
@@ -24,12 +25,25 @@ def test___init__(mock_gates_init, mock_mlflow_init, mock_postgres_init):
mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'}
minio_config = {
'endpoint_url': 'http://localhost:9000',
'access_key': 'minioadmin',
'secret_key': 'minioadmin',
'region': 'us-east-1',
'use_ssl': False,
'max_retry_attempts': 3,
'retry_mode': 'adaptive',
'connect_timeout': 10,
'read_timeout': 60,
}
logger = MagicMock()
notification_handler = MagicMock()
activities = Activities(
postgres_config=postgres_config,
mlflow_config=mlflow_config,
minio_config=minio_config,
logger=logger,
notification_handler=notification_handler,
)
@@ -62,6 +76,21 @@ def test___init__(mock_gates_init, mock_mlflow_init, mock_postgres_init):
notification_handler=notification_handler,
)
mock_minio_init.assert_called_once_with(
ANY,
endpoint_url=minio_config['endpoint_url'],
access_key=minio_config['access_key'],
secret_key=minio_config['secret_key'],
region=minio_config['region'],
use_ssl=minio_config['use_ssl'],
max_retry_attempts=minio_config['max_retry_attempts'],
retry_mode=minio_config['retry_mode'],
connect_timeout=minio_config['connect_timeout'],
read_timeout=minio_config['read_timeout'],
logger=logger,
notification_handler=notification_handler,
)
mock_gates_init.assert_called_once_with(
ANY, logger=logger, notification_handler=notification_handler
)
@@ -83,12 +112,25 @@ async def test_shutdown(_mock_mlflow_init, mock_postgres_init):
mlflow_config = {'host': 'localhost', 'port': 5000, 'username': 'mlflow', 'password': 'mlflow'}
minio_config = {
'endpoint_url': 'http://localhost:9000',
'access_key': 'minioadmin',
'secret_key': 'minioadmin',
'region': 'us-east-1',
'use_ssl': False,
'max_retry_attempts': 3,
'retry_mode': 'adaptive',
'connect_timeout': 10,
'read_timeout': 60,
}
logger = MagicMock()
notification_handler = MagicMock()
activities = Activities(
postgres_config=postgres_config,
mlflow_config=mlflow_config,
minio_config=minio_config,
logger=logger,
notification_handler=notification_handler,
)

62
tests/laborious/activities/test_gates.py
View File

@@ -117,6 +117,25 @@ async def test_input_gate_with_filter(gates_activity):
gates_activity.debug.assert_called()
@mark.asyncio
async def test_input_gate_filter_returns_false(gates_activity):
"""Test to cover line 129 branch when filter returns False (filter passes)."""
# Arrange - Use data that will NOT trigger EMPTY_DATA filter (has data)
input_data = {
**metadata,
'filters': {'EMPTY_DATA': {'policy': 'STOP', 'config': {}}},
'data': {'value': [1, 2, 3, 4, 5]}, # Has data, filter returns False
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
}
# Act
result = await gates_activity.input_gate(input_data)
# Assert - Filter returns False, so no policy is added to filter_output
assert result == (None, 0, '') # No filter triggered
gates_activity.debug.assert_called()
@mark.asyncio
async def test_mlflow_response_gate_invalid_filter(gates_activity):
# Arrange
@@ -210,6 +229,29 @@ async def test_mlflow_response_gate_with_filter(gates_activity):
gates_activity.send_notification.assert_called()
@mark.asyncio
async def test_mlflow_response_gate_filter_returns_false(gates_activity):
"""Test to cover line 208 branch when filter returns False (no API error)."""
# Arrange - Use data that will NOT trigger API_ERROR filter (success=True)
input_data = {
**metadata,
'filters': {'API_ERROR': {'policy': 'STOP'}},
'data': {
'success': True, # Success=True, filter returns False
'content': {'message': 'Operation successful', 'result': 'data'},
},
'type': 'transform',
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
}
# Act
result = await gates_activity.mlflow_response_gate(input_data)
# Assert - Filter returns False, so no policy is added to filter_output
assert result == (None, 0, '') # No filter triggered
gates_activity.debug.assert_called()
@mark.asyncio
async def test_mlflow_content_gate_invalid_filter(gates_activity):
# Arrange
@@ -304,6 +346,26 @@ async def test_mlflow_content_gate_with_filter(gates_activity):
gates_activity.send_notification.assert_called()
@mark.asyncio
async def test_mlflow_content_gate_filter_returns_false(gates_activity):
"""Test to cover line 293 branch when filter returns False (no NaN values)."""
# Arrange - Use data that will NOT trigger NAN_VALUES filter (no NaN)
input_data = {
**metadata,
'filters': {'NAN_VALUES': {'policy': 'STOP', 'config': {}}},
'data': {'value': [1.0, 2.0, 3.0, 4.0, 5.0]}, # All valid numbers, no NaN
'type': 'predict',
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'],
}
# Act
result = await gates_activity.mlflow_content_gate(input_data)
# Assert - Filter returns False, so no policy is added to filter_output
assert result == (None, 0, '') # No filter triggered
gates_activity.debug.assert_called()
def test_get_prediction_store_policy_invalid_policy(gates_activity):
# Arrange
prediction_store_policy = 'INVALID_POLICY'

328
tests/laborious/activities/test_minio.py Normal file
View File

@@ -0,0 +1,328 @@
from io import BytesIO
from unittest.mock import MagicMock, patch
from pytest import fixture, mark, raises
from model_manager.activities.minio import MinIO
@patch('model_manager.activities.minio.boto3.client')
def test___init__(mock_boto3_client):
"""Test MinIO initialization with correct configuration."""
mock_client = MagicMock()
mock_boto3_client.return_value = mock_client
logger = MagicMock()
notification_handler = MagicMock()
minio = MinIO(
endpoint_url='http://localhost:9000',
access_key='minioadmin',
secret_key='minioadmin',
region='us-east-1',
use_ssl=False,
max_retry_attempts=3,
retry_mode='adaptive',
connect_timeout=10,
read_timeout=60,
logger=logger,
notification_handler=notification_handler,
)
assert minio.endpoint_url == 'http://localhost:9000'
assert minio.access_key == 'minioadmin'
assert minio.secret_key == 'minioadmin'
assert minio.region == 'us-east-1'
assert minio.use_ssl is False
assert minio.max_retry_attempts == 3
assert minio.retry_mode == 'adaptive'
assert minio.connect_timeout == 10
assert minio.read_timeout == 60
# Verify boto3 client was created with correct parameters
mock_boto3_client.assert_called_once()
call_kwargs = mock_boto3_client.call_args[1]
assert call_kwargs['endpoint_url'] == 'http://localhost:9000'
assert call_kwargs['aws_access_key_id'] == 'minioadmin'
assert call_kwargs['aws_secret_access_key'] == 'minioadmin'
assert call_kwargs['use_ssl'] is False
@patch('model_manager.activities.minio.boto3.client')
def test___init___failure(mock_boto3_client):
"""Test MinIO initialization failure handling."""
mock_boto3_client.side_effect = Exception('Connection failed')
logger = MagicMock()
notification_handler = MagicMock()
with raises(Exception, match='Failed to initialize MinIO client'):
MinIO(
endpoint_url='http://localhost:9000',
access_key='minioadmin',
secret_key='minioadmin',
region='us-east-1',
use_ssl=False,
max_retry_attempts=3,
retry_mode='adaptive',
connect_timeout=10,
read_timeout=60,
logger=logger,
notification_handler=notification_handler,
)
@fixture
@patch('model_manager.activities.minio.boto3.client')
def minio(mock_boto3_client):
"""Fixture to create a MinIO instance for testing."""
mock_client = MagicMock()
mock_boto3_client.return_value = mock_client
logger = MagicMock()
notification_handler = MagicMock()
minio_instance = MinIO(
endpoint_url='http://localhost:9000',
access_key='minioadmin',
secret_key='minioadmin',
region='us-east-1',
use_ssl=False,
max_retry_attempts=3,
retry_mode='adaptive',
connect_timeout=10,
read_timeout=60,
logger=logger,
notification_handler=notification_handler,
)
minio_instance.send_notification = MagicMock()
minio_instance.minio_client = mock_client
return minio_instance
metadata = {
'metadata': {
'workflow_name': 'test_workflow',
'model_name': 'test_model',
'model_id': 'test_model_id',
}
}
@mark.asyncio
async def test_fetch_file_from_minio_success(minio):
"""Test successful file fetch from MinIO."""
# Arrange
test_content = b'test file content'
mock_response = {'Body': MagicMock()}
mock_response['Body'].__enter__ = MagicMock(
return_value=MagicMock(read=MagicMock(return_value=test_content))
)
mock_response['Body'].__exit__ = MagicMock(return_value=None)
minio.minio_client.get_object.return_value = mock_response
input_data = {
'metadata': metadata['metadata'],
'bucket_name': 'test-bucket',
'file_name': 'test-file.txt',
}
# Act
result = await minio.fetch_file_from_minio(input_data)
# Assert
assert isinstance(result, BytesIO)
result.seek(0)
assert result.read() == test_content
minio.minio_client.get_object.assert_called_once_with(Bucket='test-bucket', Key='test-file.txt')
@mark.asyncio
async def test_fetch_file_from_minio_file_not_found(minio):
"""Test file fetch when file doesn't exist."""
# Arrange
minio.minio_client.get_object.side_effect = Exception(
'NoSuchKey: The specified key does not exist'
)
input_data = {
'metadata': metadata['metadata'],
'bucket_name': 'test-bucket',
'file_name': 'nonexistent.txt',
}
# Act & Assert
with raises(Exception, match='Error fetching file from MinIO'):
await minio.fetch_file_from_minio(input_data)
# Verify notification was sent
minio.send_notification.assert_called_once()
call_kwargs = minio.send_notification.call_args[1]
assert call_kwargs['notification_id'] == 'FETCH_FILE_FROM_MINIO_ERROR'
assert call_kwargs['block'] == 'fetch_file_from_minio'
@mark.asyncio
async def test_fetch_file_from_minio_network_error(minio):
"""Test file fetch with network error."""
# Arrange
minio.minio_client.get_object.side_effect = Exception('Network timeout')
input_data = {
'metadata': metadata['metadata'],
'bucket_name': 'test-bucket',
'file_name': 'test-file.txt',
}
# Act & Assert
with raises(Exception, match='Error fetching file from MinIO'):
await minio.fetch_file_from_minio(input_data)
minio.send_notification.assert_called_once()
@mark.asyncio
async def test_delete_file_from_minio_success(minio):
"""Test successful file deletion from MinIO."""
# Arrange
minio.minio_client.delete_object.return_value = None
input_data = {
'metadata': metadata['metadata'],
'bucket_name': 'test-bucket',
'file_name': 'test-file.txt',
}
# Act
result = await minio.delete_file_from_minio(input_data)
# Assert
assert result is None
minio.minio_client.delete_object.assert_called_once_with(
Bucket='test-bucket', Key='test-file.txt'
)
@mark.asyncio
async def test_delete_file_from_minio_idempotent(minio):
"""Test that delete is idempotent (no error if file doesn't exist)."""
# Arrange
# MinIO delete_object is idempotent - no error if file doesn't exist
minio.minio_client.delete_object.return_value = None
input_data = {
'metadata': metadata['metadata'],
'bucket_name': 'test-bucket',
'file_name': 'nonexistent.txt',
}
# Act
result = await minio.delete_file_from_minio(input_data)
# Assert
assert result is None
minio.minio_client.delete_object.assert_called_once()
@mark.asyncio
async def test_delete_file_from_minio_access_denied(minio):
"""Test file deletion with access denied error."""
# Arrange
minio.minio_client.delete_object.side_effect = Exception('AccessDenied: Access Denied')
input_data = {
'metadata': metadata['metadata'],
'bucket_name': 'test-bucket',
'file_name': 'test-file.txt',
}
# Act & Assert
with raises(Exception, match='Error deleting file from MinIO'):
await minio.delete_file_from_minio(input_data)
# Verify notification was sent
minio.send_notification.assert_called_once()
call_kwargs = minio.send_notification.call_args[1]
assert call_kwargs['notification_id'] == 'DELETE_FILE_FROM_MINIO_ERROR'
assert call_kwargs['block'] == 'delete_file_from_minio'
@mark.asyncio
async def test_delete_file_from_minio_network_error(minio):
"""Test file deletion with network error."""
# Arrange
minio.minio_client.delete_object.side_effect = Exception('Connection timeout')
input_data = {
'metadata': metadata['metadata'],
'bucket_name': 'test-bucket',
'file_name': 'test-file.txt',
}
# Act & Assert
with raises(Exception, match='Error deleting file from MinIO'):
await minio.delete_file_from_minio(input_data)
minio.send_notification.assert_called_once()
@mark.asyncio
async def test_fetch_file_from_minio_large_file(minio):
"""Test fetching a large file from MinIO."""
# Arrange
# Simulate a 10MB file
large_content = b'x' * (10 * 1024 * 1024)
mock_response = {'Body': MagicMock()}
mock_response['Body'].__enter__ = MagicMock(
return_value=MagicMock(read=MagicMock(return_value=large_content))
)
mock_response['Body'].__exit__ = MagicMock(return_value=None)
minio.minio_client.get_object.return_value = mock_response
input_data = {
'metadata': metadata['metadata'],
'bucket_name': 'test-bucket',
'file_name': 'large-file.bin',
}
# Act
result = await minio.fetch_file_from_minio(input_data)
# Assert
assert isinstance(result, BytesIO)
result.seek(0)
assert len(result.read()) == 10 * 1024 * 1024
@mark.asyncio
async def test_fetch_file_from_minio_empty_file(minio):
"""Test fetching an empty file from MinIO."""
# Arrange
empty_content = b''
mock_response = {'Body': MagicMock()}
mock_response['Body'].__enter__ = MagicMock(
return_value=MagicMock(read=MagicMock(return_value=empty_content))
)
mock_response['Body'].__exit__ = MagicMock(return_value=None)
minio.minio_client.get_object.return_value = mock_response
input_data = {
'metadata': metadata['metadata'],
'bucket_name': 'test-bucket',
'file_name': 'empty-file.txt',
}
# Act
result = await minio.fetch_file_from_minio(input_data)
# Assert
assert isinstance(result, BytesIO)
result.seek(0)
assert result.read() == b''

60
tests/laborious/utils/repository/test_model_repository.py
View File

@@ -303,13 +303,22 @@ def test_create_model_experiment(set_experiment, sklearn, mlflow_repository):
)
@patch('model_manager.utils.repository.model_repository.path.exists')
@patch('model_manager.utils.repository.model_repository.remove')
@patch('model_manager.utils.repository.model_repository.mlflow.start_run')
@patch('model_manager.utils.repository.model_repository.mlflow.log_param')
@patch('model_manager.utils.repository.model_repository.mlflow.sklearn.log_model')
@patch('model_manager.utils.repository.model_repository.mlflow.log_artifact')
def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, mlflow_repository):
def test_perform_model_retrain(
log_artifact, log_model, log_param, start_run, mock_remove, mock_path_exists, mlflow_repository
):
# Create mock models with attributes to test the for loops (lines 268-274)
prediction_model_mock = MagicMock()
prediction_model_mock.__dict__ = {'model': 'pred_model', 'param1': 'value1', 'param2': 'value2'}
data_model_mock = MagicMock()
data_model_mock.__dict__ = {'model': 'data_model', 'param3': 'value3', 'param4': 'value4'}
experiment = 'test'
model_name = 'test'
data = MagicMock()
@@ -317,6 +326,7 @@ def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, ml
mlflow_repository.get_next_run_name = MagicMock(return_value='test-1')
run = MagicMock()
start_run.__enter__.return_value = run
mock_path_exists.return_value = True
output = mlflow_repository.perform_model_retrain(
prediction_model_mock, data_model_mock, experiment, model_name, data
@@ -338,12 +348,58 @@ def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, ml
log_artifact.assert_called_once_with('temp/raw_data_test.csv')
# Verify that model attributes were logged (excluding 'model' key)
log_param.assert_has_calls(
[
call('param1', 'value1'), # from prediction_model
call('param2', 'value2'), # from prediction_model
call('param3', 'value3'), # from data_model
call('param4', 'value4'), # from data_model
call('retrain', True),
]
],
any_order=True,
)
# Verify temp file cleanup
mock_path_exists.assert_called_once_with('temp/raw_data_test.csv')
mock_remove.assert_called_once_with('temp/raw_data_test.csv')
assert output == ('Model retrained successfully', experiment)
@patch('model_manager.utils.repository.model_repository.path.exists')
@patch('model_manager.utils.repository.model_repository.remove')
@patch('model_manager.utils.repository.model_repository.mlflow.start_run')
@patch('model_manager.utils.repository.model_repository.mlflow.log_param')
@patch('model_manager.utils.repository.model_repository.mlflow.sklearn.log_model')
@patch('model_manager.utils.repository.model_repository.mlflow.log_artifact')
def test_perform_model_retrain_file_not_exists(
log_artifact, log_model, log_param, start_run, mock_remove, mock_path_exists, mlflow_repository
):
"""Test perform_model_retrain when temp file doesn't exist (line 291->294 branch)."""
prediction_model_mock = MagicMock()
prediction_model_mock.__dict__ = {'model': 'pred_model'}
data_model_mock = MagicMock()
data_model_mock.__dict__ = {'model': 'data_model'}
experiment = 'test'
model_name = 'test'
data = MagicMock()
mlflow_repository.get_next_run_name = MagicMock(return_value='test-1')
run = MagicMock()
start_run.__enter__.return_value = run
mock_path_exists.return_value = False # File doesn't exist
output = mlflow_repository.perform_model_retrain(
prediction_model_mock, data_model_mock, experiment, model_name, data
)
# Verify temp file cleanup was checked but not executed
mock_path_exists.assert_called_once_with('temp/raw_data_test.csv')
mock_remove.assert_not_called() # Should not be called when file doesn't exist
assert output == ('Model retrained successfully', experiment)

56
tests/laborious/utils/test_connectors_config.py
View File

@@ -1,6 +1,7 @@
from os import environ
from model_manager.utils.connectors_config import (
build_minio_config,
build_mlflow_config,
build_mongodb_config,
build_postgres_config,
@@ -113,3 +114,58 @@ def test_build_mongo_db_config_with_defaults():
'database_name': 'sientia',
'ttl_index_seconds': 3600,
}
def test_build_minio_config_with_env_vars():
# Arrange
environ['MINIO_ENDPOINT_URL'] = 'http://test-minio:9000'
environ['MINIO_ACCESS_KEY'] = 'test-access-key'
environ['MINIO_SECRET_KEY'] = 'test-secret-key'
environ['MINIO_REGION'] = 'eu-west-1'
environ['MINIO_USE_SSL'] = 'true'
environ['MINIO_MAX_RETRY_ATTEMPTS'] = '5'
environ['MINIO_RETRY_MODE'] = 'standard'
environ['MINIO_CONNECT_TIMEOUT'] = '20'
environ['MINIO_READ_TIMEOUT'] = '120'
# Act
config = build_minio_config()
# Assert
assert config['endpoint_url'] == 'http://test-minio:9000'
assert config['access_key'] == 'test-access-key'
assert config['secret_key'] == 'test-secret-key'
assert config['region'] == 'eu-west-1'
assert config['use_ssl'] is True
assert config['max_retry_attempts'] == 5
assert config['retry_mode'] == 'standard'
assert config['connect_timeout'] == 20
assert config['read_timeout'] == 120
def test_build_minio_config_with_defaults():
# Arrange
# Clear any existing env vars
environ.pop('MINIO_ENDPOINT_URL', None)
environ.pop('MINIO_ACCESS_KEY', None)
environ.pop('MINIO_SECRET_KEY', None)
environ.pop('MINIO_REGION', None)
environ.pop('MINIO_USE_SSL', None)
environ.pop('MINIO_MAX_RETRY_ATTEMPTS', None)
environ.pop('MINIO_RETRY_MODE', None)
environ.pop('MINIO_CONNECT_TIMEOUT', None)
environ.pop('MINIO_READ_TIMEOUT', None)
# Act
config = build_minio_config()
# Assert
assert config['endpoint_url'] == 'http://localhost:9000'
assert config['access_key'] == 'minioadmin'
assert config['secret_key'] == 'minioadmin'
assert config['region'] == 'us-east-1'
assert config['use_ssl'] is False
assert config['max_retry_attempts'] == 3
assert config['retry_mode'] == 'adaptive'
assert config['connect_timeout'] == 10
assert config['read_timeout'] == 60