SIENTIAPDE-1248: Integrate MinIO for object storage and add related configurations
This commit introduces MinIO integration for object storage within the Model Manager system. It includes: - Added MinIO activity class for file operations (fetch, delete). - Updated Activities orchestrator to include MinIO activities. - Added MinIO configuration builder to utils/connectors_config.py. - Added environment variables for MinIO configuration in .env.example. - Added boto3 and botocore dependencies to requirements.txt. - Added unit tests for MinIO activities.
This commit is contained in:
@@ -303,13 +303,22 @@ def test_create_model_experiment(set_experiment, sklearn, mlflow_repository):
|
||||
)
|
||||
|
||||
|
||||
@patch('model_manager.utils.repository.model_repository.path.exists')
|
||||
@patch('model_manager.utils.repository.model_repository.remove')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.start_run')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.log_param')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.sklearn.log_model')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.log_artifact')
|
||||
def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, mlflow_repository):
|
||||
def test_perform_model_retrain(
|
||||
log_artifact, log_model, log_param, start_run, mock_remove, mock_path_exists, mlflow_repository
|
||||
):
|
||||
# Create mock models with attributes to test the for loops (lines 268-274)
|
||||
prediction_model_mock = MagicMock()
|
||||
prediction_model_mock.__dict__ = {'model': 'pred_model', 'param1': 'value1', 'param2': 'value2'}
|
||||
|
||||
data_model_mock = MagicMock()
|
||||
data_model_mock.__dict__ = {'model': 'data_model', 'param3': 'value3', 'param4': 'value4'}
|
||||
|
||||
experiment = 'test'
|
||||
model_name = 'test'
|
||||
data = MagicMock()
|
||||
@@ -317,6 +326,7 @@ def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, ml
|
||||
mlflow_repository.get_next_run_name = MagicMock(return_value='test-1')
|
||||
run = MagicMock()
|
||||
start_run.__enter__.return_value = run
|
||||
mock_path_exists.return_value = True
|
||||
|
||||
output = mlflow_repository.perform_model_retrain(
|
||||
prediction_model_mock, data_model_mock, experiment, model_name, data
|
||||
@@ -338,12 +348,58 @@ def test_perform_model_retrain(log_artifact, log_model, log_param, start_run, ml
|
||||
|
||||
log_artifact.assert_called_once_with('temp/raw_data_test.csv')
|
||||
|
||||
# Verify that model attributes were logged (excluding 'model' key)
|
||||
log_param.assert_has_calls(
|
||||
[
|
||||
call('param1', 'value1'), # from prediction_model
|
||||
call('param2', 'value2'), # from prediction_model
|
||||
call('param3', 'value3'), # from data_model
|
||||
call('param4', 'value4'), # from data_model
|
||||
call('retrain', True),
|
||||
]
|
||||
],
|
||||
any_order=True,
|
||||
)
|
||||
|
||||
# Verify temp file cleanup
|
||||
mock_path_exists.assert_called_once_with('temp/raw_data_test.csv')
|
||||
mock_remove.assert_called_once_with('temp/raw_data_test.csv')
|
||||
|
||||
assert output == ('Model retrained successfully', experiment)
|
||||
|
||||
|
||||
@patch('model_manager.utils.repository.model_repository.path.exists')
|
||||
@patch('model_manager.utils.repository.model_repository.remove')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.start_run')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.log_param')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.sklearn.log_model')
|
||||
@patch('model_manager.utils.repository.model_repository.mlflow.log_artifact')
|
||||
def test_perform_model_retrain_file_not_exists(
|
||||
log_artifact, log_model, log_param, start_run, mock_remove, mock_path_exists, mlflow_repository
|
||||
):
|
||||
"""Test perform_model_retrain when temp file doesn't exist (line 291->294 branch)."""
|
||||
prediction_model_mock = MagicMock()
|
||||
prediction_model_mock.__dict__ = {'model': 'pred_model'}
|
||||
|
||||
data_model_mock = MagicMock()
|
||||
data_model_mock.__dict__ = {'model': 'data_model'}
|
||||
|
||||
experiment = 'test'
|
||||
model_name = 'test'
|
||||
data = MagicMock()
|
||||
|
||||
mlflow_repository.get_next_run_name = MagicMock(return_value='test-1')
|
||||
run = MagicMock()
|
||||
start_run.__enter__.return_value = run
|
||||
mock_path_exists.return_value = False # File doesn't exist
|
||||
|
||||
output = mlflow_repository.perform_model_retrain(
|
||||
prediction_model_mock, data_model_mock, experiment, model_name, data
|
||||
)
|
||||
|
||||
# Verify temp file cleanup was checked but not executed
|
||||
mock_path_exists.assert_called_once_with('temp/raw_data_test.csv')
|
||||
mock_remove.assert_not_called() # Should not be called when file doesn't exist
|
||||
|
||||
assert output == ('Model retrained successfully', experiment)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user