diff --git a/tests/utils/repository/test_storage_repository.py b/tests/utils/repository/test_storage_repository.py new file mode 100644 index 0000000..c37b155 --- /dev/null +++ b/tests/utils/repository/test_storage_repository.py @@ -0,0 +1,465 @@ +"""Unit tests for StorageRepository class.""" + +from io import BytesIO +from unittest.mock import MagicMock, Mock, patch + +import pytest +from botocore.exceptions import ClientError + + +@pytest.fixture +def mock_logger(): + """Create a mock logger for testing.""" + logger = Mock() + logger.info = Mock() + logger.error = Mock() + logger.warning = Mock() + return logger + + +@pytest.fixture +def storage_config(): + """Create storage repository configuration.""" + return { + 'endpoint_url': 'http://localhost:9000', + 'access_key': 'test_access_key', + 'secret_key': 'test_secret_key', + 'region': 'us-east-1', + 'use_ssl': False, + 'max_retry_attempts': 3, + 'retry_mode': 'standard', + 'connect_timeout': 30, + 'read_timeout': 60, + } + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_storage_repository_initialization(mock_boto3, mock_logger, storage_config): + """Test StorageRepository initialization with correct parameters.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + mock_s3_client = Mock() + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Verify attributes are set correctly + assert repo.endpoint_url == storage_config['endpoint_url'] + assert repo.access_key == storage_config['access_key'] + assert repo.secret_key == storage_config['secret_key'] + assert repo.region == storage_config['region'] + assert repo.use_ssl == storage_config['use_ssl'] + assert repo.max_retry_attempts == storage_config['max_retry_attempts'] + assert repo.retry_mode == storage_config['retry_mode'] + assert repo.connect_timeout == storage_config['connect_timeout'] + assert repo.read_timeout == storage_config['read_timeout'] + assert repo.logger == mock_logger + + # Verify boto3 client was created + mock_boto3.client.assert_called_once() + call_args = mock_boto3.client.call_args + + assert call_args[0][0] == 's3' + assert call_args[1]['endpoint_url'] == storage_config['endpoint_url'] + assert call_args[1]['aws_access_key_id'] == storage_config['access_key'] + assert call_args[1]['aws_secret_access_key'] == storage_config['secret_key'] + assert call_args[1]['use_ssl'] == storage_config['use_ssl'] + + # Verify logger was called + mock_logger.info.assert_called() + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_storage_repository_boto_config(mock_boto3, mock_logger, storage_config): + """Test that boto3 Config is created with correct retry settings.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + mock_s3_client = Mock() + mock_boto3.client.return_value = mock_s3_client + + StorageRepository(logger=mock_logger, **storage_config) + + # Verify Config was passed with correct settings + call_args = mock_boto3.client.call_args + boto_config = call_args[1]['config'] + + assert boto_config.region_name == storage_config['region'] + assert boto_config.connect_timeout == storage_config['connect_timeout'] + assert boto_config.read_timeout == storage_config['read_timeout'] + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_fetch_file_success(mock_boto3, mock_logger, storage_config): + """Test successful file fetch from MinIO.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + # Setup mock response + file_content = b'test file content' + mock_body = Mock() + mock_body.read.return_value = file_content + mock_body.__enter__ = Mock(return_value=mock_body) + mock_body.__exit__ = Mock(return_value=False) + + mock_response = {'Body': mock_body} + + mock_s3_client = Mock() + mock_s3_client.get_object.return_value = mock_response + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Fetch file + result = repo.fetch_file('test-bucket', 'test-file.csv') + + # Verify result + assert isinstance(result, BytesIO) + assert result.getvalue() == file_content + + # Verify get_object was called correctly + mock_s3_client.get_object.assert_called_once_with(Bucket='test-bucket', Key='test-file.csv') + + # Verify logging + assert mock_logger.info.call_count >= 2 # Init + fetch + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_fetch_file_with_large_content(mock_boto3, mock_logger, storage_config): + """Test fetch file with large content.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + # Setup mock response with large content + large_content = b'x' * 1024 * 1024 # 1MB + mock_body = Mock() + mock_body.read.return_value = large_content + mock_body.__enter__ = Mock(return_value=mock_body) + mock_body.__exit__ = Mock(return_value=False) + + mock_response = {'Body': mock_body} + + mock_s3_client = Mock() + mock_s3_client.get_object.return_value = mock_response + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Fetch file + result = repo.fetch_file('test-bucket', 'large-file.bin') + + # Verify result + assert isinstance(result, BytesIO) + assert len(result.getvalue()) == 1024 * 1024 + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_fetch_file_empty_content(mock_boto3, mock_logger, storage_config): + """Test fetch file with empty content.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + # Setup mock response with empty content + mock_body = Mock() + mock_body.read.return_value = b'' + mock_body.__enter__ = Mock(return_value=mock_body) + mock_body.__exit__ = Mock(return_value=False) + + mock_response = {'Body': mock_body} + + mock_s3_client = Mock() + mock_s3_client.get_object.return_value = mock_response + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Fetch file + result = repo.fetch_file('test-bucket', 'empty-file.txt') + + # Verify result + assert isinstance(result, BytesIO) + assert result.getvalue() == b'' + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_fetch_file_not_found(mock_boto3, mock_logger, storage_config): + """Test fetch file when object doesn't exist.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + # Setup mock to raise NoSuchKey error + mock_s3_client = Mock() + mock_s3_client.get_object.side_effect = ClientError( + {'Error': {'Code': 'NoSuchKey', 'Message': 'The specified key does not exist.'}}, + 'GetObject', + ) + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Attempt to fetch non-existent file + with pytest.raises(ClientError) as exc_info: + repo.fetch_file('test-bucket', 'non-existent.csv') + + assert exc_info.value.response['Error']['Code'] == 'NoSuchKey' + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_fetch_file_access_denied(mock_boto3, mock_logger, storage_config): + """Test fetch file when access is denied.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + # Setup mock to raise AccessDenied error + mock_s3_client = Mock() + mock_s3_client.get_object.side_effect = ClientError( + {'Error': {'Code': 'AccessDenied', 'Message': 'Access Denied'}}, 'GetObject' + ) + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Attempt to fetch file without permissions + with pytest.raises(ClientError) as exc_info: + repo.fetch_file('test-bucket', 'protected-file.csv') + + assert exc_info.value.response['Error']['Code'] == 'AccessDenied' + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_fetch_file_network_error(mock_boto3, mock_logger, storage_config): + """Test fetch file when network error occurs.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + # Setup mock to raise network error + mock_s3_client = Mock() + mock_s3_client.get_object.side_effect = ConnectionError('Network unreachable') + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Attempt to fetch file with network error + with pytest.raises(ConnectionError): + repo.fetch_file('test-bucket', 'test-file.csv') + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_delete_file_success(mock_boto3, mock_logger, storage_config): + """Test successful file deletion from MinIO.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + mock_s3_client = Mock() + mock_s3_client.delete_object.return_value = {} + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Delete file + repo.delete_file('test-bucket', 'test-file.csv') + + # Verify delete_object was called correctly + mock_s3_client.delete_object.assert_called_once_with(Bucket='test-bucket', Key='test-file.csv') + + # Verify logging + assert mock_logger.info.call_count >= 2 # Init + delete + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_delete_file_non_existent(mock_boto3, mock_logger, storage_config): + """Test delete file that doesn't exist (should succeed silently in S3).""" + from model_manager.utils.repository.storage_repository import StorageRepository + + # S3/MinIO delete is idempotent - deleting non-existent file succeeds + mock_s3_client = Mock() + mock_s3_client.delete_object.return_value = {} + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Delete non-existent file (should succeed) + repo.delete_file('test-bucket', 'non-existent.csv') + + mock_s3_client.delete_object.assert_called_once() + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_delete_file_access_denied(mock_boto3, mock_logger, storage_config): + """Test delete file when access is denied.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + # Setup mock to raise AccessDenied error + mock_s3_client = Mock() + mock_s3_client.delete_object.side_effect = ClientError( + {'Error': {'Code': 'AccessDenied', 'Message': 'Access Denied'}}, 'DeleteObject' + ) + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Attempt to delete file without permissions + with pytest.raises(ClientError) as exc_info: + repo.delete_file('test-bucket', 'protected-file.csv') + + assert exc_info.value.response['Error']['Code'] == 'AccessDenied' + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_delete_file_network_error(mock_boto3, mock_logger, storage_config): + """Test delete file when network error occurs.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + # Setup mock to raise network error + mock_s3_client = Mock() + mock_s3_client.delete_object.side_effect = ConnectionError('Network unreachable') + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Attempt to delete file with network error + with pytest.raises(ConnectionError): + repo.delete_file('test-bucket', 'test-file.csv') + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_storage_repository_with_ssl(mock_boto3, mock_logger, storage_config): + """Test StorageRepository initialization with SSL enabled.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + mock_s3_client = Mock() + mock_boto3.client.return_value = mock_s3_client + + storage_config['use_ssl'] = True + repo = StorageRepository(logger=mock_logger, **storage_config) + + assert repo.use_ssl is True + + # Verify boto3 client was created with use_ssl=True + call_args = mock_boto3.client.call_args + assert call_args[1]['use_ssl'] is True + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_storage_repository_custom_timeouts(mock_boto3, mock_logger, storage_config): + """Test StorageRepository with custom timeout values.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + mock_s3_client = Mock() + mock_boto3.client.return_value = mock_s3_client + + storage_config['connect_timeout'] = 10 + storage_config['read_timeout'] = 120 + + repo = StorageRepository(logger=mock_logger, **storage_config) + + assert repo.connect_timeout == 10 + assert repo.read_timeout == 120 + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_storage_repository_custom_retry_mode(mock_boto3, mock_logger, storage_config): + """Test StorageRepository with different retry modes.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + mock_s3_client = Mock() + mock_boto3.client.return_value = mock_s3_client + + for retry_mode in ['standard', 'legacy', 'adaptive']: + storage_config['retry_mode'] = retry_mode + repo = StorageRepository(logger=mock_logger, **storage_config) + assert repo.retry_mode == retry_mode + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_storage_repository_custom_max_retries(mock_boto3, mock_logger, storage_config): + """Test StorageRepository with different max retry attempts.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + mock_s3_client = Mock() + mock_boto3.client.return_value = mock_s3_client + + storage_config['max_retry_attempts'] = 5 + repo = StorageRepository(logger=mock_logger, **storage_config) + + assert repo.max_retry_attempts == 5 + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_fetch_file_with_special_characters(mock_boto3, mock_logger, storage_config): + """Test fetch file with special characters in name.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + file_content = b'test content' + mock_body = Mock() + mock_body.read.return_value = file_content + mock_body.__enter__ = Mock(return_value=mock_body) + mock_body.__exit__ = Mock(return_value=False) + + mock_response = {'Body': mock_body} + + mock_s3_client = Mock() + mock_s3_client.get_object.return_value = mock_response + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Fetch file with special characters + special_filename = 'test file (2023-01-01) #1.csv' + result = repo.fetch_file('test-bucket', special_filename) + + assert isinstance(result, BytesIO) + mock_s3_client.get_object.assert_called_once_with(Bucket='test-bucket', Key=special_filename) + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_delete_file_with_path_separators(mock_boto3, mock_logger, storage_config): + """Test delete file with path separators in object key.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + mock_s3_client = Mock() + mock_s3_client.delete_object.return_value = {} + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + # Delete file with path separators + file_path = 'data/2023/01/test-file.csv' + repo.delete_file('test-bucket', file_path) + + mock_s3_client.delete_object.assert_called_once_with(Bucket='test-bucket', Key=file_path) + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_storage_repository_different_regions(mock_boto3, mock_logger, storage_config): + """Test StorageRepository with different AWS regions.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + mock_s3_client = Mock() + mock_boto3.client.return_value = mock_s3_client + + regions = ['us-west-1', 'eu-central-1', 'ap-southeast-1'] + + for region in regions: + storage_config['region'] = region + repo = StorageRepository(logger=mock_logger, **storage_config) + assert repo.region == region + + +@patch('model_manager.utils.repository.storage_repository.boto3') +def test_fetch_file_logs_file_size(mock_boto3, mock_logger, storage_config): + """Test that fetch_file logs the file size.""" + from model_manager.utils.repository.storage_repository import StorageRepository + + file_content = b'x' * 12345 + mock_body = Mock() + mock_body.read.return_value = file_content + mock_body.__enter__ = Mock(return_value=mock_body) + mock_body.__exit__ = Mock(return_value=False) + + mock_response = {'Body': mock_body} + + mock_s3_client = Mock() + mock_s3_client.get_object.return_value = mock_response + mock_boto3.client.return_value = mock_s3_client + + repo = StorageRepository(logger=mock_logger, **storage_config) + + repo.fetch_file('test-bucket', 'test-file.csv') + + # Verify logging includes file size + log_calls = [str(call) for call in mock_logger.info.call_args_list] + assert any('12345 bytes' in str(call) for call in log_calls) +