SIENTIAPDE-1241: Storage repository tests

tests/utils/repository/test_storage_repository.py

@@ -0,0 +1,465 @@
+"""Unit tests for StorageRepository class."""
+
+from io import BytesIO
+from unittest.mock import MagicMock, Mock, patch
+
+import pytest
+from botocore.exceptions import ClientError
+
+
+@pytest.fixture
+def mock_logger():
+    """Create a mock logger for testing."""
+    logger = Mock()
+    logger.info = Mock()
+    logger.error = Mock()
+    logger.warning = Mock()
+    return logger
+
+
+@pytest.fixture
+def storage_config():
+    """Create storage repository configuration."""
+    return {
+        'endpoint_url': 'http://localhost:9000',
+        'access_key': 'test_access_key',
+        'secret_key': 'test_secret_key',
+        'region': 'us-east-1',
+        'use_ssl': False,
+        'max_retry_attempts': 3,
+        'retry_mode': 'standard',
+        'connect_timeout': 30,
+        'read_timeout': 60,
+    }
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_storage_repository_initialization(mock_boto3, mock_logger, storage_config):
+    """Test StorageRepository initialization with correct parameters."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    mock_s3_client = Mock()
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Verify attributes are set correctly
+    assert repo.endpoint_url == storage_config['endpoint_url']
+    assert repo.access_key == storage_config['access_key']
+    assert repo.secret_key == storage_config['secret_key']
+    assert repo.region == storage_config['region']
+    assert repo.use_ssl == storage_config['use_ssl']
+    assert repo.max_retry_attempts == storage_config['max_retry_attempts']
+    assert repo.retry_mode == storage_config['retry_mode']
+    assert repo.connect_timeout == storage_config['connect_timeout']
+    assert repo.read_timeout == storage_config['read_timeout']
+    assert repo.logger == mock_logger
+
+    # Verify boto3 client was created
+    mock_boto3.client.assert_called_once()
+    call_args = mock_boto3.client.call_args
+
+    assert call_args[0][0] == 's3'
+    assert call_args[1]['endpoint_url'] == storage_config['endpoint_url']
+    assert call_args[1]['aws_access_key_id'] == storage_config['access_key']
+    assert call_args[1]['aws_secret_access_key'] == storage_config['secret_key']
+    assert call_args[1]['use_ssl'] == storage_config['use_ssl']
+
+    # Verify logger was called
+    mock_logger.info.assert_called()
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_storage_repository_boto_config(mock_boto3, mock_logger, storage_config):
+    """Test that boto3 Config is created with correct retry settings."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    mock_s3_client = Mock()
+    mock_boto3.client.return_value = mock_s3_client
+
+    StorageRepository(logger=mock_logger, **storage_config)
+
+    # Verify Config was passed with correct settings
+    call_args = mock_boto3.client.call_args
+    boto_config = call_args[1]['config']
+
+    assert boto_config.region_name == storage_config['region']
+    assert boto_config.connect_timeout == storage_config['connect_timeout']
+    assert boto_config.read_timeout == storage_config['read_timeout']
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_fetch_file_success(mock_boto3, mock_logger, storage_config):
+    """Test successful file fetch from MinIO."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    # Setup mock response
+    file_content = b'test file content'
+    mock_body = Mock()
+    mock_body.read.return_value = file_content
+    mock_body.__enter__ = Mock(return_value=mock_body)
+    mock_body.__exit__ = Mock(return_value=False)
+
+    mock_response = {'Body': mock_body}
+
+    mock_s3_client = Mock()
+    mock_s3_client.get_object.return_value = mock_response
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Fetch file
+    result = repo.fetch_file('test-bucket', 'test-file.csv')
+
+    # Verify result
+    assert isinstance(result, BytesIO)
+    assert result.getvalue() == file_content
+
+    # Verify get_object was called correctly
+    mock_s3_client.get_object.assert_called_once_with(Bucket='test-bucket', Key='test-file.csv')
+
+    # Verify logging
+    assert mock_logger.info.call_count >= 2  # Init + fetch
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_fetch_file_with_large_content(mock_boto3, mock_logger, storage_config):
+    """Test fetch file with large content."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    # Setup mock response with large content
+    large_content = b'x' * 1024 * 1024  # 1MB
+    mock_body = Mock()
+    mock_body.read.return_value = large_content
+    mock_body.__enter__ = Mock(return_value=mock_body)
+    mock_body.__exit__ = Mock(return_value=False)
+
+    mock_response = {'Body': mock_body}
+
+    mock_s3_client = Mock()
+    mock_s3_client.get_object.return_value = mock_response
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Fetch file
+    result = repo.fetch_file('test-bucket', 'large-file.bin')
+
+    # Verify result
+    assert isinstance(result, BytesIO)
+    assert len(result.getvalue()) == 1024 * 1024
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_fetch_file_empty_content(mock_boto3, mock_logger, storage_config):
+    """Test fetch file with empty content."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    # Setup mock response with empty content
+    mock_body = Mock()
+    mock_body.read.return_value = b''
+    mock_body.__enter__ = Mock(return_value=mock_body)
+    mock_body.__exit__ = Mock(return_value=False)
+
+    mock_response = {'Body': mock_body}
+
+    mock_s3_client = Mock()
+    mock_s3_client.get_object.return_value = mock_response
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Fetch file
+    result = repo.fetch_file('test-bucket', 'empty-file.txt')
+
+    # Verify result
+    assert isinstance(result, BytesIO)
+    assert result.getvalue() == b''
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_fetch_file_not_found(mock_boto3, mock_logger, storage_config):
+    """Test fetch file when object doesn't exist."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    # Setup mock to raise NoSuchKey error
+    mock_s3_client = Mock()
+    mock_s3_client.get_object.side_effect = ClientError(
+        {'Error': {'Code': 'NoSuchKey', 'Message': 'The specified key does not exist.'}},
+        'GetObject',
+    )
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Attempt to fetch non-existent file
+    with pytest.raises(ClientError) as exc_info:
+        repo.fetch_file('test-bucket', 'non-existent.csv')
+
+    assert exc_info.value.response['Error']['Code'] == 'NoSuchKey'
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_fetch_file_access_denied(mock_boto3, mock_logger, storage_config):
+    """Test fetch file when access is denied."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    # Setup mock to raise AccessDenied error
+    mock_s3_client = Mock()
+    mock_s3_client.get_object.side_effect = ClientError(
+        {'Error': {'Code': 'AccessDenied', 'Message': 'Access Denied'}}, 'GetObject'
+    )
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Attempt to fetch file without permissions
+    with pytest.raises(ClientError) as exc_info:
+        repo.fetch_file('test-bucket', 'protected-file.csv')
+
+    assert exc_info.value.response['Error']['Code'] == 'AccessDenied'
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_fetch_file_network_error(mock_boto3, mock_logger, storage_config):
+    """Test fetch file when network error occurs."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    # Setup mock to raise network error
+    mock_s3_client = Mock()
+    mock_s3_client.get_object.side_effect = ConnectionError('Network unreachable')
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Attempt to fetch file with network error
+    with pytest.raises(ConnectionError):
+        repo.fetch_file('test-bucket', 'test-file.csv')
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_delete_file_success(mock_boto3, mock_logger, storage_config):
+    """Test successful file deletion from MinIO."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    mock_s3_client = Mock()
+    mock_s3_client.delete_object.return_value = {}
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Delete file
+    repo.delete_file('test-bucket', 'test-file.csv')
+
+    # Verify delete_object was called correctly
+    mock_s3_client.delete_object.assert_called_once_with(Bucket='test-bucket', Key='test-file.csv')
+
+    # Verify logging
+    assert mock_logger.info.call_count >= 2  # Init + delete
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_delete_file_non_existent(mock_boto3, mock_logger, storage_config):
+    """Test delete file that doesn't exist (should succeed silently in S3)."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    # S3/MinIO delete is idempotent - deleting non-existent file succeeds
+    mock_s3_client = Mock()
+    mock_s3_client.delete_object.return_value = {}
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Delete non-existent file (should succeed)
+    repo.delete_file('test-bucket', 'non-existent.csv')
+
+    mock_s3_client.delete_object.assert_called_once()
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_delete_file_access_denied(mock_boto3, mock_logger, storage_config):
+    """Test delete file when access is denied."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    # Setup mock to raise AccessDenied error
+    mock_s3_client = Mock()
+    mock_s3_client.delete_object.side_effect = ClientError(
+        {'Error': {'Code': 'AccessDenied', 'Message': 'Access Denied'}}, 'DeleteObject'
+    )
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Attempt to delete file without permissions
+    with pytest.raises(ClientError) as exc_info:
+        repo.delete_file('test-bucket', 'protected-file.csv')
+
+    assert exc_info.value.response['Error']['Code'] == 'AccessDenied'
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_delete_file_network_error(mock_boto3, mock_logger, storage_config):
+    """Test delete file when network error occurs."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    # Setup mock to raise network error
+    mock_s3_client = Mock()
+    mock_s3_client.delete_object.side_effect = ConnectionError('Network unreachable')
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Attempt to delete file with network error
+    with pytest.raises(ConnectionError):
+        repo.delete_file('test-bucket', 'test-file.csv')
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_storage_repository_with_ssl(mock_boto3, mock_logger, storage_config):
+    """Test StorageRepository initialization with SSL enabled."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    mock_s3_client = Mock()
+    mock_boto3.client.return_value = mock_s3_client
+
+    storage_config['use_ssl'] = True
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    assert repo.use_ssl is True
+
+    # Verify boto3 client was created with use_ssl=True
+    call_args = mock_boto3.client.call_args
+    assert call_args[1]['use_ssl'] is True
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_storage_repository_custom_timeouts(mock_boto3, mock_logger, storage_config):
+    """Test StorageRepository with custom timeout values."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    mock_s3_client = Mock()
+    mock_boto3.client.return_value = mock_s3_client
+
+    storage_config['connect_timeout'] = 10
+    storage_config['read_timeout'] = 120
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    assert repo.connect_timeout == 10
+    assert repo.read_timeout == 120
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_storage_repository_custom_retry_mode(mock_boto3, mock_logger, storage_config):
+    """Test StorageRepository with different retry modes."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    mock_s3_client = Mock()
+    mock_boto3.client.return_value = mock_s3_client
+
+    for retry_mode in ['standard', 'legacy', 'adaptive']:
+        storage_config['retry_mode'] = retry_mode
+        repo = StorageRepository(logger=mock_logger, **storage_config)
+        assert repo.retry_mode == retry_mode
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_storage_repository_custom_max_retries(mock_boto3, mock_logger, storage_config):
+    """Test StorageRepository with different max retry attempts."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    mock_s3_client = Mock()
+    mock_boto3.client.return_value = mock_s3_client
+
+    storage_config['max_retry_attempts'] = 5
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    assert repo.max_retry_attempts == 5
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_fetch_file_with_special_characters(mock_boto3, mock_logger, storage_config):
+    """Test fetch file with special characters in name."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    file_content = b'test content'
+    mock_body = Mock()
+    mock_body.read.return_value = file_content
+    mock_body.__enter__ = Mock(return_value=mock_body)
+    mock_body.__exit__ = Mock(return_value=False)
+
+    mock_response = {'Body': mock_body}
+
+    mock_s3_client = Mock()
+    mock_s3_client.get_object.return_value = mock_response
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Fetch file with special characters
+    special_filename = 'test file (2023-01-01) #1.csv'
+    result = repo.fetch_file('test-bucket', special_filename)
+
+    assert isinstance(result, BytesIO)
+    mock_s3_client.get_object.assert_called_once_with(Bucket='test-bucket', Key=special_filename)
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_delete_file_with_path_separators(mock_boto3, mock_logger, storage_config):
+    """Test delete file with path separators in object key."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    mock_s3_client = Mock()
+    mock_s3_client.delete_object.return_value = {}
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    # Delete file with path separators
+    file_path = 'data/2023/01/test-file.csv'
+    repo.delete_file('test-bucket', file_path)
+
+    mock_s3_client.delete_object.assert_called_once_with(Bucket='test-bucket', Key=file_path)
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_storage_repository_different_regions(mock_boto3, mock_logger, storage_config):
+    """Test StorageRepository with different AWS regions."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    mock_s3_client = Mock()
+    mock_boto3.client.return_value = mock_s3_client
+
+    regions = ['us-west-1', 'eu-central-1', 'ap-southeast-1']
+
+    for region in regions:
+        storage_config['region'] = region
+        repo = StorageRepository(logger=mock_logger, **storage_config)
+        assert repo.region == region
+
+
+@patch('model_manager.utils.repository.storage_repository.boto3')
+def test_fetch_file_logs_file_size(mock_boto3, mock_logger, storage_config):
+    """Test that fetch_file logs the file size."""
+    from model_manager.utils.repository.storage_repository import StorageRepository
+
+    file_content = b'x' * 12345
+    mock_body = Mock()
+    mock_body.read.return_value = file_content
+    mock_body.__enter__ = Mock(return_value=mock_body)
+    mock_body.__exit__ = Mock(return_value=False)
+
+    mock_response = {'Body': mock_body}
+
+    mock_s3_client = Mock()
+    mock_s3_client.get_object.return_value = mock_response
+    mock_boto3.client.return_value = mock_s3_client
+
+    repo = StorageRepository(logger=mock_logger, **storage_config)
+
+    repo.fetch_file('test-bucket', 'test-file.csv')
+
+    # Verify logging includes file size
+    log_calls = [str(call) for call in mock_logger.info.call_args_list]
+    assert any('12345 bytes' in str(call) for call in log_calls)
+