SIENTIAPDE-1717: Remove list_bucket_objects method from StorageRepository and its associated test.
This commit is contained in:
@@ -118,36 +118,3 @@ class StorageRepository:
|
|||||||
)
|
)
|
||||||
|
|
||||||
return BytesIO(file_content)
|
return BytesIO(file_content)
|
||||||
|
|
||||||
def list_bucket_objects(self, bucket_name: str, max_keys: int = 1000) -> list[str]:
|
|
||||||
"""
|
|
||||||
List objects in a MinIO bucket.
|
|
||||||
|
|
||||||
This method uses the MinIO/S3 list_objects_v2 API to retrieve objects
|
|
||||||
from the specified bucket. This is optimized for cleanup operations
|
|
||||||
by using configurable pagination.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
bucket_name: Name of the bucket to list objects from.
|
|
||||||
max_keys: Maximum number of keys per page (default: 1000).
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
List[str]: List of object keys (file names).
|
|
||||||
"""
|
|
||||||
# Use list_objects_v2 for efficient pagination
|
|
||||||
paginator = self.minio_client.get_paginator('list_objects_v2')
|
|
||||||
|
|
||||||
pages = paginator.paginate(Bucket=bucket_name, MaxKeys=max_keys)
|
|
||||||
|
|
||||||
objects = []
|
|
||||||
total_count = 0
|
|
||||||
|
|
||||||
for page in pages:
|
|
||||||
if 'Contents' in page:
|
|
||||||
for obj in page['Contents']:
|
|
||||||
objects.append(obj['Key'])
|
|
||||||
total_count += 1
|
|
||||||
|
|
||||||
self.logger.info(f'Listed {total_count} objects from bucket {bucket_name}')
|
|
||||||
|
|
||||||
return objects
|
|
||||||
|
|||||||
@@ -382,37 +382,3 @@ def test_close_method(mock_boto3, mock_logger, storage_config):
|
|||||||
|
|
||||||
mock_s3_client.close.assert_called_once()
|
mock_s3_client.close.assert_called_once()
|
||||||
mock_logger.info.assert_called_with('MinIO client closed')
|
mock_logger.info.assert_called_with('MinIO client closed')
|
||||||
|
|
||||||
|
|
||||||
@patch('model_manager.utils.repository.storage_repository.boto3')
|
|
||||||
def test_list_bucket_objects_with_pagination(mock_boto3, mock_logger, storage_config):
|
|
||||||
"""Test list_bucket_objects with a paginated response."""
|
|
||||||
from model_manager.utils.repository.storage_repository import StorageRepository
|
|
||||||
|
|
||||||
mock_s3_client = Mock()
|
|
||||||
mock_paginator = Mock()
|
|
||||||
page1 = {
|
|
||||||
'Contents': [
|
|
||||||
{'Key': 'file1.txt'},
|
|
||||||
{'Key': 'file2.txt'},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
page2 = {
|
|
||||||
'Contents': [
|
|
||||||
{'Key': 'file3.txt'},
|
|
||||||
]
|
|
||||||
}
|
|
||||||
page3 = {}
|
|
||||||
|
|
||||||
mock_paginator.paginate.return_value = [page1, page2, page3]
|
|
||||||
mock_s3_client.get_paginator.return_value = mock_paginator
|
|
||||||
mock_boto3.client.return_value = mock_s3_client
|
|
||||||
|
|
||||||
repo = StorageRepository(logger=mock_logger, **storage_config)
|
|
||||||
objects = repo.list_bucket_objects('test-bucket', max_keys=2)
|
|
||||||
|
|
||||||
assert objects == ['file1.txt', 'file2.txt', 'file3.txt']
|
|
||||||
assert len(objects) == 3
|
|
||||||
mock_s3_client.get_paginator.assert_called_once_with('list_objects_v2')
|
|
||||||
mock_paginator.paginate.assert_called_once_with(Bucket='test-bucket', MaxKeys=2)
|
|
||||||
mock_logger.info.assert_any_call('Listed 3 objects from bucket test-bucket')
|
|
||||||
|
|||||||
Reference in New Issue
Block a user