SIENTIAPDE-1248: Integrate MinIO for object storage and add related configurations
This commit introduces MinIO integration for object storage within the Model Manager system. It includes: - Added MinIO activity class for file operations (fetch, delete). - Updated Activities orchestrator to include MinIO activities. - Added MinIO configuration builder to utils/connectors_config.py. - Added environment variables for MinIO configuration in .env.example. - Added boto3 and botocore dependencies to requirements.txt. - Added unit tests for MinIO activities.
This commit is contained in:
@@ -87,3 +87,38 @@ def build_mongodb_config() -> dict[str, Any]:
|
||||
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
|
||||
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600,
|
||||
}
|
||||
|
||||
|
||||
def build_minio_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build MinIO (S3-compatible) configuration from environment variables.
|
||||
|
||||
This function constructs a MinIO configuration dictionary from
|
||||
environment variables with sensible defaults for local development.
|
||||
It handles endpoint URL, authentication, connection parameters, and retry policies.
|
||||
|
||||
Environment Variables:
|
||||
MINIO_ENDPOINT_URL: MinIO server endpoint URL (default: http://localhost:9000)
|
||||
MINIO_ACCESS_KEY: MinIO access key ID (default: minioadmin)
|
||||
MINIO_SECRET_KEY: MinIO secret access key (default: minioadmin)
|
||||
MINIO_REGION: MinIO region name (default: us-east-1)
|
||||
MINIO_USE_SSL: Whether to use SSL/TLS (default: false)
|
||||
MINIO_MAX_RETRY_ATTEMPTS: Maximum number of retry attempts (default: 3)
|
||||
MINIO_RETRY_MODE: Retry mode - standard, legacy, or adaptive (default: adaptive)
|
||||
MINIO_CONNECT_TIMEOUT: Connection timeout in seconds (default: 10)
|
||||
MINIO_READ_TIMEOUT: Read timeout in seconds (default: 60)
|
||||
|
||||
Returns:
|
||||
dict: MinIO configuration dictionary with all required parameters
|
||||
"""
|
||||
return {
|
||||
'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'),
|
||||
'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'),
|
||||
'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'),
|
||||
'region': getenv('MINIO_REGION', 'us-east-1'),
|
||||
'use_ssl': getenv('MINIO_USE_SSL', 'false').lower() == 'true',
|
||||
'max_retry_attempts': int(getenv('MINIO_MAX_RETRY_ATTEMPTS', '3')),
|
||||
'retry_mode': getenv('MINIO_RETRY_MODE', 'adaptive'),
|
||||
'connect_timeout': int(getenv('MINIO_CONNECT_TIMEOUT', '10')),
|
||||
'read_timeout': int(getenv('MINIO_READ_TIMEOUT', '60')),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user