SIENTIAPDE-1248: Integrate MinIO for object storage and add related configurations

This commit introduces MinIO integration for object storage within the Model Manager system. It includes:

- Added MinIO activity class for file operations (fetch, delete).
- Updated Activities orchestrator to include MinIO activities.
- Added MinIO configuration builder to utils/connectors_config.py.
- Added environment variables for MinIO configuration in .env.example.
- Added boto3 and botocore dependencies to requirements.txt.
- Added unit tests for MinIO activities.
This commit is contained in:
Bruno Domingues
2025-10-03 21:04:27 -03:00
parent 11c25d126c
commit 9afe711075
12 changed files with 850 additions and 7 deletions

View File

@@ -87,3 +87,38 @@ def build_mongodb_config() -> dict[str, Any]:
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600,
}
def build_minio_config() -> dict[str, Any]:
"""
Build MinIO (S3-compatible) configuration from environment variables.
This function constructs a MinIO configuration dictionary from
environment variables with sensible defaults for local development.
It handles endpoint URL, authentication, connection parameters, and retry policies.
Environment Variables:
MINIO_ENDPOINT_URL: MinIO server endpoint URL (default: http://localhost:9000)
MINIO_ACCESS_KEY: MinIO access key ID (default: minioadmin)
MINIO_SECRET_KEY: MinIO secret access key (default: minioadmin)
MINIO_REGION: MinIO region name (default: us-east-1)
MINIO_USE_SSL: Whether to use SSL/TLS (default: false)
MINIO_MAX_RETRY_ATTEMPTS: Maximum number of retry attempts (default: 3)
MINIO_RETRY_MODE: Retry mode - standard, legacy, or adaptive (default: adaptive)
MINIO_CONNECT_TIMEOUT: Connection timeout in seconds (default: 10)
MINIO_READ_TIMEOUT: Read timeout in seconds (default: 60)
Returns:
dict: MinIO configuration dictionary with all required parameters
"""
return {
'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'),
'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'),
'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'),
'region': getenv('MINIO_REGION', 'us-east-1'),
'use_ssl': getenv('MINIO_USE_SSL', 'false').lower() == 'true',
'max_retry_attempts': int(getenv('MINIO_MAX_RETRY_ATTEMPTS', '3')),
'retry_mode': getenv('MINIO_RETRY_MODE', 'adaptive'),
'connect_timeout': int(getenv('MINIO_CONNECT_TIMEOUT', '10')),
'read_timeout': int(getenv('MINIO_READ_TIMEOUT', '60')),
}