SIENTIAPDE-1248: Integrate MinIO for object storage and add related configurations

This commit introduces MinIO integration for object storage within the Model Manager system. It includes:

- Added MinIO activity class for file operations (fetch, delete).
- Updated Activities orchestrator to include MinIO activities.
- Added MinIO configuration builder to utils/connectors_config.py.
- Added environment variables for MinIO configuration in .env.example.
- Added boto3 and botocore dependencies to requirements.txt.
- Added unit tests for MinIO activities.
This commit is contained in:
Bruno Domingues
2025-10-03 21:04:27 -03:00
parent 11c25d126c
commit 9afe711075
12 changed files with 850 additions and 7 deletions

View File

@@ -8,25 +8,28 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.temporal.activities.postgres import Postgres
from model_manager.activities.gates import Gates
from model_manager.activities.minio import MinIO
from model_manager.activities.mlflow import MLFlow
class Activities(Postgres, MLFlow, Gates):
class Activities(Postgres, MLFlow, MinIO, Gates):
"""
Main activities orchestrator for the Model Manager system.
This class combines functionality from multiple activity classes to provide
a unified interface for all workflow operations. It manages database connections,
MLFlow model interactions, and data quality validation.
MLFlow model interactions, MinIO storage operations, and data quality validation.
The class implements multiple inheritance to combine specialized functionality:
- Postgres: Database operations and data persistence
- MLFlow: Model inference and transformation operations
- MinIO: Object storage operations (file upload/download/delete)
- Gates: Data quality validation and filtering mechanisms
Attributes:
postgres_config (dict): PostgreSQL connection configuration
mlflow_config (dict): MLFlow server configuration
minio_config (dict): MinIO storage configuration
logger (Logger): Logging and observability instance
notification_handler (NotificationHandler): Notification management instance
"""
@@ -35,6 +38,7 @@ class Activities(Postgres, MLFlow, Gates):
self,
postgres_config: dict[str, Any],
mlflow_config: dict[str, Any],
minio_config: dict[str, Any],
logger: Logger,
notification_handler: NotificationHandler,
):
@@ -49,6 +53,8 @@ class Activities(Postgres, MLFlow, Gates):
Required keys: host, port, user, password, dbname, min_connections, max_connections
mlflow_config: MLFlow server configuration dictionary
Required keys: host, port, username, password
minio_config: MinIO storage configuration dictionary
Required keys: endpoint_url, access_key, secret_key, region, use_ssl
logger: Logger instance for observability and debugging
notification_handler: Notification handler for alerts and monitoring
@@ -79,6 +85,21 @@ class Activities(Postgres, MLFlow, Gates):
notification_handler=notification_handler,
)
MinIO.__init__(
self,
endpoint_url=minio_config['endpoint_url'],
access_key=minio_config['access_key'],
secret_key=minio_config['secret_key'],
region=minio_config['region'],
use_ssl=minio_config['use_ssl'],
max_retry_attempts=minio_config['max_retry_attempts'],
retry_mode=minio_config['retry_mode'],
connect_timeout=minio_config['connect_timeout'],
read_timeout=minio_config['read_timeout'],
logger=logger,
notification_handler=notification_handler,
)
Gates.__init__(self, logger=logger, notification_handler=notification_handler)
async def shutdown(self):