SIENTIAPDE-1248: Integrate MinIO for object storage and add related configurations
This commit introduces MinIO integration for object storage within the Model Manager system. It includes: - Added MinIO activity class for file operations (fetch, delete). - Updated Activities orchestrator to include MinIO activities. - Added MinIO configuration builder to utils/connectors_config.py. - Added environment variables for MinIO configuration in .env.example. - Added boto3 and botocore dependencies to requirements.txt. - Added unit tests for MinIO activities.
This commit is contained in:
@@ -8,25 +8,28 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.temporal.activities.postgres import Postgres
|
||||
|
||||
from model_manager.activities.gates import Gates
|
||||
from model_manager.activities.minio import MinIO
|
||||
from model_manager.activities.mlflow import MLFlow
|
||||
|
||||
|
||||
class Activities(Postgres, MLFlow, Gates):
|
||||
class Activities(Postgres, MLFlow, MinIO, Gates):
|
||||
"""
|
||||
Main activities orchestrator for the Model Manager system.
|
||||
|
||||
This class combines functionality from multiple activity classes to provide
|
||||
a unified interface for all workflow operations. It manages database connections,
|
||||
MLFlow model interactions, and data quality validation.
|
||||
MLFlow model interactions, MinIO storage operations, and data quality validation.
|
||||
|
||||
The class implements multiple inheritance to combine specialized functionality:
|
||||
- Postgres: Database operations and data persistence
|
||||
- MLFlow: Model inference and transformation operations
|
||||
- MinIO: Object storage operations (file upload/download/delete)
|
||||
- Gates: Data quality validation and filtering mechanisms
|
||||
|
||||
Attributes:
|
||||
postgres_config (dict): PostgreSQL connection configuration
|
||||
mlflow_config (dict): MLFlow server configuration
|
||||
minio_config (dict): MinIO storage configuration
|
||||
logger (Logger): Logging and observability instance
|
||||
notification_handler (NotificationHandler): Notification management instance
|
||||
"""
|
||||
@@ -35,6 +38,7 @@ class Activities(Postgres, MLFlow, Gates):
|
||||
self,
|
||||
postgres_config: dict[str, Any],
|
||||
mlflow_config: dict[str, Any],
|
||||
minio_config: dict[str, Any],
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
):
|
||||
@@ -49,6 +53,8 @@ class Activities(Postgres, MLFlow, Gates):
|
||||
Required keys: host, port, user, password, dbname, min_connections, max_connections
|
||||
mlflow_config: MLFlow server configuration dictionary
|
||||
Required keys: host, port, username, password
|
||||
minio_config: MinIO storage configuration dictionary
|
||||
Required keys: endpoint_url, access_key, secret_key, region, use_ssl
|
||||
logger: Logger instance for observability and debugging
|
||||
notification_handler: Notification handler for alerts and monitoring
|
||||
|
||||
@@ -79,6 +85,21 @@ class Activities(Postgres, MLFlow, Gates):
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
MinIO.__init__(
|
||||
self,
|
||||
endpoint_url=minio_config['endpoint_url'],
|
||||
access_key=minio_config['access_key'],
|
||||
secret_key=minio_config['secret_key'],
|
||||
region=minio_config['region'],
|
||||
use_ssl=minio_config['use_ssl'],
|
||||
max_retry_attempts=minio_config['max_retry_attempts'],
|
||||
retry_mode=minio_config['retry_mode'],
|
||||
connect_timeout=minio_config['connect_timeout'],
|
||||
read_timeout=minio_config['read_timeout'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
)
|
||||
|
||||
Gates.__init__(self, logger=logger, notification_handler=notification_handler)
|
||||
|
||||
async def shutdown(self):
|
||||
|
||||
Reference in New Issue
Block a user