Code import - branch release/SIENTIAPDE-1645
This commit is contained in:
166
model_manager/activities/activities.py
Normal file
166
model_manager/activities/activities.py
Normal file
@@ -0,0 +1,166 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.repository.minio_repository_sync import MinioRepository
|
||||
from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
|
||||
from sientia_model.model_repository.plugin_store import PluginStore
|
||||
|
||||
from model_manager.activities.cleanup import Cleanup
|
||||
from model_manager.activities.experiment_tracking import ExperimentTracking
|
||||
from model_manager.activities.training import Training
|
||||
|
||||
|
||||
class Activities(ExperimentTracking, Training, Cleanup):
|
||||
"""
|
||||
Main activities orchestrator for the Model Manager system.
|
||||
|
||||
This class combines functionality from multiple activity classes to provide
|
||||
a unified interface for all workflow operations. It manages database connections,
|
||||
MLFlow model interactions, MinIO storage operations, and cleanup operations.
|
||||
|
||||
The class implements multiple inheritance to combine specialized functionality:
|
||||
- ExperimentTracking: ML experiment lifecycle tracking and database operations
|
||||
- Training: ML model training operations with MLFlow and MinIO integration
|
||||
- Cleanup: File and directory cleanup operations for MinIO and local filesystem
|
||||
|
||||
Attributes:
|
||||
postgres_config (dict): PostgreSQL connection configuration
|
||||
mlflow_config (dict): MLFlow server configuration
|
||||
minio_config (dict): MinIO storage configuration
|
||||
logger (Logger): Logging and observability instance
|
||||
notification_handler (NotificationHandler): Notification management instance
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
postgres_config: dict[str, Any],
|
||||
mlflow_config: dict[str, Any],
|
||||
minio_config: dict[str, Any],
|
||||
plugin_store: PluginStore,
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
):
|
||||
"""
|
||||
Initialize the Activities orchestrator with all required configurations.
|
||||
|
||||
This constructor initializes all parent classes with their respective
|
||||
configurations and sets up the foundation for all activity operations.
|
||||
|
||||
Args:
|
||||
postgres_config: PostgreSQL connection configuration dictionary
|
||||
Required keys: host, port, user, password, dbname, min_connections, max_connections
|
||||
mlflow_config: MLFlow server configuration dictionary
|
||||
Required keys: host, port, username, password
|
||||
minio_config: MinIO storage configuration dictionary
|
||||
Required keys: endpoint_url, access_key, secret_key, region, use_ssl, default_bucket
|
||||
logger: Logger instance for observability and debugging
|
||||
notification_handler: Notification handler for alerts and monitoring
|
||||
|
||||
Raises:
|
||||
Exception: If any parent class initialization fails
|
||||
"""
|
||||
|
||||
ExperimentTracking.__init__(
|
||||
self,
|
||||
host=postgres_config['host'],
|
||||
port=postgres_config['port'],
|
||||
user=postgres_config['user'],
|
||||
password=postgres_config['password'],
|
||||
dbname=postgres_config['dbname'],
|
||||
min_connections=postgres_config['min_connections'],
|
||||
max_connections=postgres_config['max_connections'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
self.mlflow_repository = SientiaMLflowRepository(
|
||||
host=mlflow_config['url'],
|
||||
username=mlflow_config['username'],
|
||||
password=mlflow_config['password'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
# MinIO repository used for all object storage operations
|
||||
endpoint_url = minio_config['endpoint_url']
|
||||
# MinioRepository expects the endpoint without scheme
|
||||
if endpoint_url.startswith('http://'):
|
||||
endpoint = endpoint_url.removeprefix('http://')
|
||||
elif endpoint_url.startswith('https://'):
|
||||
endpoint = endpoint_url.removeprefix('https://')
|
||||
else:
|
||||
endpoint = endpoint_url
|
||||
|
||||
self.minio_repository = MinioRepository(
|
||||
endpoint=endpoint,
|
||||
access_key=minio_config['access_key'],
|
||||
secret_key=minio_config['secret_key'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
secure=minio_config['use_ssl'],
|
||||
bucket=minio_config['default_bucket'],
|
||||
)
|
||||
|
||||
Training.__init__(
|
||||
self,
|
||||
mlflow_repository=self.mlflow_repository,
|
||||
plugin_store=plugin_store,
|
||||
minio_repository=self.minio_repository,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
Cleanup.__init__(
|
||||
self,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
def __del__(self):
|
||||
"""
|
||||
Destructor to safely handle cleanup during garbage collection.
|
||||
|
||||
This prevents AttributeError when the parent Postgres.__del__ tries to access
|
||||
self.engine in objects with multiple inheritance. Only attempts cleanup if
|
||||
the engine attribute exists.
|
||||
"""
|
||||
# Only call parent __del__ if engine attribute exists
|
||||
# This prevents AttributeError in multiple inheritance scenarios
|
||||
if hasattr(self, 'engine'):
|
||||
try:
|
||||
# Call parent class __del__ if it exists
|
||||
if hasattr(super(), '__del__'): # pragma: no cover
|
||||
super().__del__() # pragma: no cover
|
||||
except Exception: # noqa: S110, BLE001 # pragma: no cover
|
||||
# Silently ignore errors during garbage collection
|
||||
# Logging here could cause issues if logger is already destroyed
|
||||
pass
|
||||
|
||||
def shutdown(self):
|
||||
"""
|
||||
Gracefully shutdown all activities and clean up resources.
|
||||
|
||||
This method ensures proper cleanup of all resources including:
|
||||
- PostgreSQL connection pools (via ExperimentTracking)
|
||||
- Any other resources that need explicit cleanup
|
||||
|
||||
The method should be called before the application terminates to ensure
|
||||
proper resource cleanup and prevent resource leaks.
|
||||
|
||||
Prefer calling this method explicitly rather than relying on __del__.
|
||||
"""
|
||||
ExperimentTracking.close(self)
|
||||
self.info('Postgres client closed')
|
||||
SientiaMonitoring.shutdown(self)
|
||||
Reference in New Issue
Block a user