sientia-dataops-model-manager/model_manager/activities/activities.py

from temporalio import workflow

with workflow.unsafe.imports_passed_through():
    from typing import Any

    from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
    from sientia_do.observability.logger import Logger
    from sientia_do.observability.metrics_controller import MetricsController
    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
    from sientia_do.repository.minio_repository import MinioRepository
    from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
    from sientia_model.model_repository.plugin_store import PluginStore

    from model_manager.activities.cleanup import Cleanup
    from model_manager.activities.experiment_tracking import ExperimentTracking
    from model_manager.activities.training import Training


class Activities(ExperimentTracking, Training, Cleanup):
    """
    Main activities orchestrator for the Model Manager system.

    This class combines functionality from multiple activity classes to provide
    a unified interface for all workflow operations. It manages database connections,
    MLFlow model interactions, MinIO storage operations, and cleanup operations.

    The class implements multiple inheritance to combine specialized functionality:
    - ExperimentTracking: ML experiment lifecycle tracking and database operations
    - Training: ML model training operations with MLFlow and MinIO integration
    - Cleanup: File and directory cleanup operations for MinIO and local filesystem

    Attributes:
        postgres_config (dict): PostgreSQL connection configuration
        mlflow_config (dict): MLFlow server configuration
        minio_config (dict): MinIO storage configuration
        logger (Logger): Logging and observability instance
        notification_handler (NotificationHandler): Notification management instance
    """

    def __init__(
        self,
        postgres_config: dict[str, Any],
        mlflow_config: dict[str, Any],
        minio_config: dict[str, Any],
        plugin_store: PluginStore,
        logger: Logger,
        notification_handler: NotificationHandler,
        metrics_controller: MetricsController,
    ):
        """
        Initialize the Activities orchestrator with all required configurations.

        This constructor initializes all parent classes with their respective
        configurations and sets up the foundation for all activity operations.

        Args:
            postgres_config: PostgreSQL connection configuration dictionary
                Required keys: host, port, user, password, dbname, min_connections, max_connections
            mlflow_config: MLFlow server configuration dictionary
                Required keys: host, port, username, password
            minio_config: MinIO storage configuration dictionary
                Required keys: endpoint_url, access_key, secret_key, region, use_ssl, default_bucket
            logger: Logger instance for observability and debugging
            notification_handler: Notification handler for alerts and monitoring

        Raises:
            Exception: If any parent class initialization fails
        """

        ExperimentTracking.__init__(
            self,
            host=postgres_config['host'],
            port=postgres_config['port'],
            user=postgres_config['user'],
            password=postgres_config['password'],
            dbname=postgres_config['dbname'],
            min_connections=postgres_config['min_connections'],
            max_connections=postgres_config['max_connections'],
            logger=logger,
            notification_handler=notification_handler,
            metrics_controller=metrics_controller,
        )

        self.mlflow_repository = SientiaMLflowRepository(
            host=mlflow_config['url'],
            username=mlflow_config['username'],
            password=mlflow_config['password'],
            logger=logger,
            notification_handler=notification_handler,
            metrics_controller=metrics_controller,
        )

        # MinIO repository used for all object storage operations
        endpoint_url = minio_config['endpoint_url']
        # MinioRepository expects the endpoint without scheme
        if endpoint_url.startswith('http://'):
            endpoint = endpoint_url.removeprefix('http://')
        elif endpoint_url.startswith('https://'):
            endpoint = endpoint_url.removeprefix('https://')
        else:
            endpoint = endpoint_url

        self.minio_repository = MinioRepository(
            endpoint=endpoint,
            access_key=minio_config['access_key'],
            secret_key=minio_config['secret_key'],
            logger=logger,
            notification_handler=notification_handler,
            metrics_controller=metrics_controller,
            secure=minio_config['use_ssl'],
            bucket=minio_config['default_bucket'],
        )

        Training.__init__(
            self,
            mlflow_repository=self.mlflow_repository,
            plugin_store=plugin_store,
            minio_repository=self.minio_repository,
            logger=logger,
            notification_handler=notification_handler,
            metrics_controller=metrics_controller,
        )

        Cleanup.__init__(
            self,
            logger=logger,
            notification_handler=notification_handler,
            metrics_controller=metrics_controller,
        )

    def __del__(self):
        """
        Destructor to safely handle cleanup during garbage collection.

        This prevents AttributeError when the parent Postgres.__del__ tries to access
        self.engine in objects with multiple inheritance. Only attempts cleanup if
        the engine attribute exists.
        """
        # Only call parent __del__ if engine attribute exists
        # This prevents AttributeError in multiple inheritance scenarios
        if hasattr(self, 'engine'):
            try:
                # Call parent class __del__ if it exists
                if hasattr(super(), '__del__'):  # pragma: no cover
                    super().__del__()  # pragma: no cover
            except Exception:  # noqa: S110, BLE001  # pragma: no cover
                # Silently ignore errors during garbage collection
                # Logging here could cause issues if logger is already destroyed
                pass

    def shutdown(self):
        """
        Gracefully shutdown all activities and clean up resources.

        This method ensures proper cleanup of all resources including:
        - PostgreSQL connection pools (via ExperimentTracking)
        - Any other resources that need explicit cleanup

        The method should be called before the application terminates to ensure
        proper resource cleanup and prevent resource leaks.

        Prefer calling this method explicitly rather than relying on __del__.
        """
        ExperimentTracking.close(self)
        self.info('Postgres client closed')
        SientiaMonitoring.shutdown(self)