SIENTIAPDE-1325

Refactor monitoring and metrics integration across various components

- Removed coverage options from `pyproject.toml`.
- Updated prediction metrics in `README.md` to replace `pipeline_name` with `workflow_name`.
- Upgraded `sientia-dataops-library` dependency version in `requirements-light.txt` and `requirements.txt`.
- Enhanced metrics handling in `laborious` activities, including `Activities`, `Gates`, `MLFlow`, and `OPC`, to utilize a new `MetricsController`.
- Refactored metric emission methods to improve clarity and consistency across the codebase.
- Updated tests to reflect changes in metrics handling and ensure proper functionality.

laborious/activities/mlflow.py

@@ -10,7 +10,8 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.notifications.models import NotificationLevel
     from sientia_do.observability.logger import Logger
-    from sientia_do.temporal.activities.base import BaseActivity
+    from sientia_do.observability.metrics_controller import MetricsController
+    from sientia_do.observability.sientia_monitoring import SientiaMonitoring
     from sientia_do.temporal.constants import (
         DATETIME_FORMAT,
         DATETIME_FORMAT_MS_WITH_TZ,
@@ -22,7 +23,7 @@ with workflow.unsafe.imports_passed_through():
     from laborious.utils.repository.model_repository import MLFlowRepository
 
 
-class MLFlow(BaseActivity):
+class MLFlow(SientiaMonitoring):
     """
     MLFlow integration activities for model inference operations.
 
@@ -50,6 +51,7 @@ class MLFlow(BaseActivity):
         mlflow_password: str,
         logger: Logger,
         notification_handler: NotificationHandler,
+        metrics_controller: MetricsController,
     ):
         """
         Initialize MLFlow activities with server configuration.
@@ -65,14 +67,19 @@ class MLFlow(BaseActivity):
         Raises:
             Exception: If MLFlowRepository initialization fails
         """
-        BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True)
+        SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
         self.mlflow_host = mlflow_host
         self.mlflow_port = mlflow_port
         self.mlflow_username = mlflow_username
         self.mlflow_password = mlflow_password
 
         self.model_monitoring_repository = MLFlowRepository(
-            f'{mlflow_host}:{mlflow_port}', mlflow_username, mlflow_password, logger
+            f'{mlflow_host}:{mlflow_port}',
+            mlflow_username,
+            mlflow_password,
+            logger,
+            notification_handler,
+            metrics_controller,
         )
 
         if not hasattr(self, 'minio_repository'):
@@ -87,8 +94,18 @@ class MLFlow(BaseActivity):
                 minio_secret_key=minio_config['secret_key'],
                 minio_region_name=minio_config['region_name'],
                 minio_default_bucket=minio_config['default_bucket'],
+                metrics_controller=metrics_controller,
             )
 
+    def close(self) -> None:
+        """
+        Close the MLFlow activity and clean up resources.
+        """
+        SientiaMonitoring.shutdown(self)
+
+    def __del__(self):
+        self.close()
+
     @activity.defn(name='request_transform')
     async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
@@ -143,7 +160,7 @@ class MLFlow(BaseActivity):
         self.debug(data.head(5).to_string(), metadata)
 
         # Request transformation from MLFlow model
-        response_data = self.model_monitoring_repository.transform(
+        response_data = await self.model_monitoring_repository.transform(
             model_name, data, model_config, metadata
         )
 
@@ -208,7 +225,7 @@ class MLFlow(BaseActivity):
         ).dt.strftime(DATETIME_FORMAT)
 
         # Request prediction from MLFlow model
-        response_data = self.model_monitoring_repository.predict(
+        response_data = await self.model_monitoring_repository.predict(
             model_name, data, model_config, metadata
         )
 
@@ -263,12 +280,12 @@ class MLFlow(BaseActivity):
         self.info(f'Loading retrain data from Key: {object_key}', metadata)
 
         try:
-            data = self.minio_repository.get_parquet_as_dataframe(
+            data = await self.minio_repository.get_parquet_as_dataframe(
                 object_key=object_key, metadata=metadata
             )
         except Exception as e:
             trace = traceback.format_exc()
-            self.send_notification(
+            await self.send_notification_async(
                 metadata=metadata,
                 notification_id='ERROR_LOADING_RETRAIN_DATA',
                 message=f'Error loading retrain data: {e}',
@@ -316,13 +333,13 @@ class MLFlow(BaseActivity):
 
         data.columns.name = None
 
-        retrain_output = self.model_monitoring_repository.retrain_model(
+        retrain_output = await self.model_monitoring_repository.retrain_model(
             data=data, model_name=model_name, model_config=model_config, metadata=metadata
         )
 
         if not retrain_output['success']:
             trace = retrain_output['traceback']
-            self.send_notification(
+            await self.send_notification_async(
                 metadata=metadata,
                 notification_id='RETRAIN_MODEL_ERROR',
                 message=f'Error retraining model {model_name}: {retrain_output["message"]}',
@@ -379,7 +396,7 @@ class MLFlow(BaseActivity):
         )
 
         try:
-            response = self.model_monitoring_repository.update_production_model(
+            response = await self.model_monitoring_repository.update_production_model(
                 experiment=experiment, model_name=model_name, metadata=metadata
             )
 
@@ -388,7 +405,7 @@ class MLFlow(BaseActivity):
 
         except Exception as e:
             trace = traceback.format_exc()
-            self.send_notification(
+            await self.send_notification_async(
                 metadata=metadata,
                 notification_id='UPDATE_PRODUCTION_MODEL_ERROR',
                 message=f'Error updating production model {model_name}: {e}',