SIENTIAPDE-1273

Update requirements and enhance metrics and data handling

- Updated the sientia-dataops-library dependency version in requirements.txt to 1.5.3.
- Added new metrics for model analysis, including lag, count, and error count in metrics.py.
- Implemented a new method for formatting transformed data in gates.py.
- Enhanced MLFlowRepository with methods to load artifact dataframes and calculate model metrics, including drift and performance metrics.
- Updated the prediction process to handle transformed data and ensure proper execution of related activities in format_and_export_prediction.py and prediction_process.py.

laborious/utils/repository/model_repository.py

@@ -16,6 +16,7 @@ Capabilities:
 
 import ctypes
 import gc
+from io import StringIO
 import threading
 import time
 import traceback
@@ -33,6 +34,7 @@ from sientia_do.observability.logger import Logger
 from sientia_do.observability.metrics_controller import MetricsController
 from sientia_do.observability.sientia_monitoring import SientiaMonitoring
 from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
+from sientia.ModelAnalysis import ModelAnalysis
 
 from laborious import metrics
 
@@ -259,6 +261,40 @@ class MLFlowRepository(SientiaMonitoring):
 
         return artifacts
 
+
+    async def load_artifact_dataframe(self, model_name: str, artifact_path: str,
+                                         metadata: dict[str, Any]) -> pd.DataFrame:
+        
+        """
+        Load the dataframe content of an artifact from the MLflow Model Registry.
+
+        Args:
+            model_name (str): The name of the model to download from the registry.
+            artifact_path (str): The path to the artifact to load.
+            metadata (dict[str, Any]): Metadata used for structured logging.
+
+        Returns:
+            pd.DataFrame: The dataframe content of the artifact.
+        """
+        run_id = self.get_model_run_id(model_name=model_name, stage='Production')
+        artifact_path = path.join("runs:/", run_id, artifact_path)
+        core_labels = self.get_core_labels(metadata, operation_type='load_text')
+
+        start_time = time.time()
+        try:
+            content = mlflow.artifacts.load_text(artifact_path)
+        except Exception as e:
+            await self.emit_metric(metric_object=metrics.MODEL_READ_ERROR_COUNT, tags=core_labels)
+            raise e
+
+        await self.observe_lag(start_time, metrics.MODEL_READ_LAG, core_labels)
+        await self.emit_metric(metric_object=metrics.MODEL_READ_COUNT, tags=core_labels)
+
+        dataframe = pd.read_csv(StringIO(content))
+
+        self.info(f'Loaded dataframe from {model_name}:{artifact_path}', metadata)
+        return dataframe
+
     async def load_predict_model(
         self, model_name: str, metadata: dict[str, Any], flavor: str = 'sklearn'
     ) -> Any:
@@ -1335,3 +1371,60 @@ class MLFlowRepository(SientiaMonitoring):
         metadata_result['mlflow_experiment_id'] = experiment_id
 
         return metadata_result
+
+    async def get_model_metrics(self, 
+        analyse_data: pd.DataFrame,
+        train_reference_data: pd.DataFrame,
+        test_reference_data: pd.DataFrame,
+        target_name: str,
+        drift_metrics: list[str],
+        performance_metrics: list[str],
+        metadata: dict) -> dict[str, Any]:
+
+        """
+        Calculates drift and preformance metrics for a model.
+
+        Args:
+            analyse_data (pd.DataFrame): The data to analyse.
+            train_reference_data (pd.DataFrame): The train reference data.
+            test_reference_data (pd.DataFrame): The test reference data.
+            target_name (str): The target name.
+            drift_metrics (list[str]): The drift metrics.
+            performance_metrics (list[str]): The performance metrics.
+            metadata (dict): The metadata.
+
+        Returns:
+            dict: The metrics.
+        """
+        columns = train_reference_data.drop(
+            columns=[target_name, 'timestamp', 'target', 'prediction'], errors='ignore').columns
+
+        config = {
+            "target": target_name,
+            "prediction": "prediction",
+            "timestamp": "timestamp",
+            "columns": columns,
+        }
+
+        model_analysis = ModelAnalysis(config=config)
+
+        core_labels = self.get_core_labels(metadata, operation_type='detect_univariate_drift')
+
+        start_time = time.time()
+        try:
+            model_analysis.detect_univariate_drift(
+                reference_df=train_reference_data,
+                analyse_df=analyse_data,
+                features=columns,
+                timestamp_column=config['timestamp'],
+                metrics=drift_metrics,
+                chunk_period="s"
+            )
+        except Exception as e:
+            await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
+            raise e
+
+        await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
+        await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
+
+        return {}