SIENTIAPDE-1273

Update dependencies and refactor data handling in various modules

- Updated sientia-dataops-library dependency version from 1.5.3 to 1.5.4 in requirements files.
- Updated sientia-mlops-library dependency version from 0.39.0 to 0.40.2 in requirements files.
- Refactored return types in Gates, MLFlow, and ModelMetrics classes to return dictionaries instead of DataFrames for improved compatibility with downstream systems.
- Removed the temporal_codec module as it is no longer needed for DataFrame serialization.
- Adjusted data handling in the Drift workflow to ensure proper data structure is maintained.

laborious/activities/gates.py

@@ -405,7 +405,7 @@ class Gates(SientiaMonitoring):
 
 
     @activity.defn(name='format_transformed_data')
-    async def format_transformed_data(self, input_data: dict[str, Any]) -> DataFrame:
+    async def format_transformed_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
         Format transformed data according to configured storage policies.
         """
@@ -423,10 +423,10 @@ class Gates(SientiaMonitoring):
         data = data.melt(id_vars='timestamp', var_name='variable', value_name='value')
         data['model_id'] = model_id
 
-        return data
+        return data.to_dict()
 
     @activity.defn(name='format_prediction')
-    async def format_prediction(self, input_data: dict[str, Any]) -> DataFrame:
+    async def format_prediction(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
         Format prediction data according to configured storage policies.
 
@@ -496,10 +496,10 @@ class Gates(SientiaMonitoring):
         self.info(f'Prediction formatted: {len(data)} rows', metadata)
         self.debug(f'Prediction data: {data.head(5).to_string()}', metadata)
 
-        return data
+        return data.to_dict()
 
     @activity.defn(name='format_default_prediction')
-    async def format_default_prediction(self, input_data: dict[str, Any]) -> DataFrame:
+    async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
         Create and format default prediction data for error conditions.
 
@@ -541,10 +541,10 @@ class Gates(SientiaMonitoring):
         )
 
         self.info(f'Default prediction formatted: {data.size} rows', metadata)
-        return data
+        return data.to_dict()
 
     @activity.defn(name='format_retrain_report')
-    async def format_retrain_report(self, input_data: dict[str, Any]) -> DataFrame:
+    async def format_retrain_report(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
         Format retrain report data according to configured storage policies.
         """
@@ -573,7 +573,7 @@ class Gates(SientiaMonitoring):
 
         self.debug(f'Retrain report: {report.to_csv()}', metadata)
 
-        return report
+        return report.to_dict()
 
     @activity.defn(name='get_last_timestamp')
     async def get_last_timestamp(self, input_data: dict[str, Any]) -> str:

laborious/activities/mlflow.py

@@ -423,7 +423,7 @@ class MLFlow(SientiaMonitoring):
 
 
     @activity.defn(name='get_reference_data')
-    async def get_reference_data(self, input_data: dict[str, Any]) -> DataFrame | None:
+    async def get_reference_data(self, input_data: dict[str, Any]) -> list[dict[Hashable, Any]] | None:
         """
         Get reference data from the MLflow Model Registry.
 
@@ -433,7 +433,7 @@ class MLFlow(SientiaMonitoring):
                 - model_name (str): Name of the MLFlow model to get reference data from
 
         Returns:
-            dict[Hashable, Any] | None: Reference data from the MLflow Model Registry.
+            list[dict[Hashable, Any]] | None: Reference data from the MLflow Model Registry.
 
         """
 
@@ -452,4 +452,4 @@ class MLFlow(SientiaMonitoring):
         reference_data['timestamp'] = to_datetime(reference_data['timestamp'])
         reference_data['timestamp'] = reference_data['timestamp'].dt.strftime(DATETIME_FORMAT)
 
-        return reference_data
+        return reference_data.to_dict(orient='records')

laborious/activities/model_metrics.py

@@ -138,7 +138,7 @@ class ModelMetrics(SientiaMonitoring):
 
 
     @activity.defn(name='calculate_drift')
-    async def calculate_drift(self, input_data: dict[str, Any]) -> DataFrame | dict:
+    async def calculate_drift(self, input_data: dict[str, Any]) -> list[dict[Hashable, Any]]:
         """
         Calculate drift metrics for a model.
 
@@ -217,11 +217,11 @@ class ModelMetrics(SientiaMonitoring):
                 level=NotificationLevel.ERROR,
                 attachment_content=traceback.format_exc(),
             )
-            return {}
+            return []
 
         if drift_df.empty:
             self.warning('No drift metrics found', metadata)
-            return {}
+            return []
 
         # Drop unnecessary columns
         drift_df.drop(columns=['p_value', 'chunk_start_date', 'chunk_end_date'], inplace=True)
@@ -238,7 +238,7 @@ class ModelMetrics(SientiaMonitoring):
 
         if drift_df.empty:
             self.warning('No drift metrics found after dropping rows where timestamp is not in target data', metadata)
-            return {}
+            return []
 
         # Rename columns to match database columns
         drift_df.rename(columns={
@@ -261,10 +261,12 @@ class ModelMetrics(SientiaMonitoring):
 
         self.debug(f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata)
 
+        self.debug(f'Drift dataframe: {drift_df.head(5).to_string()}', metadata)
         
-        return drift_df
+        return drift_df.to_dict(orient='records')
 
-    async def calculate_simple_metrics(self, input_data: dict[str, Any]) -> DataFrame:
+    @activity.defn(name='calculate_simple_metrics')
+    async def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict[Hashable, Any]]:
         """
         Calculate simple metrics for a model. Metrics available are:
             - rmse
@@ -342,6 +344,6 @@ class ModelMetrics(SientiaMonitoring):
 
         self.debug(f'Simple metrics dataframe: Size {data.shape} \n{data.head(5).to_string()}', metadata)
 
-        return data
+        return data.to_dict(orient='records')