SIENTIAPDE-1273

Update dependencies and refactor data handling in various modules

- Updated sientia-dataops-library dependency version from 1.5.3 to 1.5.4 in requirements files.
- Updated sientia-mlops-library dependency version from 0.39.0 to 0.40.2 in requirements files.
- Refactored return types in Gates, MLFlow, and ModelMetrics classes to return dictionaries instead of DataFrames for improved compatibility with downstream systems.
- Removed the temporal_codec module as it is no longer needed for DataFrame serialization.
- Adjusted data handling in the Drift workflow to ensure proper data structure is maintained.

laborious/activities/model_metrics.py

@@ -138,7 +138,7 @@ class ModelMetrics(SientiaMonitoring):
 
 
     @activity.defn(name='calculate_drift')
-    async def calculate_drift(self, input_data: dict[str, Any]) -> DataFrame | dict:
+    async def calculate_drift(self, input_data: dict[str, Any]) -> list[dict[Hashable, Any]]:
         """
         Calculate drift metrics for a model.
 
@@ -217,11 +217,11 @@ class ModelMetrics(SientiaMonitoring):
                 level=NotificationLevel.ERROR,
                 attachment_content=traceback.format_exc(),
             )
-            return {}
+            return []
 
         if drift_df.empty:
             self.warning('No drift metrics found', metadata)
-            return {}
+            return []
 
         # Drop unnecessary columns
         drift_df.drop(columns=['p_value', 'chunk_start_date', 'chunk_end_date'], inplace=True)
@@ -238,7 +238,7 @@ class ModelMetrics(SientiaMonitoring):
 
         if drift_df.empty:
             self.warning('No drift metrics found after dropping rows where timestamp is not in target data', metadata)
-            return {}
+            return []
 
         # Rename columns to match database columns
         drift_df.rename(columns={
@@ -261,10 +261,12 @@ class ModelMetrics(SientiaMonitoring):
 
         self.debug(f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata)
 
+        self.debug(f'Drift dataframe: {drift_df.head(5).to_string()}', metadata)
         
-        return drift_df
+        return drift_df.to_dict(orient='records')
 
-    async def calculate_simple_metrics(self, input_data: dict[str, Any]) -> DataFrame:
+    @activity.defn(name='calculate_simple_metrics')
+    async def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict[Hashable, Any]]:
         """
         Calculate simple metrics for a model. Metrics available are:
             - rmse
@@ -342,6 +344,6 @@ class ModelMetrics(SientiaMonitoring):
 
         self.debug(f'Simple metrics dataframe: Size {data.shape} \n{data.head(5).to_string()}', metadata)
 
-        return data
+        return data.to_dict(orient='records')