SIENTIAPDE-1273

Update dependencies and refactor data handling in various modules

- Updated sientia-dataops-library dependency version from 1.5.3 to 1.5.4 in requirements files.
- Updated sientia-mlops-library dependency version from 0.39.0 to 0.40.2 in requirements files.
- Refactored return types in Gates, MLFlow, and ModelMetrics classes to return dictionaries instead of DataFrames for improved compatibility with downstream systems.
- Removed the temporal_codec module as it is no longer needed for DataFrame serialization.
- Adjusted data handling in the Drift workflow to ensure proper data structure is maintained.

laborious/activities/gates.py

@@ -405,7 +405,7 @@ class Gates(SientiaMonitoring):
 
 
     @activity.defn(name='format_transformed_data')
-    async def format_transformed_data(self, input_data: dict[str, Any]) -> DataFrame:
+    async def format_transformed_data(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
         Format transformed data according to configured storage policies.
         """
@@ -423,10 +423,10 @@ class Gates(SientiaMonitoring):
         data = data.melt(id_vars='timestamp', var_name='variable', value_name='value')
         data['model_id'] = model_id
 
-        return data
+        return data.to_dict()
 
     @activity.defn(name='format_prediction')
-    async def format_prediction(self, input_data: dict[str, Any]) -> DataFrame:
+    async def format_prediction(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
         Format prediction data according to configured storage policies.
 
@@ -496,10 +496,10 @@ class Gates(SientiaMonitoring):
         self.info(f'Prediction formatted: {len(data)} rows', metadata)
         self.debug(f'Prediction data: {data.head(5).to_string()}', metadata)
 
-        return data
+        return data.to_dict()
 
     @activity.defn(name='format_default_prediction')
-    async def format_default_prediction(self, input_data: dict[str, Any]) -> DataFrame:
+    async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
         Create and format default prediction data for error conditions.
 
@@ -541,10 +541,10 @@ class Gates(SientiaMonitoring):
         )
 
         self.info(f'Default prediction formatted: {data.size} rows', metadata)
-        return data
+        return data.to_dict()
 
     @activity.defn(name='format_retrain_report')
-    async def format_retrain_report(self, input_data: dict[str, Any]) -> DataFrame:
+    async def format_retrain_report(self, input_data: dict[str, Any]) -> dict[str, Any]:
         """
         Format retrain report data according to configured storage policies.
         """
@@ -573,7 +573,7 @@ class Gates(SientiaMonitoring):
 
         self.debug(f'Retrain report: {report.to_csv()}', metadata)
 
-        return report
+        return report.to_dict()
 
     @activity.defn(name='get_last_timestamp')
     async def get_last_timestamp(self, input_data: dict[str, Any]) -> str: