SIENTIAPDE-1273

Refactor data handling in various modules to ensure DataFrame consistency

- Replaced direct DataFrame instantiation with `ensure_dataframe` utility in Gates, MLFlow, OPC, and ModelMetrics classes to standardize data handling.
- Updated return types in several asynchronous methods to return DataFrames instead of dictionaries for improved usability.
- Adjusted data export processes in workflows to convert DataFrames to dictionaries with `to_dict(orient='records')` for compatibility with downstream systems.

laborious/activities/gates.py

@@ -15,6 +15,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
 
     from laborious import metrics
+    from laborious.utils.dataframe_utils import ensure_dataframe
     from laborious.utils.filters.conditional_filters import (
         filter_empty_data,
         filter_specific_variables_null_values,
@@ -149,7 +150,7 @@ class Gates(SientiaMonitoring):
         self.info('Performing input gate...', metadata)
 
         filters = input_data['filters']
-        data = DataFrame(input_data['data'])
+        data = ensure_dataframe(input_data['data'])
         path_priority = input_data['path_priority']
 
         filter_output = []
@@ -404,7 +405,7 @@ class Gates(SientiaMonitoring):
 
 
     @activity.defn(name='format_transformed_data')
-    async def format_transformed_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+    async def format_transformed_data(self, input_data: dict[str, Any]) -> DataFrame:
         """
         Format transformed data according to configured storage policies.
         """
@@ -414,7 +415,7 @@ class Gates(SientiaMonitoring):
 
         self.info('Formatting transformed data...', metadata)
 
-        data = DataFrame(input_data['data'])
+        data = ensure_dataframe(input_data['data'])
 
         data['timestamp'] = data.index
         data = data.reset_index(drop=True)
@@ -422,10 +423,10 @@ class Gates(SientiaMonitoring):
         data = data.melt(id_vars='timestamp', var_name='variable', value_name='value')
         data['model_id'] = model_id
 
-        return data.to_dict()
+        return data
 
     @activity.defn(name='format_prediction')
-    async def format_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+    async def format_prediction(self, input_data: dict[str, Any]) -> DataFrame:
         """
         Format prediction data according to configured storage policies.
 
@@ -453,7 +454,7 @@ class Gates(SientiaMonitoring):
         prediction_store_policy = input_data['prediction_store_policy']
         self.info('Formatting prediction...', metadata)
 
-        data = DataFrame(input_data['data'])
+        data = ensure_dataframe(input_data['data'])
 
         # Create timestamp column from index and reset index
         data['timestamp'] = data.index
@@ -495,10 +496,10 @@ class Gates(SientiaMonitoring):
         self.info(f'Prediction formatted: {len(data)} rows', metadata)
         self.debug(f'Prediction data: {data.head(5).to_string()}', metadata)
 
-        return data.to_dict()
+        return data
 
     @activity.defn(name='format_default_prediction')
-    async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+    async def format_default_prediction(self, input_data: dict[str, Any]) -> DataFrame:
         """
         Create and format default prediction data for error conditions.
 
@@ -540,10 +541,10 @@ class Gates(SientiaMonitoring):
         )
 
         self.info(f'Default prediction formatted: {data.size} rows', metadata)
-        return data.to_dict()
+        return data
 
     @activity.defn(name='format_retrain_report')
-    async def format_retrain_report(self, input_data: dict[str, Any]) -> dict[Any, Any]:
+    async def format_retrain_report(self, input_data: dict[str, Any]) -> DataFrame:
         """
         Format retrain report data according to configured storage policies.
         """
@@ -572,7 +573,7 @@ class Gates(SientiaMonitoring):
 
         self.debug(f'Retrain report: {report.to_csv()}', metadata)
 
-        return report.to_dict()
+        return report
 
     @activity.defn(name='get_last_timestamp')
     async def get_last_timestamp(self, input_data: dict[str, Any]) -> str:
@@ -601,7 +602,7 @@ class Gates(SientiaMonitoring):
 
         self.info('Getting last timestamp...', metadata)
 
-        data = DataFrame(input_data['data'])
+        data = ensure_dataframe(input_data['data'])
 
         self.debug(f'Input data: {data.head(5).to_string()}', metadata)