SIENTIAPDE-1712

SIENTIAPDE-1712 Refactor debug logging for DataFrames across multiple classes. Introduced a new method to log DataFrame content conditionally based on row count in Gates, MLFlow, ModelMetrics, and MLFlowRepository classes, improving debugging capabilities while managing log output effectively.

laborious/activities/gates.py

@@ -16,6 +16,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.utils.formatters import create_sample_dict
 
     from laborious import metrics
+    from laborious.utils.dataframe_debug import build_dataframe_debug_message
     from laborious.utils.filters.conditional_filters import (
         filter_empty_data,
         filter_specific_variables_null_values,
@@ -86,6 +87,7 @@ class Gates(MinioManager):
     """
 
     minio_repository: MinioRepository | None = None
+    _MAX_DEBUG_DATAFRAME_ROWS = 100
 
     def __init__(
         self,
@@ -118,6 +120,24 @@ class Gates(MinioManager):
     def __del__(self):
         self.close()
 
+    def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None:
+        """
+        Log dataframe content only when row count is below the configured threshold
+
+        Args:
+         - message (str): Base log message to identify the dataframe in logs
+         - data (Any): Dataframe-like payload to be logged
+         - metadata (dict[str, Any]): Workflow metadata for contextual logging
+        """
+        self.debug(
+            build_dataframe_debug_message(
+                message=message,
+                data=data,
+                max_rows=self._MAX_DEBUG_DATAFRAME_ROWS,
+            ),
+            metadata,
+        )
+
     @staticmethod
     def _read_filter_entry(config: dict[str, Any]) -> tuple[str, dict[str, Any]]:
         """
@@ -179,7 +199,7 @@ class Gates(MinioManager):
 
         filter_output = []
 
-        self.debug(f'Input data: {data.head(5).to_string()}', metadata)
+        self._debug_dataframe('Input data:', data, metadata)
         self.debug(f'Filters: {filters}', metadata)
 
         # Apply each configured filter
@@ -355,7 +375,7 @@ class Gates(MinioManager):
 
         filter_output = []
 
-        self.debug(f'Input data:\n {data.head(5).to_string()}', metadata)
+        self._debug_dataframe('Input data:', data, metadata)
         self.debug(f'Filters: \n {filters}', metadata)
 
         for fil, config in filters.items():
@@ -544,7 +564,7 @@ class Gates(MinioManager):
         data = data.reset_index(drop=True)
 
         self.debug(f'Prediction store policy: {prediction_store_policy}', metadata)
-        self.debug(f'Prediction data: {data.head(5).to_string()}', metadata)
+        self._debug_dataframe('Prediction data:', data, metadata)
 
         policy_type, policy_value = self.get_prediction_store_policy(
             prediction_store_policy, metadata
@@ -581,7 +601,7 @@ class Gates(MinioManager):
         data = data.reset_index(drop=True)
 
         self.info(f'Prediction formatted: {len(data)} rows', metadata)
-        self.debug(f'Prediction data: {data.head(5).to_string()}', metadata)
+        self._debug_dataframe('Prediction data:', data, metadata)
 
         return data.to_dict()
 
@@ -696,7 +716,7 @@ class Gates(MinioManager):
             report['mlflow_run_id'] = update_report['mlflow_run_id']
             report['mlflow_experiment_id'] = update_report['mlflow_experiment_id']
 
-        self.debug(f'Retrain report: {report.to_csv()}', metadata)
+        self._debug_dataframe('Retrain report:', report, metadata)
 
         return report.to_dict()