SIENTIAPDE-1712

SIENTIAPDE-1712 Refactor debug logging for DataFrames across multiple classes. Introduced a new method to log DataFrame content conditionally based on row count in Gates, MLFlow, ModelMetrics, and MLFlowRepository classes, improving debugging capabilities while managing log output effectively.

laborious/activities/model_metrics.py

@@ -17,6 +17,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
 
     from laborious import metrics
+    from laborious.utils.dataframe_debug import build_dataframe_debug_message
 
 warnings.filterwarnings('ignore', category=RuntimeWarning, message='Degrees of freedom <= 0')
 warnings.filterwarnings(
@@ -26,6 +27,8 @@ warnings.filterwarnings(
 
 class ModelMetrics(SientiaMonitoring):
     """
+    _MAX_DEBUG_DATAFRAME_ROWS = 100
+
     Metrics activities for the Laborious system.
 
     This class provides activities for writing metrics to the Prometheus monitoring system.
@@ -48,6 +51,24 @@ class ModelMetrics(SientiaMonitoring):
     def __del__(self):
         self.close()
 
+    def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None:
+        """
+        Log dataframe content only when row count is below the configured threshold
+
+        Args:
+         - message (str): Base log message to identify the dataframe in logs
+         - data (Any): Dataframe-like payload to be logged
+         - metadata (dict[str, Any]): Workflow metadata for contextual logging
+        """
+        self.debug(
+            build_dataframe_debug_message(
+                message=message,
+                data=data,
+                max_rows=self._MAX_DEBUG_DATAFRAME_ROWS,
+            ),
+            metadata,
+        )
+
     async def get_drift_metrics(
         self,
         reference_data: DataFrame,
@@ -78,14 +99,9 @@ class ModelMetrics(SientiaMonitoring):
 
         model_analysis = ModelAnalysis(config=config)
 
-        self.debug(
-            f'Reference data: Size {reference_data.shape} \n{reference_data.head(5).to_string()}',
-            metadata,
-        )
+        self._debug_dataframe(f'Reference data: Size {reference_data.shape}', reference_data, metadata)
 
-        self.debug(
-            f'Target data: Size {target_data.shape} \n{target_data.head(5).to_string()}', metadata
-        )
+        self._debug_dataframe(f'Target data: Size {target_data.shape}', target_data, metadata)
 
         core_labels = self.get_core_labels(metadata, operation_type='detect_univariate_drift')
         start_time = time.time()
@@ -142,9 +158,7 @@ class ModelMetrics(SientiaMonitoring):
         await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
         await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
 
-        self.debug(
-            f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata
-        )
+        self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
 
         return drift_df
 
@@ -274,11 +288,7 @@ class ModelMetrics(SientiaMonitoring):
         drift_df['timestamp'] = drift_df['timestamp'].dt.tz_localize('UTC')
         drift_df['timestamp'] = drift_df['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
 
-        self.debug(
-            f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata
-        )
-
-        self.debug(f'Drift dataframe: {drift_df.head(5).to_string()}', metadata)
+        self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
 
         return drift_df.to_dict(orient='records')
 
@@ -347,8 +357,6 @@ class ModelMetrics(SientiaMonitoring):
         data['data_size'] = data_size
         data['interval_minutes'] = interval_minutes
 
-        self.debug(
-            f'Simple metrics dataframe: Size {data.shape} \n{data.head(5).to_string()}', metadata
-        )
+        self._debug_dataframe(f'Simple metrics dataframe: Size {data.shape}', data, metadata)
 
         return data.to_dict(orient='records')