SIENTIAPDE-1222

SIENTIAPDE-1222
Enhance MLFlow logging with sample dictionary for response data

- Introduced a new method to create a sample dictionary for debugging, allowing for better visualization of nested data structures in logs.
- Updated debug logging to utilize the new sampling method for raw and transformed response data, improving clarity and reducing output size.
- Adjusted logging for processed input data to display only the first few rows, enhancing readability.

laborious/activities/mlflow.py

@@ -17,6 +17,39 @@ with workflow.unsafe.imports_passed_through():
     import traceback
 
 
+def create_sample_dict(data: dict, max_items: int = 3, max_depth: int = 2) -> dict:
+    """
+    Create a sample of a dictionary for debugging purposes.
+
+    Args:
+        data: Dictionary to sample
+        max_items: Maximum number of items to show per level
+        max_depth: Maximum depth to traverse nested structures
+
+    Returns:
+        Dictionary with sampled content
+    """
+    if max_depth <= 0:
+        return {"...": "max_depth_reached"}
+
+    sample = {}
+    items = list(data.items())[:max_items]
+
+    for key, value in items:
+        if isinstance(value, dict):
+            sample[key] = create_sample_dict(value, max_items, max_depth - 1)
+        elif isinstance(value, list):
+            sample[key] = value[:max_items] if len(
+                value) > max_items else value
+        else:
+            sample[key] = value
+
+    if len(data) > max_items:
+        sample["..."] = f"({len(data) - max_items} more items)"
+
+    return sample
+
+
 class MLFlow(BaseActivity):
     """
     MLFlow integration activities for model inference operations.
@@ -138,7 +171,7 @@ class MLFlow(BaseActivity):
         model_config = input_data.get('model_config', {})
 
         self.debug("Raw input data:", metadata)
-        self.debug(data, metadata)
+        self.debug(data.head(5).to_string(), metadata)
 
         # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
         data = data.sort_values('created_at', ascending=False).drop_duplicates(
@@ -154,7 +187,7 @@ class MLFlow(BaseActivity):
         data.columns.name = None
 
         self.debug("Processed input data:", metadata)
-        self.debug(data.to_string(), metadata)
+        self.debug(data.head(5).to_string(), metadata)
 
         # Request transformation from MLFlow model
         response_data = self.model_monitoring_repository.transform(
@@ -162,7 +195,8 @@ class MLFlow(BaseActivity):
         )
 
         self.debug("Transform raw response data:", metadata)
-        self.debug(f"{response_data}", metadata)
+        self.debug(create_sample_dict(
+            response_data), metadata)
 
         if response_data['success']:
 
@@ -194,7 +228,8 @@ class MLFlow(BaseActivity):
             response_data['content'] = response_dataframe.to_dict()
 
         self.debug("Transform response data:", metadata)
-        self.debug(response_data, metadata)
+        self.debug(create_sample_dict(
+            response_data), metadata)
 
         self.info("Data transformed successfully", metadata)
 
@@ -236,7 +271,7 @@ class MLFlow(BaseActivity):
         model_name = input_data['model_name']
         model_config = input_data.get('model_config', {})
 
-        self.debug(data, metadata)
+        self.debug(data.head(5).to_string(), metadata)
 
         # Convert numpy.nan to None for model compatibility
         data.replace(np.nan, None, inplace=True)
@@ -247,7 +282,8 @@ class MLFlow(BaseActivity):
         )
 
         self.debug("Prediction response data:", metadata)
-        self.debug(json.dumps(response_data, indent=4), metadata)
+        self.debug(create_sample_dict(
+            response_data), metadata)
 
         self.info("Data predicted successfully", metadata)