From 1a9c1a31c5b7cdfd225800908ebd3b506884c52e Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Tue, 16 Sep 2025 08:16:30 -0300 Subject: [PATCH] SIENTIAPDE-1222 SIENTIAPDE-1222 Enhance MLFlow logging with sample dictionary for response data - Introduced a new method to create a sample dictionary for debugging, allowing for better visualization of nested data structures in logs. - Updated debug logging to utilize the new sampling method for raw and transformed response data, improving clarity and reducing output size. - Adjusted logging for processed input data to display only the first few rows, enhancing readability. --- laborious/activities/mlflow.py | 48 +++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 5f5b12c..c11bbaa 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -17,6 +17,39 @@ with workflow.unsafe.imports_passed_through(): import traceback +def create_sample_dict(data: dict, max_items: int = 3, max_depth: int = 2) -> dict: + """ + Create a sample of a dictionary for debugging purposes. + + Args: + data: Dictionary to sample + max_items: Maximum number of items to show per level + max_depth: Maximum depth to traverse nested structures + + Returns: + Dictionary with sampled content + """ + if max_depth <= 0: + return {"...": "max_depth_reached"} + + sample = {} + items = list(data.items())[:max_items] + + for key, value in items: + if isinstance(value, dict): + sample[key] = create_sample_dict(value, max_items, max_depth - 1) + elif isinstance(value, list): + sample[key] = value[:max_items] if len( + value) > max_items else value + else: + sample[key] = value + + if len(data) > max_items: + sample["..."] = f"({len(data) - max_items} more items)" + + return sample + + class MLFlow(BaseActivity): """ MLFlow integration activities for model inference operations. @@ -138,7 +171,7 @@ class MLFlow(BaseActivity): model_config = input_data.get('model_config', {}) self.debug("Raw input data:", metadata) - self.debug(data, metadata) + self.debug(data.head(5).to_string(), metadata) # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair data = data.sort_values('created_at', ascending=False).drop_duplicates( @@ -154,7 +187,7 @@ class MLFlow(BaseActivity): data.columns.name = None self.debug("Processed input data:", metadata) - self.debug(data.to_string(), metadata) + self.debug(data.head(5).to_string(), metadata) # Request transformation from MLFlow model response_data = self.model_monitoring_repository.transform( @@ -162,7 +195,8 @@ class MLFlow(BaseActivity): ) self.debug("Transform raw response data:", metadata) - self.debug(f"{response_data}", metadata) + self.debug(create_sample_dict( + response_data), metadata) if response_data['success']: @@ -194,7 +228,8 @@ class MLFlow(BaseActivity): response_data['content'] = response_dataframe.to_dict() self.debug("Transform response data:", metadata) - self.debug(response_data, metadata) + self.debug(create_sample_dict( + response_data), metadata) self.info("Data transformed successfully", metadata) @@ -236,7 +271,7 @@ class MLFlow(BaseActivity): model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) - self.debug(data, metadata) + self.debug(data.head(5).to_string(), metadata) # Convert numpy.nan to None for model compatibility data.replace(np.nan, None, inplace=True) @@ -247,7 +282,8 @@ class MLFlow(BaseActivity): ) self.debug("Prediction response data:", metadata) - self.debug(json.dumps(response_data, indent=4), metadata) + self.debug(create_sample_dict( + response_data), metadata) self.info("Data predicted successfully", metadata)