SIENTIAPDE-1222

Update requirements and enhance logging in Gates and MLFlow activities

- Updated the sientia-dataops-library and sientia-mlops-library dependencies in requirements.txt to the latest versions.
- Improved debug logging in the Gates activity to display a sample of input data and filters, enhancing clarity and reducing output size.
- Refactored MLFlow activity logging to utilize the create_sample_dict function for better visualization of nested data structures in logs.

laborious/activities/gates.py

@@ -8,6 +8,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.temporal.activities.base import BaseActivity
     from sientia_do.observability.logger import Logger
     from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
+    from sientia_do.formatters import create_sample_dict
     from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter
     from typing import Any
     from laborious.utils.filters.conditional_filters import (
@@ -122,15 +123,13 @@ class Gates(BaseActivity):
 
         self.info("Performing input gate...", metadata)
 
-        self.debug(f"Input data: {input_data}", metadata)
-
         filters = input_data['filters']
         data = DataFrame(input_data['data'])
         path_priority = input_data['path_priority']
 
         filter_output = []
 
-        self.debug(f"Input data:\n {data}", metadata)
+        self.debug(f"Input data: {data.head(5).to_string()}", metadata)
         self.debug(f"Filters: {filters}", metadata)
 
         # Apply each configured filter
@@ -207,7 +206,8 @@ class Gates(BaseActivity):
 
         filter_output = []
 
-        self.debug(f"Input data:\n {data}", metadata)
+        self.debug(create_sample_dict(
+            data, max_items=5, max_depth=2), metadata)
         self.debug(f"Filters: {filters}", metadata)
 
         comments = []
@@ -292,8 +292,8 @@ class Gates(BaseActivity):
 
         filter_output = []
 
-        self.debug(f"Input data:\n {data}", metadata)
-        self.debug(f"Filters: {filters}", metadata)
+        self.debug(f"Input data:\n {data.head(5).to_string()}", metadata)
+        self.debug(create_sample_dict(filters), metadata)
 
         for fil, config in filters.items():
             if fil not in mlflow_content_filter_functions:
@@ -410,7 +410,7 @@ class Gates(BaseActivity):
 
         self.debug(
             f"Prediction store policy: {prediction_store_policy}", metadata)
-        self.debug(f"Prediction data: {data.to_string()}", metadata)
+        self.debug(f"Prediction data: {data.head(5).to_string()}", metadata)
 
         policy_type, policy_value = self.get_prediction_store_policy(
             prediction_store_policy, metadata)
@@ -445,7 +445,7 @@ class Gates(BaseActivity):
         data = data.reset_index(drop=True)
 
         self.info(f"Prediction formatted: {len(data)} rows", metadata)
-        self.debug(f"Prediction data: {data.to_string()}", metadata)
+        self.debug(f"Prediction data: {data.head(5).to_string()}", metadata)
 
         return data.to_dict()
 
@@ -521,7 +521,7 @@ class Gates(BaseActivity):
 
         data = DataFrame(input_data['data'])
 
-        self.debug(f"Input data: {data.to_string()}", metadata)
+        self.debug(f"Input data: {data.head(5).to_string()}", metadata)
 
         if data.empty:
             return now().strftime(DATETIME_FORMAT_WITH_TZ)