SIENTIAPDE-1110

Implement store_data_package method in Redis activity for debug data storage. Update CoreScouter workflow to conditionally call store_data_package based on debug flag. Enhance tests for store_data_package functionality and error handling.

scouter/activities/redis.py

@@ -93,7 +93,7 @@ class Redis(RedisBase):
     @activity.defn(name="group_and_hold_data")
     async def group_and_hold_data(self, input_data: dict[str, Any]):
         """
-        Groups and holds data in redis. Keep a copy of the most recent 
+        Groups and holds data in redis. Keep a copy of the most recent
         received data for a given pipeline and schedule. This activity updates
         the data in redis and return the full keeped data.
 
@@ -169,3 +169,37 @@ class Redis(RedisBase):
         )
 
         return data_hold_melted.to_dict()
+
+    async def store_data_package(self, input_data: dict[str, Any]):
+        """
+        Stores the data package in redis. It's a debug feature and must be toggled on.
+        input_data:
+            metadata: The metadata of the workflow.
+            workflow_name: The name of the workflow.
+            schedule_name: The name of the schedule.
+            held_data: The final scouter output.
+            data: The data used to collect the data.
+        """
+        metadata = input_data['metadata']
+        key = f"data_package_{input_data['workflow_name']}_{input_data['schedule_name']}_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}"
+
+        data = DataFrame(input_data['data'])
+        held_data = DataFrame(input_data['held_data'])
+
+        cache = {
+            'data': data.to_dict(),
+            'held_data': held_data.to_dict()
+        }
+
+        try:
+            self.set(key, cache, ttl=120)
+        except Exception as e:
+            self.send_notification(
+                metadata=metadata,
+                notification_id="REDIS_SET_ERROR",
+                message=f"Error setting data package: {e}",
+                block="store_data_package",
+                level=NotificationLevel.ERROR,
+                attachment_content=traceback.format_exc()
+            )
+            raise e