SIENTIAPDE-1110

SIENTIAPDE-1110 Add MongoDB integration and enhance Redis activity with timestamp management functions.

scouter/activities/redis.py

@@ -18,6 +18,47 @@ class Redis(RedisBase):
         RedisBase.__init__(self, host, port, username,
                            password, logger, notification_handler)
 
+    @activity.defn(name="get_last_data_timestamp")
+    async def get_last_data_timestamp(self, input_data: dict[str, Any]) -> dict[str, Any]:
+        """
+        Gets the last data timestamp from redis.
+        """
+        metadata = input_data['metadata']
+        key = f"{input_data['workflow_name']}_{input_data['schedule_name']}"
+
+        data_hold = self.get(key)
+
+        self.debug(
+            f"Last collected timestamp: {data_hold}",
+            metadata=metadata
+        )
+
+        if not data_hold:
+            return None
+
+        return data_hold
+
+    @activity.defn(name="put_last_data_timestamp")
+    async def put_last_data_timestamp(self, input_data: dict[str, Any]):
+        """
+        Puts the last data timestamp into redis.
+        """
+        metadata = input_data['metadata']
+        key = f"{input_data['workflow_name']}_{input_data['schedule_name']}"
+
+        data = DataFrame(input_data['data'])
+
+        last_data_timestamp = data['inserted_at'].max()
+
+        self.debug(
+            f"Last collected timestamp to insert: {last_data_timestamp}",
+            metadata=metadata
+        )
+
+        self.set(key, last_data_timestamp, ttl=input_data['retention_time'])
+
+        return last_data_timestamp
+
     @activity.defn(name="group_and_hold_data")
     async def group_and_hold_data(self, input_data: dict[str, Any]):
         """