SIENTIAPDE-1193

Update sientia-dataops-library version to 1.4.3 and refactor timestamp handling in MongoDB and Redis activities to use DATETIME_FORMAT constants for improved consistency. Enhance unit tests to reflect these changes.

scouter/activities/mongodb.py

@@ -10,6 +10,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.notifications.models import NotificationLevel
     from sientia_do.temporal.activities.base import BaseActivity
     from sientia_do.observability.logger import Logger
+    from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
 
 
 def clear_mongo_id(docs: list) -> list:
@@ -99,7 +100,7 @@ class MongoDB(BaseActivity):
             else:
                 data_filter = {
                     "inserted_at": {
-                        "$gt": datetime.strptime(last_data_timestamp, "%Y-%m-%d %H:%M:%S.%f")
+                        "$gt": datetime.strptime(last_data_timestamp, DATETIME_FORMAT_MS_WITH_TZ)
                     }
                 }
 
@@ -120,7 +121,7 @@ class MongoDB(BaseActivity):
 
             for item in data:
                 item['inserted_at'] = item['inserted_at'].strftime(
-                    "%Y-%m-%d %H:%M:%S.%f")
+                    DATETIME_FORMAT_MS_WITH_TZ)
 
             self.info(
                 f"Loaded {len(data)} documents from MongoDB",

scouter/activities/redis.py

@@ -11,6 +11,7 @@ with workflow.unsafe.imports_passed_through():
     from pandas import DataFrame
     from datetime import datetime
     from scouter import metrics
+    from sientia_do.temporal.constants import DATETIME_FORMAT
 
 
 class Redis(RedisBase):
@@ -169,7 +170,7 @@ class Redis(RedisBase):
                     (row['name'], value))
 
             data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \
-                datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                data_hold['timestamp']
 
             self.set(key, data_hold, ttl=retention_time)
 
@@ -224,7 +225,7 @@ class Redis(RedisBase):
             data: The data used to collect the data.
         """
         metadata = input_data['metadata']
-        key = f"data_package_{input_data['workflow_name']}_{input_data['schedule_name']}_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}"
+        key = f"data_package_{input_data['workflow_name']}_{input_data['schedule_name']}_{datetime.now().strftime(DATETIME_FORMAT)}"
 
         data = DataFrame(input_data['data'])
         held_data = DataFrame(input_data['held_data'])