From 96a28e91e5f2a4f42a176feb3c10b160e5d16b70 Mon Sep 17 00:00:00 2001
From: vitor-aignosi <vitor.santos@aignosi.com.br>
Date: Fri, 22 Aug 2025 14:05:41 -0300
Subject: [PATCH] SIENTIAPDE-1193  Enhance timestamp handling in Redis and
 CoreScouter workflows by updating to DATETIME_FORMAT_MS_WITH_TZ. Modify tests
 to reflect new timestamp format, ensuring consistency across data structures
 and improving overall datetime management.

---
 scouter/activities/redis.py                       |  1 +
 scouter/workflow/sub_workflows/core_scouter.py    |  4 ++--
 tests/activities/test_mongo.py                    | 14 +++++++-------
 tests/workflow/sub_workflows/test_core_scouter.py |  8 +++++++-
 4 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/scouter/activities/redis.py b/scouter/activities/redis.py
index 56a4313..e7443a1 100644
--- a/scouter/activities/redis.py
+++ b/scouter/activities/redis.py
@@ -149,6 +149,7 @@ class Redis(RedisBase):
 
             # Remove possibly removed tags
             tags = list(model_tags.keys())
+            tags.append('timestamp')
             self.debug(
                 f"Tags to keep: {tags}",
                 metadata=metadata
diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py
index 2209b72..3beb4d9 100644
--- a/scouter/workflow/sub_workflows/core_scouter.py
+++ b/scouter/workflow/sub_workflows/core_scouter.py
@@ -5,7 +5,7 @@ with workflow.unsafe.imports_passed_through():
     from typing import Any
     from datetime import timedelta
     from sientia_do.temporal.policies import retry_policy
-    from sientia_do.temporal.constants import DATETIME_FORMAT
+    from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
 
 
 @workflow.defn(name="core_scouter")
@@ -85,7 +85,7 @@ class CoreScouter:
                 'data': held_data,
                 'timestamp_conversion': {
                     'column': 'timestamp',
-                    'format': DATETIME_FORMAT
+                    'format': DATETIME_FORMAT_MS_WITH_TZ
                 }
             },
             retry_policy=retry_policy,
diff --git a/tests/activities/test_mongo.py b/tests/activities/test_mongo.py
index 8b8cd87..5f91e1c 100644
--- a/tests/activities/test_mongo.py
+++ b/tests/activities/test_mongo.py
@@ -96,7 +96,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
             'name': 'test1',
             'value': 1,
             'inserted_at': datetime.strptime(
-                '2023-01-01 12:00:00.000000', DATETIME_FORMAT_MS_WITH_TZ)
+                '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ)
         }
     ]
 
@@ -122,7 +122,7 @@ async def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
             0: 1
         },
         'inserted_at': {
-            0: '2023-01-01 12:00:00.000000'
+            0: '2023-01-01 12:00:00.000000+0000'
         }
     }
 
@@ -138,14 +138,14 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
             'name': 'test1',
             'value': 1,
             'inserted_at': datetime.strptime(
-                '2023-01-01 12:00:00.000000', DATETIME_FORMAT_MS_WITH_TZ)
+                '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ)
         }
     ]
 
     result = await mongodb_activity.load_latest_data({
         'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
         'collection_name': 'test_collection',
-        'last_data_timestamp': '2023-01-01 12:00:00.000000'
+        'last_data_timestamp': '2023-01-01 12:00:00.000000+0000'
     })
 
     mongodb_activity.database.__getitem__.assert_called_once_with(
@@ -155,7 +155,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
         {
             'inserted_at': {
                 '$gt': datetime.strptime(
-                    '2023-01-01 12:00:00.000000', DATETIME_FORMAT_MS_WITH_TZ)
+                    '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ)
             }
         },
         {"_id": 0}
@@ -169,7 +169,7 @@ async def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
             0: 1
         },
         'inserted_at': {
-            0: '2023-01-01 12:00:00.000000'
+            0: '2023-01-01 12:00:00.000000+0000'
         }
     }
 
@@ -187,7 +187,7 @@ async def test_load_latest_data_error(mongodb_activity):
         await mongodb_activity.load_latest_data({
             'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
             'collection_name': 'test_collection',
-            'last_data_timestamp': '2023-01-01 12:00:00.000000'
+            'last_data_timestamp': '2023-01-01 12:00:00.000000+0000'
         })
     except Exception as e:
         assert str(e) == 'test'
diff --git a/tests/workflow/sub_workflows/test_core_scouter.py b/tests/workflow/sub_workflows/test_core_scouter.py
index 5e88c5b..f574325 100644
--- a/tests/workflow/sub_workflows/test_core_scouter.py
+++ b/tests/workflow/sub_workflows/test_core_scouter.py
@@ -2,6 +2,7 @@ from unittest.mock import AsyncMock, patch, call, ANY
 import pytest
 from scouter.workflow.sub_workflows.core_scouter import CoreScouter
 from scouter.activities.activities import Activities
+from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
 
 
 @pytest.fixture
@@ -94,7 +95,12 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
                 **expected_metadata,
                 'schema': 'test_schema',
                 'table_name': 'test_table',
-                'data': 'held_data'},
+                'data': 'held_data',
+                'timestamp_conversion': {
+                    'column': 'timestamp',
+                    'format': DATETIME_FORMAT_MS_WITH_TZ
+                }
+            },
             retry_policy=ANY,
             start_to_close_timeout=ANY
         )