SIENTIAPDE-1193

Update requirements and core scouter workflow: comment out old dataops library dependency, change default namespace in worker.py, and enhance timestamp handling in core_scouter.py with DATETIME_FORMAT.

requirements.txt

@@ -5,7 +5,8 @@ asyncua
 redis
 aiokafka
 pymongo
-git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1
+# git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1
+/home/grezewave/Documents/projects/sientia/sientia-dataops-library
 git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5
 pydruid[pandas]
 prometheus-client

scouter/worker/worker.py

@@ -86,7 +86,7 @@ async def main():
 
     temporal_client = await client.Client.connect(
         target_host=host,
-        namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'),
+        namespace=os.getenv('TEMPORAL_NAMESPACE', 'scouter'),
         runtime=new_runtime
     )
 

scouter/workflow/sub_workflows/core_scouter.py

@@ -5,6 +5,7 @@ with workflow.unsafe.imports_passed_through():
     from typing import Any
     from datetime import timedelta
     from sientia_do.temporal.policies import retry_policy
+    from sientia_do.temporal.constants import DATETIME_FORMAT
 
 
 @workflow.defn(name="core_scouter")
@@ -81,7 +82,11 @@ class CoreScouter:
                 **metadata,
                 'schema': input_data['schema'],
                 'table_name': input_data['table_name'],
-                'data': held_data
+                'data': held_data,
+                'timestamp_conversion': {
+                    'column': 'timestamp',
+                    'format': DATETIME_FORMAT
+                }
             },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(seconds=60)
@@ -104,7 +109,7 @@ class CoreScouter:
                     'data': input_data['data'],
                     'held_data': held_data,
                     'workflow_name': input_data['workflow_name'],
-                    'schedule_name': input_data['schedule_name']
+                    'schedule_name': input_data['schedule_name'],
                 },
                 retry_policy=retry_policy,
                 start_to_close_timeout=timedelta(seconds=60)