From fad10bc0d0a49755088c17083edf99fc16a5ba8f Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Thu, 21 Aug 2025 12:16:06 -0300 Subject: [PATCH] SIENTIAPDE-1193 Update requirements and core scouter workflow: comment out old dataops library dependency, change default namespace in worker.py, and enhance timestamp handling in core_scouter.py with DATETIME_FORMAT. --- requirements.txt | 3 ++- scouter/worker/worker.py | 2 +- scouter/workflow/sub_workflows/core_scouter.py | 9 +++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 91864f7..3d33f78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,8 @@ asyncua redis aiokafka pymongo -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1 +# git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.4.1 +/home/grezewave/Documents/projects/sientia/sientia-dataops-library git+ssh://git@github.com/Aignosi/sientia-mlops-library.git@0.38.5 pydruid[pandas] prometheus-client \ No newline at end of file diff --git a/scouter/worker/worker.py b/scouter/worker/worker.py index f636d59..95af238 100644 --- a/scouter/worker/worker.py +++ b/scouter/worker/worker.py @@ -86,7 +86,7 @@ async def main(): temporal_client = await client.Client.connect( target_host=host, - namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'), + namespace=os.getenv('TEMPORAL_NAMESPACE', 'scouter'), runtime=new_runtime ) diff --git a/scouter/workflow/sub_workflows/core_scouter.py b/scouter/workflow/sub_workflows/core_scouter.py index d6c83f4..2209b72 100644 --- a/scouter/workflow/sub_workflows/core_scouter.py +++ b/scouter/workflow/sub_workflows/core_scouter.py @@ -5,6 +5,7 @@ with workflow.unsafe.imports_passed_through(): from typing import Any from datetime import timedelta from sientia_do.temporal.policies import retry_policy + from sientia_do.temporal.constants import DATETIME_FORMAT @workflow.defn(name="core_scouter") @@ -81,7 +82,11 @@ class CoreScouter: **metadata, 'schema': input_data['schema'], 'table_name': input_data['table_name'], - 'data': held_data + 'data': held_data, + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT + } }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60) @@ -104,7 +109,7 @@ class CoreScouter: 'data': input_data['data'], 'held_data': held_data, 'workflow_name': input_data['workflow_name'], - 'schedule_name': input_data['schedule_name'] + 'schedule_name': input_data['schedule_name'], }, retry_policy=retry_policy, start_to_close_timeout=timedelta(seconds=60)