gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1110

Browse Source 
SIENTIAPDE-1110 Add MongoDB integration and enhance Redis activity with timestamp management functions.
This commit is contained in:
vitor-aignosi
2025-07-01 15:10:32 -03:00
parent e7b05ebf3a
commit 53337de74a
6 changed files with 227 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
scouter/workflow/scouter.py
View File

@@ -41,14 +41,48 @@ class Scouter:
}
}
data = await workflow.execute_activity_method(
Activities.load_from_kafka,
# data = await workflow.execute_activity_method(
# Activities.load_from_kafka,
# {
# **metadata,
# 'topic': input_data['topic']
# },
# retry_policy=retry_policy,
# start_to_close_timeout=timedelta(seconds=60)
# )
last_data_timestamp = await workflow.execute_local_activity_method(
Activities.get_last_data_timestamp,
{
**metadata,
'topic': input_data['topic']
'workflow_name': input_data['workflow_name'],
'schedule_name': input_data['schedule_name']
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
retry_policy=retry_policy
)
data = await workflow.execute_local_activity_method(
Activities.load_latest_data,
{
**metadata,
'collection_name': f"raw_{input_data['schedule_name']}",
'last_data_timestamp': last_data_timestamp
},
start_to_close_timeout=timedelta(seconds=60),
retry_policy=retry_policy
)
await workflow.execute_activity_method(
Activities.put_last_data_timestamp,
{
**metadata,
'data': data,
'workflow_name': input_data['workflow_name'],
'schedule_name': input_data['schedule_name']
},
start_to_close_timeout=timedelta(seconds=60),
retry_policy=retry_policy
)
if data == {}: