gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1110

Browse Source 
Refactor Scouter workflow to update data loading method, changing from load_latest_druid_data to load_latest_data. Adjusted parameters for consistency in collection naming and improved clarity in data retrieval process.
This commit is contained in:
vitor-aignosi
2025-07-03 14:04:10 -03:00
parent 3ee7e3687a
commit 1dcba6b48e
1 changed files with 13 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
scouter/workflow/scouter.py
View File

@@ -62,28 +62,28 @@ class Scouter:
retry_policy=retry_policy retry_policy=retry_policy
) )
# data = await workflow.execute_local_activity_method(
# Activities.load_latest_data,
# {
# **metadata,
# 'collection_name': f"raw_{input_data['schedule_name']}",
# 'last_data_timestamp': last_data_timestamp
# },
# start_to_close_timeout=timedelta(seconds=60),
# retry_policy=retry_policy
# )
data = await workflow.execute_local_activity_method( data = await workflow.execute_local_activity_method(
Activities.load_latest_druid_data, Activities.load_latest_data,
{ {
**metadata, **metadata,
'schedule_name': input_data['schedule_name'], 'collection_name': f"raw_{input_data['schedule_name']}",
'last_data_timestamp': last_data_timestamp 'last_data_timestamp': last_data_timestamp
}, },
start_to_close_timeout=timedelta(seconds=60), start_to_close_timeout=timedelta(seconds=60),
retry_policy=retry_policy retry_policy=retry_policy
) )
# data = await workflow.execute_local_activity_method(
# Activities.load_latest_druid_data,
# {
# **metadata,
# 'schedule_name': input_data['schedule_name'],
# 'last_data_timestamp': last_data_timestamp
# },
# start_to_close_timeout=timedelta(seconds=60),
# retry_policy=retry_policy
# )
await workflow.execute_activity_method( await workflow.execute_activity_method(
Activities.put_last_data_timestamp, Activities.put_last_data_timestamp,
{ {