gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1005

Browse Source 
Enhance README and configuration files; add Redis authentication and update workflows
This commit is contained in:
vitor-aignosi
2025-05-16 16:30:43 -03:00
parent e252ca7962
commit df8a0fc706
10 changed files with 350 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scouter/workflow/fake_data.py
View File

@@ -17,8 +17,6 @@ class FakeData:
Args:
workflow_input (dict[str, Any]): The input data containing:
topic (str): The Kafka topic to send data to
num_messages (int, optional): Number of messages to generate.
Defaults to random.randint(1, len(self.tags)).
"""
await workflow.execute_activity_method(
Faker.generate_and_send_data,

16
scouter/workflow/scouter.py
View File

@@ -18,7 +18,6 @@ class Scouter:
Args:
input_data (dict[str, Any]): The data to process. Contains:
topic (str): The topic to load data from.
workflow_name (str): The name of the workflow.
schedule_name (str): The name of the schedule.
model_name (str): The name of the model.
model_id (str): The id of the model.
@@ -27,9 +26,22 @@ class Scouter:
schema (str): The schema of the table to export data to.
table_name (str): The name of the table to export data to.
retention_time (int): The retention time for data in redis in seconds.
model_tags (dict[str, Any]): The tags of the model. And it's respective configuration.
model_tags (dict[str, Any]): The tags of the model.
And it's respective configuration.
"""
input_data['workflow_name'] = 'scouter'
await workflow.execute_local_activity_method(
Activities.prepare_activity,
{
'workflow_name': input_data['workflow_name'],
'schedule_name': input_data['schedule_name'],
'model_name': input_data['model_name'],
'model_id': input_data['model_id']
}
)
data = await workflow.execute_activity_method(
Activities.load_from_kafka,
{