gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1110

Browse Source 
Update dependencies and enhance logging in activities; bump version in requirements and values.yaml
This commit is contained in:
vitor-aignosi
2025-06-27 09:17:24 -03:00
parent cefef0b1e9
commit 98ea2a7f75
10 changed files with 87 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
laborious/workflows/predictions_batch.py
View File

@@ -40,27 +40,28 @@ class PredictionsBatch():
Exception: If any of the required parameters are missing or if the workflow fails.
"""
await workflow.execute_local_activity_method(
Activities.prepare_activity,
{
metadata = {
'metadata': {
'schedule_name': input_data['schedule_name'],
'model_name': input_data['model_name'],
'model_id': input_data['model_id'],
'workflow_name': 'predictions_batch'
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
)
}
}
data = await workflow.execute_local_activity_method(
Activities.load_custom_query,
input_data['query'],
{
**metadata,
'query': input_data['query'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
)
# Prepare input for prediction_process workflow
prediction_input = {
**metadata,
'data': data,
'schema': input_data['schema'],
'table_name': input_data['table_name'],

5
laborious/workflows/sub_workflows/format_and_export_prediction.py
View File

@@ -36,6 +36,7 @@ class FormatAndExportPrediction():
Returns:
bool: True if the workflow was successful, False otherwise.
"""
metadata = input_data['metadata']
path_flag = input_data['path_flag']
data = input_data['data']
prediction_confidence = input_data['prediction_confidence']
@@ -45,6 +46,7 @@ class FormatAndExportPrediction():
prediction = await workflow.execute_local_activity_method(
Activities.format_prediction,
{
**metadata,
'data': data,
'timestamp': input_data['timestamp'],
'model_id': input_data['model_id'],
@@ -59,6 +61,7 @@ class FormatAndExportPrediction():
prediction = await workflow.execute_local_activity_method(
Activities.format_default_prediction,
{
**metadata,
'timestamp': input_data['timestamp'],
'model_id': input_data['model_id'],
'prediction_confidence': prediction_confidence,
@@ -72,6 +75,7 @@ class FormatAndExportPrediction():
prediction = await workflow.execute_activity_method(
Activities.write_opc_data,
{
**metadata,
'opc_output_config': input_data['opc_output_config'],
'data': prediction
},
@@ -83,6 +87,7 @@ class FormatAndExportPrediction():
await workflow.execute_activity_method(
Activities.export_data_to_postgres,
{
**metadata,
'schema': input_data['schema'],
'table_name': input_data['table_name'],
'data': prediction,

15
laborious/workflows/sub_workflows/prediction_process.py
View File

@@ -41,6 +41,7 @@ class PredictionProcess():
Exception: If any of the required parameters are missing or if the workflow fails.
"""
metadata = input_data['metadata']
data = input_data['data']
model_id = input_data['model_id']
model_name = input_data['model_name']
@@ -49,6 +50,7 @@ class PredictionProcess():
last_timestamp = await workflow.execute_local_activity_method(
Activities.get_last_timestamp,
{
**metadata,
'data': data
},
retry_policy=retry_policy,
@@ -58,6 +60,7 @@ class PredictionProcess():
path_flag, confidence, comment = await workflow.execute_local_activity_method(
Activities.input_gate,
{
**metadata,
'filters': input_data['input_filters'],
'data': data,
'path_priority': input_data['path_priority']
@@ -74,6 +77,7 @@ class PredictionProcess():
response_data = await workflow.execute_local_activity_method(
Activities.request_transform,
{
**metadata,
'data': data,
'model_name': model_name,
'model_retention': model_retention
@@ -85,6 +89,7 @@ class PredictionProcess():
path_flag, confidence, comment = await workflow.execute_local_activity_method(
Activities.mlflow_response_gate,
{
**metadata,
'filters': input_data['mlflow_transform_filters'],
'data': response_data,
'type': 'transform',
@@ -104,6 +109,7 @@ class PredictionProcess():
path_flag, confidence, comment = await workflow.execute_local_activity_method(
Activities.mlflow_content_gate,
{
**metadata,
'filters': input_data['mlflow_transform_filters'],
'data': transformed_data,
'type': 'transform',
@@ -121,6 +127,7 @@ class PredictionProcess():
response_data = await workflow.execute_local_activity_method(
Activities.request_predict,
{
**metadata,
'data': transformed_data,
'model_name': model_name,
'model_retention': model_retention
@@ -132,6 +139,7 @@ class PredictionProcess():
path_flag, confidence, comment = await workflow.execute_local_activity_method(
Activities.mlflow_response_gate,
{
**metadata,
'filters': input_data['mlflow_predict_filters'],
'data': response_data,
'type': 'predict',
@@ -149,6 +157,7 @@ class PredictionProcess():
await workflow.execute_child_workflow(
'format_and_export_prediction',
{
**metadata,
'path_flag': path_flag,
'data': response_data['content'],
'prediction_confidence': confidence,
@@ -187,13 +196,15 @@ class PredictionProcess():
bool: True if the prediction should be stopped, False otherwise.
"""
metadata = input_data['metadata']
schema = input_data['schema']
table_name = input_data['table_name']
model_id = input_data['model_id']
model_name = input_data['model_name']
model_retention = input_data['model_retention']
path_flag = path_flag.upper() if path_flag else None
path_flag = path_flag.upper() if path_flag else ''
if path_flag == 'STOP':
return True
@@ -203,6 +214,7 @@ class PredictionProcess():
await workflow.execute_activity_method(
Activities.repeat_last_prediction,
{
**metadata,
'schema': schema,
'table_name': table_name,
'model': model_id,
@@ -218,6 +230,7 @@ class PredictionProcess():
await workflow.execute_child_workflow(
'format_and_export_prediction',
{
**metadata,
'path_flag': path_flag,
'data': data,
'prediction_confidence': confidence,