gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1273

Browse Source 
Refactor data export in Drift, SimpleMetrics, and FormatAndExportPrediction workflows to improve data handling. Changed the export format from a list of dictionaries to direct objects for enhanced clarity and compatibility with downstream systems.
This commit is contained in:
vitor-aignosi
2025-11-17 10:38:44 -03:00
parent 08f9522695
commit c9b1ddfd15
3 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
laborious/workflows/drift.py
View File

@@ -52,6 +52,7 @@ class Drift:
**metadata, **metadata,
'query': gathering_query, 'query': gathering_query,
'datetime_columns': ['timestamp', 'created_at'], 'datetime_columns': ['timestamp', 'created_at'],
'orient': 'records',
}, },
retry_policy=retry_policy, retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=300), start_to_close_timeout=timedelta(seconds=300),

3
laborious/workflows/simple_metrics.py
View File

@@ -53,6 +53,7 @@ class SimpleMetrics:
**metadata, **metadata,
'query': query, 'query': query,
'datetime_columns': ['timestamp'], 'datetime_columns': ['timestamp'],
'orient': 'records',
}, },
retry_policy=retry_policy, retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=300), start_to_close_timeout=timedelta(seconds=300),
@@ -82,7 +83,7 @@ class SimpleMetrics:
Activities.export_data_to_postgres, Activities.export_data_to_postgres,
{ {
**metadata, **metadata,
'data': simple_metrics.to_dict(orient='records'), 'data': simple_metrics,
'schema': input_data['schema'], 'schema': input_data['schema'],
'table_name': input_data['target_table_name'], 'table_name': input_data['target_table_name'],
'timestamp_conversion': { 'timestamp_conversion': {

4
laborious/workflows/sub_workflows/format_and_export_prediction.py
View File

@@ -107,7 +107,7 @@ class FormatAndExportPrediction:
**metadata, **metadata,
'schema': input_data['schema'], 'schema': input_data['schema'],
'table_name': input_data['transform_table_name'], 'table_name': input_data['transform_table_name'],
'data': transformed.to_dict(orient='records'), 'data': transformed,
'timestamp_conversion': { 'timestamp_conversion': {
'column': 'timestamp', 'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ, 'format': DATETIME_FORMAT_WITH_TZ,
@@ -156,7 +156,7 @@ class FormatAndExportPrediction:
**metadata, **metadata,
'schema': input_data['schema'], 'schema': input_data['schema'],
'table_name': input_data['table_name'], 'table_name': input_data['table_name'],
'data': prediction.to_dict(orient='records'), 'data': prediction,
'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ}, 'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ},
}, },
retry_policy=retry_policy, retry_policy=retry_policy,