gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1193

Browse Source 
Update Gates activity and format_and_export_prediction workflow to use DATETIME_FORMAT_WITH_TZ for consistent timestamp handling

- Modified Gates activity to correctly handle the maximum timestamp without formatting it prematurely.
- Updated format_and_export_prediction workflow to utilize DATETIME_FORMAT_WITH_TZ for timestamp conversion.
- Enhanced tests to ensure timestamp conversion is applied consistently across workflows.
This commit is contained in:
vitor-aignosi
2025-08-22 14:17:54 -03:00
parent 2301dd63c7
commit ae6d28df54
3 changed files with 14 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
laborious/activities/gates.py
View File

@@ -322,7 +322,7 @@ class Gates(BaseActivity):
return now().strftime(DATETIME_FORMAT_WITH_TZ) return now().strftime(DATETIME_FORMAT_WITH_TZ)
max_timestamp = max( max_timestamp = max(
data['timestamp'].values.tolist()).strftime(DATETIME_FORMAT_WITH_TZ) data['timestamp'].values.tolist())
self.info( self.info(
f"Last timestamp: {max_timestamp}", metadata) f"Last timestamp: {max_timestamp}", metadata)

4
laborious/workflows/sub_workflows/format_and_export_prediction.py
View File

@@ -5,7 +5,7 @@ with workflow.unsafe.imports_passed_through():
from typing import Any from typing import Any
from datetime import timedelta from datetime import timedelta
from sientia_do.temporal.policies import retry_policy from sientia_do.temporal.policies import retry_policy
from sientia_do.temporal.constants import DATETIME_FORMAT from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
@workflow.defn(name="format_and_export_prediction") @workflow.defn(name="format_and_export_prediction")
@@ -94,7 +94,7 @@ class FormatAndExportPrediction():
'data': prediction, 'data': prediction,
'timestamp_conversion': { 'timestamp_conversion': {
'column': 'timestamp', 'column': 'timestamp',
'format': DATETIME_FORMAT 'format': DATETIME_FORMAT_WITH_TZ
} }
}, },
retry_policy=retry_policy, retry_policy=retry_policy,

13
tests/laborious/workflows/subworkflows/test_format_and_export_prediction.py
View File

@@ -3,6 +3,7 @@ from pytest import mark, fixture
from laborious.activities.activities import Activities from laborious.activities.activities import Activities
from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction from laborious.workflows.sub_workflows.format_and_export_prediction import FormatAndExportPrediction
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
@fixture @fixture
@@ -73,7 +74,11 @@ async def test_run_none_path_flag(workflow_mock, format_and_export_prediction):
'schema': input_data['schema'], 'schema': input_data['schema'],
'table_name': input_data['table_name'], 'table_name': input_data['table_name'],
'data': workflow_mock.execute_activity_method.return_value, 'data': workflow_mock.execute_activity_method.return_value,
**metadata **metadata,
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ
}
}, },
retry_policy=ANY, retry_policy=ANY,
start_to_close_timeout=ANY start_to_close_timeout=ANY
@@ -138,7 +143,11 @@ async def test_run_default_path_flag(workflow_mock, format_and_export_prediction
'schema': input_data['schema'], 'schema': input_data['schema'],
'table_name': input_data['table_name'], 'table_name': input_data['table_name'],
'data': workflow_mock.execute_activity_method.return_value, 'data': workflow_mock.execute_activity_method.return_value,
**metadata **metadata,
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ
}
}, },
retry_policy=ANY, retry_policy=ANY,
start_to_close_timeout=ANY start_to_close_timeout=ANY