From 5b4903fffa669eb036c3af17a0553fae90592907 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 25 Mar 2026 12:18:27 -0300 Subject: [PATCH] SIENTIAPDE-1712 Update Gates and FormatAndExportPrediction classes to use 'last_timestamp' for improved data handling - Modified the Gates class to utilize 'last_timestamp' when only one row is present, ensuring accurate timestamp assignment. - Updated the FormatAndExportPrediction class to replace 'timestamp' with 'last_timestamp' in the output data structure. --- laborious/activities/gates.py | 11 +++++++++-- .../sub_workflows/format_and_export_prediction.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 1d561c5..0e8dc82 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -514,6 +514,8 @@ class Gates(MinioManager): and ensures data consistency before persistence. The method supports multiple storage policies for flexible data retention strategies. + If only one row is present, we use the last timestamp as the timestamp + Storage Policies: - 'lts:N': Latest timestamp - retains N most recent predictions - 'erl:N': Earliest timestamp - retains N oldest predictions @@ -521,7 +523,7 @@ class Gates(MinioManager): Args: input_data (dict): Input data containing: - data (dict[str, Any]): Raw prediction data to format - - timestamp (str): Default timestamp if data lacks timestamp column + - last_timestamp (str): Last timestamp of the data - model_id (str): Unique identifier for the ML model - prediction_confidence (float): Confidence score for the prediction - prediction_store_policy (str): Storage policy in format 'type:value' @@ -530,6 +532,7 @@ class Gates(MinioManager): dict: Formatted prediction data ready for storage and export """ metadata = input_data['metadata'] + last_timestamp = input_data['last_timestamp'] prediction_store_policy = input_data['prediction_store_policy'] self.info('Formatting prediction...', metadata) @@ -564,7 +567,11 @@ class Gates(MinioManager): self.error(f'Invalid policy type: {policy_type}, using default policy', metadata) raise ValueError(f'Invalid policy type: {policy_type}') - data = data.head(int(policy_value)) + int_policy_value = int(policy_value) + + data = data.head(int_policy_value) + if int_policy_value == 1: + data['timestamp'] = last_timestamp data['model_id'] = input_data['model_id'] data['prediction_confidence'] = input_data['prediction_confidence'] diff --git a/laborious/workflows/sub_workflows/format_and_export_prediction.py b/laborious/workflows/sub_workflows/format_and_export_prediction.py index a9cda2a..1cd2be1 100644 --- a/laborious/workflows/sub_workflows/format_and_export_prediction.py +++ b/laborious/workflows/sub_workflows/format_and_export_prediction.py @@ -101,7 +101,7 @@ class FormatAndExportPrediction: { **metadata, 'data': data, - 'timestamp': input_data['timestamp'], + 'last_timestamp': input_data['last_timestamp'], 'model_id': input_data['model_id'], 'prediction_confidence': prediction_confidence, 'prediction_store_policy': input_data['prediction_store_policy'],