gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1712

Browse Source 
Update Gates and FormatAndExportPrediction classes to use 'last_timestamp' for improved data handling

- Modified the Gates class to utilize 'last_timestamp' when only one row is present, ensuring accurate timestamp assignment.
- Updated the FormatAndExportPrediction class to replace 'timestamp' with 'last_timestamp' in the output data structure.
This commit is contained in:
vitor-aignosi
2026-03-25 12:18:27 -03:00
parent 1f5a14be3f
commit 5b4903fffa
2 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
laborious/activities/gates.py
View File

@@ -514,6 +514,8 @@ class Gates(MinioManager):
and ensures data consistency before persistence. The method supports and ensures data consistency before persistence. The method supports
multiple storage policies for flexible data retention strategies. multiple storage policies for flexible data retention strategies.
If only one row is present, we use the last timestamp as the timestamp
Storage Policies: Storage Policies:
- 'lts:N': Latest timestamp - retains N most recent predictions - 'lts:N': Latest timestamp - retains N most recent predictions
- 'erl:N': Earliest timestamp - retains N oldest predictions - 'erl:N': Earliest timestamp - retains N oldest predictions
@@ -521,7 +523,7 @@ class Gates(MinioManager):
Args: Args:
input_data (dict): Input data containing: input_data (dict): Input data containing:
- data (dict[str, Any]): Raw prediction data to format - data (dict[str, Any]): Raw prediction data to format
- timestamp (str): Default timestamp if data lacks timestamp column - last_timestamp (str): Last timestamp of the data
- model_id (str): Unique identifier for the ML model - model_id (str): Unique identifier for the ML model
- prediction_confidence (float): Confidence score for the prediction - prediction_confidence (float): Confidence score for the prediction
- prediction_store_policy (str): Storage policy in format 'type:value' - prediction_store_policy (str): Storage policy in format 'type:value'
@@ -530,6 +532,7 @@ class Gates(MinioManager):
dict: Formatted prediction data ready for storage and export dict: Formatted prediction data ready for storage and export
""" """
metadata = input_data['metadata'] metadata = input_data['metadata']
last_timestamp = input_data['last_timestamp']
prediction_store_policy = input_data['prediction_store_policy'] prediction_store_policy = input_data['prediction_store_policy']
self.info('Formatting prediction...', metadata) self.info('Formatting prediction...', metadata)
@@ -564,7 +567,11 @@ class Gates(MinioManager):
self.error(f'Invalid policy type: {policy_type}, using default policy', metadata) self.error(f'Invalid policy type: {policy_type}, using default policy', metadata)
raise ValueError(f'Invalid policy type: {policy_type}') raise ValueError(f'Invalid policy type: {policy_type}')
data = data.head(int(policy_value)) int_policy_value = int(policy_value)
data = data.head(int_policy_value)
if int_policy_value == 1:
data['timestamp'] = last_timestamp
data['model_id'] = input_data['model_id'] data['model_id'] = input_data['model_id']
data['prediction_confidence'] = input_data['prediction_confidence'] data['prediction_confidence'] = input_data['prediction_confidence']

2
laborious/workflows/sub_workflows/format_and_export_prediction.py
View File

@@ -101,7 +101,7 @@ class FormatAndExportPrediction:
{ {
**metadata, **metadata,
'data': data, 'data': data,
'timestamp': input_data['timestamp'], 'last_timestamp': input_data['last_timestamp'],
'model_id': input_data['model_id'], 'model_id': input_data['model_id'],
'prediction_confidence': prediction_confidence, 'prediction_confidence': prediction_confidence,
'prediction_store_policy': input_data['prediction_store_policy'], 'prediction_store_policy': input_data['prediction_store_policy'],