SIENTIAPDE-1110

Enhance MLFlow activity to sort data by created_at and remove duplicates; update tests to reflect changes
This commit is contained in:
vitor-aignosi
2025-06-25 15:33:31 -03:00
parent 2be78b1aea
commit cefef0b1e9
2 changed files with 20 additions and 4 deletions

View File

@@ -44,6 +44,11 @@ class MLFlow(BaseActivity):
self.logger.debug("Raw input data:")
self.logger.debug(data)
# Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
data = data.sort_values('created_at', ascending=False).drop_duplicates(
subset=['variable', 'timestamp'], keep='first'
)
data = data.pivot(
index='timestamp', columns='variable',
values='value')