SIENTIAPDE-1110
Enhance MLFlow activity to sort data by created_at and remove duplicates; update tests to reflect changes
This commit is contained in:
@@ -44,6 +44,11 @@ class MLFlow(BaseActivity):
|
||||
self.logger.debug("Raw input data:")
|
||||
self.logger.debug(data)
|
||||
|
||||
# Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
|
||||
data = data.sort_values('created_at', ascending=False).drop_duplicates(
|
||||
subset=['variable', 'timestamp'], keep='first'
|
||||
)
|
||||
|
||||
data = data.pivot(
|
||||
index='timestamp', columns='variable',
|
||||
values='value')
|
||||
|
||||
Reference in New Issue
Block a user