From cefef0b1e97da5fd151a4be1160bcb4ad214b9f9 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 25 Jun 2025 15:33:31 -0300 Subject: [PATCH] SIENTIAPDE-1110 Enhance MLFlow activity to sort data by created_at and remove duplicates; update tests to reflect changes --- laborious/activities/mlflow.py | 5 +++++ tests/laborious/activities/test_mlflow.py | 19 +++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index 9b8a58a..b336f45 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -44,6 +44,11 @@ class MLFlow(BaseActivity): self.logger.debug("Raw input data:") self.logger.debug(data) + # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair + data = data.sort_values('created_at', ascending=False).drop_duplicates( + subset=['variable', 'timestamp'], keep='first' + ) + data = data.pivot( index='timestamp', columns='variable', values='value') diff --git a/tests/laborious/activities/test_mlflow.py b/tests/laborious/activities/test_mlflow.py index 5834cb4..53dd018 100644 --- a/tests/laborious/activities/test_mlflow.py +++ b/tests/laborious/activities/test_mlflow.py @@ -47,10 +47,18 @@ async def test_request_transform(mock_max, mock_dataframe, mlflow): # Mock input data input_data = { 'data': [ - {'timestamp': '2024-01-01', 'variable': 'var1', 'value': 1.0}, - {'timestamp': '2024-01-01', 'variable': 'var2', 'value': 2.0}, - {'timestamp': '2024-01-02', 'variable': 'var1', 'value': 3.0}, - {'timestamp': '2024-01-02', 'variable': 'var2', 'value': 4.0} + {'timestamp': '2024-01-01', 'variable': 'var1', + 'value': 1.0, 'created_at': '2024-01-01 12:00:00'}, + {'timestamp': '2024-01-01', 'variable': 'var2', + 'value': 2.0, 'created_at': '2024-01-01 12:00:00'}, + {'timestamp': '2024-01-02', 'variable': 'var1', + 'value': 3.0, 'created_at': '2024-01-02 12:00:00'}, + {'timestamp': '2024-01-02', 'variable': 'var2', + 'value': 4.0, 'created_at': '2024-01-02 12:00:00'}, + {'timestamp': '2024-01-02', 'variable': 'var1', + 'value': 1.0, 'created_at': '2024-01-01 12:00:00'}, + {'timestamp': '2024-01-02', 'variable': 'var2', + 'value': 1.0, 'created_at': '2024-01-01 12:00:00'} ], 'model_name': 'test_model', 'model_retention': 30 @@ -60,6 +68,9 @@ async def test_request_transform(mock_max, mock_dataframe, mlflow): expected_response = {'prediction': [0.5, 0.6]} mlflow.model_monitoring_repository.transform.return_value = expected_response + mock_dataframe.return_value.sort_values.return_value = mock_dataframe.return_value + mock_dataframe.return_value.drop_duplicates.return_value = mock_dataframe.return_value + # Call the method response_data = await mlflow.request_transform(input_data)