gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1110

Browse Source 
Enhance MLFlow activity to sort data by created_at and remove duplicates; update tests to reflect changes
This commit is contained in:
vitor-aignosi
2025-06-25 15:33:31 -03:00
parent 2be78b1aea
commit cefef0b1e9
2 changed files with 20 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
laborious/activities/mlflow.py
View File

@@ -44,6 +44,11 @@ class MLFlow(BaseActivity):
self.logger.debug("Raw input data:")
self.logger.debug(data)
# Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
data = data.sort_values('created_at', ascending=False).drop_duplicates(
subset=['variable', 'timestamp'], keep='first'
)
data = data.pivot(
index='timestamp', columns='variable',
values='value')

19
tests/laborious/activities/test_mlflow.py
View File

@@ -47,10 +47,18 @@ async def test_request_transform(mock_max, mock_dataframe, mlflow):
# Mock input data
input_data = {
'data': [
{'timestamp': '2024-01-01', 'variable': 'var1', 'value': 1.0},
{'timestamp': '2024-01-01', 'variable': 'var2', 'value': 2.0},
{'timestamp': '2024-01-02', 'variable': 'var1', 'value': 3.0},
{'timestamp': '2024-01-02', 'variable': 'var2', 'value': 4.0}
{'timestamp': '2024-01-01', 'variable': 'var1',
'value': 1.0, 'created_at': '2024-01-01 12:00:00'},
{'timestamp': '2024-01-01', 'variable': 'var2',
'value': 2.0, 'created_at': '2024-01-01 12:00:00'},
{'timestamp': '2024-01-02', 'variable': 'var1',
'value': 3.0, 'created_at': '2024-01-02 12:00:00'},
{'timestamp': '2024-01-02', 'variable': 'var2',
'value': 4.0, 'created_at': '2024-01-02 12:00:00'},
{'timestamp': '2024-01-02', 'variable': 'var1',
'value': 1.0, 'created_at': '2024-01-01 12:00:00'},
{'timestamp': '2024-01-02', 'variable': 'var2',
'value': 1.0, 'created_at': '2024-01-01 12:00:00'}
],
'model_name': 'test_model',
'model_retention': 30
@@ -60,6 +68,9 @@ async def test_request_transform(mock_max, mock_dataframe, mlflow):
expected_response = {'prediction': [0.5, 0.6]}
mlflow.model_monitoring_repository.transform.return_value = expected_response
mock_dataframe.return_value.sort_values.return_value = mock_dataframe.return_value
mock_dataframe.return_value.drop_duplicates.return_value = mock_dataframe.return_value
# Call the method
response_data = await mlflow.request_transform(input_data)