gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1110

Browse Source 
Enhance MLFlow activity to sort data by created_at and remove duplicates; update tests to reflect changes
This commit is contained in:
vitor-aignosi
2025-06-25 15:33:31 -03:00
parent 2be78b1aea
commit cefef0b1e9
2 changed files with 20 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
laborious/activities/mlflow.py
View File

@@ -44,6 +44,11 @@ class MLFlow(BaseActivity):
self.logger.debug("Raw input data:") self.logger.debug("Raw input data:")
self.logger.debug(data) self.logger.debug(data)
# Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
data = data.sort_values('created_at', ascending=False).drop_duplicates(
subset=['variable', 'timestamp'], keep='first'
)
data = data.pivot( data = data.pivot(
index='timestamp', columns='variable', index='timestamp', columns='variable',
values='value') values='value')

19
tests/laborious/activities/test_mlflow.py
View File

@@ -47,10 +47,18 @@ async def test_request_transform(mock_max, mock_dataframe, mlflow):
# Mock input data # Mock input data
input_data = { input_data = {
'data': [ 'data': [
{'timestamp': '2024-01-01', 'variable': 'var1', 'value': 1.0}, {'timestamp': '2024-01-01', 'variable': 'var1',
{'timestamp': '2024-01-01', 'variable': 'var2', 'value': 2.0}, 'value': 1.0, 'created_at': '2024-01-01 12:00:00'},
{'timestamp': '2024-01-02', 'variable': 'var1', 'value': 3.0}, {'timestamp': '2024-01-01', 'variable': 'var2',
{'timestamp': '2024-01-02', 'variable': 'var2', 'value': 4.0} 'value': 2.0, 'created_at': '2024-01-01 12:00:00'},
{'timestamp': '2024-01-02', 'variable': 'var1',
'value': 3.0, 'created_at': '2024-01-02 12:00:00'},
{'timestamp': '2024-01-02', 'variable': 'var2',
'value': 4.0, 'created_at': '2024-01-02 12:00:00'},
{'timestamp': '2024-01-02', 'variable': 'var1',
'value': 1.0, 'created_at': '2024-01-01 12:00:00'},
{'timestamp': '2024-01-02', 'variable': 'var2',
'value': 1.0, 'created_at': '2024-01-01 12:00:00'}
], ],
'model_name': 'test_model', 'model_name': 'test_model',
'model_retention': 30 'model_retention': 30
@@ -60,6 +68,9 @@ async def test_request_transform(mock_max, mock_dataframe, mlflow):
expected_response = {'prediction': [0.5, 0.6]} expected_response = {'prediction': [0.5, 0.6]}
mlflow.model_monitoring_repository.transform.return_value = expected_response mlflow.model_monitoring_repository.transform.return_value = expected_response
mock_dataframe.return_value.sort_values.return_value = mock_dataframe.return_value
mock_dataframe.return_value.drop_duplicates.return_value = mock_dataframe.return_value
# Call the method # Call the method
response_data = await mlflow.request_transform(input_data) response_data = await mlflow.request_transform(input_data)