SIENTIAPDE-1110

Update sientia-dataops-library version to 1.2.0 in requirements.txt; refactor logging in activities to use a unified Logger instance and include metadata in log messages across various activities.
This commit is contained in:
vitor-aignosi
2025-06-26 15:15:31 -03:00
parent 9355af1709
commit 4a90b77786
15 changed files with 238 additions and 111 deletions

View File

@@ -14,6 +14,16 @@ def gates_fixture():
return Gates(logger=logger, notification_handler=notification_handler)
metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
@pytest.mark.asyncio
async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
"""Test data_quality_gate with NULL_VALUES_FILTER and DISCARD policy."""
@@ -31,7 +41,8 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
'tag1': {'data_range': [0, 100]},
'tag2': {'data_range': [0, 100]},
'tag3': {'data_range': [0, 100]}
}
},
**metadata
}
# Execute
@@ -60,7 +71,8 @@ async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture):
'tag1': {'data_range': [0, 100]},
'tag2': {'data_range': [0, 100]},
'tag3': {'data_range': [0, 100]}
}
},
**metadata
}
# Mock the out_of_bounds_filter to return rows with out of bounds values
@@ -95,7 +107,8 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture):
'tag2': {'data_range': [0, 100]},
'tag3': {'data_range': [0, 100]},
'tag4': {'data_range': [0, 100]}
}
},
**metadata
}
result = await gates_fixture.data_quality_gate(input_data)
@@ -111,6 +124,7 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture):
async def test_data_quality_gate_with_unknown_filter(gates_fixture):
"""Test data_quality_gate with an unknown filter."""
# Setup test data with unknown filter
gates_fixture.warning = MagicMock()
input_data = {
'filters': {
'UNKNOWN_FILTER': 'DISCARD'
@@ -123,7 +137,8 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture):
},
'model_tags': {
'tag1': {'data_range': [0, 100]}
}
},
**metadata
}
# Execute
@@ -131,8 +146,10 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture):
# Verify data is unchanged and warning is logged
assert len(result['tag']) == 1
gates_fixture.logger.warning.assert_called_once_with(
"Filter UNKNOWN_FILTER not found")
gates_fixture.warning.assert_called_once_with(
"Filter UNKNOWN_FILTER not found",
metadata=metadata['metadata']
)
@pytest.mark.asyncio
@@ -151,7 +168,8 @@ async def test_data_quality_gate_with_filter_error(gates_fixture):
},
'model_tags': {
'tag1': {'data_range': [0, 100]}
}
},
**metadata
}
# Mock the filter to raise an exception
@@ -187,7 +205,8 @@ async def test_data_quality_gate_with_empty_data(gates_fixture):
'value': [],
'timestamp': []
},
'model_tags': {}
'model_tags': {},
**metadata
}
# Execute
@@ -212,7 +231,8 @@ async def test_data_quality_gate_with_no_filters(gates_fixture):
},
'model_tags': {
'tag1': {'data_range': [0, 100]}
}
},
**metadata
}
# Execute
@@ -272,7 +292,8 @@ async def test_aggregate_data(gates_fixture):
'model_tags': {
'name1': {'aggr_function': 'avg'},
'name2': {'aggr_function': 'max'},
}
},
**metadata
}
# Expected result
@@ -309,7 +330,8 @@ async def test_aggregate_data_with_continue(gates_fixture):
'model_tags': {
'name1': {'aggr_function': 'avg'},
'name2': {'aggr_function': 'max'},
}
},
**metadata
}
# Expected result
@@ -343,7 +365,8 @@ async def test_aggregate_data_raise_exception(gates_fixture):
'model_tags': {
'name1': {'aggr_function': 'avg'},
'name2': {'aggr_function': 'max'},
}
},
**metadata
}
try: