SIENTIAPDE-1110
Update sientia-dataops-library version to 1.2.0 in requirements.txt; refactor logging in activities to use a unified Logger instance and include metadata in log messages across various activities.
This commit is contained in:
@@ -14,6 +14,16 @@ def gates_fixture():
|
||||
return Gates(logger=logger, notification_handler=notification_handler)
|
||||
|
||||
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'model_id': 'test_model_id',
|
||||
'model_name': 'test_model',
|
||||
'schedule_name': 'test_schedule',
|
||||
'workflow_name': 'scouter'
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
|
||||
"""Test data_quality_gate with NULL_VALUES_FILTER and DISCARD policy."""
|
||||
@@ -31,7 +41,8 @@ async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
|
||||
'tag1': {'data_range': [0, 100]},
|
||||
'tag2': {'data_range': [0, 100]},
|
||||
'tag3': {'data_range': [0, 100]}
|
||||
}
|
||||
},
|
||||
**metadata
|
||||
}
|
||||
|
||||
# Execute
|
||||
@@ -60,7 +71,8 @@ async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture):
|
||||
'tag1': {'data_range': [0, 100]},
|
||||
'tag2': {'data_range': [0, 100]},
|
||||
'tag3': {'data_range': [0, 100]}
|
||||
}
|
||||
},
|
||||
**metadata
|
||||
}
|
||||
|
||||
# Mock the out_of_bounds_filter to return rows with out of bounds values
|
||||
@@ -95,7 +107,8 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture):
|
||||
'tag2': {'data_range': [0, 100]},
|
||||
'tag3': {'data_range': [0, 100]},
|
||||
'tag4': {'data_range': [0, 100]}
|
||||
}
|
||||
},
|
||||
**metadata
|
||||
}
|
||||
|
||||
result = await gates_fixture.data_quality_gate(input_data)
|
||||
@@ -111,6 +124,7 @@ async def test_data_quality_gate_with_multiple_filters(gates_fixture):
|
||||
async def test_data_quality_gate_with_unknown_filter(gates_fixture):
|
||||
"""Test data_quality_gate with an unknown filter."""
|
||||
# Setup test data with unknown filter
|
||||
gates_fixture.warning = MagicMock()
|
||||
input_data = {
|
||||
'filters': {
|
||||
'UNKNOWN_FILTER': 'DISCARD'
|
||||
@@ -123,7 +137,8 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture):
|
||||
},
|
||||
'model_tags': {
|
||||
'tag1': {'data_range': [0, 100]}
|
||||
}
|
||||
},
|
||||
**metadata
|
||||
}
|
||||
|
||||
# Execute
|
||||
@@ -131,8 +146,10 @@ async def test_data_quality_gate_with_unknown_filter(gates_fixture):
|
||||
|
||||
# Verify data is unchanged and warning is logged
|
||||
assert len(result['tag']) == 1
|
||||
gates_fixture.logger.warning.assert_called_once_with(
|
||||
"Filter UNKNOWN_FILTER not found")
|
||||
gates_fixture.warning.assert_called_once_with(
|
||||
"Filter UNKNOWN_FILTER not found",
|
||||
metadata=metadata['metadata']
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -151,7 +168,8 @@ async def test_data_quality_gate_with_filter_error(gates_fixture):
|
||||
},
|
||||
'model_tags': {
|
||||
'tag1': {'data_range': [0, 100]}
|
||||
}
|
||||
},
|
||||
**metadata
|
||||
}
|
||||
|
||||
# Mock the filter to raise an exception
|
||||
@@ -187,7 +205,8 @@ async def test_data_quality_gate_with_empty_data(gates_fixture):
|
||||
'value': [],
|
||||
'timestamp': []
|
||||
},
|
||||
'model_tags': {}
|
||||
'model_tags': {},
|
||||
**metadata
|
||||
}
|
||||
|
||||
# Execute
|
||||
@@ -212,7 +231,8 @@ async def test_data_quality_gate_with_no_filters(gates_fixture):
|
||||
},
|
||||
'model_tags': {
|
||||
'tag1': {'data_range': [0, 100]}
|
||||
}
|
||||
},
|
||||
**metadata
|
||||
}
|
||||
|
||||
# Execute
|
||||
@@ -272,7 +292,8 @@ async def test_aggregate_data(gates_fixture):
|
||||
'model_tags': {
|
||||
'name1': {'aggr_function': 'avg'},
|
||||
'name2': {'aggr_function': 'max'},
|
||||
}
|
||||
},
|
||||
**metadata
|
||||
}
|
||||
|
||||
# Expected result
|
||||
@@ -309,7 +330,8 @@ async def test_aggregate_data_with_continue(gates_fixture):
|
||||
'model_tags': {
|
||||
'name1': {'aggr_function': 'avg'},
|
||||
'name2': {'aggr_function': 'max'},
|
||||
}
|
||||
},
|
||||
**metadata
|
||||
}
|
||||
|
||||
# Expected result
|
||||
@@ -343,7 +365,8 @@ async def test_aggregate_data_raise_exception(gates_fixture):
|
||||
'model_tags': {
|
||||
'name1': {'aggr_function': 'avg'},
|
||||
'name2': {'aggr_function': 'max'},
|
||||
}
|
||||
},
|
||||
**metadata
|
||||
}
|
||||
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user