gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1097

Browse Source 
Refactor filter configuration keys from 'POLICY' to 'policy' for consistency
This commit is contained in:
vitor-aignosi
2025-06-10 11:51:27 -03:00
parent 98a931393b
commit 948fecc444
4 changed files with 11 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
laborious/activities/gates.py
View File

@@ -84,7 +84,7 @@ class Gates(BaseActivity):
if input_filter_functions[fil](data, config['config']): if input_filter_functions[fil](data, config['config']):
self.logger.debug( self.logger.debug(
f"Data not passed the input filter {fil}:{config}") f"Data not passed the input filter {fil}:{config}")
filter_output.append(config['POLICY']) filter_output.append(config['policy'])
except Exception as e: except Exception as e:
trace = traceback.format_exc() trace = traceback.format_exc()
self.notification_handler.build_and_send_notification( self.notification_handler.build_and_send_notification(
@@ -201,7 +201,7 @@ class Gates(BaseActivity):
continue continue
try: try:
if mlflow_content_filter_functions[fil](data, config): if mlflow_content_filter_functions[fil](data, config):
filter_output.append(config['POLICY']) filter_output.append(config['policy'])
self.notification_handler.build_and_send_notification( self.notification_handler.build_and_send_notification(
notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}", notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}",
message=f"Data not passed the content filter {fil}:{config}", message=f"Data not passed the content filter {fil}:{config}",

10
tests/laborious/activities/test_gates.py
View File

@@ -42,7 +42,7 @@ async def test_input_gate_filter_exception(mock_input_filter_functions, gates_ac
side_effect=Exception("Test error")) side_effect=Exception("Test error"))
input_data = { input_data = {
'filters': { 'filters': {
'EMPTY_DATA': {'POLICY': 'STOP'} 'EMPTY_DATA': {'policy': 'STOP', 'config': {}}
}, },
'data': {'value': []}, 'data': {'value': []},
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] 'path_priority': ['STOP', 'CONTINUE', 'REPEAT']
@@ -55,7 +55,7 @@ async def test_input_gate_filter_exception(mock_input_filter_functions, gates_ac
assert result == (None, 0, "") assert result == (None, 0, "")
gates_activity.notification_handler.build_and_send_notification.assert_called_once_with( gates_activity.notification_handler.build_and_send_notification.assert_called_once_with(
notification_id="INTPUT_GATE_ERROR__EMPTY_DATA", notification_id="INTPUT_GATE_ERROR__EMPTY_DATA",
message="Error in filter EMPTY_DATA:{'POLICY': 'STOP'}: \n Test error", message="Error in filter EMPTY_DATA:{'policy': 'STOP', 'config': {}}: \n Test error",
block="input_gate", block="input_gate",
level=NotificationLevel.ERROR, level=NotificationLevel.ERROR,
attachment_content=ANY attachment_content=ANY
@@ -84,7 +84,7 @@ async def test_input_gate_with_filter(gates_activity):
# Arrange # Arrange
input_data = { input_data = {
'filters': { 'filters': {
'EMPTY_DATA': {'POLICY': 'STOP'} 'EMPTY_DATA': {'policy': 'STOP', 'config': {}}
}, },
'data': {'value': []}, 'data': {'value': []},
'path_priority': ['STOP', 'CONTINUE', 'REPEAT'] 'path_priority': ['STOP', 'CONTINUE', 'REPEAT']
@@ -171,7 +171,7 @@ async def test_mlflow_response_gate_with_filter(gates_activity):
# Arrange # Arrange
input_data = { input_data = {
'filters': { 'filters': {
'API_ERROR': {'POLICY': 'STOP'} 'API_ERROR': {'policy': 'STOP'}
}, },
'data': { 'data': {
'success': False, 'success': False,
@@ -273,7 +273,7 @@ async def test_mlflow_content_gate_with_filter(gates_activity):
# Arrange # Arrange
input_data = { input_data = {
'filters': { 'filters': {
'NAN_VALUES': {'POLICY': 'STOP'} 'NAN_VALUES': {'policy': 'STOP', 'config': {}}
}, },
'data': {'value': [None, None, None]}, 'data': {'value': [None, None, None]},
'type': 'test', 'type': 'test',

4
tests/laborious/utils/filters/test_conditional_filters.py
View File

@@ -10,14 +10,14 @@ def test_filter_specific_variables_null_values():
assert filter_specific_variables_null_values( assert filter_specific_variables_null_values(
DataFrame( DataFrame(
{'variable': ['variable1', 'variable2'], 'value': [1, 2]}), {'variable': ['variable1', 'variable2'], 'value': [1, 2]}),
config={'VARIABLES': ['variable2']}) is False config={'variables': ['variable2']}) is False
def test_filter_specific_variables_null_values_with_null_values(): def test_filter_specific_variables_null_values_with_null_values():
assert filter_specific_variables_null_values( assert filter_specific_variables_null_values(
DataFrame( DataFrame(
{'variable': ['variable1', 'variable2'], 'value': [1, None]}), {'variable': ['variable1', 'variable2'], 'value': [1, None]}),
config={'VARIABLES': ['variable2']}) is True config={'variables': ['variable2']}) is True
def test_filter_empty_data(): def test_filter_empty_data():

3
tests/laborious/workflows/subworkflows/test_prediction_process.py
View File

@@ -426,7 +426,8 @@ async def test_path_flag_handler_repeat(workflow_mock, prediction_process):
{ {
'schema': schema, 'schema': schema,
'table_name': table_name, 'table_name': table_name,
'model_id': model 'model': model,
'last_timestamp': last_timestamp,
}, },
retry_policy=ANY, retry_policy=ANY,
start_to_close_timeout=ANY start_to_close_timeout=ANY