gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-994

Browse Source 
Refactor activity methods and update requirements.txt to enhance functionality and remove deprecated filters. Added detailed docstrings for clarity and improved error handling in data processing workflows.
This commit is contained in:
vitor-aignosi
2025-05-09 16:27:57 -03:00
parent 43f19ed93a
commit d09fb6ac5e
22 changed files with 1435 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
laborious/activities/gates.py
View File

@@ -14,15 +14,29 @@ with workflow.unsafe.imports_passed_through():
input_filter_functions = {
'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values,
'EMPTY_DATA': filter_empty_data
'EMPTY_DATA': filter_empty_data,
'path_confidence': {
'stop': -1,
'continue': 2,
'repeat': -1
}
}
transform_filter_functions = {
'response_filter': {
'API_ERROR': api_error_filter,
mlflow_response_filter_functions = {
'API_ERROR': api_error_filter,
'path_confidence': {
'stop': -1,
'continue': 10,
'repeat': -1
},
'content_filter': {
'NAN_VALUES': nan_values_filter,
}
mlflow_content_filter_functions = {
'NAN_VALUES': nan_values_filter,
'path_confidence': {
'stop': -1,
'continue': 18,
'repeat': -1
}
}
@@ -40,13 +54,13 @@ class Gates(BaseActivity):
input_data (dict): The input data. Contains:
filters (dict): The filters to apply.
data (dict[str, Any]): The data to filter.
path_priority (list[str]): The path priority.
Returns:
tuple[str, int]: ('stop', -1) if some filter policy is 'stop', ('continue', 2)
if no filter policy is 'stop' and some filter policy is 'continue',
None if no filter is applied.
tuple[str, int]: (policy, confidence) based in priority list and filter configuration and functions.
"""
filters = input_data['filters']
data = DataFrame(input_data['data'])
path_priority = input_data['path_priority']
filter_output = []
for fil, config in filters.items():
@@ -63,23 +77,34 @@ class Gates(BaseActivity):
attachment_content=trace
)
if 'stop' in filter_output:
return 'stop', -1
elif 'continue' in filter_output:
return 'continue', 2
for path_flag in path_priority:
if path_flag in filter_output:
return path_flag, input_filter_functions['path_confidence'][path_flag]
return None, 0
@activity.defn(name="mlflow_gate")
async def mlflow_gate(self, input_data: dict[str, Any]) -> tuple[str, int]:
@activity.defn(name="mlflow_response_gate")
async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str, int]:
"""
Filters the data based on the mlflow response filters. The return value is a tuple with the first element
being the policy and the second element being the confidence status.
Args:
input_data (dict): The input data. Contains:
filters (dict): The filter configuration to apply.
data (dict[str, Any]): The data to filter.
path_priority (list[str]): The path priority list.
type (str): The type of the gate.
Returns:
tuple[str, int]: (policy, confidence) based in priority list and filter configuration and functions.
"""
filters = input_data['filters']
data = DataFrame(input_data['data'])
data = input_data['data']
gate_type = input_data['type']
path_priority = input_data['path_priority']
filter_output = []
for fil, config in filters.items():
if transform_filter_functions['response_filter'][fil](data, config):
if mlflow_response_filter_functions[fil](data, config):
filter_output.append(config['POLICY'])
self.notification_handler.build_and_send_notification(
notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}",
@@ -89,18 +114,36 @@ class Gates(BaseActivity):
attachment_content=data['content']['traceback']
)
if 'stop' in filter_output:
return 'stop', -1
elif 'continue' in filter_output:
return 'continue', 10
for path_flag in path_priority:
if path_flag in filter_output:
return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag]
if gate_type == 'predict':
return None, 0
return None, 0
data = DataFrame(data['content'])
@activity.defn(name="mlflow_content_gate")
async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str, int]:
"""
Filters the data based on the mlflow content filters. The return value is a tuple with the first element
being the policy and the second element being the confidence status.
Args:
input_data (dict): The input data. Contains:
filters (dict): The filter configuration to apply.
data (dict[str, Any]): The data to filter.
path_priority (list[str]): The path priority list.
type (str): The type of the gate.
Returns:
tuple[str, int]: (policy, confidence) based in priority list and filter configuration and functions.
"""
filters = input_data['filters']
data = DataFrame(input_data['data'])
gate_type = input_data['type']
path_priority = input_data['path_priority']
filter_output = []
for fil, config in filters.items():
if transform_filter_functions['content_filter'][fil](data, config):
if mlflow_content_filter_functions[fil](data, config):
filter_output.append(config['POLICY'])
self.notification_handler.build_and_send_notification(
notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}",
@@ -110,17 +153,26 @@ class Gates(BaseActivity):
attachment_content=data.to_string()
)
if 'stop' in filter_output:
return 'stop', -1
elif 'continue' in filter_output:
return 'continue', 18
for path_flag in path_priority:
if path_flag in filter_output:
return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag]
return None, 0
@activity.defn(name="format_prediction")
async def format_prediction(self, input_data: dict[str, Any]) -> str:
async def format_prediction(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Formats the prediction data.
Args:
input_data (dict): The input data. Contains:
data (dict[str, Any]): The data to format.
timestamp (str): The timestamp of the data.
model_id (str): The id of the model.
prediction_confidence (float): The confidence of the prediction.
Returns:
dict: The formatted data.
"""
data = DataFrame(input_data['data'])
data['timestamp'] = input_data['timestamp']
data['model_id'] = input_data['model_id']
data['prediction_confidence'] = input_data['prediction_confidence']
@@ -131,7 +183,21 @@ class Gates(BaseActivity):
return data.to_dict()
@activity.defn(name="format_default_prediction")
async def format_default_prediction(self, input_data: dict[str, Any]) -> str:
async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Creates and formats the default prediction data, with zero value in prediction,
and usefull information in the other fields.
Args:
input_data (dict): The input data. Contains:
timestamp (str): The timestamp of the data.
model_id (str): The id of the model.
prediction_confidence (float): The confidence of the prediction.
comment (str): The comment of the prediction.
Returns:
dict: The formatted data.
"""
return DataFrame({
'prediction': [0],
'response_time': [0],