gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-994

Browse Source 
Refactor and enhance the laborious workflow and utilities

- Removed outdated test file `test_predictions_batch.py` from workflows.
- Added `input_sample.json` for standardized input configuration.
- Introduced `connectors_config.py` to manage database and service configurations.
- Implemented a logging utility in `logger.py` for consistent logging across the application.
- Created `policies.py` to define retry policies for workflows.
- Developed comprehensive tests for `MLFlowRepository` in `test_model_repository.py`.
- Added extensive tests for `OpcRepository` in `test_opc_repository.py`.
- Updated `test_predictions_batch.py` to reflect new workflow structure and testing methodology.
This commit is contained in:
vitor-aignosi
2025-05-23 17:34:47 -03:00
parent 5fe552410b
commit 67fe4afaa6
30 changed files with 1385 additions and 765 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
laborious/activities/gates.py
View File

@@ -5,67 +5,85 @@ with workflow.unsafe.imports_passed_through():
import traceback
from logging import Logger
from sientia_do.notifications.handlers import NotificationHandler
from laborious.activities.base import BaseActivity
from typing import Any
from laborious.utils.filters.conditional_filters import filter_empty_data, filter_specific_variables_null_values
from pandas import DataFrame
from sientia_do.notifications.models import NotificationLevel
from laborious.activities.base import BaseActivity
from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter
from typing import Any
from laborious.utils.filters.conditional_filters import (
filter_empty_data,
filter_specific_variables_null_values
)
from pandas import DataFrame
from datetime import datetime
input_filter_functions = {
'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values,
'EMPTY_DATA': filter_empty_data,
'path_confidence': {
'stop': -1,
'continue': 2,
'repeat': -1
'STOP': -1,
'CONTINUE': 2,
'REPEAT': -1
}
}
mlflow_response_filter_functions = {
'API_ERROR': api_error_filter,
'path_confidence': {
'stop': -1,
'continue': 10,
'repeat': -1
'STOP': -1,
'CONTINUE': 10,
'REPEAT': -1
},
}
mlflow_content_filter_functions = {
'NAN_VALUES': nan_values_filter,
'path_confidence': {
'stop': -1,
'continue': 18,
'repeat': -1
'STOP': -1,
'CONTINUE': 18,
'REPEAT': -1
}
}
class Gates(BaseActivity):
def __init__(self, logger: Logger, notification_handler: NotificationHandler):
super().__init__(logger, notification_handler)
BaseActivity.__init__(self, logger, notification_handler)
@activity.defn(name="input_gate")
async def input_gate(self, input_data: dict[str, Any]) -> tuple[str, int]:
async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Filters the data based on the filters. The return value is a tuple with the first element
being the policy and the second element being the confidence status.
Args:
input_data (dict): The input data. Contains:
filters (dict): The filters to apply.
The key is the filter name and the value is the filter configuration.
data (dict[str, Any]): The data to filter.
path_priority (list[str]): The path priority.
Returns:
tuple[str, int]: (policy, confidence) based in priority list and filter configuration and functions.
tuple[str | None, int, str]: (policy, confidence) based in priority
list and filter configuration and functions.
"""
self.logger.debug("Performing input gate...")
filters = input_data['filters']
data = DataFrame(input_data['data'])
path_priority = input_data['path_priority']
filter_output = []
self.logger.debug(f"Input data:\n {data.to_string()}")
self.logger.debug(f"Filters: {filters}")
for fil, config in filters.items():
if fil not in input_filter_functions:
self.logger.error(f"Filter {fil} not found")
continue
try:
if input_filter_functions[fil](data, config):
self.logger.debug(
f"Data not passed the input filter {fil}:{config}")
filter_output.append(config['POLICY'])
except Exception as e:
trace = traceback.format_exc()
@@ -79,14 +97,18 @@ class Gates(BaseActivity):
for path_flag in path_priority:
if path_flag in filter_output:
return path_flag, input_filter_functions['path_confidence'][path_flag]
self.logger.debug(f"Input gate result: {path_flag}")
return path_flag, input_filter_functions['path_confidence'][path_flag], \
"Input data with bad quality"
return None, 0
self.logger.debug("Nothing was filtered by the input gate")
return None, 0, ""
@activity.defn(name="mlflow_response_gate")
async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str, int]:
async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Filters the data based on the mlflow response filters. The return value is a tuple with the first element
Filters the data based on the mlflow response filters.
The return value is a tuple with the first element
being the policy and the second element being the confidence status.
Args:
input_data (dict): The input data. Contains:
@@ -95,17 +117,29 @@ class Gates(BaseActivity):
path_priority (list[str]): The path priority list.
type (str): The type of the gate.
Returns:
tuple[str, int]: (policy, confidence) based in priority list and filter configuration and functions.
tuple[str | None, int, str]: (policy, confidence) based in priority list
and filter configuration and functions.
"""
self.logger.debug("Performing mlflow response gate...")
filters = input_data['filters']
data = input_data['data']
gate_type = input_data['type']
path_priority = input_data['path_priority']
filter_output = []
self.logger.debug(f"Input data:\n {data}")
self.logger.debug(f"Filters: {filters}")
comments = []
for fil, config in filters.items():
if fil not in mlflow_response_filter_functions:
continue
if mlflow_response_filter_functions[fil](data, config):
filter_output.append(config['POLICY'])
comments.append(data['content']['message'])
self.notification_handler.build_and_send_notification(
notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}",
message=data['content']['message'],
@@ -116,14 +150,18 @@ class Gates(BaseActivity):
for path_flag in path_priority:
if path_flag in filter_output:
return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag]
self.logger.debug(f"Mlflow response gate result: {path_flag}")
return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag], \
", ".join(comments)
return None, 0
self.logger.debug("Nothing was filtered by the mlflow response gate")
return None, 0, ""
@activity.defn(name="mlflow_content_gate")
async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str, int]:
async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Filters the data based on the mlflow content filters. The return value is a tuple with the first element
Filters the data based on the mlflow content filters.
The return value is a tuple with the first element
being the policy and the second element being the confidence status.
Args:
input_data (dict): The input data. Contains:
@@ -132,9 +170,12 @@ class Gates(BaseActivity):
path_priority (list[str]): The path priority list.
type (str): The type of the gate.
Returns:
tuple[str, int]: (policy, confidence) based in priority list and filter configuration and functions.
tuple[str | None, int, str]: (policy, confidence) based in priority
list and filter configuration and functions.
"""
self.logger.debug("Performing mlflow content gate...")
filters = input_data['filters']
data = DataFrame(input_data['data'])
gate_type = input_data['type']
@@ -142,7 +183,12 @@ class Gates(BaseActivity):
filter_output = []
self.logger.debug(f"Input data:\n {data}")
self.logger.debug(f"Filters: {filters}")
for fil, config in filters.items():
if fil not in mlflow_content_filter_functions:
continue
if mlflow_content_filter_functions[fil](data, config):
filter_output.append(config['POLICY'])
self.notification_handler.build_and_send_notification(
@@ -155,9 +201,12 @@ class Gates(BaseActivity):
for path_flag in path_priority:
if path_flag in filter_output:
return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag]
self.logger.debug(f"Mlflow content gate result: {path_flag}")
return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag], \
"Transformed data not passed the content filter"
return None, 0
self.logger.debug("Nothing was filtered by the mlflow content gate")
return None, 0, ""
@activity.defn(name="format_prediction")
async def format_prediction(self, input_data: dict[str, Any]) -> dict[str, Any]:
@@ -172,12 +221,14 @@ class Gates(BaseActivity):
Returns:
dict: The formatted data.
"""
self.logger.debug("Formatting prediction...")
data = DataFrame(input_data['data'])
data['timestamp'] = input_data['timestamp']
data['model_id'] = input_data['model_id']
data['prediction_confidence'] = input_data['prediction_confidence']
data['prediction_status'] = 'Good'
data['comment'] = ""
data['comments'] = ""
data.sort_values(by='timestamp', inplace=True)
return data.to_dict()
@@ -198,6 +249,8 @@ class Gates(BaseActivity):
dict: The formatted data.
"""
self.logger.debug("Formatting default prediction...")
return DataFrame({
'prediction': [0],
'response_time': [0],
@@ -205,7 +258,7 @@ class Gates(BaseActivity):
'model_id': [input_data['model_id']],
'prediction_confidence': [input_data['prediction_confidence']],
'prediction_status': ['Bad'],
'comment': [input_data['comment']]
'comments': [input_data['comment']]
}).to_dict()
@activity.defn(name="get_last_timestamp")
@@ -219,4 +272,6 @@ class Gates(BaseActivity):
str: The last timestamp of the data.
"""
data = DataFrame(input_data['data'])
if data.empty:
return datetime.now().strftime('%Y-%m-%d %H:%M:%S')
return max(data['timestamp'].values.tolist())