diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py new file mode 100644 index 0000000..af7268a --- /dev/null +++ b/laborious/activities/activities.py @@ -0,0 +1,54 @@ +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + from laborious.activities.postgres import Postgres + from laborious.activities.mlflow import MLFlow + from laborious.activities.gates import Gates + from laborious.activities.opc import OPC + from typing import Any + from logging import Logger + from sientia_do.notifications.handlers import NotificationHandler + + +class Activities(Postgres, MLFlow, Gates, OPC): + + def __init__(self, + postgres_config: dict[str, Any], + mlflow_config: dict[str, Any], + opc_config: dict[str, Any], + logger: Logger, notification_handler: NotificationHandler): + + # Initialize parent classes + Postgres.__init__(self, host=postgres_config['host'], + port=postgres_config['port'], + user=postgres_config['user'], + password=postgres_config['password'], + dbname=postgres_config['dbname'], + min_connections=postgres_config['min_connections'], + max_connections=postgres_config['max_connections'], + logger=logger, + notification_handler=notification_handler) + + MLFlow.__init__(self, mlflow_host=mlflow_config['host'], + mlflow_port=mlflow_config['port'], + mlflow_username=mlflow_config['username'], + mlflow_password=mlflow_config['password'], + logger=logger, + notification_handler=notification_handler) + + Gates.__init__(self, logger=logger, + notification_handler=notification_handler) + + OPC.__init__(self, + name=opc_config['name'], + url=opc_config['url'], + server_uri=opc_config['server_uri'], + cert_path=opc_config['cert_path'], + private_key_path=opc_config['private_key_path'], + server_cert_path=opc_config['server_cert_path'], + logger=logger, + notification_handler=notification_handler) + + @activity.defn(name="prepare_activity") + async def prepare_activity(self, schedule_name: str, model_name: str, model_id: str): + await super().prepare_activity(schedule_name, model_name, model_id) diff --git a/laborious/activities/base.py b/laborious/activities/base.py index c195f8a..8c6fba8 100644 --- a/laborious/activities/base.py +++ b/laborious/activities/base.py @@ -8,10 +8,9 @@ class BaseActivity: self.logger = logger self.notification_handler = notification_handler - @activity.defn(name="prepare_notification_handler") - async def prepare_notification_handler(self, schedule_name: str, - model_name: str, - model_id: str): + def prepare_activity(self, schedule_name: str, + model_name: str, + model_id: str): self.notification_handler.base_notification.schedule_name = schedule_name self.notification_handler.base_notification.model_name = model_name self.notification_handler.base_notification.model_id = model_id diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 73cd49a..ac07fbf 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -1,19 +1,31 @@ from temporalio import activity, workflow + with workflow.unsafe.imports_passed_through(): + import traceback from logging import Logger from sientia_do.notifications.handlers import NotificationHandler from laborious.activities.base import BaseActivity from typing import Any from laborious.utils.filters.conditional_filters import filter_empty_data, filter_specific_variables_null_values from pandas import DataFrame + from sientia_do.notifications.models import NotificationLevel + from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter - -filter_functions = { +input_filter_functions = { 'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values, 'EMPTY_DATA': filter_empty_data } +transform_filter_functions = { + 'response_filter': { + 'API_ERROR': api_error_filter, + }, + 'content_filter': { + 'NAN_VALUES': nan_values_filter, + } +} + class Gates(BaseActivity): def __init__(self, logger: Logger, notification_handler: NotificationHandler): @@ -25,7 +37,9 @@ class Gates(BaseActivity): Filters the data based on the filters. The return value is a tuple with the first element being the policy and the second element being the confidence status. Args: - input_data (dict): The input data. + input_data (dict): The input data. Contains: + filters (dict): The filters to apply. + data (dict[str, Any]): The data to filter. Returns: tuple[str, int]: ('stop', -1) if some filter policy is 'stop', ('continue', 2) if no filter policy is 'stop' and some filter policy is 'continue', @@ -36,8 +50,18 @@ class Gates(BaseActivity): filter_output = [] for fil, config in filters.items(): - if filter_functions[fil](data, config): - filter_output.append(config['POLICY']) + try: + if input_filter_functions[fil](data, config): + filter_output.append(config['POLICY']) + except Exception as e: + trace = traceback.format_exc() + self.notification_handler.build_and_send_notification( + notification_id=f"INTPUT_GATE_ERROR__{fil}", + message=f"Error in filter {fil}:{config}: \n {e}", + block="input_gate", + level=NotificationLevel.ERROR, + attachment_content=trace + ) if 'stop' in filter_output: return 'stop', -1 @@ -45,3 +69,75 @@ class Gates(BaseActivity): return 'continue', 2 return None, 0 + + @activity.defn(name="mlflow_gate") + async def mlflow_gate(self, input_data: dict[str, Any]) -> tuple[str, int]: + + filters = input_data['filters'] + data = DataFrame(input_data['data']) + gate_type = input_data['type'] + + filter_output = [] + for fil, config in filters.items(): + if transform_filter_functions['response_filter'][fil](data, config): + filter_output.append(config['POLICY']) + self.notification_handler.build_and_send_notification( + notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}", + message=data['content']['message'], + block="mlflow_gate", + level=NotificationLevel.WARNING, + attachment_content=data['content']['traceback'] + ) + + if 'stop' in filter_output: + return 'stop', -1 + elif 'continue' in filter_output: + return 'continue', 10 + + if gate_type == 'predict': + return None, 0 + + data = DataFrame(data['content']) + + for fil, config in filters.items(): + if transform_filter_functions['content_filter'][fil](data, config): + filter_output.append(config['POLICY']) + self.notification_handler.build_and_send_notification( + notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}", + message=f"Data not passed the content filter {fil}:{config}", + block="mlflow_gate", + level=NotificationLevel.WARNING, + attachment_content=data.to_string() + ) + + if 'stop' in filter_output: + return 'stop', -1 + elif 'continue' in filter_output: + return 'continue', 18 + + return None, 0 + + @activity.defn(name="format_prediction") + async def format_prediction(self, input_data: dict[str, Any]) -> str: + data = DataFrame(input_data['data']) + + data['timestamp'] = input_data['timestamp'] + data['model_id'] = input_data['model_id'] + data['prediction_confidence'] = input_data['prediction_confidence'] + data['prediction_status'] = 'Good' + data['comment'] = "" + data.sort_values(by='timestamp', inplace=True) + + return data.to_dict() + + @activity.defn(name="format_default_prediction") + async def format_default_prediction(self, input_data: dict[str, Any]) -> str: + return DataFrame({ + 'prediction': [0], + 'response_time': [0], + 'timestamp': [input_data['timestamp']], + 'model_id': [input_data['model_id']], + 'prediction_confidence': [input_data['prediction_confidence']], + 'prediction_status': ['Bad'], + 'comment': [input_data['comment']] + }).to_dict() diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index e69de29..af3408a 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -0,0 +1,65 @@ +import numpy as np +from pandas import DataFrame +from temporalio import activity, workflow + + +with workflow.unsafe.imports_passed_through(): + from laborious.activities.base import BaseActivity + from typing import Any + from logging import Logger + from sientia_do.notifications.handlers import NotificationHandler + from laborious.utils.model_repository import ModelMonitoringRepository + from sientia_do.notifications.models import NotificationLevel + + +class MLFlow(BaseActivity): + def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str, + mlflow_password: str, logger: Logger, notification_handler: NotificationHandler): + super().__init__(logger, notification_handler) + self.mlflow_host = mlflow_host + self.mlflow_port = mlflow_port + self.mlflow_username = mlflow_username + self.mlflow_password = mlflow_password + + self.model_monitoring_repository = ModelMonitoringRepository( + f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password + ) + + @activity.defn(name="request_transform") + async def request_transform(self, input_data: dict[str, Any]) -> tuple[dict[str, Any], str]: + self.logger.info('Transforming data...') + data = DataFrame(input_data['data']) + model_name = input_data['model_name'] + model_retention = input_data['model_retention'] + + self.logger.debug(data) + + data = data.pivot( + index='timestamp', columns='variable', + values='value') + data.fillna(np.nan, inplace=True) + data.reset_index(inplace=True) + data.columns.name = None + + response_data = self.model_monitoring_repository.transform( + model_name, data, model_retention) + + timestamp = max(data['timestamp'].values.tolist()) + + return response_data, timestamp + + @activity.defn(name="request_predict") + async def request_predict(self, input_data: dict[str, Any]) -> tuple[dict[str, Any], str]: + self.logger.info('Predicting data...') + data = DataFrame(input_data['data']) + model_name = input_data['model_name'] + model_retention = input_data['model_retention'] + + self.logger.debug(data) + + data.replace(np.nan, None, inplace=True) + + response_data = self.model_monitoring_repository.predict( + model_name, data, model_retention) + + return response_data diff --git a/laborious/activities/opc.py b/laborious/activities/opc.py index e69de29..eae202f 100644 --- a/laborious/activities/opc.py +++ b/laborious/activities/opc.py @@ -0,0 +1,76 @@ +import traceback +from pandas import DataFrame +from temporalio import activity, workflow + + +with workflow.unsafe.imports_passed_through(): + from logging import Logger + from sientia_do.notifications.handlers import NotificationHandler + from laborious.activities.base import BaseActivity + from laborious.utils.repository.opc_repository import OpcRepository + from typing import Any + from sientia_do.notifications.models import NotificationLevel + + +class OPC(BaseActivity): + def __init__(self, + name: str, url: str, server_uri: str, + cert_path: str, private_key_path: str, server_cert_path: str, + logger: Logger, notification_handler: NotificationHandler): + + self.logger = logger + self.notification_handler = notification_handler + self.name = name + self.url = url + self.server_uri = server_uri + self.cert_path = cert_path + self.private_key_path = private_key_path + self.server_cert_path = server_cert_path + + self.opc_repository = OpcRepository( + name=self.name, + url=self.url, + logger=self.logger, + server_uri=self.server_uri, + cert_path=self.cert_path, + private_key_path=self.private_key_path, + server_cert_path=self.server_cert_path + ) + + self.opc_repository.connect() + + @activity.defn(name='write_opc_data') + async def write_opc_data(self, input_data: dict[str, Any]): + data = DataFrame(input_data['data']) + _opc_servers = input_data['opc_servers'] + opc_output_config = input_data['opc_output_config'] + + for tag, config in opc_output_config['prediction_tags'].items(): + try: + self.opc_repository.write_data( + tag, data.head(1)['prediction'].values[0], config['data_type']) + except Exception as e: + trace = traceback.format_exc() + self.notification_handler.build_and_send_notification( + notification_id="WRITE_OPC_PREDICTION_ERROR", + message=f"Error writing data to OPC server: {e}", + block="write_opc_data", + level=NotificationLevel.ERROR, + attachment_content=trace + ) + self.logger.error(trace) + + for tag, config in opc_output_config['confidence_tags'].items(): + try: + self.opc_repository.write_data( + tag, data.head(1)['prediction_confidence'].values[0], config['data_type']) + except Exception as e: + trace = traceback.format_exc() + self.notification_handler.build_and_send_notification( + notification_id="WRITE_OPC_CONFIDENCE_ERROR", + message=f"Error writing data to OPC server: {e}", + block="write_opc_data", + level=NotificationLevel.ERROR, + attachment_content=trace + ) + self.logger.error(trace) diff --git a/laborious/activities/postgres.py b/laborious/activities/postgres.py index 0fc8cdd..7ef2309 100644 --- a/laborious/activities/postgres.py +++ b/laborious/activities/postgres.py @@ -83,7 +83,10 @@ class Postgres(BaseActivity): Repeats the last prediction for a given model. Args: - query_items (dict[str, str]): The query items. + query_items (dict[str, str]): The query items. Contains: + schema (str): The schema of the table. + table_name (str): The name of the table. + model (str): The model to repeat the prediction for. Returns: None diff --git a/laborious/utils/filters/api_filters.py b/laborious/utils/filters/api_filters.py deleted file mode 100644 index 4e4bdd2..0000000 --- a/laborious/utils/filters/api_filters.py +++ /dev/null @@ -1,69 +0,0 @@ -import numpy as np -from pandas import DataFrame -from laborious.utils.filters.base_filter import Filter - - -class ApiErrorFilter(Filter): - def __init__(self, policy): - self.policy = policy - super().__init__('API_FILTER') - - def method(self, response: dict, prediction_confidence: int): - """ - Processes the API response and determines the next action based on the response and prediction confidence. - Args: - response (dict): The API response to be processed. - prediction_confidence (int): The confidence level of the prediction. - Returns: - str: 'stop' if the policy is to stop on captured errors, 'continue' if the policy is to continue on captured errors. - Raises: - KeyError: If 'success' or 'content' keys are missing in the response dictionary. - """ - - captured = False - if not response and prediction_confidence == 10: - self.warning('No valid response.') - captured = True - - else: - if not response['success']: - message = response['content']["message"] - self.warning( - f'Model repository error: {message}') - captured = True - if captured and self.policy == 'stop': - return 'stop' - elif captured and self.policy == 'continue': - return 'continue' - - -class NaNValuesFilter(Filter): - def __init__(self, policy): - self.policy = policy - super().__init__('NAN_VALUES') - - def method(self, predictions: DataFrame, prediction_confidence: int): - """ - Processes the given predictions DataFrame by replacing None values with NaN, - dropping the 'timestamp' column if it exists, and checking for NaN values. - Args: - predictions (pd.DataFrame): The DataFrame containing prediction data. - prediction_confidence (float): The confidence level of the predictions. - Returns: - float or int or bool: Returns the prediction confidence if the DataFrame - is not entirely NaN. If all values are NaN and the policy is 'stop', - returns False. If all values are NaN and the policy is 'continue', - returns 18. - """ - - data = predictions.replace({None: np.nan}).drop( - columns=['timestamp'], errors='ignore') - - if data.isna().all().all(): - if self.policy == 'stop': - self.warning('All values are NaN.') - return False - elif self.policy == 'continue': - return 18 - - return prediction_confidence diff --git a/laborious/utils/filters/mlflow_filters.py b/laborious/utils/filters/mlflow_filters.py new file mode 100644 index 0000000..3f5db3b --- /dev/null +++ b/laborious/utils/filters/mlflow_filters.py @@ -0,0 +1,23 @@ +import numpy as np +from pandas import DataFrame +from laborious.utils.filters.base_filter import Filter + + +def api_error_filter(response: dict, _config: dict): + if not response: + return True + + if not response['success']: + return True + + return False + + +def nan_values_filter(predictions: DataFrame, _config: dict): + data = predictions.replace({None: np.nan}).drop( + columns=['timestamp'], errors='ignore') + + if data.isna().all().all(): + return True + + return False diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py new file mode 100644 index 0000000..5efe83b --- /dev/null +++ b/laborious/utils/repository/model_repository.py @@ -0,0 +1,296 @@ +""" +Model Monitoring Repository + +This module contains the ModelMonitoringRepository class, which is responsible for handling the communication with the Model Monitoring API. + +It includes the methods that are used to answer ModelMonitoringService requests using the Model Monitoring API functions. + +By Monitoring we mean the evaluation of the performance of models, the generation of reports. + +""" +from datetime import datetime +import traceback +import mlflow +import pandas as pd +from sientia.ModelServing import ModelServing +from pathlib import Path + + +class ModelMonitoringRepository(): + def __init__(self, host, username, password): + + self.model_serving = ModelServing(tracking_uri=host, + username=username, password=password) + + def get_current_data_df(self, current_data: pd.DataFrame, model_name: str, target: str): + """ + Get the current data as a DataFrame and update the prediction and target columns + + Parameters: + current_data (pd.DataFrame): the current data + model_name (str): the name of the model + target (str): the target column + + Returns: + DataFrame: the current data as a DataFrame + + + """ + predictions = current_data['prediction'] + + target = current_data[target] + current_data = self.model_serving.get_transformed_data( + model_name, current_data, by='model') + current_data['prediction'] = predictions + current_data['target'] = target + + return pd.DataFrame(current_data).dropna() + + def get_artifact(self, destination: str, search_by: str, run_id: str = None, + model_name: str = None, artifact_name: str = None) -> None: + """ + Get an artifact in MLflow by experiment or model and save it to a destination path using API. + If the artifact is searched by model, the latest production version will be used. + + Args: + destination: The destination path to save the artifact. + search_by: The way to search for the artifact ('experiment' or 'model'). + run_id: The run ID of the experiment (if search_by is "experiment"). + model_name: The name of the model (if search_by is "model"). + artifact_name: The path of the artifact to download. + + Returns: + artifact: The artifact(.csv) downloaded from MLflow. + """ + + self.model_serving.get_artifact(destination=destination, search_by=search_by, + run_id=run_id, model_name=model_name, artifact_name=artifact_name) + + def calculate_model_metrics(self, real_data, predictions, flag): + """ + Function to calculate the metrics of a model using API + + Parameters: + real_data (array): the real data + predictions (array): the predictions + + Returns: + dict: the metrics of the model including MSE and R2 + """ + return self.model_serving.get_model_metrics(reference_data=None, real_data=real_data, predictions=predictions, type_flag=flag) + + def get_experiment_by_run_id(self, run_id: str) -> dict: + # Get the run information using the run_id + run = mlflow.get_run(run_id) + + # Extract the experiment ID from the run + experiment_id = run.info.experiment_id + + # Get the experiment details using the experiment ID + experiment = mlflow.get_experiment(experiment_id) + experiment_name = experiment.name + return experiment_name + + def get_next_run_name(self, model_name: str) -> str: + """ + Function to get the next run number of a specific model + + Parameters: + model_name (str): the name of the model + + Returns: + str: the next run number + """ + + runs = mlflow.search_runs( + experiment_names=[model_name], order_by=["start_time desc"]) + next_run_number = len(runs) + 1 + return f"{model_name}-{next_run_number}" + + def retrain_model(self, data: pd.DataFrame, model_name: str) -> tuple: + """ + Retrain a model with new data. + + Parameters: + data (pandas.DataFrame): The new data to use for retraining. + model_name (str): The name of the model to retrain. + metrics_list (list): The metrics to be used to compare the models. + compare_metrics (bool): If True, the retrain will only be considered if the new model is better than the current one. + If False, the retrain will always be considered. + split_dataset (bool): If True, the data will be split into X and Y and into training and testing sets. + If False, the data will be used as a unique block for retraining. + update_report (bool): If True, a report will be created with the data of the retrained model. + update_transformation (bool): If True, the model will be updated in the MLflow tracking server. + update_prediction (bool): If True, the prediction model will be updated in the MLflow tracking server. + shuffle_data (bool): If True, the data will be shuffled before splitting. + model_type (str): The type of model to get metrics for. Ex: 'regression', 'classification'. + + + Returns: + mlflow.sklearn.Model: The retrained prediction model. + mlflow.sklearn.Model: The retrained data model. + mse (float): The mean squared error of the retrained model. + r2 (float): The R-squared score of the retrained model. + """ + + # load predictor model + predictor_uri = f"models:/{model_name}/production" + # load transform model + latest_production_id = self.model_serving.get_model_run_id( + model_name, stage="Production" + ) + transform_uri = self.model_serving.get_model_uri( + latest_production_id, prediction=False + ) + # load + data_model = mlflow.sklearn.load_model(transform_uri) + prediction_model = mlflow.sklearn.load_model(predictor_uri) + data_model = data_model.fit(data) + treated_data = data_model.predict(data) + # align target column with treated_data + target_name = data_model.target_variable + y = data[target_name] + treated_data = pd.merge( + treated_data, y, left_index=True, right_index=True) + prediction_model = prediction_model.fit(treated_data) + # Example usage + experiment = self.get_experiment_by_run_id(latest_production_id) + pred_model_atributes = vars(prediction_model) # load class attributes + data_model_atributes = vars(data_model) # load class attributes + mlflow.set_experiment(experiment) + experiment_description = "Retrain model {model_name} with new data" + current_run_name = self.get_next_run_name(experiment) + with mlflow.start_run( + run_name=current_run_name, description=experiment_description + ) as _run: + # update transfomation model + # fixed parameters + for name_atribute, val_atribute in pred_model_atributes.items(): + if name_atribute != "model": + mlflow.log_param(name_atribute, val_atribute) + # update prediction model + for name_atribute, val_atribute in data_model_atributes.items(): + if name_atribute != "model": + mlflow.log_param(name_atribute, val_atribute) + # dynamic parameters, including model itself + mlflow.sklearn.log_model(data_model, "data_model") + file_path = f"laborious/data/raw_data_{model_name}.csv" + data.to_csv( + f"laborious/data/raw_data_{model_name}.csv", index=True) + # log the data raw + mlflow.log_artifact(file_path) + + # dynamic parameters, including model itself + mlflow.sklearn.log_model(prediction_model, "prediction_model") + mlflow.log_param("retrain", True) + + return "Model retrained successfully", experiment + + def get_experiment(self, experiment_name: str) -> int: + experiment = mlflow.get_experiment_by_name(experiment_name) + + if experiment is None: + raise ValueError(f'Experiment {experiment_name} not found') + + return int(experiment.experiment_id) + + def get_experiment_last_run(self, experiment_id: int) -> str: + runs = mlflow.search_runs( + experiment_ids=[experiment_id], + filter_string="", # Sem filtro no MLflow ainda + output_format="pandas" + ) + + # Filtrar apenas as runs onde params.retrain == True + filtered_runs = runs[runs["params.retrain"] == 'True'] + + # Converter a coluna 'end_time' para datetime + filtered_runs['end_time'] = pd.to_datetime(filtered_runs['end_time']) + + # Ordenar o DataFrame de forma descendente pela coluna 'end_time' + filtered_runs = filtered_runs.sort_values( + by='end_time', ascending=False) + + # Pegar a última run_id do DataFrame filtrado e ordenado + latest_run_id = filtered_runs.iloc[0]['run_id'] + + return latest_run_id + + def update_production_model_by_run_id(self, run_id: str, model_name: str) -> dict: + # Registrar o modelo + # Aqui estamos assumindo que você já tem um modelo salvo, caso contrário você precisará treiná-lo e salvá-lo primeiro. + # Se o modelo já está registrado, você pode usar o método register_model() ou pyfunc.load_model() para isso. + mlflow.register_model( + f"runs:/{run_id}/prediction_model", model_name) + + # Colocar a versão do modelo em produção + # Depois de registrar o modelo, precisamos pegar a versão mais recente do modelo e movê-lo para o estágio 'Production' + client = mlflow.tracking.MlflowClient() + + # Obter a versão mais recente registrada do modelo + model_versions = client.get_registered_model( + model_name).latest_versions + max_version = max(model_versions, key=lambda x: int(x.version)).version + + # Mover a versão mais recente do modelo para o estágio de 'Production' + client.transition_model_version_stage( + name=model_name, + version=max_version, + stage="Production", + archive_existing_versions=True + ) + + return { + 'model_name': model_name, + 'version': max_version, + 'mlflow_run_id': run_id + } + + def update_production_model(self, experiment: str, model_name: str) -> dict: + + experiment_id = self.get_experiment(experiment) + run_id = self.get_experiment_last_run(experiment_id) + metadata = self.update_production_model_by_run_id(run_id, model_name) + + metadata['mlflow_experiment_id'] = experiment_id + + return metadata + + def transform(self, model_name: str, data: pd.DataFrame, model_retention: int): + try: + return { + 'success': True, + 'content': self.model_serving.get_cached_transform(model_name, data, model_retention).to_dict() + } + + except Exception as e: + return { + 'success': False, + 'content': { + 'message': str(e), + 'traceback': traceback.format_exc() + } + } + + def predict(self, model_name: str, data: pd.DataFrame, model_retention: int): + try: + start_time = datetime.now() + data = self.model_serving.get_cached_predict( + model_name, data, model_retention) + end_time = datetime.now() + data = pd.DataFrame(data, columns=['prediction']) + data['response_time'] = (end_time - start_time).total_seconds() + + return { + 'success': True, + 'content': data.to_dict() + } + + except Exception as e: + return { + 'success': False, + 'content': { + 'message': str(e), + 'traceback': traceback.format_exc() + } + } diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py new file mode 100644 index 0000000..99fbc08 --- /dev/null +++ b/laborious/utils/repository/opc_repository.py @@ -0,0 +1,112 @@ +from pathlib import Path +from asyncua.sync import Client +from asyncua.crypto.security_policies import SecurityPolicyBasic256 +from asyncua.ua import DataValue, Variant, VariantType +from datetime import datetime +from time import sleep +from logging import Logger +from statistics import mean, median +from typing import Callable + +from sientia_do.notifications.models import NotificationLevel + +data_type_map = { + 'float': VariantType.Float, + 'double': VariantType.Double, + 'int': VariantType.Int32, + 'bool': VariantType.Boolean, + 'str': VariantType.String, + 'datetime': VariantType.DateTime, +} + + +class OpcRepository(): + def __init__(self, name: str, url: str, logger: Logger, server_uri: str, + cert_path: str = None, private_key_path: str = None, server_cert_path: str = None): + self.url = url + self.name = name + self.server_uri = server_uri + self.cert_path = cert_path + self.private_key_path = private_key_path + self.server_cert_path = server_cert_path + self.logger = logger + self.non_receive_count = 0 + + self.client = None + + def set_security(self): + """ + Configures the security settings for the OPC UA client. + This method sets up the security policy, certificates, and timeouts + required for establishing a secure connection with the OPC UA server. + Raises: + ValueError: If either the certificate path or private key path is not provided. + Attributes: + cert_path (str): Path to the client's certificate file. + private_key_path (str): Path to the client's private key file. + server_cert_path (str, optional): Path to the server's certificate file. + server_uri (str): The URI of the server to be used as the application URI. + client (opcua.Client): The OPC UA client instance. + logger (logging.Logger): Logger instance for logging information. + Security Settings: + - Security Policy: Basic256 + - Secure Channel Timeout: 10,000,000 ms + - Session Timeout: 10,000,000 ms + """ + + if not all([self.cert_path, self.private_key_path]): + raise ValueError( + "Certificate and private key paths must be provided for secure connection.") + cert = Path(self.cert_path) + private_key = Path(self.private_key_path) + server_cert = Path( + self.server_cert_path) if self.server_cert_path else None + + self.client.application_uri = self.server_uri + self.logger.info('Setting security...') + self.client.set_security( + SecurityPolicyBasic256, + certificate=str(cert), + private_key=str(private_key), + server_certificate=str(server_cert) + ) + self.client.secure_channel_timeout = 10000000 + self.client.session_timeout = 10000000 + + def connect(self): + self.client = Client(self.url) + if self.security: + self.set_security() + self.logger.info('Starting connection...') + self.client.connect() + + def connect(self): + """ + Establishes a connection to the OPC server. + This method initializes the OPC client using the provided URL and + sets up security if a certificate path is specified. It then + attempts to connect to the server and logs the connection status. + Raises: + Exception: If the connection to the OPC server fails. + """ + + self.client = Client(self.url) + if self.cert_path: + self.set_security() + self.logger.info('Starting connection...') + self.client.connect() + + def disconnect(self): + self.client.disconnect() + self.client = None + self.logger.info('Disconnected from OPC server') + + def __del__(self): + self.disconnect() + + def write_data(self, node, value, data_type, logger): + node = self.client.get_node(node) + data = float(value) + logger.info(f'Writing {data} - {type(data)} to {node}') + ua_data = DataValue(Variant(data, data_type_map[data_type])) + node.write_value(ua_data) diff --git a/laborious/workflows/predictions_batch.py b/laborious/workflows/predictions_batch.py new file mode 100644 index 0000000..c3c54b0 --- /dev/null +++ b/laborious/workflows/predictions_batch.py @@ -0,0 +1,85 @@ +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from laborious.activities.postgres import Postgres + from laborious.activities.mlflow import MLFlow + from laborious.activities.gates import Gates + from laborious.activities.opc import OPC + from typing import Any + + +@workflow.defn(name="predictions_batch") +class PredictionsBatch(): + @workflow.run + async def run(self, input_data: dict[str, Any]): + + await workflow.execute_activity_method( + Postgres.prepare_activity, + { + 'schedule_name': input_data['schedule_name'], + 'model_name': input_data['model_name'], + 'model_id': input_data['model_id'] + } + ) + + data = await workflow.execute_activity_method( + Postgres.load_custom_query, + input_data['query'] + ) + + path_flag, confidence = await workflow.execute_activity_method( + Gates.input_gate, + { + 'filters': input_data['filters'], + 'data': data + } + ) + + if path_flag == 'stop': + return + + if path_flag == 'continue': + # repeat last prediction + await workflow.execute_activity_method( + Postgres.repeat_last_prediction, + { + 'schema': input_data['schema'], + 'table_name': input_data['table_name'], + 'model': input_data['model'] + } + ) + return + + response_data, last_timestamp = await workflow.execute_activity_method( + MLFlow.transform_data, + { + 'data': data, + 'model_name': input_data['model_name'], + 'model_retention': input_data['model_retention'] + } + ) + + path_flag, confidence = await workflow.execute_activity_method( + Gates.mlflow_gate, + { + 'filters': input_data['filters'], + 'data': response_data, + 'type': 'transform' + } + ) + + if path_flag == 'stop': + return + + if path_flag is None: + # procced with prediction + response_data = await workflow.execute_activity_method( + MLFlow.request_predict, + { + 'data': response_data, + 'model_name': input_data['model_name'], + 'model_retention': input_data['model_retention'] + } + ) + + path_flag diff --git a/laborious/workflows/sub_workflows/format_and_export_prediction.py b/laborious/workflows/sub_workflows/format_and_export_prediction.py new file mode 100644 index 0000000..8591f8a --- /dev/null +++ b/laborious/workflows/sub_workflows/format_and_export_prediction.py @@ -0,0 +1,60 @@ +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from laborious.activities.activities import Activities + from typing import Any + + +@workflow.defn(name="format_and_export_prediction") +class FormatAndExportPrediction(): + @workflow.run + async def run(self, input_data: dict[str, Any]): + path_flag = input_data['path_flag'] + data = input_data['data'] + confidence = input_data['confidence'] + + if path_flag is None: + # proceed with formatting and exporting + prediction = await workflow.execute_activity_method( + Activities.format_prediction, + { + 'data': data, + 'timestamp': input_data['timestamp'], + 'model_id': input_data['model_id'], + 'prediction_confidence': confidence, + } + ) + + else: + # create default prediction + prediction = await workflow.execute_activity_method( + Activities.format_default_prediction, + { + 'timestamp': input_data['timestamp'], + 'model_id': input_data['model_id'], + 'prediction_confidence': confidence, + 'comment': input_data['comment'] + } + ) + + # write to postgres + postgres_holder = workflow.execute_activity_method( + Activities.export_data_to_postgres, + { + 'schema': input_data['schema'], + 'table_name': input_data['table_name'], + 'data': prediction + } + ) + + opc_holder = workflow.execute_activity_method( + Activities.write_opc_data, + { + 'opc_servers': input_data['opc_servers'], + 'opc_output_config': input_data['opc_output_config'], + 'data': prediction + } + ) + + await postgres_holder + await opc_holder diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py new file mode 100644 index 0000000..5b40362 --- /dev/null +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -0,0 +1,59 @@ +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from laborious.activities.postgres import Postgres + from laborious.activities.mlflow import MLFlow + from laborious.activities.gates import Gates + from laborious.activities.opc import OPC + from typing import Any + + +@workflow.defn(name="prediction_process") +class PredictionProcess(): + @workflow.run + async def run(self, input_data: dict[str, Any]): + data = input_data['data'] + + path_flag, _confidence = await workflow.execute_activity_method( + Gates.input_gate, + { + 'filters': input_data['filters'], + 'data': data + } + ) + + if path_flag == 'stop': + return + + if path_flag == 'continue': + # repeat last prediction + await workflow.execute_activity_method( + Postgres.repeat_last_prediction, + { + 'schema': input_data['schema'], + 'table_name': input_data['table_name'], + 'model': input_data['model'] + } + ) + return + + response_data, last_timestamp = await workflow.execute_activity_method( + MLFlow.transform_data, + { + 'data': data, + 'model_name': input_data['model_name'], + 'model_retention': input_data['model_retention'] + } + ) + + path_flag, confidence = await workflow.execute_activity_method( + Gates.mlflow_gate, + { + 'filters': input_data['filters'], + 'data': response_data, + 'type': 'transform' + } + ) + + if path_flag == 'stop': + return diff --git a/requirements.txt b/requirements.txt index 0b537a6..a03f80a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,24 @@ temporalio psycopg2-binary -pandas -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git \ No newline at end of file +asyncua +mlflow==2.10.1 +scikit-learn==1.4.2 +scipy==1.13.0 +shap==0.46.0 +catboost==1.2.5 +hyperopt==0.2.7 +kaleido==0.2.1 +xgboost==2.0.2 +cloudpickle==3.0.0 +pathspec +kafka-python +sshtunnel +git+ssh://git@github.com/Aignosi/sientia-dataops-library.git +git+ssh://git@github.com/Aignosi/sientia-mlops-library.git +async-timeout +kubernetes +boto3 +scikit-optimize==0.10.2 +pandas==2.2.2 +PyYAML==6.0.1 +numpy==1.26.4 \ No newline at end of file diff --git a/tests/laborious/activities/test_gates.py b/tests/laborious/activities/test_gates.py index 28f83ca..8edca83 100644 --- a/tests/laborious/activities/test_gates.py +++ b/tests/laborious/activities/test_gates.py @@ -1,8 +1,9 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import ANY, MagicMock, patch from pandas import DataFrame from pytest import fixture, mark from laborious.activities.gates import Gates +from sientia_do.notifications.models import NotificationLevel @fixture @@ -272,3 +273,36 @@ async def test_input_gate_no_filtered( assert empty_data_input_args[0][0].equals(DataFrame( {'variable': ['variable1', 'variable2'], 'value': [1, 2]})) assert empty_data_input_args[0][1] == input_data['filters']['EMPTY_DATA'] + + +@mark.asyncio +@patch('laborious.activities.gates.filter_functions') +async def test_input_gate_error( + filter_functions_mock, + gates +): + filter_functions_mock.__getitem__.side_effect = KeyError('test') + + input_data = { + 'filters': { + 'SPECIFIC_VARIABLES_NULL_VALUES': { + 'POLICY': 'stop', + 'VARIABLES': ['variable2'] + } + }, + 'data': { + 'variable': ['variable1', 'variable2'], + 'value': [1, 2] + } + } + + result = await gates.input_gate(input_data) + assert result == (None, 0) + + gates.notification_handler.build_and_send_notification.assert_called_once_with( + notification_id='INTPUT_GATE_ERROR__SPECIFIC_VARIABLES_NULL_VALUES', + message="Error in filter SPECIFIC_VARIABLES_NULL_VALUES:{'POLICY': 'stop', 'VARIABLES': ['variable2']}: \n 'test'", + block='input_gate', + level=NotificationLevel.ERROR, + attachment_content=ANY + )