SIENTIAPDE-994

Update requirements.txt with new dependencies and refactor activity methods for improved functionality and error handling

laborious/activities/activities.py

@@ -0,0 +1,54 @@
+from temporalio import activity, workflow
+
+with workflow.unsafe.imports_passed_through():
+    from laborious.activities.postgres import Postgres
+    from laborious.activities.mlflow import MLFlow
+    from laborious.activities.gates import Gates
+    from laborious.activities.opc import OPC
+    from typing import Any
+    from logging import Logger
+    from sientia_do.notifications.handlers import NotificationHandler
+
+
+class Activities(Postgres, MLFlow, Gates, OPC):
+
+    def __init__(self,
+                 postgres_config: dict[str, Any],
+                 mlflow_config: dict[str, Any],
+                 opc_config: dict[str, Any],
+                 logger: Logger, notification_handler: NotificationHandler):
+
+        # Initialize parent classes
+        Postgres.__init__(self, host=postgres_config['host'],
+                          port=postgres_config['port'],
+                          user=postgres_config['user'],
+                          password=postgres_config['password'],
+                          dbname=postgres_config['dbname'],
+                          min_connections=postgres_config['min_connections'],
+                          max_connections=postgres_config['max_connections'],
+                          logger=logger,
+                          notification_handler=notification_handler)
+
+        MLFlow.__init__(self, mlflow_host=mlflow_config['host'],
+                        mlflow_port=mlflow_config['port'],
+                        mlflow_username=mlflow_config['username'],
+                        mlflow_password=mlflow_config['password'],
+                        logger=logger,
+                        notification_handler=notification_handler)
+
+        Gates.__init__(self, logger=logger,
+                       notification_handler=notification_handler)
+
+        OPC.__init__(self,
+                     name=opc_config['name'],
+                     url=opc_config['url'],
+                     server_uri=opc_config['server_uri'],
+                     cert_path=opc_config['cert_path'],
+                     private_key_path=opc_config['private_key_path'],
+                     server_cert_path=opc_config['server_cert_path'],
+                     logger=logger,
+                     notification_handler=notification_handler)
+
+    @activity.defn(name="prepare_activity")
+    async def prepare_activity(self, schedule_name: str, model_name: str, model_id: str):
+        await super().prepare_activity(schedule_name, model_name, model_id)

laborious/activities/base.py

@@ -8,10 +8,9 @@ class BaseActivity:
         self.logger = logger
         self.notification_handler = notification_handler
 
-    @activity.defn(name="prepare_notification_handler")
+    def prepare_activity(self, schedule_name: str,
-    async def prepare_notification_handler(self, schedule_name: str,
+                         model_name: str,
-                                           model_name: str,
+                         model_id: str):
-                                           model_id: str):
         self.notification_handler.base_notification.schedule_name = schedule_name
         self.notification_handler.base_notification.model_name = model_name
         self.notification_handler.base_notification.model_id = model_id

laborious/activities/gates.py

@@ -1,19 +1,31 @@
 from temporalio import activity, workflow
 
+
 with workflow.unsafe.imports_passed_through():
+    import traceback
     from logging import Logger
     from sientia_do.notifications.handlers import NotificationHandler
     from laborious.activities.base import BaseActivity
     from typing import Any
     from laborious.utils.filters.conditional_filters import filter_empty_data, filter_specific_variables_null_values
     from pandas import DataFrame
+    from sientia_do.notifications.models import NotificationLevel
+    from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter
 
-
+input_filter_functions = {
-filter_functions = {
     'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values,
     'EMPTY_DATA': filter_empty_data
 }
 
+transform_filter_functions = {
+    'response_filter': {
+        'API_ERROR': api_error_filter,
+    },
+    'content_filter': {
+        'NAN_VALUES': nan_values_filter,
+    }
+}
+
 
 class Gates(BaseActivity):
     def __init__(self, logger: Logger, notification_handler: NotificationHandler):
@@ -25,7 +37,9 @@ class Gates(BaseActivity):
         Filters the data based on the filters. The return value is a tuple with the first element
         being the policy and the second element being the confidence status.
         Args:
-            input_data (dict): The input data.
+            input_data (dict): The input data. Contains:
+                filters (dict): The filters to apply.
+                data (dict[str, Any]): The data to filter.
         Returns:
             tuple[str, int]: ('stop', -1) if some filter policy is 'stop', ('continue', 2)
             if no filter policy is 'stop' and some filter policy is 'continue',
@@ -36,8 +50,18 @@ class Gates(BaseActivity):
 
         filter_output = []
         for fil, config in filters.items():
-            if filter_functions[fil](data, config):
+            try:
-                filter_output.append(config['POLICY'])
+                if input_filter_functions[fil](data, config):
+                    filter_output.append(config['POLICY'])
+            except Exception as e:
+                trace = traceback.format_exc()
+                self.notification_handler.build_and_send_notification(
+                    notification_id=f"INTPUT_GATE_ERROR__{fil}",
+                    message=f"Error in filter {fil}:{config}: \n {e}",
+                    block="input_gate",
+                    level=NotificationLevel.ERROR,
+                    attachment_content=trace
+                )
 
         if 'stop' in filter_output:
             return 'stop', -1
@@ -45,3 +69,75 @@ class Gates(BaseActivity):
             return 'continue', 2
 
         return None, 0
+
+    @activity.defn(name="mlflow_gate")
+    async def mlflow_gate(self, input_data: dict[str, Any]) -> tuple[str, int]:
+
+        filters = input_data['filters']
+        data = DataFrame(input_data['data'])
+        gate_type = input_data['type']
+
+        filter_output = []
+        for fil, config in filters.items():
+            if transform_filter_functions['response_filter'][fil](data, config):
+                filter_output.append(config['POLICY'])
+                self.notification_handler.build_and_send_notification(
+                    notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}",
+                    message=data['content']['message'],
+                    block="mlflow_gate",
+                    level=NotificationLevel.WARNING,
+                    attachment_content=data['content']['traceback']
+                )
+
+        if 'stop' in filter_output:
+            return 'stop', -1
+        elif 'continue' in filter_output:
+            return 'continue', 10
+
+        if gate_type == 'predict':
+            return None, 0
+
+        data = DataFrame(data['content'])
+
+        for fil, config in filters.items():
+            if transform_filter_functions['content_filter'][fil](data, config):
+                filter_output.append(config['POLICY'])
+                self.notification_handler.build_and_send_notification(
+                    notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}",
+                    message=f"Data not passed the content filter {fil}:{config}",
+                    block="mlflow_gate",
+                    level=NotificationLevel.WARNING,
+                    attachment_content=data.to_string()
+                )
+
+        if 'stop' in filter_output:
+            return 'stop', -1
+        elif 'continue' in filter_output:
+            return 'continue', 18
+
+        return None, 0
+
+    @activity.defn(name="format_prediction")
+    async def format_prediction(self, input_data: dict[str, Any]) -> str:
+        data = DataFrame(input_data['data'])
+
+        data['timestamp'] = input_data['timestamp']
+        data['model_id'] = input_data['model_id']
+        data['prediction_confidence'] = input_data['prediction_confidence']
+        data['prediction_status'] = 'Good'
+        data['comment'] = ""
+        data.sort_values(by='timestamp', inplace=True)
+
+        return data.to_dict()
+
+    @activity.defn(name="format_default_prediction")
+    async def format_default_prediction(self, input_data: dict[str, Any]) -> str:
+        return DataFrame({
+            'prediction': [0],
+            'response_time': [0],
+            'timestamp': [input_data['timestamp']],
+            'model_id': [input_data['model_id']],
+            'prediction_confidence': [input_data['prediction_confidence']],
+            'prediction_status': ['Bad'],
+            'comment': [input_data['comment']]
+        }).to_dict()

laborious/activities/mlflow.py

@@ -0,0 +1,65 @@
+import numpy as np
+from pandas import DataFrame
+from temporalio import activity, workflow
+
+
+with workflow.unsafe.imports_passed_through():
+    from laborious.activities.base import BaseActivity
+    from typing import Any
+    from logging import Logger
+    from sientia_do.notifications.handlers import NotificationHandler
+    from laborious.utils.model_repository import ModelMonitoringRepository
+    from sientia_do.notifications.models import NotificationLevel
+
+
+class MLFlow(BaseActivity):
+    def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str,
+                 mlflow_password: str, logger: Logger, notification_handler: NotificationHandler):
+        super().__init__(logger, notification_handler)
+        self.mlflow_host = mlflow_host
+        self.mlflow_port = mlflow_port
+        self.mlflow_username = mlflow_username
+        self.mlflow_password = mlflow_password
+
+        self.model_monitoring_repository = ModelMonitoringRepository(
+            f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password
+        )
+
+    @activity.defn(name="request_transform")
+    async def request_transform(self, input_data: dict[str, Any]) -> tuple[dict[str, Any], str]:
+        self.logger.info('Transforming data...')
+        data = DataFrame(input_data['data'])
+        model_name = input_data['model_name']
+        model_retention = input_data['model_retention']
+
+        self.logger.debug(data)
+
+        data = data.pivot(
+            index='timestamp', columns='variable',
+            values='value')
+        data.fillna(np.nan, inplace=True)
+        data.reset_index(inplace=True)
+        data.columns.name = None
+
+        response_data = self.model_monitoring_repository.transform(
+            model_name, data, model_retention)
+
+        timestamp = max(data['timestamp'].values.tolist())
+
+        return response_data, timestamp
+
+    @activity.defn(name="request_predict")
+    async def request_predict(self, input_data: dict[str, Any]) -> tuple[dict[str, Any], str]:
+        self.logger.info('Predicting data...')
+        data = DataFrame(input_data['data'])
+        model_name = input_data['model_name']
+        model_retention = input_data['model_retention']
+
+        self.logger.debug(data)
+
+        data.replace(np.nan, None, inplace=True)
+
+        response_data = self.model_monitoring_repository.predict(
+            model_name, data, model_retention)
+
+        return response_data

laborious/activities/opc.py

@@ -0,0 +1,76 @@
+import traceback
+from pandas import DataFrame
+from temporalio import activity, workflow
+
+
+with workflow.unsafe.imports_passed_through():
+    from logging import Logger
+    from sientia_do.notifications.handlers import NotificationHandler
+    from laborious.activities.base import BaseActivity
+    from laborious.utils.repository.opc_repository import OpcRepository
+    from typing import Any
+    from sientia_do.notifications.models import NotificationLevel
+
+
+class OPC(BaseActivity):
+    def __init__(self,
+                 name: str, url: str, server_uri: str,
+                 cert_path: str, private_key_path: str, server_cert_path: str,
+                 logger: Logger, notification_handler: NotificationHandler):
+
+        self.logger = logger
+        self.notification_handler = notification_handler
+        self.name = name
+        self.url = url
+        self.server_uri = server_uri
+        self.cert_path = cert_path
+        self.private_key_path = private_key_path
+        self.server_cert_path = server_cert_path
+
+        self.opc_repository = OpcRepository(
+            name=self.name,
+            url=self.url,
+            logger=self.logger,
+            server_uri=self.server_uri,
+            cert_path=self.cert_path,
+            private_key_path=self.private_key_path,
+            server_cert_path=self.server_cert_path
+        )
+
+        self.opc_repository.connect()
+
+    @activity.defn(name='write_opc_data')
+    async def write_opc_data(self, input_data: dict[str, Any]):
+        data = DataFrame(input_data['data'])
+        _opc_servers = input_data['opc_servers']
+        opc_output_config = input_data['opc_output_config']
+
+        for tag, config in opc_output_config['prediction_tags'].items():
+            try:
+                self.opc_repository.write_data(
+                    tag, data.head(1)['prediction'].values[0], config['data_type'])
+            except Exception as e:
+                trace = traceback.format_exc()
+                self.notification_handler.build_and_send_notification(
+                    notification_id="WRITE_OPC_PREDICTION_ERROR",
+                    message=f"Error writing data to OPC server: {e}",
+                    block="write_opc_data",
+                    level=NotificationLevel.ERROR,
+                    attachment_content=trace
+                )
+                self.logger.error(trace)
+
+        for tag, config in opc_output_config['confidence_tags'].items():
+            try:
+                self.opc_repository.write_data(
+                    tag, data.head(1)['prediction_confidence'].values[0], config['data_type'])
+            except Exception as e:
+                trace = traceback.format_exc()
+                self.notification_handler.build_and_send_notification(
+                    notification_id="WRITE_OPC_CONFIDENCE_ERROR",
+                    message=f"Error writing data to OPC server: {e}",
+                    block="write_opc_data",
+                    level=NotificationLevel.ERROR,
+                    attachment_content=trace
+                )
+                self.logger.error(trace)

laborious/activities/postgres.py

@@ -83,7 +83,10 @@ class Postgres(BaseActivity):
         Repeats the last prediction for a given model.
 
         Args:
-            query_items (dict[str, str]): The query items.
+            query_items (dict[str, str]): The query items. Contains:
+                schema (str): The schema of the table.
+                table_name (str): The name of the table.
+                model (str): The model to repeat the prediction for.
 
         Returns:
             None

laborious/utils/filters/api_filters.py

@@ -1,69 +0,0 @@
-import numpy as np
-from pandas import DataFrame
-from laborious.utils.filters.base_filter import Filter
-
-
-class ApiErrorFilter(Filter):
-    def __init__(self, policy):
-        self.policy = policy
-        super().__init__('API_FILTER')
-
-    def method(self, response: dict, prediction_confidence: int):
-        """
-        Processes the API response and determines the next action based on the response and prediction confidence.
-        Args:
-            response (dict): The API response to be processed.
-            prediction_confidence (int): The confidence level of the prediction.
-        Returns:
-            str: 'stop' if the policy is to stop on captured errors, 'continue' if the policy is to continue on captured errors.
-        Raises:
-            KeyError: If 'success' or 'content' keys are missing in the response dictionary.
-        """
-
-        captured = False
-        if not response and prediction_confidence == 10:
-            self.warning('No valid response.')
-            captured = True
-
-        else:
-            if not response['success']:
-                message = response['content']["message"]
-                self.warning(
-                    f'Model repository error: {message}')
-                captured = True
-        if captured and self.policy == 'stop':
-            return 'stop'
-        elif captured and self.policy == 'continue':
-            return 'continue'
-
-
-class NaNValuesFilter(Filter):
-    def __init__(self, policy):
-        self.policy = policy
-        super().__init__('NAN_VALUES')
-
-    def method(self, predictions: DataFrame, prediction_confidence: int):
-        """
-        Processes the given predictions DataFrame by replacing None values with NaN,
-        dropping the 'timestamp' column if it exists, and checking for NaN values.
-        Args:
-            predictions (pd.DataFrame): The DataFrame containing prediction data.
-            prediction_confidence (float): The confidence level of the predictions.
-        Returns:
-            float or int or bool: Returns the prediction confidence if the DataFrame
-            is not entirely NaN. If all values are NaN and the policy is 'stop', 
-            returns False. If all values are NaN and the policy is 'continue', 
-            returns 18.
-        """
-
-        data = predictions.replace({None: np.nan}).drop(
-            columns=['timestamp'], errors='ignore')
-
-        if data.isna().all().all():
-            if self.policy == 'stop':
-                self.warning('All values are NaN.')
-                return False
-            elif self.policy == 'continue':
-                return 18
-
-        return prediction_confidence

laborious/utils/filters/mlflow_filters.py

@@ -0,0 +1,23 @@
+import numpy as np
+from pandas import DataFrame
+from laborious.utils.filters.base_filter import Filter
+
+
+def api_error_filter(response: dict, _config: dict):
+    if not response:
+        return True
+
+    if not response['success']:
+        return True
+
+    return False
+
+
+def nan_values_filter(predictions: DataFrame, _config: dict):
+    data = predictions.replace({None: np.nan}).drop(
+        columns=['timestamp'], errors='ignore')
+
+    if data.isna().all().all():
+        return True
+
+    return False

laborious/utils/repository/model_repository.py

@@ -0,0 +1,296 @@
+"""
+Model Monitoring Repository
+
+This module contains the ModelMonitoringRepository class, which is responsible for handling the communication with the Model Monitoring API.
+
+It includes the methods that are used to answer ModelMonitoringService requests using the Model Monitoring API functions.
+
+By Monitoring we mean the evaluation of the performance of models, the generation of reports.
+
+"""
+from datetime import datetime
+import traceback
+import mlflow
+import pandas as pd
+from sientia.ModelServing import ModelServing
+from pathlib import Path
+
+
+class ModelMonitoringRepository():
+    def __init__(self, host, username, password):
+
+        self.model_serving = ModelServing(tracking_uri=host,
+                                          username=username, password=password)
+
+    def get_current_data_df(self, current_data: pd.DataFrame, model_name: str, target: str):
+        """
+        Get the current data as a DataFrame and update the prediction and target columns
+
+        Parameters:
+        current_data (pd.DataFrame): the current data
+        model_name (str): the name of the model
+        target (str): the target column
+
+        Returns:
+        DataFrame: the current data as a DataFrame
+
+
+        """
+        predictions = current_data['prediction']
+
+        target = current_data[target]
+        current_data = self.model_serving.get_transformed_data(
+            model_name, current_data, by='model')
+        current_data['prediction'] = predictions
+        current_data['target'] = target
+
+        return pd.DataFrame(current_data).dropna()
+
+    def get_artifact(self, destination: str, search_by: str, run_id: str = None,
+                     model_name: str = None, artifact_name: str = None) -> None:
+        """
+        Get an artifact in MLflow by experiment or model and save it to a destination path using API.
+        If the artifact is searched by model, the latest production version will be used.
+
+        Args:
+            destination: The destination path to save the artifact.
+            search_by: The way to search for the artifact ('experiment' or 'model').
+            run_id: The run ID of the experiment (if search_by is "experiment").
+            model_name: The name of the model (if search_by is "model").
+            artifact_name: The path of the artifact to download.
+
+        Returns:
+            artifact: The artifact(.csv) downloaded from MLflow.
+        """
+
+        self.model_serving.get_artifact(destination=destination, search_by=search_by,
+                                        run_id=run_id, model_name=model_name, artifact_name=artifact_name)
+
+    def calculate_model_metrics(self, real_data, predictions, flag):
+        """
+        Function to calculate the metrics of a model using API
+
+        Parameters:
+        real_data (array): the real data
+        predictions (array): the predictions
+
+        Returns:
+        dict: the metrics of the model including MSE and R2
+        """
+        return self.model_serving.get_model_metrics(reference_data=None, real_data=real_data, predictions=predictions, type_flag=flag)
+
+    def get_experiment_by_run_id(self, run_id: str) -> dict:
+        # Get the run information using the run_id
+        run = mlflow.get_run(run_id)
+
+        # Extract the experiment ID from the run
+        experiment_id = run.info.experiment_id
+
+        # Get the experiment details using the experiment ID
+        experiment = mlflow.get_experiment(experiment_id)
+        experiment_name = experiment.name
+        return experiment_name
+
+    def get_next_run_name(self, model_name: str) -> str:
+        """ 
+        Function to get the next run number of a specific model
+
+        Parameters:
+        model_name (str): the name of the model
+
+        Returns:
+        str: the next run number
+        """
+
+        runs = mlflow.search_runs(
+            experiment_names=[model_name], order_by=["start_time desc"])
+        next_run_number = len(runs) + 1
+        return f"{model_name}-{next_run_number}"
+
+    def retrain_model(self, data: pd.DataFrame, model_name: str) -> tuple:
+        """
+        Retrain a model with new data.
+
+        Parameters:
+            data (pandas.DataFrame): The new data to use for retraining.
+            model_name (str): The name of the model to retrain.
+            metrics_list (list): The metrics to be used to compare the models.
+            compare_metrics (bool): If True, the retrain will only be considered if the new model is better than the current one.
+                                If False, the retrain will always be considered.
+            split_dataset (bool): If True, the data will be split into X and Y and into training and testing sets.
+                If False, the data will be used as a unique block for retraining.
+            update_report (bool): If True, a report will be created with the data of the retrained model.
+            update_transformation (bool): If True, the model will be updated in the MLflow tracking server.
+            update_prediction (bool): If True, the prediction model will be updated in the MLflow tracking server.
+            shuffle_data (bool): If True, the data will be shuffled before splitting.
+            model_type (str): The type of model to get metrics for. Ex: 'regression', 'classification'.
+
+
+        Returns:
+            mlflow.sklearn.Model: The retrained prediction model.
+            mlflow.sklearn.Model: The retrained data model.
+            mse (float): The mean squared error of the retrained model.
+            r2 (float): The R-squared score of the retrained model.
+        """
+
+        # load predictor model
+        predictor_uri = f"models:/{model_name}/production"
+        # load transform model
+        latest_production_id = self.model_serving.get_model_run_id(
+            model_name, stage="Production"
+        )
+        transform_uri = self.model_serving.get_model_uri(
+            latest_production_id, prediction=False
+        )
+        # load
+        data_model = mlflow.sklearn.load_model(transform_uri)
+        prediction_model = mlflow.sklearn.load_model(predictor_uri)
+        data_model = data_model.fit(data)
+        treated_data = data_model.predict(data)
+        # align target column with treated_data
+        target_name = data_model.target_variable
+        y = data[target_name]
+        treated_data = pd.merge(
+            treated_data, y, left_index=True, right_index=True)
+        prediction_model = prediction_model.fit(treated_data)
+        # Example usage
+        experiment = self.get_experiment_by_run_id(latest_production_id)
+        pred_model_atributes = vars(prediction_model)  # load class attributes
+        data_model_atributes = vars(data_model)  # load class attributes
+        mlflow.set_experiment(experiment)
+        experiment_description = "Retrain model {model_name} with new data"
+        current_run_name = self.get_next_run_name(experiment)
+        with mlflow.start_run(
+            run_name=current_run_name, description=experiment_description
+        ) as _run:
+            # update transfomation model
+            # fixed parameters
+            for name_atribute, val_atribute in pred_model_atributes.items():
+                if name_atribute != "model":
+                    mlflow.log_param(name_atribute, val_atribute)
+            # update prediction model
+            for name_atribute, val_atribute in data_model_atributes.items():
+                if name_atribute != "model":
+                    mlflow.log_param(name_atribute, val_atribute)
+            # dynamic parameters, including model itself
+            mlflow.sklearn.log_model(data_model, "data_model")
+            file_path = f"laborious/data/raw_data_{model_name}.csv"
+            data.to_csv(
+                f"laborious/data/raw_data_{model_name}.csv", index=True)
+            # log the data raw
+            mlflow.log_artifact(file_path)
+
+            # dynamic parameters, including model itself
+            mlflow.sklearn.log_model(prediction_model, "prediction_model")
+            mlflow.log_param("retrain", True)
+
+            return "Model retrained successfully", experiment
+
+    def get_experiment(self, experiment_name: str) -> int:
+        experiment = mlflow.get_experiment_by_name(experiment_name)
+
+        if experiment is None:
+            raise ValueError(f'Experiment {experiment_name} not found')
+
+        return int(experiment.experiment_id)
+
+    def get_experiment_last_run(self, experiment_id: int) -> str:
+        runs = mlflow.search_runs(
+            experiment_ids=[experiment_id],
+            filter_string="",  # Sem filtro no MLflow ainda
+            output_format="pandas"
+        )
+
+        # Filtrar apenas as runs onde params.retrain == True
+        filtered_runs = runs[runs["params.retrain"] == 'True']
+
+        # Converter a coluna 'end_time' para datetime
+        filtered_runs['end_time'] = pd.to_datetime(filtered_runs['end_time'])
+
+        # Ordenar o DataFrame de forma descendente pela coluna 'end_time'
+        filtered_runs = filtered_runs.sort_values(
+            by='end_time', ascending=False)
+
+        # Pegar a última run_id do DataFrame filtrado e ordenado
+        latest_run_id = filtered_runs.iloc[0]['run_id']
+
+        return latest_run_id
+
+    def update_production_model_by_run_id(self, run_id: str, model_name: str) -> dict:
+        # Registrar o modelo
+        # Aqui estamos assumindo que você já tem um modelo salvo, caso contrário você precisará treiná-lo e salvá-lo primeiro.
+        # Se o modelo já está registrado, você pode usar o método register_model() ou pyfunc.load_model() para isso.
+        mlflow.register_model(
+            f"runs:/{run_id}/prediction_model", model_name)
+
+        # Colocar a versão do modelo em produção
+        # Depois de registrar o modelo, precisamos pegar a versão mais recente do modelo e movê-lo para o estágio 'Production'
+        client = mlflow.tracking.MlflowClient()
+
+        # Obter a versão mais recente registrada do modelo
+        model_versions = client.get_registered_model(
+            model_name).latest_versions
+        max_version = max(model_versions, key=lambda x: int(x.version)).version
+
+        # Mover a versão mais recente do modelo para o estágio de 'Production'
+        client.transition_model_version_stage(
+            name=model_name,
+            version=max_version,
+            stage="Production",
+            archive_existing_versions=True
+        )
+
+        return {
+            'model_name': model_name,
+            'version': max_version,
+            'mlflow_run_id': run_id
+        }
+
+    def update_production_model(self, experiment: str, model_name: str) -> dict:
+
+        experiment_id = self.get_experiment(experiment)
+        run_id = self.get_experiment_last_run(experiment_id)
+        metadata = self.update_production_model_by_run_id(run_id, model_name)
+
+        metadata['mlflow_experiment_id'] = experiment_id
+
+        return metadata
+
+    def transform(self, model_name: str, data: pd.DataFrame, model_retention: int):
+        try:
+            return {
+                'success': True,
+                'content': self.model_serving.get_cached_transform(model_name, data, model_retention).to_dict()
+            }
+
+        except Exception as e:
+            return {
+                'success': False,
+                'content': {
+                    'message': str(e),
+                    'traceback': traceback.format_exc()
+                }
+            }
+
+    def predict(self, model_name: str, data: pd.DataFrame, model_retention: int):
+        try:
+            start_time = datetime.now()
+            data = self.model_serving.get_cached_predict(
+                model_name, data, model_retention)
+            end_time = datetime.now()
+            data = pd.DataFrame(data, columns=['prediction'])
+            data['response_time'] = (end_time - start_time).total_seconds()
+
+            return {
+                'success': True,
+                'content': data.to_dict()
+            }
+
+        except Exception as e:
+            return {
+                'success': False,
+                'content': {
+                    'message': str(e),
+                    'traceback': traceback.format_exc()
+                }
+            }

laborious/utils/repository/opc_repository.py

@@ -0,0 +1,112 @@
+from pathlib import Path
+from asyncua.sync import Client
+from asyncua.crypto.security_policies import SecurityPolicyBasic256
+from asyncua.ua import DataValue, Variant, VariantType
+from datetime import datetime
+from time import sleep
+from logging import Logger
+from statistics import mean, median
+from typing import Callable
+
+from sientia_do.notifications.models import NotificationLevel
+
+data_type_map = {
+    'float': VariantType.Float,
+    'double': VariantType.Double,
+    'int': VariantType.Int32,
+    'bool': VariantType.Boolean,
+    'str': VariantType.String,
+    'datetime': VariantType.DateTime,
+}
+
+
+class OpcRepository():
+    def __init__(self, name: str, url: str, logger: Logger, server_uri: str,
+                 cert_path: str = None, private_key_path: str = None, server_cert_path: str = None):
+        self.url = url
+        self.name = name
+        self.server_uri = server_uri
+        self.cert_path = cert_path
+        self.private_key_path = private_key_path
+        self.server_cert_path = server_cert_path
+        self.logger = logger
+        self.non_receive_count = 0
+
+        self.client = None
+
+    def set_security(self):
+        """
+        Configures the security settings for the OPC UA client.
+        This method sets up the security policy, certificates, and timeouts
+        required for establishing a secure connection with the OPC UA server.
+        Raises:
+            ValueError: If either the certificate path or private key path is not provided.
+        Attributes:
+            cert_path (str): Path to the client's certificate file.
+            private_key_path (str): Path to the client's private key file.
+            server_cert_path (str, optional): Path to the server's certificate file.
+            server_uri (str): The URI of the server to be used as the application URI.
+            client (opcua.Client): The OPC UA client instance.
+            logger (logging.Logger): Logger instance for logging information.
+        Security Settings:
+            - Security Policy: Basic256
+            - Secure Channel Timeout: 10,000,000 ms
+            - Session Timeout: 10,000,000 ms
+        """
+
+        if not all([self.cert_path, self.private_key_path]):
+            raise ValueError(
+                "Certificate and private key paths must be provided for secure connection.")
+        cert = Path(self.cert_path)
+        private_key = Path(self.private_key_path)
+        server_cert = Path(
+            self.server_cert_path) if self.server_cert_path else None
+
+        self.client.application_uri = self.server_uri
+        self.logger.info('Setting security...')
+        self.client.set_security(
+            SecurityPolicyBasic256,
+            certificate=str(cert),
+            private_key=str(private_key),
+            server_certificate=str(server_cert)
+        )
+        self.client.secure_channel_timeout = 10000000
+        self.client.session_timeout = 10000000
+
+    def connect(self):
+        self.client = Client(self.url)
+        if self.security:
+            self.set_security()
+        self.logger.info('Starting connection...')
+        self.client.connect()
+
+    def connect(self):
+        """
+        Establishes a connection to the OPC server.
+        This method initializes the OPC client using the provided URL and
+        sets up security if a certificate path is specified. It then
+        attempts to connect to the server and logs the connection status.
+        Raises:
+            Exception: If the connection to the OPC server fails.
+        """
+
+        self.client = Client(self.url)
+        if self.cert_path:
+            self.set_security()
+        self.logger.info('Starting connection...')
+        self.client.connect()
+
+    def disconnect(self):
+        self.client.disconnect()
+        self.client = None
+        self.logger.info('Disconnected from OPC server')
+
+    def __del__(self):
+        self.disconnect()
+
+    def write_data(self, node, value, data_type, logger):
+        node = self.client.get_node(node)
+        data = float(value)
+        logger.info(f'Writing {data} - {type(data)} to {node}')
+        ua_data = DataValue(Variant(data, data_type_map[data_type]))
+        node.write_value(ua_data)

laborious/workflows/predictions_batch.py

@@ -0,0 +1,85 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from laborious.activities.postgres import Postgres
+    from laborious.activities.mlflow import MLFlow
+    from laborious.activities.gates import Gates
+    from laborious.activities.opc import OPC
+    from typing import Any
+
+
+@workflow.defn(name="predictions_batch")
+class PredictionsBatch():
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+
+        await workflow.execute_activity_method(
+            Postgres.prepare_activity,
+            {
+                'schedule_name': input_data['schedule_name'],
+                'model_name': input_data['model_name'],
+                'model_id': input_data['model_id']
+            }
+        )
+
+        data = await workflow.execute_activity_method(
+            Postgres.load_custom_query,
+            input_data['query']
+        )
+
+        path_flag, confidence = await workflow.execute_activity_method(
+            Gates.input_gate,
+            {
+                'filters': input_data['filters'],
+                'data': data
+            }
+        )
+
+        if path_flag == 'stop':
+            return
+
+        if path_flag == 'continue':
+            # repeat last prediction
+            await workflow.execute_activity_method(
+                Postgres.repeat_last_prediction,
+                {
+                    'schema': input_data['schema'],
+                    'table_name': input_data['table_name'],
+                    'model': input_data['model']
+                }
+            )
+            return
+
+        response_data, last_timestamp = await workflow.execute_activity_method(
+            MLFlow.transform_data,
+            {
+                'data': data,
+                'model_name': input_data['model_name'],
+                'model_retention': input_data['model_retention']
+            }
+        )
+
+        path_flag, confidence = await workflow.execute_activity_method(
+            Gates.mlflow_gate,
+            {
+                'filters': input_data['filters'],
+                'data': response_data,
+                'type': 'transform'
+            }
+        )
+
+        if path_flag == 'stop':
+            return
+
+        if path_flag is None:
+            # procced with prediction
+            response_data = await workflow.execute_activity_method(
+                MLFlow.request_predict,
+                {
+                    'data': response_data,
+                    'model_name': input_data['model_name'],
+                    'model_retention': input_data['model_retention']
+                }
+            )
+
+            path_flag

laborious/workflows/sub_workflows/format_and_export_prediction.py

@@ -0,0 +1,60 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from laborious.activities.activities import Activities
+    from typing import Any
+
+
+@workflow.defn(name="format_and_export_prediction")
+class FormatAndExportPrediction():
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+        path_flag = input_data['path_flag']
+        data = input_data['data']
+        confidence = input_data['confidence']
+
+        if path_flag is None:
+            # proceed with formatting and exporting
+            prediction = await workflow.execute_activity_method(
+                Activities.format_prediction,
+                {
+                    'data': data,
+                    'timestamp': input_data['timestamp'],
+                    'model_id': input_data['model_id'],
+                    'prediction_confidence': confidence,
+                }
+            )
+
+        else:
+            # create default prediction
+            prediction = await workflow.execute_activity_method(
+                Activities.format_default_prediction,
+                {
+                    'timestamp': input_data['timestamp'],
+                    'model_id': input_data['model_id'],
+                    'prediction_confidence': confidence,
+                    'comment': input_data['comment']
+                }
+            )
+
+        # write to postgres
+        postgres_holder = workflow.execute_activity_method(
+            Activities.export_data_to_postgres,
+            {
+                'schema': input_data['schema'],
+                'table_name': input_data['table_name'],
+                'data': prediction
+            }
+        )
+
+        opc_holder = workflow.execute_activity_method(
+            Activities.write_opc_data,
+            {
+                'opc_servers': input_data['opc_servers'],
+                'opc_output_config': input_data['opc_output_config'],
+                'data': prediction
+            }
+        )
+
+        await postgres_holder
+        await opc_holder

laborious/workflows/sub_workflows/prediction_process.py

@@ -0,0 +1,59 @@
+from temporalio import workflow
+
+with workflow.unsafe.imports_passed_through():
+    from laborious.activities.postgres import Postgres
+    from laborious.activities.mlflow import MLFlow
+    from laborious.activities.gates import Gates
+    from laborious.activities.opc import OPC
+    from typing import Any
+
+
+@workflow.defn(name="prediction_process")
+class PredictionProcess():
+    @workflow.run
+    async def run(self, input_data: dict[str, Any]):
+        data = input_data['data']
+
+        path_flag, _confidence = await workflow.execute_activity_method(
+            Gates.input_gate,
+            {
+                'filters': input_data['filters'],
+                'data': data
+            }
+        )
+
+        if path_flag == 'stop':
+            return
+
+        if path_flag == 'continue':
+            # repeat last prediction
+            await workflow.execute_activity_method(
+                Postgres.repeat_last_prediction,
+                {
+                    'schema': input_data['schema'],
+                    'table_name': input_data['table_name'],
+                    'model': input_data['model']
+                }
+            )
+            return
+
+        response_data, last_timestamp = await workflow.execute_activity_method(
+            MLFlow.transform_data,
+            {
+                'data': data,
+                'model_name': input_data['model_name'],
+                'model_retention': input_data['model_retention']
+            }
+        )
+
+        path_flag, confidence = await workflow.execute_activity_method(
+            Gates.mlflow_gate,
+            {
+                'filters': input_data['filters'],
+                'data': response_data,
+                'type': 'transform'
+            }
+        )
+
+        if path_flag == 'stop':
+            return

requirements.txt

@@ -1,4 +1,24 @@
 temporalio
 psycopg2-binary
-pandas
+asyncua
+mlflow==2.10.1
+scikit-learn==1.4.2
+scipy==1.13.0
+shap==0.46.0
+catboost==1.2.5
+hyperopt==0.2.7
+kaleido==0.2.1
+xgboost==2.0.2
+cloudpickle==3.0.0
+pathspec
+kafka-python
+sshtunnel
 git+ssh://git@github.com/Aignosi/sientia-dataops-library.git
+git+ssh://git@github.com/Aignosi/sientia-mlops-library.git
+async-timeout
+kubernetes
+boto3
+scikit-optimize==0.10.2
+pandas==2.2.2
+PyYAML==6.0.1
+numpy==1.26.4

tests/laborious/activities/test_gates.py

@@ -1,8 +1,9 @@
-from unittest.mock import MagicMock, patch
+from unittest.mock import ANY, MagicMock, patch
 from pandas import DataFrame
 from pytest import fixture, mark
 
 from laborious.activities.gates import Gates
+from sientia_do.notifications.models import NotificationLevel
 
 
 @fixture
@@ -272,3 +273,36 @@ async def test_input_gate_no_filtered(
     assert empty_data_input_args[0][0].equals(DataFrame(
         {'variable': ['variable1', 'variable2'], 'value': [1, 2]}))
     assert empty_data_input_args[0][1] == input_data['filters']['EMPTY_DATA']
+
+
+@mark.asyncio
+@patch('laborious.activities.gates.filter_functions')
+async def test_input_gate_error(
+    filter_functions_mock,
+    gates
+):
+    filter_functions_mock.__getitem__.side_effect = KeyError('test')
+
+    input_data = {
+        'filters': {
+            'SPECIFIC_VARIABLES_NULL_VALUES': {
+                'POLICY': 'stop',
+                'VARIABLES': ['variable2']
+            }
+        },
+        'data': {
+            'variable': ['variable1', 'variable2'],
+            'value': [1, 2]
+        }
+    }
+
+    result = await gates.input_gate(input_data)
+    assert result == (None, 0)
+
+    gates.notification_handler.build_and_send_notification.assert_called_once_with(
+        notification_id='INTPUT_GATE_ERROR__SPECIFIC_VARIABLES_NULL_VALUES',
+        message="Error in filter SPECIFIC_VARIABLES_NULL_VALUES:{'POLICY': 'stop', 'VARIABLES': ['variable2']}: \n 'test'",
+        block='input_gate',
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY
+    )