SIENTIAPDE-994

Remove unused utility files and update requirements.txt to include new dependencies for data processing and database interaction.

laborious/utils/filters/api_filters.py

@@ -0,0 +1,69 @@
+import numpy as np
+from pandas import DataFrame
+from laborious.utils.filters.base_filter import Filter
+
+
+class ApiErrorFilter(Filter):
+    def __init__(self, policy):
+        self.policy = policy
+        super().__init__('API_FILTER')
+
+    def method(self, response: dict, prediction_confidence: int):
+        """
+        Processes the API response and determines the next action based on the response and prediction confidence.
+        Args:
+            response (dict): The API response to be processed.
+            prediction_confidence (int): The confidence level of the prediction.
+        Returns:
+            str: 'stop' if the policy is to stop on captured errors, 'continue' if the policy is to continue on captured errors.
+        Raises:
+            KeyError: If 'success' or 'content' keys are missing in the response dictionary.
+        """
+
+        captured = False
+        if not response and prediction_confidence == 10:
+            self.warning('No valid response.')
+            captured = True
+
+        else:
+            if not response['success']:
+                message = response['content']["message"]
+                self.warning(
+                    f'Model repository error: {message}')
+                captured = True
+        if captured and self.policy == 'stop':
+            return 'stop'
+        elif captured and self.policy == 'continue':
+            return 'continue'
+
+
+class NaNValuesFilter(Filter):
+    def __init__(self, policy):
+        self.policy = policy
+        super().__init__('NAN_VALUES')
+
+    def method(self, predictions: DataFrame, prediction_confidence: int):
+        """
+        Processes the given predictions DataFrame by replacing None values with NaN,
+        dropping the 'timestamp' column if it exists, and checking for NaN values.
+        Args:
+            predictions (pd.DataFrame): The DataFrame containing prediction data.
+            prediction_confidence (float): The confidence level of the predictions.
+        Returns:
+            float or int or bool: Returns the prediction confidence if the DataFrame
+            is not entirely NaN. If all values are NaN and the policy is 'stop', 
+            returns False. If all values are NaN and the policy is 'continue', 
+            returns 18.
+        """
+
+        data = predictions.replace({None: np.nan}).drop(
+            columns=['timestamp'], errors='ignore')
+
+        if data.isna().all().all():
+            if self.policy == 'stop':
+                self.warning('All values are NaN.')
+                return False
+            elif self.policy == 'continue':
+                return 18
+
+        return prediction_confidence

laborious/utils/filters/base_filter.py

@@ -0,0 +1,13 @@
+from pandas import DataFrame
+
+class Filter:
+    def __init__(self, id: str):
+        self.id = id
+        self.warnings = []
+
+    @staticmethod
+    def method(df: DataFrame) -> DataFrame:
+        raise NotImplementedError
+    
+    def warning(self, message: str):
+        self.warnings.append(f'[{self.id}] - {message}')

laborious/utils/filters/conditional_filters.py

@@ -0,0 +1,19 @@
+from typing import List
+
+from laborious.utils.filters.base_filter import Filter
+from pandas import DataFrame
+
+
+def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool:
+    """
+    Returns True if the data is empty, False otherwise.
+    """
+    return data[
+        data['variable'].isin(config['VARIABLES']) & data['value'].isna()].empty
+
+
+def filter_empty_data(data: DataFrame, _config: dict) -> bool:
+    """
+    Returns True if the data is empty, False otherwise.
+    """
+    return data.empty

laborious/utils/git_clone.py

@@ -0,0 +1,30 @@
+import os
+from git import Repo
+from urllib.parse import quote
+
+# Lê variáveis de ambiente
+GIT_TOKEN = os.getenv("GIT_TOKEN")
+GIT_EMAIL = os.getenv("GIT_EMAIL")
+REPO_URL = os.getenv("REPO_URL")  # ex: "github.com/usuario/repositorio.git"
+CLONE_DIR = os.getenv("CLONE_DIR", "./repo_clonado")
+
+if not GIT_TOKEN or not GIT_EMAIL or not REPO_URL:
+    raise EnvironmentError("As variáveis GIT_TOKEN, GIT_EMAIL e REPO_URL devem estar definidas.")
+
+# Escapa o token (caso contenha caracteres especiais)
+safe_token = quote(GIT_TOKEN)
+
+# Monta URL com autenticação via token
+repo_url_with_auth = f"https://{safe_token}@{REPO_URL}"
+
+# Clona o repositório
+print(f"Clonando repositório em {CLONE_DIR}...")
+Repo.clone_from(repo_url_with_auth, CLONE_DIR)
+print("Repositório clonado com sucesso.")
+
+# Opcional: configura o e-mail globalmente no Git (ou dentro do repo)
+repo = Repo(CLONE_DIR)
+with repo.config_writer() as git_config:
+    git_config.set_value("user", "email", GIT_EMAIL)
+
+print(f"E-mail configurado como {GIT_EMAIL}.")