SIENTIAPDE-994

Refactor tests for Postgres activities and improve error handling

- Updated test_postgres.py to enhance the testing of load_custom_query method, including cases for None data and date conversion.
- Refactored repeat_last_prediction tests to use mocks for SQLAlchemy session execution.
- Added tests for export_data_to_postgres method, covering both success and error scenarios.
- Improved the initialization tests for Activities class to ensure proper instantiation of dependencies.
- Enhanced test coverage for OPC repository connection validation.
- Updated tests for prediction workflows to streamline input handling and improve clarity.
- Introduced tests for connectors configuration to validate environment variable handling for MLFlow, OPC, and Postgres.
- Added tests for logger utility to ensure default settings are correctly applied.

laborious/activities/gates.py

@@ -137,15 +137,25 @@ class Gates(BaseActivity):
         for fil, config in filters.items():
             if fil not in mlflow_response_filter_functions:
                 continue
-            if mlflow_response_filter_functions[fil](data, config):
-                filter_output.append(config['POLICY'])
-                comments.append(data['content']['message'])
+            try:
+                if mlflow_response_filter_functions[fil](data, config):
+                    filter_output.append(config['POLICY'])
+                    comments.append(data['content']['message'])
+                    self.notification_handler.build_and_send_notification(
+                        notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}",
+                        message=data['content']['message'],
+                        block="mlflow_gate",
+                        level=NotificationLevel.WARNING,
+                        attachment_content=data['content']['traceback']
+                    )
+            except Exception as e:
+                trace = traceback.format_exc()
                 self.notification_handler.build_and_send_notification(
-                    notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}",
-                    message=data['content']['message'],
+                    notification_id=f"MLFLOW_GATE_RESPONSE_FILTER__{fil}",
+                    message=f"Error in filter {fil}:{config}: \n {e}",
                     block="mlflow_gate",
-                    level=NotificationLevel.WARNING,
-                    attachment_content=data['content']['traceback']
+                    level=NotificationLevel.ERROR,
+                    attachment_content=trace
                 )
 
         for path_flag in path_priority:
@@ -189,14 +199,24 @@ class Gates(BaseActivity):
         for fil, config in filters.items():
             if fil not in mlflow_content_filter_functions:
                 continue
-            if mlflow_content_filter_functions[fil](data, config):
-                filter_output.append(config['POLICY'])
+            try:
+                if mlflow_content_filter_functions[fil](data, config):
+                    filter_output.append(config['POLICY'])
+                    self.notification_handler.build_and_send_notification(
+                        notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}",
+                        message=f"Data not passed the content filter {fil}:{config}",
+                        block="mlflow_gate",
+                        level=NotificationLevel.WARNING,
+                        attachment_content=data.to_string()
+                    )
+            except Exception as e:
+                trace = traceback.format_exc()
                 self.notification_handler.build_and_send_notification(
-                    notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}",
-                    message=f"Data not passed the content filter {fil}:{config}",
+                    notification_id=f"MLFLOW_GATE_CONTENT_FILTER__{fil}",
+                    message=f"Error in filter {fil}:{config}: \n {e}",
                     block="mlflow_gate",
-                    level=NotificationLevel.WARNING,
-                    attachment_content=data.to_string()
+                    level=NotificationLevel.ERROR,
+                    attachment_content=trace
                 )
 
         for path_flag in path_priority:

laborious/utils/filters/mlflow_filters.py

@@ -14,7 +14,7 @@ def api_error_filter(response: dict, _config: dict):
 
 def nan_values_filter(predictions: DataFrame, _config: dict):
     data = predictions.replace({None: np.nan}).drop(
-        columns=['timestamp'], errors='ignore')
+        columns=['timestamp'], errors='ignore').infer_objects(copy=False)
 
     if data.isna().all().all():
         return True

laborious/utils/repository/opc_repository.py

@@ -122,12 +122,17 @@ class OpcRepository():
             return False
 
     def disconnect(self):
+        if self.client is None:
+            return
         self.client.disconnect()
         self.client = None
         self.logger.info('Disconnected from OPC server')
 
     def __del__(self):
-        self.disconnect()
+        try:
+            self.disconnect()
+        except Exception as e:
+            self.logger.error(f"Error in destructor: {e}")
 
     def validate_connection(self):
         if self.client is None: