SIENTIAPDE-1273

SIENTIAPDE-1273
Enhance security analysis and SQL injection handling

- Added skip for potential SQL injection false positives in Bandit configuration.
- Updated validate.sh to use the pyproject.toml configuration for Bandit security analysis.
- Refactored code to replace ensure_dataframe utility with direct DataFrame usage in multiple activities, improving clarity and reducing dependencies.
- Removed the deprecated dataframe_utils module to streamline the codebase.

laborious/utils/dataframe_utils.py

@@ -1,31 +0,0 @@
-"""
-DataFrame utility functions for handling serialized DataFrames.
-
-This module provides helper functions to work with DataFrames that may
-come from Temporal serialization (already as DataFrame) or from legacy
-code (as dict).
-"""
-
-from typing import Any
-
-from pandas import DataFrame
-
-
-def ensure_dataframe(data: Any) -> DataFrame:
-    """
-    Ensure that data is a DataFrame, converting from dict if necessary.
-
-    This function handles both cases:
-    - Data already deserialized as DataFrame (from Temporal codec)
-    - Data as dict (legacy format or non-DataFrame serialization)
-
-    Args:
-        data: Data that should be a DataFrame (can be DataFrame or dict)
-
-    Returns:
-        DataFrame: The data as a pandas DataFrame
-    """
-    if isinstance(data, DataFrame):
-        return data
-    return DataFrame(data)
-

laborious/utils/repository/model_repository.py

@@ -16,11 +16,11 @@ Capabilities:
 
 import ctypes
 import gc
-from io import StringIO
 import threading
 import time
 import traceback
 from datetime import datetime, timedelta
+from io import StringIO
 from os import environ, makedirs, path
 from shutil import rmtree
 from typing import Any, Literal, overload
@@ -34,7 +34,6 @@ from sientia_do.observability.logger import Logger
 from sientia_do.observability.metrics_controller import MetricsController
 from sientia_do.observability.sientia_monitoring import SientiaMonitoring
 from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
-from sientia.ModelAnalysis import ModelAnalysis
 
 from laborious import metrics
 
@@ -217,8 +216,9 @@ class MLFlowRepository(SientiaMonitoring):
         run_info = mlflow.get_run(run_id)
         return run_info.data.params
 
-    def check_artifact_exists(self, run_id: str,
-    artifact_path: str, metadata: dict[str, Any]) -> bool:
+    def check_artifact_exists(
+        self, run_id: str, artifact_path: str, metadata: dict[str, Any]
+    ) -> bool:
         """
         Check if an artifact exists in the MLflow Model Registry.
 
@@ -233,8 +233,9 @@ class MLFlowRepository(SientiaMonitoring):
 
         self.debug(f'Artifacts of {run_id}: \n{artifacts}', metadata)
         self.debug(f'Looking for artifact {artifact_path} in {run_id}', metadata)
-        
+
         return any(artifact.path == artifact_path for artifact in artifacts)
+
     """
     Functions related to download and load models
     """
@@ -279,10 +280,9 @@ class MLFlowRepository(SientiaMonitoring):
 
         return artifacts
 
-
-    async def load_artifact_dataframe(self, model_name: str, artifact_path: str,
-                                         metadata: dict[str, Any]) -> pd.DataFrame | None:
-        
+    async def load_artifact_dataframe(
+        self, model_name: str, artifact_path: str, metadata: dict[str, Any]
+    ) -> pd.DataFrame | None:
         """
         Load the dataframe content of an artifact from the MLflow Model Registry.
 
@@ -300,7 +300,7 @@ class MLFlowRepository(SientiaMonitoring):
         if not self.check_artifact_exists(run_id, artifact_path, metadata):
             return None
 
-        artifact_path = path.join("runs:/", run_id, artifact_path)
+        artifact_path = path.join('runs:/', run_id, artifact_path)
 
         start_time = time.time()
         try:
@@ -704,8 +704,9 @@ class MLFlowRepository(SientiaMonitoring):
     Functions related to model retraining
     """
 
-    def get_prediction_data(self, prediction_model: Any, retrain_dataset: pd.DataFrame,
-    target_name: str) -> pd.DataFrame:
+    def get_prediction_data(
+        self, prediction_model: Any, retrain_dataset: pd.DataFrame, target_name: str
+    ) -> pd.DataFrame:
         """
         Get prediction data from prediction model.
         """
@@ -714,7 +715,6 @@ class MLFlowRepository(SientiaMonitoring):
         prediction_data = prediction_model.predict(retrain_dataset)
 
         if isinstance(prediction_data, pd.DataFrame):
-
             prediction_data.columns = pd.Index(['prediction'])
 
         else:
@@ -724,7 +724,8 @@ class MLFlowRepository(SientiaMonitoring):
 
         # Merge prediction data with retrain_dataset on index
         prediction_data = pd.merge(
-            retrain_dataset, prediction_data, left_index=True, right_index=True, how='left')
+            retrain_dataset, prediction_data, left_index=True, right_index=True, how='left'
+        )
 
         # Rename column "target_name" to "target"
         prediction_data.rename(columns={target_name: 'target'}, inplace=True)
@@ -733,9 +734,7 @@ class MLFlowRepository(SientiaMonitoring):
 
         prediction_data.reset_index(drop=True, inplace=True)
 
-        prediction_data.sort_values(
-            by='timestamp', ascending=True, inplace=True
-        )
+        prediction_data.sort_values(by='timestamp', ascending=True, inplace=True)
 
         return prediction_data
 
@@ -859,8 +858,7 @@ class MLFlowRepository(SientiaMonitoring):
         prediction_model.fit(retrain_dataset)
 
         # get prediction data
-        prediction_data = self.get_prediction_data(
-            prediction_model, retrain_dataset, target_name)
+        prediction_data = self.get_prediction_data(prediction_model, retrain_dataset, target_name)
 
         self.info(f'Model experiment creation completed successfully for {model_name}', metadata)
 
@@ -1439,4 +1437,3 @@ class MLFlowRepository(SientiaMonitoring):
         metadata_result['mlflow_experiment_id'] = experiment_id
 
         return metadata_result
-