SIENTIAPDE-1273

SIENTIAPDE-1273
Enhance security analysis and SQL injection handling

- Added skip for potential SQL injection false positives in Bandit configuration.
- Updated validate.sh to use the pyproject.toml configuration for Bandit security analysis.
- Refactored code to replace ensure_dataframe utility with direct DataFrame usage in multiple activities, improving clarity and reducing dependencies.
- Removed the deprecated dataframe_utils module to streamline the codebase.

laborious/workflows/drift.py

@@ -4,10 +4,10 @@ with workflow.unsafe.imports_passed_through():
     from datetime import timedelta
     from typing import Any
 
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
     from sientia_do.temporal.policies import retry_policy
 
     from laborious.activities.activities import Activities
-    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
 
 
 @workflow.defn(name='drift')
@@ -39,10 +39,10 @@ class Drift:
 
         gathering_query = f"""
             SELECT *
-            FROM {input_data['schema']}.{input_data['source_table_name']}
+            FROM "{input_data['schema']}"."{input_data['source_table_name']}"
             WHERE
-                model_id = {input_data['model_id']} AND
-                timestamp > NOW() - INTERVAL '{input_data['interval']} minutes'
+                model_id = '{input_data['model_id']}' AND
+                timestamp > NOW() - INTERVAL {input_data['interval']} minutes
             ORDER BY timestamp ASC
         """
 
@@ -60,10 +60,7 @@ class Drift:
 
         reference_data_handler = workflow.start_local_activity_method(
             Activities.get_reference_data,
-            {
-                **metadata,
-                'model_name': input_data['model_name']
-            },
+            {**metadata, 'model_name': input_data['model_name']},
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(seconds=300),
         )
@@ -83,8 +80,9 @@ class Drift:
                 'model_name': input_data['model_name'],
                 'model_id': input_data['model_id'],
                 'target_name': target_name,
-                'drift_metrics': input_data.get('drift_metrics',
-                    ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']),
+                'drift_metrics': input_data.get(
+                    'drift_metrics', ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']
+                ),
                 'chunk_period': input_data.get('chunk_period', 'min'),
             },
             retry_policy=retry_policy,
@@ -99,8 +97,11 @@ class Drift:
                     'data': drift_data,
                     'schema': input_data['schema'],
                     'table_name': input_data['target_table_name'],
-                    'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ},
+                    'timestamp_conversion': {
+                        'column': 'timestamp',
+                        'format': DATETIME_FORMAT_WITH_TZ,
+                    },
                 },
                 retry_policy=retry_policy,
                 start_to_close_timeout=timedelta(seconds=300),
-            )
+            )

laborious/workflows/simple_metrics.py

@@ -4,10 +4,10 @@ with workflow.unsafe.imports_passed_through():
     from datetime import timedelta
     from typing import Any
 
+    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
     from sientia_do.temporal.policies import retry_policy
 
     from laborious.activities.activities import Activities
-    from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
 
 
 @workflow.defn(name='simple_metrics')
@@ -19,10 +19,10 @@ class SimpleMetrics:
         """
         metadata = {
             'metadata': {
-                'schedule_name': input_data['schedule_name'],
-                'model_name': input_data['model_name'],
                 'model_id': input_data['model_id'],
+                'model_name': input_data['model_name'],
                 'workflow_name': 'simple_metrics',
+                'schedule_name': input_data['schedule_name'],
             }
         }
 
@@ -34,15 +34,15 @@ class SimpleMetrics:
 
         query = f"""
             select p."timestamp", p.prediction, ld.value as "target"
-            from {input_data['schema']}.{input_data['predictions_table_name']} p
-            inner join {input_data['schema']}.{input_data['data_table_name']} ld
+            from "{input_data['schema']}"."{input_data['predictions_table_name']}" p
+            inner join "{input_data['schema']}"."{input_data['data_table_name']}" ld
                 on p."timestamp" = ld."timestamp"
             where
-                p.model_id = {model_id} and
+                p.model_id = '{model_id}' and
                 p.prediction is not null and
                 ld.variable = '{target_name}' and
                 ld.value is not null and
-                p."timestamp" >= NOW() - INTERVAL '{interval_minutes} minutes'
+                p."timestamp" >= NOW() - INTERVAL {interval_minutes} minutes
             order by
                 p."timestamp" desc;
         """
@@ -74,7 +74,6 @@ class SimpleMetrics:
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(seconds=300),
         )
-        
 
         if not simple_metrics:
             return
@@ -93,4 +92,4 @@ class SimpleMetrics:
             },
             retry_policy=retry_policy,
             start_to_close_timeout=timedelta(seconds=300),
-        )
+        )

laborious/workflows/sub_workflows/format_and_export_prediction.py

@@ -89,7 +89,6 @@ class FormatAndExportPrediction:
             )
 
             if transformed_data is not None:
-
                 transformed = await workflow.execute_local_activity_method(
                     Activities.format_transformed_data,
                     {