gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1273

Browse Source 
SIENTIAPDE-1273
Enhance security analysis and SQL injection handling

- Added skip for potential SQL injection false positives in Bandit configuration.
- Updated validate.sh to use the pyproject.toml configuration for Bandit security analysis.
- Refactored code to replace ensure_dataframe utility with direct DataFrame usage in multiple activities, improving clarity and reducing dependencies.
- Removed the deprecated dataframe_utils module to streamline the codebase.
This commit is contained in:
vitor-aignosi
2025-11-17 16:04:54 -03:00
parent 1014c33dd9
commit a88a15c60a
18 changed files with 516 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
laborious/activities/activities.py
View File

@@ -9,9 +9,10 @@ with workflow.unsafe.imports_passed_through():
from laborious.activities.gates import Gates
from laborious.activities.mlflow import MLFlow
from laborious.activities.model_metrics import ModelMetrics
from laborious.activities.opc import OPC
from laborious.activities.storage import Storage
from laborious.activities.model_metrics import ModelMetrics
class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics):
"""
@@ -131,4 +132,4 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics):
MLFlow.close(self)
Gates.close(self)
await OPC.close(self)
ModelMetrics.close(self)
ModelMetrics.close(self)

12
laborious/activities/gates.py
View File

@@ -15,7 +15,6 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
from laborious import metrics
from laborious.utils.dataframe_utils import ensure_dataframe
from laborious.utils.filters.conditional_filters import (
filter_empty_data,
filter_specific_variables_null_values,
@@ -150,7 +149,7 @@ class Gates(SientiaMonitoring):
self.info('Performing input gate...', metadata)
filters = input_data['filters']
data = ensure_dataframe(input_data['data'])
data = DataFrame(input_data['data'])
path_priority = input_data['path_priority']
filter_output = []
@@ -403,19 +402,18 @@ class Gates(SientiaMonitoring):
return policy_type, int(policy_value)
@activity.defn(name='format_transformed_data')
async def format_transformed_data(self, input_data: dict[str, Any]) -> dict:
"""
Format transformed data according to configured storage policies.
"""
metadata = input_data['metadata']
model_id = input_data['model_id']
self.info('Formatting transformed data...', metadata)
data = ensure_dataframe(input_data['data'])
data = DataFrame(input_data['data'])
data['timestamp'] = data.index
data = data.reset_index(drop=True)
@@ -454,7 +452,7 @@ class Gates(SientiaMonitoring):
prediction_store_policy = input_data['prediction_store_policy']
self.info('Formatting prediction...', metadata)
data = ensure_dataframe(input_data['data'])
data = DataFrame(input_data['data'])
# Create timestamp column from index and reset index
data['timestamp'] = data.index
@@ -602,7 +600,7 @@ class Gates(SientiaMonitoring):
self.info('Getting last timestamp...', metadata)
data = ensure_dataframe(input_data['data'])
data = DataFrame(input_data['data'])
self.debug(f'Input data: {data.head(5).to_string()}', metadata)

11
laborious/activities/mlflow.py
View File

@@ -1,4 +1,3 @@
from typing import Hashable
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
@@ -20,7 +19,6 @@ with workflow.unsafe.imports_passed_through():
now,
)
from laborious.utils.dataframe_utils import ensure_dataframe
from laborious.utils.repository.minio_repository import MinioRepository
from laborious.utils.repository.model_repository import MLFlowRepository
@@ -140,7 +138,7 @@ class MLFlow(SientiaMonitoring):
"""
metadata = input_data['metadata']
self.info('Transforming data...', metadata)
data = ensure_dataframe(input_data['data'])
data = DataFrame(input_data['data'])
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
@@ -212,7 +210,7 @@ class MLFlow(SientiaMonitoring):
"""
metadata = input_data['metadata']
self.info('Predicting data...', metadata)
data = ensure_dataframe(input_data['data'])
data = DataFrame(input_data['data'])
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
@@ -421,7 +419,6 @@ class MLFlow(SientiaMonitoring):
self.error(trace, metadata=metadata)
raise e
@activity.defn(name='get_reference_data')
async def get_reference_data(self, input_data: dict[str, Any]) -> list[dict] | None:
"""
@@ -439,7 +436,7 @@ class MLFlow(SientiaMonitoring):
metadata = input_data['metadata']
model_name = input_data['model_name']
artifact = "evaluation_data.csv"
artifact = 'evaluation_data.csv'
reference_data = await self.model_monitoring_repository.load_artifact_dataframe(
model_name=model_name, artifact_path=artifact, metadata=metadata
@@ -452,4 +449,4 @@ class MLFlow(SientiaMonitoring):
reference_data['timestamp'] = to_datetime(reference_data['timestamp'])
reference_data['timestamp'] = reference_data['timestamp'].dt.strftime(DATETIME_FORMAT)
return reference_data.to_dict(orient='records')
return reference_data.to_dict(orient='records')

139
laborious/activities/model_metrics.py
View File

@@ -1,25 +1,28 @@
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import time
import traceback
import warnings
from typing import Any
import numpy as np
from pandas import DataFrame, Index, to_datetime
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.logger import Logger
from sientia.ModelAnalysis import ModelAnalysis
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia.ModelAnalysis import ModelAnalysis
from laborious import metrics
import time
import numpy as np
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
import warnings
import traceback
from laborious import metrics
warnings.filterwarnings('ignore', category=RuntimeWarning, message='Degrees of freedom <= 0')
warnings.filterwarnings('ignore', category=RuntimeWarning, message='invalid value encountered in scalar divide')
warnings.filterwarnings(
'ignore', category=RuntimeWarning, message='invalid value encountered in scalar divide'
)
class ModelMetrics(SientiaMonitoring):
"""
@@ -28,12 +31,12 @@ class ModelMetrics(SientiaMonitoring):
This class provides activities for writing metrics to the Prometheus monitoring system.
"""
def __init__(self,
def __init__(
self,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
def close(self) -> None:
@@ -42,11 +45,11 @@ class ModelMetrics(SientiaMonitoring):
"""
SientiaMonitoring.shutdown(self)
def __del__(self):
self.close()
async def get_drift_metrics(self,
async def get_drift_metrics(
self,
reference_data: DataFrame,
target_data: DataFrame,
target_name: str,
@@ -73,34 +76,37 @@ class ModelMetrics(SientiaMonitoring):
'features': reference_columns,
}
model_analysis = ModelAnalysis(config=config)
self.debug(f'Reference data: Size {reference_data.shape} \n{reference_data.head(5).to_string()}', metadata)
self.debug(
f'Reference data: Size {reference_data.shape} \n{reference_data.head(5).to_string()}',
metadata,
)
self.debug(f'Target data: Size {target_data.shape} \n{target_data.head(5).to_string()}', metadata)
self.debug(
f'Target data: Size {target_data.shape} \n{target_data.head(5).to_string()}', metadata
)
core_labels = self.get_core_labels(metadata, operation_type='detect_univariate_drift')
start_time = time.time()
try:
univariate_drift = model_analysis.detect_univariate_drift(
reference_df=reference_data,
analysis_df=target_data,
features=reference_columns,
timestamp_col=config['timestamp'],
methods=drift_metrics,
chunk_period=chunk_period
chunk_period=chunk_period,
)
except Exception as e:
self.error(f'Error detecting univariate drift: {e}', metadata)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
core_labels = self.get_core_labels(metadata, operation_type='detect_multivariate_drift')
start_time = time.time()
try:
@@ -109,11 +115,13 @@ class ModelMetrics(SientiaMonitoring):
analysis_df=target_data,
features=reference_columns,
timestamp_col=config['timestamp'],
chunk_period=chunk_period
chunk_period=chunk_period,
)
except Exception as e:
self.error(f'Error detecting multivariate drift: {e}', metadata)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
@@ -127,15 +135,18 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error getting drift metrics: {e}', metadata)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self.debug(f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata)
return drift_df
self.debug(
f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata
)
return drift_df
@activity.defn(name='calculate_drift')
async def calculate_drift(self, input_data: dict[str, Any]) -> list[dict]:
@@ -194,8 +205,8 @@ class ModelMetrics(SientiaMonitoring):
)
reference_columns = reference_data.drop(
columns=[target_name, 'timestamp', 'target', 'prediction'],
errors='ignore').columns
columns=[target_name, 'timestamp', 'target', 'prediction'], errors='ignore'
).columns
try:
drift_df = await self.get_drift_metrics(
@@ -224,12 +235,11 @@ class ModelMetrics(SientiaMonitoring):
return []
# Drop unnecessary columns
drift_df.drop(columns=['p_value', 'chunk_start_date', 'chunk_end_date'], inplace=True)
drift_df.drop(columns=['p_value'], inplace=True)
# Extract timestamps only until minutes
if chunk_period == 'min':
target_timestamps = target_data['timestamp'].apply(
lambda x: x[:16])
target_timestamps = target_data['timestamp'].apply(lambda x: x[:16])
else:
target_timestamps = target_data['timestamp']
@@ -237,32 +247,39 @@ class ModelMetrics(SientiaMonitoring):
drift_df = drift_df[drift_df['timestamp'].isin(target_timestamps)]
if drift_df.empty:
self.warning('No drift metrics found after dropping rows where timestamp is not in target data', metadata)
self.warning(
'No drift metrics found after dropping rows where timestamp is not in target data',
metadata,
)
return []
# Rename columns to match database columns
drift_df.rename(columns={
'metric': 'method',
'statistic': 'value',
}, inplace=True)
drift_df.rename(
columns={
'metric': 'method',
'statistic': 'value',
},
inplace=True,
)
# Drop duplicates
drift_df.drop_duplicates(
subset=['timestamp', 'method', 'feature'],
keep='first', inplace=True)
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
)
drift_df['model_id'] = model_id
drift_df['accurate'] = accurate
drift_df['timestamp'] = to_datetime(drift_df['timestamp'])
drift_df['timestamp'] = drift_df['timestamp'].dt.tz_localize('UTC')
drift_df['timestamp'] = drift_df['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
self.debug(f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata)
self.debug(
f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata
)
self.debug(f'Drift dataframe: {drift_df.head(5).to_string()}', metadata)
return drift_df.to_dict(orient='records')
@activity.defn(name='calculate_simple_metrics')
@@ -298,52 +315,40 @@ class ModelMetrics(SientiaMonitoring):
output_data = []
diff = target_data['target'] - target_data['prediction']
diff_squared = diff ** 2
diff_squared = diff**2
self.info(f'Calculating simple metrics for model {model_id}: {metrics}', metadata)
for metric in metrics:
if metric == 'rmse':
output_data.append({
'metric': 'rmse',
'value': np.sqrt(np.mean(diff_squared))
})
output_data.append({'metric': 'rmse', 'value': np.sqrt(np.mean(diff_squared))})
elif metric == 'mse':
output_data.append({
'metric': 'mse',
'value': np.mean(diff_squared)
})
output_data.append({'metric': 'mse', 'value': np.mean(diff_squared)})
elif metric == 'mae':
output_data.append({
'metric': 'mae',
'value': np.mean(np.abs(diff))
})
output_data.append({'metric': 'mae', 'value': np.mean(np.abs(diff))})
elif metric == 'r2':
y_true = target_data['target']
y_mean = np.mean(y_true)
ss_res = np.sum(diff_squared)
ss_tot = np.sum((y_true - y_mean) ** 2)
# Evita divisão por zero
if ss_tot == 0:
r2_score = 0.0
else:
r2_score = 1 - (ss_res / ss_tot)
output_data.append({
'metric': 'r2',
'value': r2_score
})
output_data.append({'metric': 'r2', 'value': r2_score})
data = DataFrame(output_data)
data['model_id'] = model_id
data['timestamp'] = target_data['timestamp'].max()
data['data_size'] = data_size
data['interval_minutes'] = interval_minutes
self.debug(f'Simple metrics dataframe: Size {data.shape} \n{data.head(5).to_string()}', metadata)
self.debug(
f'Simple metrics dataframe: Size {data.shape} \n{data.head(5).to_string()}', metadata
)
return data.to_dict(orient='records')

6
laborious/activities/opc.py
View File

@@ -1,4 +1,5 @@
from typing import Hashable
from collections.abc import Hashable
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
@@ -12,7 +13,6 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from laborious.utils.dataframe_utils import ensure_dataframe
from laborious.utils.repository.opc_repository import OpcRepository
OPC_WRITTING_ERROR_CONFIDENCE = 12
@@ -297,7 +297,7 @@ class OPC(SientiaMonitoring):
"""
metadata = input_data['metadata']
self.info('Writing data to OPC servers...', metadata)
data = ensure_dataframe(input_data['data'])
data = DataFrame(input_data['data'])
opc_output_config = input_data['opc_output_config']
self.info(f'Data to write: {data.size} rows', metadata)