gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1273

Browse Source 
SIENTIAPDE-1273
Enhance security analysis and SQL injection handling

- Added skip for potential SQL injection false positives in Bandit configuration.
- Updated validate.sh to use the pyproject.toml configuration for Bandit security analysis.
- Refactored code to replace ensure_dataframe utility with direct DataFrame usage in multiple activities, improving clarity and reducing dependencies.
- Removed the deprecated dataframe_utils module to streamline the codebase.
This commit is contained in:
vitor-aignosi
2025-11-17 16:04:54 -03:00
parent 1014c33dd9
commit a88a15c60a
18 changed files with 516 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
laborious/activities/model_metrics.py
View File

@@ -1,25 +1,28 @@
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import time
import traceback
import warnings
from typing import Any
import numpy as np
from pandas import DataFrame, Index, to_datetime
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.logger import Logger
from sientia.ModelAnalysis import ModelAnalysis
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia.ModelAnalysis import ModelAnalysis
from laborious import metrics
import time
import numpy as np
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
import warnings
import traceback
from laborious import metrics
warnings.filterwarnings('ignore', category=RuntimeWarning, message='Degrees of freedom <= 0')
warnings.filterwarnings('ignore', category=RuntimeWarning, message='invalid value encountered in scalar divide')
warnings.filterwarnings(
'ignore', category=RuntimeWarning, message='invalid value encountered in scalar divide'
)
class ModelMetrics(SientiaMonitoring):
"""
@@ -28,12 +31,12 @@ class ModelMetrics(SientiaMonitoring):
This class provides activities for writing metrics to the Prometheus monitoring system.
"""
def __init__(self,
def __init__(
self,
logger: Logger,
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
):
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
def close(self) -> None:
@@ -42,11 +45,11 @@ class ModelMetrics(SientiaMonitoring):
"""
SientiaMonitoring.shutdown(self)
def __del__(self):
self.close()
async def get_drift_metrics(self,
async def get_drift_metrics(
self,
reference_data: DataFrame,
target_data: DataFrame,
target_name: str,
@@ -73,34 +76,37 @@ class ModelMetrics(SientiaMonitoring):
'features': reference_columns,
}
model_analysis = ModelAnalysis(config=config)
self.debug(f'Reference data: Size {reference_data.shape} \n{reference_data.head(5).to_string()}', metadata)
self.debug(
f'Reference data: Size {reference_data.shape} \n{reference_data.head(5).to_string()}',
metadata,
)
self.debug(f'Target data: Size {target_data.shape} \n{target_data.head(5).to_string()}', metadata)
self.debug(
f'Target data: Size {target_data.shape} \n{target_data.head(5).to_string()}', metadata
)
core_labels = self.get_core_labels(metadata, operation_type='detect_univariate_drift')
start_time = time.time()
try:
univariate_drift = model_analysis.detect_univariate_drift(
reference_df=reference_data,
analysis_df=target_data,
features=reference_columns,
timestamp_col=config['timestamp'],
methods=drift_metrics,
chunk_period=chunk_period
chunk_period=chunk_period,
)
except Exception as e:
self.error(f'Error detecting univariate drift: {e}', metadata)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
core_labels = self.get_core_labels(metadata, operation_type='detect_multivariate_drift')
start_time = time.time()
try:
@@ -109,11 +115,13 @@ class ModelMetrics(SientiaMonitoring):
analysis_df=target_data,
features=reference_columns,
timestamp_col=config['timestamp'],
chunk_period=chunk_period
chunk_period=chunk_period,
)
except Exception as e:
self.error(f'Error detecting multivariate drift: {e}', metadata)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
@@ -127,15 +135,18 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error getting drift metrics: {e}', metadata)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self.debug(f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata)
return drift_df
self.debug(
f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata
)
return drift_df
@activity.defn(name='calculate_drift')
async def calculate_drift(self, input_data: dict[str, Any]) -> list[dict]:
@@ -194,8 +205,8 @@ class ModelMetrics(SientiaMonitoring):
)
reference_columns = reference_data.drop(
columns=[target_name, 'timestamp', 'target', 'prediction'],
errors='ignore').columns
columns=[target_name, 'timestamp', 'target', 'prediction'], errors='ignore'
).columns
try:
drift_df = await self.get_drift_metrics(
@@ -224,12 +235,11 @@ class ModelMetrics(SientiaMonitoring):
return []
# Drop unnecessary columns
drift_df.drop(columns=['p_value', 'chunk_start_date', 'chunk_end_date'], inplace=True)
drift_df.drop(columns=['p_value'], inplace=True)
# Extract timestamps only until minutes
if chunk_period == 'min':
target_timestamps = target_data['timestamp'].apply(
lambda x: x[:16])
target_timestamps = target_data['timestamp'].apply(lambda x: x[:16])
else:
target_timestamps = target_data['timestamp']
@@ -237,32 +247,39 @@ class ModelMetrics(SientiaMonitoring):
drift_df = drift_df[drift_df['timestamp'].isin(target_timestamps)]
if drift_df.empty:
self.warning('No drift metrics found after dropping rows where timestamp is not in target data', metadata)
self.warning(
'No drift metrics found after dropping rows where timestamp is not in target data',
metadata,
)
return []
# Rename columns to match database columns
drift_df.rename(columns={
'metric': 'method',
'statistic': 'value',
}, inplace=True)
drift_df.rename(
columns={
'metric': 'method',
'statistic': 'value',
},
inplace=True,
)
# Drop duplicates
drift_df.drop_duplicates(
subset=['timestamp', 'method', 'feature'],
keep='first', inplace=True)
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
)
drift_df['model_id'] = model_id
drift_df['accurate'] = accurate
drift_df['timestamp'] = to_datetime(drift_df['timestamp'])
drift_df['timestamp'] = drift_df['timestamp'].dt.tz_localize('UTC')
drift_df['timestamp'] = drift_df['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
self.debug(f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata)
self.debug(
f'Drift dataframe: Size {drift_df.shape} \n{drift_df.head(5).to_string()}', metadata
)
self.debug(f'Drift dataframe: {drift_df.head(5).to_string()}', metadata)
return drift_df.to_dict(orient='records')
@activity.defn(name='calculate_simple_metrics')
@@ -298,52 +315,40 @@ class ModelMetrics(SientiaMonitoring):
output_data = []
diff = target_data['target'] - target_data['prediction']
diff_squared = diff ** 2
diff_squared = diff**2
self.info(f'Calculating simple metrics for model {model_id}: {metrics}', metadata)
for metric in metrics:
if metric == 'rmse':
output_data.append({
'metric': 'rmse',
'value': np.sqrt(np.mean(diff_squared))
})
output_data.append({'metric': 'rmse', 'value': np.sqrt(np.mean(diff_squared))})
elif metric == 'mse':
output_data.append({
'metric': 'mse',
'value': np.mean(diff_squared)
})
output_data.append({'metric': 'mse', 'value': np.mean(diff_squared)})
elif metric == 'mae':
output_data.append({
'metric': 'mae',
'value': np.mean(np.abs(diff))
})
output_data.append({'metric': 'mae', 'value': np.mean(np.abs(diff))})
elif metric == 'r2':
y_true = target_data['target']
y_mean = np.mean(y_true)
ss_res = np.sum(diff_squared)
ss_tot = np.sum((y_true - y_mean) ** 2)
# Evita divisão por zero
if ss_tot == 0:
r2_score = 0.0
else:
r2_score = 1 - (ss_res / ss_tot)
output_data.append({
'metric': 'r2',
'value': r2_score
})
output_data.append({'metric': 'r2', 'value': r2_score})
data = DataFrame(output_data)
data['model_id'] = model_id
data['timestamp'] = target_data['timestamp'].max()
data['data_size'] = data_size
data['interval_minutes'] = interval_minutes
self.debug(f'Simple metrics dataframe: Size {data.shape} \n{data.head(5).to_string()}', metadata)
self.debug(
f'Simple metrics dataframe: Size {data.shape} \n{data.head(5).to_string()}', metadata
)
return data.to_dict(orient='records')