SIENTIAPDE-1646
Update E2E test report and enhance drift analysis handling - Updated the E2E test report metrics to reflect the latest test results, showing 47 collected tests with all passing. - Removed outdated sections related to failed tests and their causes, streamlining the report. - Implemented a regression fix in the drift analysis to handle empty merged frames, ensuring workflows skip export when no drift metrics are available. - Enhanced the `insert_sample_data` and `insert_sample_prediction` functions to allow customizable timestamps for better test accuracy. - Refactored E2E tests to improve clarity and maintainability, particularly in handling repeat scenarios with distinct timestamps.
This commit is contained in:
vitor-aignosi
@@ -15,7 +15,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_model.analytics.drift_analysis import DriftAnalysis
|
||||
from sientia_model.analytics.drift_analysis import DriftAnalysis, DriftInsufficientDataError
|
||||
|
||||
from laborious import metrics
|
||||
from laborious.utils.dataframe_debug import build_dataframe_debug_message
|
||||
@@ -76,6 +76,25 @@ class ModelMetrics(SientiaMonitoring):
|
||||
metadata,
|
||||
)
|
||||
|
||||
def _drift_analyze_stage_error(
|
||||
self,
|
||||
exc: Exception,
|
||||
context: str,
|
||||
metadata: dict[str, Any],
|
||||
core_labels: dict[str, Any],
|
||||
) -> None:
|
||||
"""
|
||||
Log analyzer failure for a drift stage and increment the analyze error metric.
|
||||
|
||||
Args:
|
||||
- exc (Exception): Failure raised by ``sientia_model``.
|
||||
- context (str): Short label for the log line (e.g. univariate detection).
|
||||
- metadata (dict[str, Any]): Workflow metadata for logging.
|
||||
- core_labels (dict[str, Any]): Tags from ``get_core_labels`` for metrics.
|
||||
"""
|
||||
self.error(f'{context}: {exc}', metadata)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
|
||||
|
||||
def get_drift_metrics(
|
||||
self,
|
||||
reference_data: DataFrame,
|
||||
@@ -107,12 +126,15 @@ class ModelMetrics(SientiaMonitoring):
|
||||
``method`` / ``value`` (and optional ``threshold``, ``drift_type``), ready for
|
||||
activity-level formatting before Postgres export.
|
||||
"""
|
||||
# ``DriftAnalysis`` uses truthiness checks on ``features`` (e.g. ``if not features``);
|
||||
# a pandas ``Index`` is ambiguous in boolean context — normalize to a list.
|
||||
feature_names: list[str] = list(reference_columns)
|
||||
|
||||
config = {
|
||||
'target': target_name,
|
||||
'prediction': 'prediction',
|
||||
'timestamp': 'timestamp',
|
||||
'features': reference_columns,
|
||||
'features': feature_names,
|
||||
}
|
||||
|
||||
drift_analysis = DriftAnalysis(config=config)
|
||||
@@ -129,15 +151,18 @@ class ModelMetrics(SientiaMonitoring):
|
||||
univariate_drift = drift_analysis.detect_univariate_drift(
|
||||
reference_df=reference_data,
|
||||
analysis_df=target_data,
|
||||
features=reference_columns,
|
||||
features=feature_names,
|
||||
timestamp_col=config['timestamp'],
|
||||
methods=drift_metrics,
|
||||
chunk_period=chunk_period,
|
||||
)
|
||||
except Exception as e:
|
||||
self.error(f'Error detecting univariate drift: {e}', metadata)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
if isinstance(e, DriftInsufficientDataError):
|
||||
raise
|
||||
self._drift_analyze_stage_error(
|
||||
e, 'Error detecting univariate drift', metadata, core_labels
|
||||
)
|
||||
raise
|
||||
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
|
||||
|
||||
@@ -147,14 +172,17 @@ class ModelMetrics(SientiaMonitoring):
|
||||
multivariate_drift = drift_analysis.detect_multivariate_drift(
|
||||
reference_df=reference_data,
|
||||
analysis_df=target_data,
|
||||
features=reference_columns,
|
||||
features=feature_names,
|
||||
timestamp_col=config['timestamp'],
|
||||
chunk_period=chunk_period,
|
||||
)
|
||||
except Exception as e:
|
||||
self.error(f'Error detecting multivariate drift: {e}', metadata)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
if isinstance(e, DriftInsufficientDataError):
|
||||
raise
|
||||
self._drift_analyze_stage_error(
|
||||
e, 'Error detecting multivariate drift', metadata, core_labels
|
||||
)
|
||||
raise
|
||||
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
|
||||
|
||||
@@ -166,14 +194,15 @@ class ModelMetrics(SientiaMonitoring):
|
||||
multivariate_drift=multivariate_drift,
|
||||
)
|
||||
except Exception as e:
|
||||
self.error(f'Error getting drift metrics: {e}', metadata)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
|
||||
raise e
|
||||
if isinstance(e, DriftInsufficientDataError):
|
||||
raise
|
||||
self._drift_analyze_stage_error(
|
||||
e, 'Error building drift metrics dataframe', metadata, core_labels
|
||||
)
|
||||
raise
|
||||
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
|
||||
|
||||
self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
|
||||
|
||||
return drift_df
|
||||
|
||||
@staticmethod
|
||||
@@ -273,25 +302,27 @@ class ModelMetrics(SientiaMonitoring):
|
||||
metadata=metadata,
|
||||
)
|
||||
except Exception as e:
|
||||
self.error(f'Error getting drift metrics: {e}', metadata)
|
||||
if isinstance(e, DriftInsufficientDataError):
|
||||
self.error(str(e), metadata)
|
||||
notification_id = e.notification_id
|
||||
notification_message = str(e)
|
||||
else:
|
||||
self.error(f'Error getting drift metrics: {e}', metadata)
|
||||
notification_id = 'MODEL_METRICS_GET_DRIFT_METRICS_ERROR'
|
||||
notification_message = f'Error getting drift metrics: {e}'
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='MODEL_METRICS_GET_DRIFT_METRICS_ERROR',
|
||||
message=f'Error getting drift metrics: {e}',
|
||||
notification_id=notification_id,
|
||||
message=notification_message,
|
||||
block='model_metrics',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=traceback.format_exc(),
|
||||
)
|
||||
return []
|
||||
raise
|
||||
|
||||
if drift_df.empty:
|
||||
self.warning('No drift metrics found', metadata)
|
||||
return []
|
||||
|
||||
# Defense-in-depth: drop chunks whose floored timestamp does not appear
|
||||
# in the analysis window. ``DriftAnalysis`` already chunks only over
|
||||
# ``analysis_df`` so this only excludes rows injected by upstream
|
||||
# callers that pre-merge reference data into the result.
|
||||
# Drop chunks whose floored timestamp does not appear in the analysis window.
|
||||
# ``DriftAnalysis`` chunks over ``analysis_df``; this only excludes rows that
|
||||
# do not belong to the current target window (e.g. stray merged reference rows).
|
||||
target_floor = self._to_naive_utc(target_data['timestamp']).dt.floor(chunk_period)
|
||||
drift_floor = self._to_naive_utc(drift_df['timestamp']).dt.floor(chunk_period)
|
||||
drift_df = drift_df[drift_floor.isin(target_floor)]
|
||||
@@ -356,7 +387,7 @@ class ModelMetrics(SientiaMonitoring):
|
||||
metadata = input_data['metadata']
|
||||
model_id = input_data['model_id']
|
||||
target_data = DataFrame(input_data['target_data'])
|
||||
metrics = input_data['metrics']
|
||||
metric_names = input_data['metrics']
|
||||
interval_minutes = input_data['interval_minutes']
|
||||
|
||||
data_size = target_data.shape[0]
|
||||
@@ -366,9 +397,9 @@ class ModelMetrics(SientiaMonitoring):
|
||||
diff = target_data['target'] - target_data['prediction']
|
||||
diff_squared = diff**2
|
||||
|
||||
self.info(f'Calculating simple metrics for model {model_id}: {metrics}', metadata)
|
||||
self.info(f'Calculating simple metrics for model {model_id}: {metric_names}', metadata)
|
||||
|
||||
for metric in metrics:
|
||||
for metric in metric_names:
|
||||
if metric == 'rmse':
|
||||
output_data.append({'metric': 'rmse', 'value': np.sqrt(np.mean(diff_squared))})
|
||||
elif metric == 'mse':
|
||||
|
||||
Reference in New Issue
Block a user