gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Refactor ModelMetrics to utilize DriftAnalysis for drift detection

- Replaced ModelAnalysis with DriftAnalysis in the ModelMetrics class to enhance drift detection capabilities.
- Updated method signatures and documentation to reflect the changes in target_name and return values.
- Adjusted data handling to ensure compatibility with the new analysis methods and improved clarity in the drift metrics dataframe preparation.
This commit is contained in:
vitor-aignosi
2026-05-08 16:39:12 -03:00
parent e6018af23f
commit 10c7e292b9
29 changed files with 843 additions and 1266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
laborious/activities/model_metrics.py
View File

@@ -15,7 +15,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_model.analytics.model_analysis import ModelAnalysis
from sientia_model.analytics.drift_analysis import DriftAnalysis
from laborious import metrics
from laborious.utils.dataframe_debug import build_dataframe_debug_message
@@ -96,14 +96,16 @@ class ModelMetrics(SientiaMonitoring):
Args:
- reference_data (DataFrame): Baseline dataset representing expected behavior.
- target_data (DataFrame): Current analysis dataset to compare against reference.
- target_name (str): Target column name used by ``ModelAnalysis`` config.
- target_name (str): Target column name used by ``DriftAnalysis`` config.
- reference_columns (Index): Feature columns evaluated for drift.
- drift_metrics (list[str]): Enabled univariate methods.
- chunk_period (str): Time bucket granularity used by analysis methods.
- metadata (dict[str, Any]): Workflow metadata for logs and notifications.
Return:
DataFrame: Consolidated drift dataframe ready for downstream formatting/persistence.
DataFrame: Consolidated drift dataframe from ``get_drift_metrics_dataframe`` using
``method`` / ``value`` (and optional ``threshold``, ``drift_type``), ready for
activity-level formatting before Postgres export.
"""
config = {
@@ -113,7 +115,7 @@ class ModelMetrics(SientiaMonitoring):
'features': reference_columns,
}
model_analysis = ModelAnalysis(config=config)
drift_analysis = DriftAnalysis(config=config)
self._debug_dataframe(
f'Reference data: Size {reference_data.shape}', reference_data, metadata
@@ -124,7 +126,7 @@ class ModelMetrics(SientiaMonitoring):
core_labels = self.get_core_labels(metadata, operation_type='detect_univariate_drift')
start_time = time.time()
try:
univariate_drift = model_analysis.detect_univariate_drift(
univariate_drift = drift_analysis.detect_univariate_drift(
reference_df=reference_data,
analysis_df=target_data,
features=reference_columns,
@@ -142,7 +144,7 @@ class ModelMetrics(SientiaMonitoring):
core_labels = self.get_core_labels(metadata, operation_type='detect_multivariate_drift')
start_time = time.time()
try:
multivariate_drift = model_analysis.detect_multivariate_drift(
multivariate_drift = drift_analysis.detect_multivariate_drift(
reference_df=reference_data,
analysis_df=target_data,
features=reference_columns,
@@ -159,7 +161,7 @@ class ModelMetrics(SientiaMonitoring):
start_time = time.time()
core_labels = self.get_core_labels(metadata, operation_type='get_drift_metrics_dataframe')
try:
drift_df = model_analysis.get_drift_metrics_dataframe(
drift_df = drift_analysis.get_drift_metrics_dataframe(
univariate_drift=univariate_drift,
multivariate_drift=multivariate_drift,
)
@@ -179,12 +181,12 @@ class ModelMetrics(SientiaMonitoring):
"""
Parse ``series`` as datetime and return a TZ-naive UTC copy.
``sientia_model.analytics.model_analysis.ModelAnalysis`` preserves the
``sientia_model.analytics.drift_analysis.DriftAnalysis`` preserves the
timezone of the input dataframe in its outputs, while target rows
loaded from PostgreSQL come in with ``+00:00``. Forcing both sides of
a comparison to TZ-naive UTC keeps ``isin`` / ``floor`` operations
deterministic regardless of how the analyzer (or a test double)
constructs its timestamps.
deterministic regardless of how the analyzer constructs its
timestamps.
Args:
- series (Series): Input series containing datetime-parseable values.
@@ -228,7 +230,7 @@ class ModelMetrics(SientiaMonitoring):
target_data = target_data.pivot(index='timestamp', columns='variable', values='value')
target_data['timestamp'] = target_data.index
# Keep timestamps as datetime: ModelAnalysis._chunk_dataframe relies on
# Keep timestamps as datetime: DriftAnalysis._chunk_dataframe relies on
# ``pd.Grouper(freq=...)`` which rejects string timestamp columns.
target_data['timestamp'] = to_datetime(target_data['timestamp'])
target_data = target_data.reset_index(drop=True)
@@ -287,7 +289,7 @@ class ModelMetrics(SientiaMonitoring):
return []
# Defense-in-depth: drop chunks whose floored timestamp does not appear
# in the analysis window. ``ModelAnalysis`` already chunks only over
# in the analysis window. ``DriftAnalysis`` already chunks only over
# ``analysis_df`` so this only excludes rows injected by upstream
# callers that pre-merge reference data into the result.
target_floor = self._to_naive_utc(target_data['timestamp']).dt.floor(chunk_period)
@@ -301,41 +303,30 @@ class ModelMetrics(SientiaMonitoring):
)
return []
# Map ``sientia_model.analytics.model_analysis`` schema onto the drift
# table columns: ``metric -> method``, ``statistic -> value``,
# ``alert -> drift``, ``chunk_index -> chunk``,
# ``chunk_end_date -> timestamp_end``. ``p_value`` and
# ``chunk_start_date`` are not persisted.
drift_df = drift_df.rename(
columns={
'metric': 'method',
'statistic': 'value',
'alert': 'drift',
'chunk_index': 'chunk',
'chunk_end_date': 'timestamp_end',
}
)
drift_df.drop(columns=['p_value', 'chunk_start_date'], inplace=True, errors='ignore')
# Analyzer emits diagnostic columns that are not stored in ``sientia_data.drift_metrics``.
drift_df = drift_df.drop(columns=['threshold', 'drift_type'], errors='ignore')
# Drop duplicates
drift_df.drop_duplicates(
subset=['timestamp', 'method', 'feature'], keep='first', inplace=True
)
drift_df['model_id'] = model_id
drift_df['model_id'] = str(model_id)
drift_df['accurate'] = accurate
drift_df['timestamp'] = self._to_naive_utc(drift_df['timestamp'])
drift_df['timestamp'] = drift_df['timestamp'].dt.tz_localize('UTC')
drift_df['timestamp'] = drift_df['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
# ``timestamp_end`` may carry nanosecond precision (beyond
# ``timestamptz`` microseconds), so serialize as ISO text for the
# ``text`` Postgres column.
drift_df['timestamp_end'] = drift_df['timestamp_end'].apply(
lambda value: pd.Timestamp(value).isoformat() if pd.notna(value) else None
# ``timestamp`` is overridden with the most recent target instant so
# every persisted row shares a single business timestamp (the run's
# logical "now"), matching what downstream consumers expect.
latest_target_timestamp = self._to_naive_utc(target_data['timestamp']).max()
drift_df['timestamp'] = (
pd.Timestamp(latest_target_timestamp)
.tz_localize('UTC')
.strftime(DATETIME_FORMAT_WITH_TZ)
)
# ``chunk_start_date`` / ``chunk_end_date`` may carry nanosecond
# precision (beyond ``timestamptz`` microseconds), so serialize as ISO
# text for the ``text`` Postgres columns.
for column in ('chunk_start_date', 'chunk_end_date'):
drift_df[column] = drift_df[column].apply(
lambda value: pd.Timestamp(value).isoformat() if pd.notna(value) else None
)
self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
return drift_df.to_dict(orient='records')