e6018af23f
Update README, requirements, and E2E tests for improved configuration and functionality - Enhanced the README with updated model configuration examples, including the addition of an alias for production. - Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`. - Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity. - Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs. - Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.
787 lines
27 KiB
Python
787 lines
27 KiB
Python
"""
|
|
End-to-end tests for the Drift workflow.
|
|
|
|
Coverage focus:
|
|
|
|
- Full pipeline persists drift rows with **all** columns expected by the
|
|
``predictions_schema.drift`` table (model_id, feature, method, value, drift,
|
|
chunk, timestamp, timestamp_end, accurate, plus DB-managed id/created_at/updated_at).
|
|
- ``get_reference_data`` happy path (CSV downloaded from MLflow stub) and
|
|
fallback path (30% of target data when reference is unavailable).
|
|
- ``calculate_drift`` invariants: ``p_value`` dropped, duplicates removed, rows
|
|
outside target timestamps filtered, default ``drift_metrics`` propagated.
|
|
- Failure paths: ``ModelAnalysis.get_drift_metrics_dataframe`` raises ⇒
|
|
workflow completes without writes; empty target data ⇒ workflow short-circuits;
|
|
invalid ``chunk_period`` ⇒ activity raises and workflow surfaces the error.
|
|
"""
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
import pandas as pd
|
|
import pytest
|
|
from sqlalchemy import text
|
|
from temporalio.testing import WorkflowEnvironment
|
|
from temporalio.worker import Worker
|
|
|
|
from e2e.helpers import (
|
|
build_drift_dataframe,
|
|
insert_target_data_for_drift,
|
|
load_scenario_input,
|
|
make_workflow_id,
|
|
start_and_await_workflow,
|
|
)
|
|
from laborious.activities.activities import Activities
|
|
from laborious.workflows.drift import Drift
|
|
|
|
# Drift columns that must be present (and non-null where required) on every row.
|
|
EXPECTED_DRIFT_COLUMNS = [
|
|
'id',
|
|
'model_id',
|
|
'feature',
|
|
'method',
|
|
'value',
|
|
'drift',
|
|
'chunk',
|
|
'timestamp',
|
|
'timestamp_end',
|
|
'accurate',
|
|
'created_at',
|
|
'updated_at',
|
|
]
|
|
|
|
|
|
def _drift_input(model_id: int, **overrides) -> dict:
|
|
"""Load the base drift scenario JSON and apply ad-hoc overrides."""
|
|
input_data = load_scenario_input('drift_base.json', model_id=model_id)
|
|
input_data.update(overrides)
|
|
return input_data
|
|
|
|
|
|
def _five_minute_window(offset_minutes: int = 6) -> tuple[list[str], list[str]]:
|
|
"""
|
|
Build five consecutive UTC minute timestamps positioned in the recent past.
|
|
|
|
The Drift workflow filters target rows with ``timestamp > NOW() - INTERVAL``,
|
|
so timestamps must be recent for tests to retrieve any data. We snap to
|
|
minute precision and back off ``offset_minutes`` minutes so all chunks land
|
|
well inside the default 60-minute interval defined in ``drift_base.json``.
|
|
|
|
Args:
|
|
- offset_minutes: How many minutes ago the most recent chunk should be.
|
|
|
|
Return:
|
|
Tuple ``(target_timestamps, chunk_timestamps)``:
|
|
|
|
- ``target_timestamps``: ISO strings with ``+0000`` used as ``timestamp``
|
|
and ``created_at`` columns when inserting target rows.
|
|
- ``chunk_timestamps``: ``YYYY-MM-DD HH:MM`` truncations matching what
|
|
``calculate_drift`` filters on for ``chunk_period='min'``.
|
|
"""
|
|
base = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(
|
|
minutes=offset_minutes
|
|
)
|
|
target_timestamps = [
|
|
(base + timedelta(minutes=i)).strftime('%Y-%m-%d %H:%M:%S%z') for i in range(5)
|
|
]
|
|
chunk_timestamps = [
|
|
(base + timedelta(minutes=i)).strftime('%Y-%m-%d %H:%M') for i in range(5)
|
|
]
|
|
return target_timestamps, chunk_timestamps
|
|
|
|
|
|
def _configure_reference_csv(mlflow_repository_stub, reference_rows: pd.DataFrame) -> None:
|
|
"""
|
|
Wire ``mlflow_repository_stub`` so ``get_reference_data`` returns
|
|
``reference_rows`` by writing them to ``dst_path/evaluation_data.csv``.
|
|
"""
|
|
|
|
def _download(run_id: str, artifact_path: str, dst_path: str, metadata=None):
|
|
target = Path(dst_path) / 'evaluation_data.csv'
|
|
reference_rows.to_csv(target, index=False)
|
|
|
|
mlflow_repository_stub._client.get_model_version_by_alias.return_value = MagicMock(
|
|
run_id='fake-reference-run'
|
|
)
|
|
mlflow_repository_stub.download_artifacts.side_effect = _download
|
|
|
|
|
|
def _force_reference_unavailable(mlflow_repository_stub) -> None:
|
|
"""Make ``get_reference_data`` return ``None`` by failing alias resolution."""
|
|
mlflow_repository_stub._client.get_model_version_by_alias.side_effect = Exception(
|
|
'no production alias registered'
|
|
)
|
|
|
|
|
|
def _assert_drift_columns_complete(rows, *, expected_count: int) -> None:
|
|
"""Validate row count and that the canonical drift columns are populated."""
|
|
assert len(rows) == expected_count, (
|
|
f'Expected {expected_count} drift rows persisted, got {len(rows)}'
|
|
)
|
|
seen_columns = set(rows[0]._mapping.keys()) if rows else set()
|
|
for column in EXPECTED_DRIFT_COLUMNS:
|
|
assert column in seen_columns, f'Missing drift column in postgres: {column}'
|
|
|
|
for row in rows:
|
|
mapping = dict(row._mapping)
|
|
for column in EXPECTED_DRIFT_COLUMNS:
|
|
if column == 'value':
|
|
# ``value`` is nullable in the table; skip null check, only ensure key exists.
|
|
continue
|
|
assert mapping[column] is not None, f"Column '{column}' is NULL in {mapping}"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_drift_happy_path_persists_all_columns_with_reference_data(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_drift: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
mlflow_repository_stub,
|
|
model_analysis_stub,
|
|
):
|
|
"""
|
|
Scenario D.1.1: Happy path with reference data downloaded from MLflow.
|
|
|
|
Validates the full drift pipeline produces one row per (chunk, feature, method)
|
|
combination plus the multivariate row block, with **all** columns required by
|
|
``predictions_schema.drift`` populated. Uses a deterministic mocked drift
|
|
dataframe so the postgres assertions remain stable across runs.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 411
|
|
|
|
target_timestamps, chunk_timestamps = _five_minute_window()
|
|
insert_target_data_for_drift(
|
|
postgres_engine,
|
|
model_id=model_id,
|
|
timestamps=target_timestamps,
|
|
variables_values={
|
|
'sensor_1': [10.0, 11.0, 12.0, 13.0, 14.0],
|
|
'sensor_2': [20.0, 21.0, 22.0, 23.0, 24.0],
|
|
},
|
|
)
|
|
|
|
reference_df = pd.DataFrame(
|
|
{
|
|
'timestamp': ['2023-12-31 11:00:00+00:00', '2023-12-31 11:01:00+00:00'],
|
|
'sensor_1': [9.5, 9.6],
|
|
'sensor_2': [19.5, 19.6],
|
|
}
|
|
)
|
|
_configure_reference_csv(mlflow_repository_stub, reference_df)
|
|
|
|
features = ['sensor_1', 'sensor_2']
|
|
methods = ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']
|
|
drift_flags = {
|
|
('sensor_1', 'wasserstein'): True,
|
|
('sensor_2', 'wasserstein'): True,
|
|
('sensor_2', 'jensen_shannon'): True,
|
|
}
|
|
|
|
model_analysis_stub.set_drift_dataframe(
|
|
lambda univariate, multivariate: build_drift_dataframe(
|
|
timestamps=chunk_timestamps,
|
|
features=features,
|
|
methods=methods,
|
|
statistic=0.42,
|
|
drift_flags=drift_flags,
|
|
multivariate_value=15.5,
|
|
multivariate_drift=True,
|
|
)
|
|
)
|
|
|
|
input_data = _drift_input(model_id)
|
|
await start_and_await_workflow(
|
|
client, Drift.run, input_data, make_workflow_id('test-drift-happy-path')
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
rows = (
|
|
conn.execute(
|
|
text(
|
|
'SELECT * FROM predictions_schema.drift '
|
|
'WHERE model_id = :model_id ORDER BY chunk, feature, method'
|
|
),
|
|
{'model_id': model_id},
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
|
|
expected_rows = len(chunk_timestamps) * (len(features) * len(methods) + 1) # univariate + multivariate
|
|
assert len(rows) == expected_rows
|
|
for column in EXPECTED_DRIFT_COLUMNS:
|
|
assert column in rows[0], f'Missing drift column in postgres: {column}'
|
|
|
|
for row in rows:
|
|
for column in EXPECTED_DRIFT_COLUMNS:
|
|
if column == 'value':
|
|
continue
|
|
assert row[column] is not None, f"Column '{column}' is NULL in {dict(row)}"
|
|
|
|
multivariate_rows = [r for r in rows if r['feature'] == 'multivariate']
|
|
univariate_rows = [r for r in rows if r['feature'] != 'multivariate']
|
|
|
|
assert len(multivariate_rows) == len(chunk_timestamps)
|
|
assert all(r['method'] == 'multivariate' for r in multivariate_rows)
|
|
assert all(r['drift'] is True for r in multivariate_rows)
|
|
assert all(Decimal(str(r['value'])) == Decimal('15.5') for r in multivariate_rows)
|
|
|
|
assert len(univariate_rows) == len(features) * len(methods) * len(chunk_timestamps)
|
|
assert {r['method'] for r in univariate_rows} == set(methods)
|
|
assert {r['feature'] for r in univariate_rows} == set(features)
|
|
|
|
drift_pairs = {(r['feature'], r['method']): r['drift'] for r in univariate_rows}
|
|
for (feature, method), expected_drift in drift_flags.items():
|
|
assert drift_pairs[(feature, method)] is expected_drift
|
|
|
|
assert all(r['accurate'] is True for r in rows), 'reference path should mark rows as accurate'
|
|
|
|
assert all(
|
|
r['timestamp_end'].endswith(':59.999999999') and 'T' in r['timestamp_end']
|
|
for r in rows
|
|
), 'timestamp_end should preserve the high-precision ISO string from ModelAnalysis'
|
|
|
|
assert all('p_value' not in r for r in rows), 'p_value must be dropped before postgres'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_drift_uses_30pct_fallback_when_reference_unavailable(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_drift: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
mlflow_repository_stub,
|
|
model_analysis_stub,
|
|
notification_inserts,
|
|
):
|
|
"""
|
|
Scenario D.1.2: ``get_reference_data`` returns ``None`` (production alias missing),
|
|
so ``calculate_drift`` falls back to using the first 30% of target rows as
|
|
reference. Persisted rows must report ``accurate=False`` and a warning
|
|
notification must be emitted to mongo.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 412
|
|
|
|
target_timestamps, chunk_timestamps = _five_minute_window()
|
|
insert_target_data_for_drift(
|
|
postgres_engine,
|
|
model_id=model_id,
|
|
timestamps=target_timestamps,
|
|
variables_values={
|
|
'sensor_1': [10.0, 10.1, 10.2, 10.3, 10.4],
|
|
'sensor_2': [20.0, 20.1, 20.2, 20.3, 20.4],
|
|
},
|
|
)
|
|
|
|
_force_reference_unavailable(mlflow_repository_stub)
|
|
|
|
model_analysis_stub.set_drift_dataframe(
|
|
lambda univariate, multivariate: build_drift_dataframe(
|
|
timestamps=chunk_timestamps,
|
|
features=['sensor_1'],
|
|
methods=['kolmogorov_smirnov'],
|
|
statistic=0.7,
|
|
include_multivariate=False,
|
|
)
|
|
)
|
|
|
|
input_data = _drift_input(model_id)
|
|
|
|
await start_and_await_workflow(
|
|
client, Drift.run, input_data, make_workflow_id('test-drift-fallback')
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
rows = (
|
|
conn.execute(
|
|
text('SELECT * FROM predictions_schema.drift WHERE model_id = :m'),
|
|
{'m': model_id},
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
|
|
assert len(rows) == len(chunk_timestamps)
|
|
assert all(r['accurate'] is False for r in rows), 'fallback path must mark rows as inaccurate'
|
|
assert all(r['feature'] == 'sensor_1' for r in rows)
|
|
|
|
fallback_warnings = [
|
|
call
|
|
for call in notification_inserts.call_args_list
|
|
if call.args
|
|
and isinstance(call.args[0], dict)
|
|
and call.args[0].get('notification_id') == 'MODEL_METRICS_REFERENCE_DATA_WARNING'
|
|
]
|
|
assert len(fallback_warnings) >= 1, 'expected reference fallback warning notification'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_drift_drops_p_value_and_dedupes_by_timestamp_method_feature(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_drift: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
mlflow_repository_stub,
|
|
model_analysis_stub,
|
|
):
|
|
"""
|
|
Scenario D.2.1: ModelAnalysis returns duplicate (timestamp, method, feature)
|
|
rows and a populated ``p_value`` column. ``calculate_drift`` must deduplicate
|
|
keeping the first occurrence and the persisted rows must not contain
|
|
``p_value``.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 421
|
|
|
|
target_timestamps, chunk_timestamps = _five_minute_window()
|
|
insert_target_data_for_drift(
|
|
postgres_engine,
|
|
model_id=model_id,
|
|
timestamps=target_timestamps,
|
|
variables_values={
|
|
'sensor_1': [1.0, 2.0, 3.0, 4.0, 5.0],
|
|
'sensor_2': [10.0, 20.0, 30.0, 40.0, 50.0],
|
|
},
|
|
)
|
|
|
|
duplicates = [
|
|
# Re-emit the first chunk for sensor_1/kolmogorov_smirnov with a different value
|
|
# so we can prove the dedup keeps the FIRST row.
|
|
{
|
|
'timestamp': pd.Timestamp(chunk_timestamps[0]),
|
|
'feature': 'sensor_1',
|
|
'metric': 'kolmogorov_smirnov',
|
|
'statistic': 0.99,
|
|
'p_value': 0.02,
|
|
'alert': True,
|
|
'chunk_index': 0,
|
|
'chunk_start_date': pd.Timestamp(chunk_timestamps[0]),
|
|
'chunk_end_date': pd.Timestamp(f'{chunk_timestamps[0]}:59.999999999'),
|
|
}
|
|
]
|
|
|
|
model_analysis_stub.set_drift_dataframe(
|
|
lambda univariate, multivariate: build_drift_dataframe(
|
|
timestamps=chunk_timestamps,
|
|
features=['sensor_1'],
|
|
methods=['kolmogorov_smirnov'],
|
|
statistic=0.5,
|
|
include_multivariate=False,
|
|
extra_rows=duplicates,
|
|
)
|
|
)
|
|
_force_reference_unavailable(mlflow_repository_stub)
|
|
|
|
input_data = _drift_input(model_id)
|
|
await start_and_await_workflow(
|
|
client, Drift.run, input_data, make_workflow_id('test-drift-dedupe')
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
rows = (
|
|
conn.execute(
|
|
text(
|
|
'SELECT chunk, feature, method, value FROM predictions_schema.drift '
|
|
'WHERE model_id = :m ORDER BY chunk'
|
|
),
|
|
{'m': model_id},
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
|
|
assert len(rows) == len(chunk_timestamps), 'duplicates must be removed before persistence'
|
|
|
|
first_chunk_rows = [r for r in rows if r['chunk'] == 0]
|
|
assert len(first_chunk_rows) == 1
|
|
assert Decimal(str(first_chunk_rows[0]['value'])) == Decimal('0.5'), (
|
|
'dedup must keep the FIRST occurrence (statistic=0.5), not the duplicate (statistic=0.99)'
|
|
)
|
|
|
|
columns = set(rows[0].keys())
|
|
assert 'p_value' not in columns
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_drift_drops_rows_outside_target_timestamps(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_drift: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
mlflow_repository_stub,
|
|
model_analysis_stub,
|
|
):
|
|
"""
|
|
Scenario D.2.2: Drift rows whose timestamps are not present in the target
|
|
data (e.g. came from the reference distribution) must be discarded so the
|
|
saved drift only reflects analysis chunks.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 422
|
|
|
|
target_timestamps, chunk_timestamps = _five_minute_window()
|
|
insert_target_data_for_drift(
|
|
postgres_engine,
|
|
model_id=model_id,
|
|
timestamps=target_timestamps,
|
|
variables_values={
|
|
'sensor_1': [1.0, 2.0, 3.0, 4.0, 5.0],
|
|
'sensor_2': [10.0, 20.0, 30.0, 40.0, 50.0],
|
|
},
|
|
)
|
|
|
|
out_of_range_timestamp = pd.Timestamp('2099-12-31 23:59')
|
|
extra = [
|
|
{
|
|
'timestamp': out_of_range_timestamp,
|
|
'feature': 'sensor_1',
|
|
'metric': 'kolmogorov_smirnov',
|
|
'statistic': 0.5,
|
|
'p_value': 0.0,
|
|
'alert': True,
|
|
'chunk_index': 99,
|
|
'chunk_start_date': out_of_range_timestamp,
|
|
'chunk_end_date': pd.Timestamp('2099-12-31 23:59:59.999999999'),
|
|
}
|
|
]
|
|
|
|
model_analysis_stub.set_drift_dataframe(
|
|
lambda univariate, multivariate: build_drift_dataframe(
|
|
timestamps=chunk_timestamps,
|
|
features=['sensor_1'],
|
|
methods=['kolmogorov_smirnov'],
|
|
include_multivariate=False,
|
|
extra_rows=extra,
|
|
)
|
|
)
|
|
_force_reference_unavailable(mlflow_repository_stub)
|
|
|
|
input_data = _drift_input(model_id)
|
|
await start_and_await_workflow(
|
|
client, Drift.run, input_data, make_workflow_id('test-drift-tts-filter')
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
chunks = [
|
|
r[0]
|
|
for r in conn.execute(
|
|
text(
|
|
'SELECT chunk FROM predictions_schema.drift WHERE model_id = :m ORDER BY chunk'
|
|
),
|
|
{'m': model_id},
|
|
).all()
|
|
]
|
|
|
|
assert chunks == [0, 1, 2, 3, 4], 'out-of-range timestamps must be filtered out'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_drift_empty_target_data_short_circuits_workflow(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_drift: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
mlflow_repository_stub,
|
|
model_analysis_stub,
|
|
):
|
|
"""
|
|
Scenario D.3.1: When ``load_custom_query`` returns no rows the workflow must
|
|
return early without invoking ModelAnalysis or writing any drift rows.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 431
|
|
|
|
with postgres_engine.begin() as conn:
|
|
conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}'))
|
|
|
|
_force_reference_unavailable(mlflow_repository_stub)
|
|
|
|
drift_calls = {'count': 0}
|
|
|
|
def _factory(univariate, multivariate):
|
|
drift_calls['count'] += 1
|
|
return build_drift_dataframe(
|
|
timestamps=['2024-01-01 12:00'],
|
|
features=['sensor_1'],
|
|
methods=['kolmogorov_smirnov'],
|
|
include_multivariate=False,
|
|
)
|
|
|
|
model_analysis_stub.set_drift_dataframe(_factory)
|
|
|
|
input_data = _drift_input(model_id)
|
|
await start_and_await_workflow(
|
|
client, Drift.run, input_data, make_workflow_id('test-drift-empty-target')
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
count = conn.execute(
|
|
text('SELECT COUNT(*) FROM predictions_schema.drift WHERE model_id = :m'),
|
|
{'m': model_id},
|
|
).scalar()
|
|
assert count == 0
|
|
assert drift_calls['count'] == 0, 'ModelAnalysis must not be invoked when target data is empty'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_drift_model_analysis_failure_keeps_workflow_alive_no_writes(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_drift: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
mlflow_repository_stub,
|
|
model_analysis_stub,
|
|
notification_inserts,
|
|
):
|
|
"""
|
|
Scenario D.3.2: ``ModelAnalysis.get_drift_metrics_dataframe`` raises. The
|
|
activity must catch the error, send an error notification and return ``[]``
|
|
so the workflow completes without persisting drift rows.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 432
|
|
|
|
target_timestamps, _ = _five_minute_window()
|
|
insert_target_data_for_drift(
|
|
postgres_engine,
|
|
model_id=model_id,
|
|
timestamps=target_timestamps,
|
|
variables_values={
|
|
'sensor_1': [1.0, 2.0, 3.0, 4.0, 5.0],
|
|
'sensor_2': [10.0, 20.0, 30.0, 40.0, 50.0],
|
|
},
|
|
)
|
|
|
|
_force_reference_unavailable(mlflow_repository_stub)
|
|
model_analysis_stub.raise_on_get_drift_metrics_dataframe(
|
|
RuntimeError('drift analyzer crashed')
|
|
)
|
|
|
|
input_data = _drift_input(model_id)
|
|
await start_and_await_workflow(
|
|
client, Drift.run, input_data, make_workflow_id('test-drift-analyzer-error')
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
count = conn.execute(
|
|
text('SELECT COUNT(*) FROM predictions_schema.drift WHERE model_id = :m'),
|
|
{'m': model_id},
|
|
).scalar()
|
|
assert count == 0
|
|
|
|
error_notifications = [
|
|
call
|
|
for call in notification_inserts.call_args_list
|
|
if call.args
|
|
and isinstance(call.args[0], dict)
|
|
and call.args[0].get('notification_id') == 'MODEL_METRICS_GET_DRIFT_METRICS_ERROR'
|
|
]
|
|
assert len(error_notifications) >= 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_drift_default_drift_metrics_propagated_to_analyzer(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_drift: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
mlflow_repository_stub,
|
|
model_analysis_stub,
|
|
):
|
|
"""
|
|
Scenario D.4.1: When ``drift_metrics`` is omitted from input the workflow
|
|
must default to ``['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']``
|
|
and forward exactly that list to ``ModelAnalysis.detect_univariate_drift``.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 441
|
|
|
|
target_timestamps, chunk_timestamps = _five_minute_window()
|
|
insert_target_data_for_drift(
|
|
postgres_engine,
|
|
model_id=model_id,
|
|
timestamps=target_timestamps,
|
|
variables_values={
|
|
'sensor_1': [1.0, 2.0, 3.0, 4.0, 5.0],
|
|
'sensor_2': [10.0, 20.0, 30.0, 40.0, 50.0],
|
|
},
|
|
)
|
|
|
|
_force_reference_unavailable(mlflow_repository_stub)
|
|
model_analysis_stub.set_drift_dataframe(
|
|
lambda univariate, multivariate: build_drift_dataframe(
|
|
timestamps=chunk_timestamps,
|
|
features=['sensor_1'],
|
|
methods=['kolmogorov_smirnov'],
|
|
include_multivariate=False,
|
|
)
|
|
)
|
|
|
|
input_data = _drift_input(model_id)
|
|
input_data.pop('drift_metrics', None)
|
|
|
|
await start_and_await_workflow(
|
|
client, Drift.run, input_data, make_workflow_id('test-drift-default-methods')
|
|
)
|
|
|
|
instance = model_analysis_stub.last_instance
|
|
assert instance is not None, 'ModelAnalysis must have been instantiated'
|
|
assert len(instance.detect_univariate_drift_calls) == 1
|
|
forwarded_methods = instance.detect_univariate_drift_calls[0]['methods']
|
|
assert forwarded_methods == ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_drift_invalid_chunk_period_raises_value_error(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_drift: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
mlflow_repository_stub,
|
|
):
|
|
"""
|
|
Scenario D.4.2: ``calculate_drift`` validates ``chunk_period`` and rejects
|
|
anything other than ``min`` / ``s``. The workflow must surface the
|
|
``ValueError`` to the caller and persist nothing.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 442
|
|
|
|
target_timestamps, _ = _five_minute_window()
|
|
insert_target_data_for_drift(
|
|
postgres_engine,
|
|
model_id=model_id,
|
|
timestamps=target_timestamps,
|
|
variables_values={
|
|
'sensor_1': [1.0, 2.0, 3.0, 4.0, 5.0],
|
|
'sensor_2': [10.0, 20.0, 30.0, 40.0, 50.0],
|
|
},
|
|
)
|
|
_force_reference_unavailable(mlflow_repository_stub)
|
|
|
|
input_data = _drift_input(model_id, chunk_period='hour')
|
|
|
|
with pytest.raises(Exception) as excinfo:
|
|
await start_and_await_workflow(
|
|
client,
|
|
Drift.run,
|
|
input_data,
|
|
make_workflow_id('test-drift-bad-chunk-period'),
|
|
)
|
|
|
|
# Temporal wraps the activity ValueError in WorkflowFailureError; the message
|
|
# may live on ``.message`` or ``str(exc)`` depending on the SDK error class.
|
|
cause_descriptions = []
|
|
current: BaseException | None = excinfo.value
|
|
while current is not None:
|
|
cause_descriptions.append(
|
|
getattr(current, 'message', None) or str(current) or repr(current)
|
|
)
|
|
current = current.__cause__
|
|
assert any('Invalid chunk period' in msg for msg in cause_descriptions), (
|
|
f'Expected ValueError about chunk period in chain, got: {cause_descriptions}'
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
count = conn.execute(
|
|
text('SELECT COUNT(*) FROM predictions_schema.drift WHERE model_id = :m'),
|
|
{'m': model_id},
|
|
).scalar()
|
|
assert count == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.integration
|
|
async def test_drift_chunk_period_seconds_preserves_seconds_in_timestamp_column(
|
|
temporal_test_env: WorkflowEnvironment,
|
|
temporal_worker_drift: Worker,
|
|
test_activities: Activities,
|
|
postgres_engine,
|
|
mlflow_repository_stub,
|
|
model_analysis_stub,
|
|
):
|
|
"""
|
|
Scenario D.4.3: With ``chunk_period='s'`` the persisted ``timestamp``
|
|
column must preserve second-level precision instead of being flattened to
|
|
the start of the minute, and ``chunk_period`` must be propagated to the
|
|
analyzer so it actually chunks at second granularity.
|
|
"""
|
|
client = temporal_test_env.client
|
|
model_id = 443
|
|
|
|
base = datetime.now(timezone.utc).replace(second=0, microsecond=0) - timedelta(minutes=2)
|
|
target_timestamps = [
|
|
base.strftime('%Y-%m-%d %H:%M:%S%z'),
|
|
(base + timedelta(seconds=30)).strftime('%Y-%m-%d %H:%M:%S%z'),
|
|
(base + timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S%z'),
|
|
]
|
|
chunk_timestamps_full = [
|
|
base.strftime('%Y-%m-%d %H:%M:%S'),
|
|
(base + timedelta(seconds=30)).strftime('%Y-%m-%d %H:%M:%S'),
|
|
(base + timedelta(minutes=1)).strftime('%Y-%m-%d %H:%M:%S'),
|
|
]
|
|
|
|
insert_target_data_for_drift(
|
|
postgres_engine,
|
|
model_id=model_id,
|
|
timestamps=target_timestamps,
|
|
variables_values={
|
|
'sensor_1': [1.0, 2.0, 3.0],
|
|
'sensor_2': [10.0, 20.0, 30.0],
|
|
},
|
|
)
|
|
_force_reference_unavailable(mlflow_repository_stub)
|
|
|
|
model_analysis_stub.set_drift_dataframe(
|
|
lambda univariate, multivariate: build_drift_dataframe(
|
|
timestamps=chunk_timestamps_full,
|
|
features=['sensor_1'],
|
|
methods=['kolmogorov_smirnov'],
|
|
include_multivariate=False,
|
|
)
|
|
)
|
|
|
|
input_data = _drift_input(model_id, chunk_period='s')
|
|
await start_and_await_workflow(
|
|
client,
|
|
Drift.run,
|
|
input_data,
|
|
make_workflow_id('test-drift-chunk-seconds'),
|
|
)
|
|
|
|
with postgres_engine.connect() as conn:
|
|
rows = (
|
|
conn.execute(
|
|
text(
|
|
'SELECT chunk, timestamp FROM predictions_schema.drift '
|
|
'WHERE model_id = :m ORDER BY chunk'
|
|
),
|
|
{'m': model_id},
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
|
|
assert [r['chunk'] for r in rows] == [0, 1, 2]
|
|
seconds_present = {r['timestamp'].second for r in rows}
|
|
assert seconds_present == {0, 30}, (
|
|
f'expected seconds 0 and 30 to be preserved, got {seconds_present}'
|
|
)
|
|
|
|
instance = model_analysis_stub.last_instance
|
|
assert instance is not None
|
|
assert instance.detect_univariate_drift_calls[0]['chunk_period'] == 's'
|