SIENTIAPDE-1646
Refactor ModelMetrics to utilize DriftAnalysis for drift detection - Replaced ModelAnalysis with DriftAnalysis in the ModelMetrics class to enhance drift detection capabilities. - Updated method signatures and documentation to reflect the changes in target_name and return values. - Adjusted data handling to ensure compatibility with the new analysis methods and improved clarity in the drift metrics dataframe preparation.
This commit is contained in:
@@ -7,9 +7,14 @@ test container; we only validate that the workflow:
|
||||
- Loads training data via ``load_query_with_minio_offload``.
|
||||
- Calls ``retrain_model`` with a payload pointing at MinIO.
|
||||
- Calls ``update_production_model`` only when retrain succeeds.
|
||||
- Persists ``retrain_reports`` rows with all required columns; success rows
|
||||
carry the new ``version`` / ``mlflow_run_id`` / ``mlflow_experiment_id``
|
||||
while failure rows leave them ``NULL``.
|
||||
- Persists ``sientia_data.log_retrain`` rows with all required columns;
|
||||
success rows carry the new ``version`` / ``mlflow_run_id`` /
|
||||
``mlflow_experiment_id`` while failure rows leave them ``NULL``.
|
||||
|
||||
The production DDL drops the legacy ``id`` / ``created_at`` columns and
|
||||
moves ``mlflow_experiment_id`` to ``int8`` and ``model_id`` to ``text``.
|
||||
The stubs used here therefore emit ``experiment_id`` as an integer to fit
|
||||
the new column type.
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
@@ -30,16 +35,18 @@ from e2e.helpers import (
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.workflows.minimal_retrain import MinimalRetrain
|
||||
|
||||
# Columns defined by the production DDL for ``sientia_data.log_retrain``.
|
||||
# The legacy ``retrain_reports`` table had ``id`` and ``created_at``; the new
|
||||
# DDL drops both. ``mlflow_experiment_id`` is ``int8`` and ``model_id`` is
|
||||
# ``text``.
|
||||
EXPECTED_RETRAIN_REPORT_COLUMNS = [
|
||||
'id',
|
||||
'mlflow_experiment_id',
|
||||
'mlflow_run_id',
|
||||
'model_id',
|
||||
'model_name',
|
||||
'timestamp',
|
||||
'status',
|
||||
'timestamp',
|
||||
'version',
|
||||
'mlflow_run_id',
|
||||
'mlflow_experiment_id',
|
||||
'created_at',
|
||||
]
|
||||
|
||||
|
||||
@@ -106,7 +113,9 @@ def _configure_retrain_happy_path(mlflow_repository_stub) -> None:
|
||||
def fake_start_run(**kwargs):
|
||||
run_info = MagicMock()
|
||||
run_info.run_id = 'retrain-run-id'
|
||||
run_info.experiment_id = 'experiment-id'
|
||||
# ``mlflow_experiment_id`` is ``int8`` in the new DDL, so we feed an
|
||||
# integer-compatible id from the stubbed run info.
|
||||
run_info.experiment_id = 4242
|
||||
yield run_info
|
||||
|
||||
mlflow_repository_stub.start_run.side_effect = fake_start_run
|
||||
@@ -125,10 +134,10 @@ async def test_minimal_retrain_happy_path_writes_success_report(
|
||||
mlflow_repository_stub,
|
||||
):
|
||||
"""
|
||||
Scenario MR.1.1: Retrain succeeds. ``retrain_reports`` must contain a
|
||||
success row with version/mlflow_run_id/mlflow_experiment_id populated and
|
||||
the registry must have been told to promote the new version to the
|
||||
configured alias.
|
||||
Scenario MR.1.1: Retrain succeeds. ``sientia_data.log_retrain`` must
|
||||
contain a success row with version/mlflow_run_id/mlflow_experiment_id
|
||||
populated and the registry must have been told to promote the new version
|
||||
to the configured alias.
|
||||
"""
|
||||
client = temporal_test_env.client
|
||||
model_id = 711
|
||||
@@ -152,10 +161,10 @@ async def test_minimal_retrain_happy_path_writes_success_report(
|
||||
rows = (
|
||||
conn.execute(
|
||||
text(
|
||||
'SELECT * FROM predictions_schema.retrain_reports '
|
||||
'SELECT * FROM sientia_data.log_retrain '
|
||||
'WHERE model_id = :m'
|
||||
),
|
||||
{'m': model_id},
|
||||
{'m': str(model_id)},
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
@@ -163,17 +172,19 @@ async def test_minimal_retrain_happy_path_writes_success_report(
|
||||
|
||||
assert len(rows) == 1
|
||||
for column in EXPECTED_RETRAIN_REPORT_COLUMNS:
|
||||
assert column in rows[0], f'Missing retrain report column: {column}'
|
||||
assert column in rows[0], f'Missing log_retrain column: {column}'
|
||||
|
||||
row = rows[0]
|
||||
assert row['model_id'] == model_id
|
||||
# ``model_id`` is now ``text``; compare against the stringified id.
|
||||
assert row['model_id'] == str(model_id)
|
||||
assert row['model_name'] == 'test_model'
|
||||
assert row['status'] == 'Model retrained successfully.'
|
||||
assert row['version'] == '7'
|
||||
assert row['mlflow_run_id'] == 'retrain-run-id'
|
||||
assert row['mlflow_experiment_id'] == 'experiment-id'
|
||||
# ``mlflow_experiment_id`` is now ``int8``; assert the integer value
|
||||
# provided by the stubbed run info.
|
||||
assert row['mlflow_experiment_id'] == 4242
|
||||
assert row['timestamp'] is not None
|
||||
assert row['created_at'] is not None
|
||||
|
||||
mlflow_repository_stub.promote_to_alias.assert_called_once()
|
||||
promote_kwargs = mlflow_repository_stub.promote_to_alias.call_args.kwargs
|
||||
@@ -220,10 +231,10 @@ async def test_minimal_retrain_failure_writes_report_without_version_columns(
|
||||
rows = (
|
||||
conn.execute(
|
||||
text(
|
||||
'SELECT * FROM predictions_schema.retrain_reports '
|
||||
'SELECT * FROM sientia_data.log_retrain '
|
||||
'WHERE model_id = :m'
|
||||
),
|
||||
{'m': model_id},
|
||||
{'m': str(model_id)},
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
@@ -231,7 +242,7 @@ async def test_minimal_retrain_failure_writes_report_without_version_columns(
|
||||
|
||||
assert len(rows) == 1
|
||||
row = rows[0]
|
||||
assert row['model_id'] == model_id
|
||||
assert row['model_id'] == str(model_id)
|
||||
assert row['model_name'] == 'test_model'
|
||||
assert 'training did not converge' in row['status']
|
||||
assert row['version'] is None
|
||||
@@ -275,10 +286,10 @@ async def test_minimal_retrain_missing_target_writes_failure_report(
|
||||
row = (
|
||||
conn.execute(
|
||||
text(
|
||||
'SELECT * FROM predictions_schema.retrain_reports '
|
||||
'SELECT * FROM sientia_data.log_retrain '
|
||||
'WHERE model_id = :m'
|
||||
),
|
||||
{'m': model_id},
|
||||
{'m': str(model_id)},
|
||||
)
|
||||
.mappings()
|
||||
.first()
|
||||
@@ -313,7 +324,7 @@ async def test_minimal_retrain_no_training_data_does_not_persist_report(
|
||||
model_id = 731
|
||||
|
||||
with postgres_engine.begin() as conn:
|
||||
conn.execute(text(f'DELETE FROM predictions_schema.laborious_data WHERE model_id = {model_id}'))
|
||||
conn.execute(text(f'DELETE FROM sientia_data.laborious_data WHERE model_id = {model_id}'))
|
||||
|
||||
_configure_retrain_happy_path(mlflow_repository_stub)
|
||||
|
||||
@@ -329,7 +340,7 @@ async def test_minimal_retrain_no_training_data_does_not_persist_report(
|
||||
|
||||
with postgres_engine.connect() as conn:
|
||||
count = conn.execute(
|
||||
text('SELECT COUNT(*) FROM predictions_schema.retrain_reports WHERE model_id = :m'),
|
||||
{'m': model_id},
|
||||
text('SELECT COUNT(*) FROM sientia_data.log_retrain WHERE model_id = :m'),
|
||||
{'m': str(model_id)},
|
||||
).scalar()
|
||||
assert count == 0
|
||||
|
||||
Reference in New Issue
Block a user