Code import - branch release/SIENTIAPDE-1645
This commit is contained in:
0
tests/sientia/__init__.py
Normal file
0
tests/sientia/__init__.py
Normal file
9
tests/sientia/test_exceptions.py
Normal file
9
tests/sientia/test_exceptions.py
Normal file
@@ -0,0 +1,9 @@
|
||||
"""Unit tests for custom exception aliases."""
|
||||
|
||||
from mlflow.exceptions import MlflowException
|
||||
|
||||
from model_manager.sientia.exceptions import SientiaMlException
|
||||
|
||||
|
||||
def test_sientia_ml_exception_is_mlflow_exception_alias():
|
||||
assert SientiaMlException is MlflowException
|
||||
481
tests/sientia/test_metrics.py
Normal file
481
tests/sientia/test_metrics.py
Normal file
@@ -0,0 +1,481 @@
|
||||
"""Unit tests for sientia metrics module."""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from model_manager.sientia.metrics import (
|
||||
mae,
|
||||
mse,
|
||||
r2,
|
||||
rce_drift,
|
||||
rce_test,
|
||||
rce_train,
|
||||
silverman_radius,
|
||||
)
|
||||
|
||||
|
||||
def test_mse_perfect_predictions():
|
||||
"""Test MSE with perfect predictions returns 0.0."""
|
||||
real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
predictions = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
|
||||
result = mse(real_data, predictions)
|
||||
|
||||
assert result == 0.0
|
||||
|
||||
|
||||
def test_mse_with_errors():
|
||||
"""Test MSE calculation with prediction errors."""
|
||||
real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
predictions = pd.Series([1.5, 2.5, 3.5, 4.5, 5.5])
|
||||
|
||||
result = mse(real_data, predictions)
|
||||
|
||||
# MSE = mean((0.5^2, 0.5^2, 0.5^2, 0.5^2, 0.5^2)) = 0.25
|
||||
assert result == 0.25
|
||||
|
||||
|
||||
def test_mse_with_integer_input():
|
||||
"""Test MSE handles integer input and converts to float64."""
|
||||
real_data = pd.Series([1, 2, 3, 4, 5])
|
||||
predictions = pd.Series([2, 3, 4, 5, 6])
|
||||
|
||||
result = mse(real_data, predictions)
|
||||
|
||||
# MSE = mean((1^2, 1^2, 1^2, 1^2, 1^2)) = 1.0
|
||||
assert result == 1.0
|
||||
|
||||
|
||||
def test_mse_with_large_errors():
|
||||
"""Test MSE with large prediction errors."""
|
||||
real_data = pd.Series([10.0, 20.0, 30.0])
|
||||
predictions = pd.Series([5.0, 15.0, 25.0])
|
||||
|
||||
result = mse(real_data, predictions)
|
||||
|
||||
# MSE = mean((25, 25, 25)) = 25.0
|
||||
assert result == 25.0
|
||||
|
||||
|
||||
def test_mse_rounds_to_two_decimals():
|
||||
"""Test MSE rounds result to 2 decimal places."""
|
||||
real_data = pd.Series([1.111, 2.222, 3.333])
|
||||
predictions = pd.Series([1.222, 2.333, 3.444])
|
||||
|
||||
result = mse(real_data, predictions)
|
||||
|
||||
# Result should be rounded to 2 decimals
|
||||
assert isinstance(result, float)
|
||||
assert len(str(result).split('.')[-1]) <= 2
|
||||
|
||||
|
||||
def test_mae_perfect_predictions():
|
||||
"""Test MAE with perfect predictions returns 0.0."""
|
||||
real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
predictions = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
|
||||
result = mae(real_data, predictions)
|
||||
|
||||
assert result == 0.0
|
||||
|
||||
|
||||
def test_mae_with_errors():
|
||||
"""Test MAE calculation with prediction errors."""
|
||||
real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
predictions = pd.Series([1.5, 2.5, 3.5, 4.5, 5.5])
|
||||
|
||||
result = mae(real_data, predictions)
|
||||
|
||||
# MAE = mean(|0.5|, |0.5|, |0.5|, |0.5|, |0.5|) = 0.5
|
||||
assert result == 0.5
|
||||
|
||||
|
||||
def test_mae_with_integer_input():
|
||||
"""Test MAE handles integer input and converts to float64."""
|
||||
real_data = pd.Series([1, 2, 3, 4, 5])
|
||||
predictions = pd.Series([2, 3, 4, 5, 6])
|
||||
|
||||
result = mae(real_data, predictions)
|
||||
|
||||
# MAE = mean(|1|, |1|, |1|, |1|, |1|) = 1.0
|
||||
assert result == 1.0
|
||||
|
||||
|
||||
def test_mae_with_negative_errors():
|
||||
"""Test MAE with negative prediction errors (absolute value)."""
|
||||
real_data = pd.Series([10.0, 20.0, 30.0])
|
||||
predictions = pd.Series([15.0, 25.0, 35.0])
|
||||
|
||||
result = mae(real_data, predictions)
|
||||
|
||||
# MAE = mean(|5|, |5|, |5|) = 5.0
|
||||
assert result == 5.0
|
||||
|
||||
|
||||
def test_mae_rounds_to_two_decimals():
|
||||
"""Test MAE rounds result to 2 decimal places."""
|
||||
real_data = pd.Series([1.111, 2.222, 3.333])
|
||||
predictions = pd.Series([1.222, 2.333, 3.444])
|
||||
|
||||
result = mae(real_data, predictions)
|
||||
|
||||
# Result should be rounded to 2 decimals
|
||||
assert isinstance(result, float)
|
||||
assert len(str(result).split('.')[-1]) <= 2
|
||||
|
||||
|
||||
def test_r2_perfect_predictions():
|
||||
"""Test R2 with perfect predictions returns 1.0."""
|
||||
real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
predictions = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
|
||||
result = r2(real_data, predictions)
|
||||
|
||||
assert result == 1.0
|
||||
|
||||
|
||||
def test_r2_with_good_predictions():
|
||||
"""Test R2 calculation with good predictions."""
|
||||
real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
predictions = pd.Series([1.1, 2.1, 2.9, 4.1, 4.9])
|
||||
|
||||
result = r2(real_data, predictions)
|
||||
|
||||
# R2 should be close to 1.0 for good predictions
|
||||
assert result > 0.9
|
||||
assert result <= 1.0
|
||||
|
||||
|
||||
def test_r2_with_integer_input():
|
||||
"""Test R2 handles integer input and converts to float64."""
|
||||
real_data = pd.Series([1, 2, 3, 4, 5])
|
||||
predictions = pd.Series([1, 2, 3, 4, 5])
|
||||
|
||||
result = r2(real_data, predictions)
|
||||
|
||||
assert result == 1.0
|
||||
|
||||
|
||||
def test_r2_with_poor_predictions():
|
||||
"""Test R2 with poor predictions returns low score."""
|
||||
real_data = pd.Series([1.0, 2.0, 3.0, 4.0, 5.0])
|
||||
predictions = pd.Series([5.0, 4.0, 3.0, 2.0, 1.0])
|
||||
|
||||
result = r2(real_data, predictions)
|
||||
|
||||
# R2 should be negative for predictions worse than mean
|
||||
assert result < 0
|
||||
|
||||
|
||||
def test_r2_rounds_to_two_decimals():
|
||||
"""Test R2 rounds result to 2 decimal places."""
|
||||
real_data = pd.Series([1.111, 2.222, 3.333, 4.444, 5.555])
|
||||
predictions = pd.Series([1.222, 2.333, 3.444, 4.555, 5.666])
|
||||
|
||||
result = r2(real_data, predictions)
|
||||
|
||||
# Result should be rounded to 2 decimals
|
||||
assert isinstance(result, float)
|
||||
assert len(str(result).split('.')[-1]) <= 2
|
||||
|
||||
|
||||
def test_mse_with_mixed_positive_negative():
|
||||
"""Test MSE with mixed positive and negative values."""
|
||||
real_data = pd.Series([-5.0, -2.0, 0.0, 3.0, 7.0])
|
||||
predictions = pd.Series([-4.0, -1.0, 1.0, 4.0, 8.0])
|
||||
|
||||
result = mse(real_data, predictions)
|
||||
|
||||
# MSE = mean((1^2, 1^2, 1^2, 1^2, 1^2)) = 1.0
|
||||
assert result == 1.0
|
||||
|
||||
|
||||
def test_mae_with_mixed_positive_negative():
|
||||
"""Test MAE with mixed positive and negative values."""
|
||||
real_data = pd.Series([-5.0, -2.0, 0.0, 3.0, 7.0])
|
||||
predictions = pd.Series([-4.0, -1.0, 1.0, 4.0, 8.0])
|
||||
|
||||
result = mae(real_data, predictions)
|
||||
|
||||
# MAE = mean(|1|, |1|, |1|, |1|, |1|) = 1.0
|
||||
assert result == 1.0
|
||||
|
||||
|
||||
def test_r2_with_mixed_positive_negative():
|
||||
"""Test R2 with mixed positive and negative values."""
|
||||
real_data = pd.Series([-5.0, -2.0, 0.0, 3.0, 7.0])
|
||||
predictions = pd.Series([-5.0, -2.0, 0.0, 3.0, 7.0])
|
||||
|
||||
result = r2(real_data, predictions)
|
||||
|
||||
assert result == 1.0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests for silverman_radius
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def test_silverman_radius_basic():
|
||||
"""Test silverman_radius returns a positive float."""
|
||||
data = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0])
|
||||
|
||||
result = silverman_radius(data)
|
||||
|
||||
assert isinstance(result, float)
|
||||
assert result > 0
|
||||
|
||||
|
||||
def test_silverman_radius_uniform_data():
|
||||
"""Test silverman_radius with uniformly distributed data."""
|
||||
data = np.linspace(0, 100, 50)
|
||||
|
||||
result = silverman_radius(data)
|
||||
|
||||
assert result > 0
|
||||
assert np.isfinite(result)
|
||||
|
||||
|
||||
def test_silverman_radius_normal_distribution():
|
||||
"""Test silverman_radius with normally distributed data."""
|
||||
np.random.seed(42)
|
||||
data = np.random.normal(loc=50, scale=10, size=100)
|
||||
|
||||
result = silverman_radius(data)
|
||||
|
||||
assert result > 0
|
||||
assert np.isfinite(result)
|
||||
|
||||
|
||||
def test_silverman_radius_small_dataset():
|
||||
"""Test silverman_radius with small dataset."""
|
||||
data = np.array([1.0, 2.0, 3.0])
|
||||
|
||||
result = silverman_radius(data)
|
||||
|
||||
assert result > 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests for rce_train
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def test_rce_train_returns_dataframe():
|
||||
"""Test rce_train returns a DataFrame."""
|
||||
training_set = pd.DataFrame({'a': [1.0, 2.0, 3.0, 4.0, 5.0], 'b': [2.0, 3.0, 4.0, 5.0, 6.0]})
|
||||
|
||||
result = rce_train(training_set, 0.1)
|
||||
|
||||
assert isinstance(result, pd.DataFrame)
|
||||
|
||||
|
||||
def test_rce_train_includes_first_vector():
|
||||
"""Test rce_train always includes the first vector as a prototype."""
|
||||
training_set = pd.DataFrame({'a': [1.0, 2.0, 3.0], 'b': [1.0, 2.0, 3.0]})
|
||||
|
||||
result = rce_train(training_set, 0.1)
|
||||
|
||||
assert len(result) >= 1
|
||||
assert result.iloc[0].tolist() == [1.0, 1.0]
|
||||
|
||||
|
||||
def test_rce_train_with_identical_vectors():
|
||||
"""Test rce_train with identical vectors returns single prototype."""
|
||||
training_set = pd.DataFrame({'a': [1.0, 1.0, 1.0], 'b': [2.0, 2.0, 2.0]})
|
||||
|
||||
result = rce_train(training_set, 0.1)
|
||||
|
||||
# All vectors are identical, so only one prototype should be created
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
def test_rce_train_with_distant_vectors():
|
||||
"""Test rce_train with very distant vectors creates multiple prototypes."""
|
||||
training_set = pd.DataFrame({'a': [0.0, 100.0, 200.0], 'b': [0.0, 100.0, 200.0]})
|
||||
|
||||
result = rce_train(training_set, 0.1)
|
||||
|
||||
# Distant vectors should create multiple prototypes
|
||||
assert len(result) >= 1
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests for rce_test
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def test_rce_test_returns_series():
|
||||
"""Test rce_test returns a pandas Series."""
|
||||
test_set = pd.DataFrame({'a': [1.5, 2.5], 'b': [1.5, 2.5]})
|
||||
prototypes = pd.DataFrame({'a': [1.0, 3.0], 'b': [1.0, 3.0]})
|
||||
|
||||
result = rce_test(test_set, prototypes)
|
||||
|
||||
assert isinstance(result, pd.Series)
|
||||
assert len(result) == len(test_set)
|
||||
|
||||
|
||||
def test_rce_test_with_exact_match():
|
||||
"""Test rce_test with test vector matching a prototype."""
|
||||
test_set = pd.DataFrame({'a': [1.0], 'b': [2.0]})
|
||||
prototypes = pd.DataFrame({'a': [1.0], 'b': [2.0]})
|
||||
|
||||
result = rce_test(test_set, prototypes)
|
||||
|
||||
# Distance should be 0 for exact match
|
||||
assert result.iloc[0] == 0.0
|
||||
|
||||
|
||||
def test_rce_test_multiple_prototypes():
|
||||
"""Test rce_test finds closest prototype."""
|
||||
test_set = pd.DataFrame({'a': [1.1], 'b': [1.1]})
|
||||
prototypes = pd.DataFrame({'a': [1.0, 10.0], 'b': [1.0, 10.0]})
|
||||
|
||||
result = rce_test(test_set, prototypes)
|
||||
|
||||
# Should find the closest prototype (1.0, 1.0)
|
||||
assert len(result) == 1
|
||||
assert np.isfinite(result.iloc[0])
|
||||
|
||||
|
||||
def test_rce_test_signed_distances():
|
||||
"""Test rce_test returns signed distances."""
|
||||
test_set = pd.DataFrame({'a': [0.0, 5.0], 'b': [0.0, 5.0]})
|
||||
prototypes = pd.DataFrame({'a': [2.0], 'b': [2.0]})
|
||||
|
||||
result = rce_test(test_set, prototypes)
|
||||
|
||||
assert len(result) == 2
|
||||
# First test vector (0,0) is less than prototype (2,2) - should be negative
|
||||
# Second test vector (5,5) is greater than prototype (2,2) - should be positive
|
||||
assert result.iloc[0] < 0
|
||||
assert result.iloc[1] > 0
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Tests for rce_drift
|
||||
# ============================================================================
|
||||
|
||||
|
||||
def test_rce_drift_returns_series():
|
||||
"""Test rce_drift returns a pandas Series."""
|
||||
reference_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [1.0, 2.0, 3.0, 4.0, 5.0],
|
||||
'feature2': [2.0, 3.0, 4.0, 5.0, 6.0],
|
||||
'target': [10.0, 20.0, 30.0, 40.0, 50.0],
|
||||
'prediction': [11.0, 21.0, 31.0, 41.0, 51.0],
|
||||
}
|
||||
)
|
||||
real_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [1.5, 2.5],
|
||||
'feature2': [2.5, 3.5],
|
||||
'target': [15.0, 25.0],
|
||||
'prediction': [16.0, 26.0],
|
||||
}
|
||||
)
|
||||
|
||||
result = rce_drift(reference_data, real_data, 'target')
|
||||
|
||||
assert isinstance(result, pd.Series)
|
||||
assert len(result) == len(real_data)
|
||||
|
||||
|
||||
def test_rce_drift_with_target_column():
|
||||
"""Test rce_drift using target column (drops prediction)."""
|
||||
reference_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [1.0, 2.0, 3.0],
|
||||
'target': [10.0, 20.0, 30.0],
|
||||
'prediction': [11.0, 21.0, 31.0],
|
||||
}
|
||||
)
|
||||
real_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [1.5],
|
||||
'target': [15.0],
|
||||
'prediction': [16.0],
|
||||
}
|
||||
)
|
||||
|
||||
result = rce_drift(reference_data, real_data, 'target')
|
||||
|
||||
assert isinstance(result, pd.Series)
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
def test_rce_drift_with_prediction_column():
|
||||
"""Test rce_drift using prediction column (drops target)."""
|
||||
reference_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [1.0, 2.0, 3.0],
|
||||
'target': [10.0, 20.0, 30.0],
|
||||
'prediction': [11.0, 21.0, 31.0],
|
||||
}
|
||||
)
|
||||
real_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [1.5],
|
||||
'target': [15.0],
|
||||
'prediction': [16.0],
|
||||
}
|
||||
)
|
||||
|
||||
result = rce_drift(reference_data, real_data, 'prediction')
|
||||
|
||||
assert isinstance(result, pd.Series)
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
def test_rce_drift_normalized_output():
|
||||
"""Test rce_drift returns normalized distances."""
|
||||
reference_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [1.0, 2.0, 3.0, 4.0, 5.0],
|
||||
'target': [10.0, 20.0, 30.0, 40.0, 50.0],
|
||||
'prediction': [10.0, 20.0, 30.0, 40.0, 50.0],
|
||||
}
|
||||
)
|
||||
real_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [2.5, 3.5],
|
||||
'target': [25.0, 35.0],
|
||||
'prediction': [25.0, 35.0],
|
||||
}
|
||||
)
|
||||
|
||||
result = rce_drift(reference_data, real_data, 'target')
|
||||
|
||||
# Result should be a Series with same length as real_data
|
||||
assert isinstance(result, pd.Series)
|
||||
assert len(result) == len(real_data)
|
||||
|
||||
|
||||
def test_rce_drift_handles_common_columns():
|
||||
"""Test rce_drift correctly handles common columns between datasets."""
|
||||
reference_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [1.0, 2.0, 3.0],
|
||||
'feature2': [2.0, 3.0, 4.0],
|
||||
'extra_ref': [100.0, 200.0, 300.0],
|
||||
'target': [10.0, 20.0, 30.0],
|
||||
'prediction': [11.0, 21.0, 31.0],
|
||||
}
|
||||
)
|
||||
real_data = pd.DataFrame(
|
||||
{
|
||||
'feature1': [1.5],
|
||||
'feature2': [2.5],
|
||||
'extra_real': [150.0],
|
||||
'target': [15.0],
|
||||
'prediction': [16.0],
|
||||
}
|
||||
)
|
||||
|
||||
result = rce_drift(reference_data, real_data, 'target')
|
||||
|
||||
# Should work with only common columns
|
||||
assert isinstance(result, pd.Series)
|
||||
assert len(result) == 1
|
||||
437
tests/sientia/test_reports.py
Normal file
437
tests/sientia/test_reports.py
Normal file
@@ -0,0 +1,437 @@
|
||||
import os
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
try:
|
||||
from model_manager.sientia import reports
|
||||
except ImportError as exc:
|
||||
pytest.skip(
|
||||
f'reports requires Evidently API matching production pin: {exc}',
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub_color_options(monkeypatch):
|
||||
def fake_color_options(**kwargs):
|
||||
return dict(kwargs)
|
||||
|
||||
monkeypatch.setattr(reports, 'ColorOptions', fake_color_options)
|
||||
|
||||
|
||||
def test_load_html_from_file_success(tmp_path):
|
||||
sample_file = tmp_path / 'sample.html'
|
||||
sample_file.write_text('<p>Hello</p>', encoding='utf-8')
|
||||
|
||||
content = reports.load_html_from_file(str(sample_file))
|
||||
|
||||
assert content == '<p>Hello</p>'
|
||||
|
||||
|
||||
def test_load_html_from_file_missing_file():
|
||||
with pytest.raises(FileNotFoundError):
|
||||
reports.load_html_from_file('non-existent.html')
|
||||
|
||||
|
||||
def test_load_html_from_file_os_error(monkeypatch):
|
||||
def fake_open(*_args, **_kwargs):
|
||||
raise OSError('boom')
|
||||
|
||||
monkeypatch.setattr('builtins.open', fake_open)
|
||||
|
||||
with pytest.raises(OSError, match='boom'):
|
||||
reports.load_html_from_file('path.html')
|
||||
|
||||
|
||||
def test_inject_content_replaces_section():
|
||||
main_html = "<html><body><div id='target'>old</div></body></html>"
|
||||
content = '<span>new</span>'
|
||||
|
||||
result = reports.inject_content(main_html, 'target', content)
|
||||
|
||||
soup = BeautifulSoup(result, 'html.parser')
|
||||
section = soup.find(id='target')
|
||||
assert section is not None
|
||||
assert section.find('span').text == 'new'
|
||||
|
||||
|
||||
def test_inject_content_missing_section():
|
||||
main_html = "<html><body><div id='other'>keep</div></body></html>"
|
||||
|
||||
result = reports.inject_content(main_html, 'missing', '<p>ignored</p>')
|
||||
|
||||
# Content should be unchanged when section is missing
|
||||
soup = BeautifulSoup(result, 'html.parser')
|
||||
assert soup.find(id='other') is not None
|
||||
assert soup.find(id='other').text == 'keep'
|
||||
|
||||
|
||||
def test_reports_init_sets_defaults(stub_color_options):
|
||||
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
||||
|
||||
assert report.metrics == []
|
||||
assert isinstance(report.options, list) and len(report.options) == 1
|
||||
assert report.sections == {}
|
||||
assert report.base_path is None
|
||||
|
||||
|
||||
def test_add_data_quality_section_without_run(monkeypatch, stub_color_options):
|
||||
monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: 'summary')
|
||||
monkeypatch.setattr(
|
||||
reports,
|
||||
'generate_column_metrics',
|
||||
lambda *args, **kwargs: ('columns', kwargs),
|
||||
)
|
||||
monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: 'conflict')
|
||||
monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: 'correlations')
|
||||
|
||||
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
||||
report.add_data_quality_section(columns=['col'], run=False)
|
||||
|
||||
assert report.metrics[-4:] == [
|
||||
'summary',
|
||||
('columns', {'columns': ['col'], 'skip_id_column': True}),
|
||||
'conflict',
|
||||
'correlations',
|
||||
]
|
||||
assert 'data_quality' not in report.sections
|
||||
|
||||
|
||||
def test_add_data_quality_section_with_run(monkeypatch, tmp_path, stub_color_options):
|
||||
summary = object()
|
||||
column_metrics = object()
|
||||
conflict = object()
|
||||
correlations = object()
|
||||
monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: summary)
|
||||
|
||||
def fake_generate_column_metrics(*_args, **kwargs):
|
||||
return column_metrics
|
||||
|
||||
monkeypatch.setattr(reports, 'generate_column_metrics', fake_generate_column_metrics)
|
||||
monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: conflict)
|
||||
monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: correlations)
|
||||
|
||||
report_instance = MagicMock()
|
||||
report_instance.as_dict.return_value = {'result': 'data_quality'}
|
||||
ReportMock = MagicMock(return_value=report_instance)
|
||||
monkeypatch.setattr(reports, 'Report', ReportMock)
|
||||
|
||||
report = reports.Reports(
|
||||
reference_data='ref', current_data='cur', target_name='target', base_path=str(tmp_path)
|
||||
)
|
||||
report.add_data_quality_section(columns=['c1'], run=True)
|
||||
|
||||
assert report.metrics[-4:] == [summary, column_metrics, conflict, correlations]
|
||||
assert report.sections['data_quality'] == {'result': 'data_quality'}
|
||||
ReportMock.assert_called_once_with(
|
||||
metrics=[summary, column_metrics, conflict, correlations], options=report.options
|
||||
)
|
||||
run_kwargs = report_instance.run.call_args.kwargs
|
||||
assert run_kwargs['reference_data'] == 'ref'
|
||||
assert run_kwargs['current_data'] == 'cur'
|
||||
assert run_kwargs['column_mapping'].target == 'target'
|
||||
report_instance.save_html.assert_called_once_with(
|
||||
os.path.join(str(tmp_path), 'data_quality.html')
|
||||
)
|
||||
|
||||
|
||||
def test_add_data_quality_section_run_without_base_path(monkeypatch, stub_color_options):
|
||||
summary = object()
|
||||
column_metrics = object()
|
||||
conflict = object()
|
||||
correlations = object()
|
||||
monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: summary)
|
||||
monkeypatch.setattr(
|
||||
reports,
|
||||
'generate_column_metrics',
|
||||
lambda *args, **kwargs: column_metrics,
|
||||
)
|
||||
monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: conflict)
|
||||
monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: correlations)
|
||||
|
||||
report_instance = MagicMock()
|
||||
report_instance.as_dict.return_value = {'result': 'quality'}
|
||||
ReportMock = MagicMock(return_value=report_instance)
|
||||
monkeypatch.setattr(reports, 'Report', ReportMock)
|
||||
|
||||
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
||||
report.add_data_quality_section(run=True)
|
||||
|
||||
assert report.sections['data_quality'] == {'result': 'quality'}
|
||||
report_instance.save_html.assert_not_called()
|
||||
|
||||
|
||||
def test_add_data_quality_section_non_default_target_keeps_conflict_metric(
|
||||
monkeypatch, stub_color_options
|
||||
):
|
||||
summary = object()
|
||||
column_metrics = object()
|
||||
conflict = object()
|
||||
correlations = object()
|
||||
|
||||
monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: summary)
|
||||
monkeypatch.setattr(
|
||||
reports,
|
||||
'generate_column_metrics',
|
||||
lambda *args, **kwargs: column_metrics,
|
||||
)
|
||||
monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: conflict)
|
||||
monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: correlations)
|
||||
|
||||
report = reports.Reports(reference_data='ref', current_data='cur', target_name='sales')
|
||||
report.add_data_quality_section(columns=['c1'], run=False)
|
||||
|
||||
assert report.metrics[-4:] == [
|
||||
summary,
|
||||
column_metrics,
|
||||
conflict,
|
||||
correlations,
|
||||
]
|
||||
|
||||
|
||||
def test_add_data_drift_section_paths(monkeypatch, tmp_path, stub_color_options):
|
||||
drift_instances = [object(), object(), object()]
|
||||
DataDriftPresetMock = MagicMock(side_effect=drift_instances)
|
||||
monkeypatch.setattr(reports, 'DataDriftPreset', DataDriftPresetMock)
|
||||
|
||||
report_instance = MagicMock()
|
||||
report_instance.as_dict.return_value = {'result': 'data_drift'}
|
||||
ReportMock = MagicMock(return_value=report_instance)
|
||||
monkeypatch.setattr(reports, 'Report', ReportMock)
|
||||
|
||||
report = reports.Reports(
|
||||
reference_data='ref', current_data='cur', target_name='target', base_path=str(tmp_path)
|
||||
)
|
||||
report.add_data_drift_section(columns=['c1'], run=False)
|
||||
assert report.metrics[-1] == drift_instances[0]
|
||||
assert 'data_drift' not in report.sections
|
||||
|
||||
report.add_data_drift_section(columns=['c1'], run=True)
|
||||
assert report.sections['data_drift'] == {'result': 'data_drift'}
|
||||
ReportMock.assert_called_with(metrics=[drift_instances[2]], options=report.options)
|
||||
run_kwargs = report_instance.run.call_args.kwargs
|
||||
assert run_kwargs['reference_data'] == 'ref'
|
||||
assert run_kwargs['current_data'] == 'cur'
|
||||
assert run_kwargs['column_mapping'].target == 'target'
|
||||
report_instance.save_html.assert_called_with(os.path.join(str(tmp_path), 'data_drift.html'))
|
||||
|
||||
|
||||
def test_add_data_drift_section_run_without_base_path(monkeypatch, stub_color_options):
|
||||
drift_instances = [object(), object(), object()]
|
||||
DataDriftPresetMock = MagicMock(side_effect=drift_instances)
|
||||
monkeypatch.setattr(reports, 'DataDriftPreset', DataDriftPresetMock)
|
||||
|
||||
report_instance = MagicMock()
|
||||
report_instance.as_dict.return_value = {'result': 'drift'}
|
||||
ReportMock = MagicMock(return_value=report_instance)
|
||||
monkeypatch.setattr(reports, 'Report', ReportMock)
|
||||
|
||||
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
||||
report.add_data_drift_section(run=True)
|
||||
|
||||
assert report.sections['data_drift'] == {'result': 'drift'}
|
||||
report_instance.save_html.assert_not_called()
|
||||
|
||||
|
||||
def test_add_regression_section(monkeypatch, tmp_path, stub_color_options):
|
||||
regression_metrics = [object() for _ in range(7)]
|
||||
monkeypatch.setattr(reports, 'RegressionPerformanceMetrics', lambda: regression_metrics[0])
|
||||
monkeypatch.setattr(reports, 'RegressionDummyMetric', lambda: regression_metrics[1])
|
||||
monkeypatch.setattr(
|
||||
reports, 'RegressionPredictedVsActualScatter', lambda: regression_metrics[2]
|
||||
)
|
||||
monkeypatch.setattr(reports, 'RegressionPredictedVsActualPlot', lambda: regression_metrics[3])
|
||||
monkeypatch.setattr(reports, 'RegressionErrorPlot', lambda: regression_metrics[4])
|
||||
monkeypatch.setattr(reports, 'RegressionAbsPercentageErrorPlot', lambda: regression_metrics[5])
|
||||
monkeypatch.setattr(reports, 'RegressionErrorDistribution', lambda: regression_metrics[6])
|
||||
|
||||
report_instance = MagicMock()
|
||||
report_instance.as_dict.return_value = {'result': 'regression'}
|
||||
ReportMock = MagicMock(return_value=report_instance)
|
||||
monkeypatch.setattr(reports, 'Report', ReportMock)
|
||||
|
||||
report = reports.Reports(
|
||||
reference_data='ref', current_data='cur', target_name='target', base_path=str(tmp_path)
|
||||
)
|
||||
|
||||
report.add_regression_section(run=False)
|
||||
assert report.metrics[-7:] == regression_metrics
|
||||
assert 'regression' not in report.sections
|
||||
|
||||
report.add_regression_section(run=True)
|
||||
assert report.sections['regression'] == {'result': 'regression'}
|
||||
ReportMock.assert_called_with(metrics=regression_metrics, options=report.options)
|
||||
report_instance.run.assert_called_with(
|
||||
reference_data='ref',
|
||||
current_data='cur',
|
||||
column_mapping=report_instance.run.call_args.kwargs['column_mapping'],
|
||||
)
|
||||
report_instance.save_html.assert_called_with(os.path.join(str(tmp_path), 'regression.html'))
|
||||
|
||||
|
||||
def test_add_regression_section_run_without_base_path(monkeypatch, stub_color_options):
|
||||
regression_metrics = [object() for _ in range(7)]
|
||||
monkeypatch.setattr(reports, 'RegressionPerformanceMetrics', lambda: regression_metrics[0])
|
||||
monkeypatch.setattr(reports, 'RegressionDummyMetric', lambda: regression_metrics[1])
|
||||
monkeypatch.setattr(
|
||||
reports, 'RegressionPredictedVsActualScatter', lambda: regression_metrics[2]
|
||||
)
|
||||
monkeypatch.setattr(reports, 'RegressionPredictedVsActualPlot', lambda: regression_metrics[3])
|
||||
monkeypatch.setattr(reports, 'RegressionErrorPlot', lambda: regression_metrics[4])
|
||||
monkeypatch.setattr(reports, 'RegressionAbsPercentageErrorPlot', lambda: regression_metrics[5])
|
||||
monkeypatch.setattr(reports, 'RegressionErrorDistribution', lambda: regression_metrics[6])
|
||||
|
||||
report_instance = MagicMock()
|
||||
report_instance.as_dict.return_value = {'result': 'reg'}
|
||||
ReportMock = MagicMock(return_value=report_instance)
|
||||
monkeypatch.setattr(reports, 'Report', ReportMock)
|
||||
|
||||
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
||||
report.add_regression_section(run=True)
|
||||
|
||||
assert report.sections['regression'] == {'result': 'reg'}
|
||||
report_instance.save_html.assert_not_called()
|
||||
|
||||
|
||||
def test_set_color_options_appends(monkeypatch):
|
||||
calls = []
|
||||
|
||||
def color_options_mock(**kwargs):
|
||||
calls.append(kwargs)
|
||||
return kwargs
|
||||
|
||||
monkeypatch.setattr(reports, 'ColorOptions', color_options_mock)
|
||||
|
||||
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
||||
report.set_color_options(primary_color='#111', secondary_color='#222')
|
||||
|
||||
options = report.options
|
||||
assert options is not None
|
||||
assert len(options) == 2
|
||||
assert calls[0]['primary_color'] == '#0F4C81'
|
||||
assert calls[1]['primary_color'] == '#111'
|
||||
assert options[1]['secondary_color'] == '#222'
|
||||
|
||||
|
||||
def test_save_all_sections_html_requires_base_path(stub_color_options):
|
||||
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
report.save_all_sections_html('output/report.html')
|
||||
|
||||
|
||||
def test_save_all_sections_html_requires_template_path(stub_color_options, tmp_path):
|
||||
report = reports.Reports(
|
||||
reference_data='ref',
|
||||
current_data='cur',
|
||||
target_name='target',
|
||||
base_path=str(tmp_path),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match='template_path is required'):
|
||||
report.save_all_sections_html('output/report.html')
|
||||
|
||||
|
||||
def test_save_all_sections_html_writes_output(tmp_path, stub_color_options):
|
||||
base_dir = tmp_path / 'templates'
|
||||
base_dir.mkdir()
|
||||
(base_dir / 'header.html').write_text(
|
||||
"<html><body><div id='data_drift'></div><div id='data_quality'></div><div id='regression'></div></body></html>",
|
||||
encoding='utf-8',
|
||||
)
|
||||
(base_dir / 'data_drift.html').write_text('<p>Drift</p>', encoding='utf-8')
|
||||
(base_dir / 'data_quality.html').write_text('<p>Quality</p>', encoding='utf-8')
|
||||
(base_dir / 'regression.html').write_text('<p>Regression</p>', encoding='utf-8')
|
||||
|
||||
report = reports.Reports(
|
||||
reference_data='ref',
|
||||
current_data='cur',
|
||||
target_name='target',
|
||||
base_path=str(base_dir),
|
||||
template_path=str(base_dir),
|
||||
)
|
||||
output_path = tmp_path / 'reports' / 'combined.html'
|
||||
|
||||
report.save_all_sections_html(str(output_path))
|
||||
|
||||
assert output_path.exists()
|
||||
content = output_path.read_text(encoding='utf-8')
|
||||
assert '<p>Drift</p>' in content
|
||||
assert '<p>Quality</p>' in content
|
||||
assert '<p>Regression</p>' in content
|
||||
|
||||
|
||||
def test_save_all_sections_html_creates_directory(monkeypatch, tmp_path, stub_color_options):
|
||||
base_dir = tmp_path / 'templates'
|
||||
base_dir.mkdir()
|
||||
(base_dir / 'header.html').write_text(
|
||||
"<html><body><div id='data_drift'></div><div id='data_quality'></div><div id='regression'></div></body></html>",
|
||||
encoding='utf-8',
|
||||
)
|
||||
(base_dir / 'data_drift.html').write_text('<p>Drift</p>', encoding='utf-8')
|
||||
(base_dir / 'data_quality.html').write_text('<p>Quality</p>', encoding='utf-8')
|
||||
(base_dir / 'regression.html').write_text('<p>Regression</p>', encoding='utf-8')
|
||||
|
||||
make_dirs_called = []
|
||||
report = reports.Reports(
|
||||
reference_data='ref',
|
||||
current_data='cur',
|
||||
target_name='target',
|
||||
base_path=str(base_dir),
|
||||
template_path=str(base_dir),
|
||||
)
|
||||
output_path = tmp_path / 'nested' / 'report.html'
|
||||
output_dir = str(output_path.parent)
|
||||
|
||||
original_exists = os.path.exists
|
||||
original_makedirs = os.makedirs
|
||||
|
||||
def fake_exists(path):
|
||||
if path == output_dir:
|
||||
return False
|
||||
return original_exists(path)
|
||||
|
||||
def fake_makedirs(path, exist_ok=False):
|
||||
make_dirs_called.append((path, exist_ok))
|
||||
return original_makedirs(path, exist_ok=exist_ok)
|
||||
|
||||
monkeypatch.setattr(os.path, 'exists', fake_exists)
|
||||
monkeypatch.setattr(os, 'makedirs', fake_makedirs)
|
||||
|
||||
report.save_all_sections_html(str(output_path))
|
||||
|
||||
assert make_dirs_called == [(str(output_path.parent), True)]
|
||||
|
||||
|
||||
def test_save_all_sections_html_no_directory_needed(monkeypatch, tmp_path, stub_color_options):
|
||||
base_dir = tmp_path / 'templates'
|
||||
base_dir.mkdir()
|
||||
(base_dir / 'header.html').write_text(
|
||||
"<html><body><div id='data_drift'></div><div id='data_quality'></div><div id='regression'></div></body></html>",
|
||||
encoding='utf-8',
|
||||
)
|
||||
(base_dir / 'data_drift.html').write_text('<p>Drift</p>', encoding='utf-8')
|
||||
(base_dir / 'data_quality.html').write_text('<p>Quality</p>', encoding='utf-8')
|
||||
(base_dir / 'regression.html').write_text('<p>Regression</p>', encoding='utf-8')
|
||||
|
||||
mk_calls = []
|
||||
|
||||
def fake_makedirs(path, exist_ok=False):
|
||||
mk_calls.append((path, exist_ok))
|
||||
|
||||
monkeypatch.setattr(os, 'makedirs', fake_makedirs)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
report = reports.Reports(
|
||||
reference_data='ref',
|
||||
current_data='cur',
|
||||
target_name='target',
|
||||
base_path=str(base_dir),
|
||||
template_path=str(base_dir),
|
||||
)
|
||||
report.save_all_sections_html('report.html')
|
||||
|
||||
assert mk_calls == []
|
||||
assert (tmp_path / 'report.html').exists()
|
||||
Reference in New Issue
Block a user