d1f9394879
- Added a new fixture to manage runtime report artifacts in a writable temp directory during E2E tests, addressing permission issues in local CI/dev environments. - Updated `conftest.py` to include a requirements.txt file in the model packaging path for training activities. - Refactored existing fixtures to use `pytest.fixture` instead of `pytest_asyncio.fixture` for better compatibility. - Enhanced the `Reports` class to include a target alias for report metrics, ensuring compatibility with Evidently's reporting requirements. - Introduced new test scenarios to validate the handling of missing and whitespace-only `date_column` inputs in the training workflow. These changes improve the robustness of the E2E testing framework and enhance the clarity of model reporting metrics.
438 lines
16 KiB
Python
438 lines
16 KiB
Python
import os
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
from bs4 import BeautifulSoup
|
|
|
|
try:
|
|
from model_manager.sientia import reports
|
|
except ImportError as exc:
|
|
pytest.skip(
|
|
f'reports requires Evidently API matching production pin: {exc}',
|
|
allow_module_level=True,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def stub_color_options(monkeypatch):
|
|
def fake_color_options(**kwargs):
|
|
return dict(kwargs)
|
|
|
|
monkeypatch.setattr(reports, 'ColorOptions', fake_color_options)
|
|
|
|
|
|
def test_load_html_from_file_success(tmp_path):
|
|
sample_file = tmp_path / 'sample.html'
|
|
sample_file.write_text('<p>Hello</p>', encoding='utf-8')
|
|
|
|
content = reports.load_html_from_file(str(sample_file))
|
|
|
|
assert content == '<p>Hello</p>'
|
|
|
|
|
|
def test_load_html_from_file_missing_file():
|
|
with pytest.raises(FileNotFoundError):
|
|
reports.load_html_from_file('non-existent.html')
|
|
|
|
|
|
def test_load_html_from_file_os_error(monkeypatch):
|
|
def fake_open(*_args, **_kwargs):
|
|
raise OSError('boom')
|
|
|
|
monkeypatch.setattr('builtins.open', fake_open)
|
|
|
|
with pytest.raises(OSError, match='boom'):
|
|
reports.load_html_from_file('path.html')
|
|
|
|
|
|
def test_inject_content_replaces_section():
|
|
main_html = "<html><body><div id='target'>old</div></body></html>"
|
|
content = '<span>new</span>'
|
|
|
|
result = reports.inject_content(main_html, 'target', content)
|
|
|
|
soup = BeautifulSoup(result, 'html.parser')
|
|
section = soup.find(id='target')
|
|
assert section is not None
|
|
assert section.find('span').text == 'new'
|
|
|
|
|
|
def test_inject_content_missing_section():
|
|
main_html = "<html><body><div id='other'>keep</div></body></html>"
|
|
|
|
result = reports.inject_content(main_html, 'missing', '<p>ignored</p>')
|
|
|
|
# Content should be unchanged when section is missing
|
|
soup = BeautifulSoup(result, 'html.parser')
|
|
assert soup.find(id='other') is not None
|
|
assert soup.find(id='other').text == 'keep'
|
|
|
|
|
|
def test_reports_init_sets_defaults(stub_color_options):
|
|
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
|
|
|
assert report.metrics == []
|
|
assert isinstance(report.options, list) and len(report.options) == 1
|
|
assert report.sections == {}
|
|
assert report.base_path is None
|
|
|
|
|
|
def test_add_data_quality_section_without_run(monkeypatch, stub_color_options):
|
|
monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: 'summary')
|
|
monkeypatch.setattr(
|
|
reports,
|
|
'generate_column_metrics',
|
|
lambda *args, **kwargs: ('columns', kwargs),
|
|
)
|
|
monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: 'conflict')
|
|
monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: 'correlations')
|
|
|
|
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
|
report.add_data_quality_section(columns=['col'], run=False)
|
|
|
|
assert report.metrics[-4:] == [
|
|
'summary',
|
|
('columns', {'columns': ['col'], 'skip_id_column': True}),
|
|
'conflict',
|
|
'correlations',
|
|
]
|
|
assert 'data_quality' not in report.sections
|
|
|
|
|
|
def test_add_data_quality_section_with_run(monkeypatch, tmp_path, stub_color_options):
|
|
summary = object()
|
|
column_metrics = object()
|
|
conflict = object()
|
|
correlations = object()
|
|
monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: summary)
|
|
|
|
def fake_generate_column_metrics(*_args, **kwargs):
|
|
return column_metrics
|
|
|
|
monkeypatch.setattr(reports, 'generate_column_metrics', fake_generate_column_metrics)
|
|
monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: conflict)
|
|
monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: correlations)
|
|
|
|
report_instance = MagicMock()
|
|
report_instance.as_dict.return_value = {'result': 'data_quality'}
|
|
ReportMock = MagicMock(return_value=report_instance)
|
|
monkeypatch.setattr(reports, 'Report', ReportMock)
|
|
|
|
report = reports.Reports(
|
|
reference_data='ref', current_data='cur', target_name='target', base_path=str(tmp_path)
|
|
)
|
|
report.add_data_quality_section(columns=['c1'], run=True)
|
|
|
|
assert report.metrics[-4:] == [summary, column_metrics, conflict, correlations]
|
|
assert report.sections['data_quality'] == {'result': 'data_quality'}
|
|
ReportMock.assert_called_once_with(
|
|
metrics=[summary, column_metrics, conflict, correlations], options=report.options
|
|
)
|
|
run_kwargs = report_instance.run.call_args.kwargs
|
|
assert run_kwargs['reference_data'] == 'ref'
|
|
assert run_kwargs['current_data'] == 'cur'
|
|
assert run_kwargs['column_mapping'].target == 'target'
|
|
report_instance.save_html.assert_called_once_with(
|
|
os.path.join(str(tmp_path), 'data_quality.html')
|
|
)
|
|
|
|
|
|
def test_add_data_quality_section_run_without_base_path(monkeypatch, stub_color_options):
|
|
summary = object()
|
|
column_metrics = object()
|
|
conflict = object()
|
|
correlations = object()
|
|
monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: summary)
|
|
monkeypatch.setattr(
|
|
reports,
|
|
'generate_column_metrics',
|
|
lambda *args, **kwargs: column_metrics,
|
|
)
|
|
monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: conflict)
|
|
monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: correlations)
|
|
|
|
report_instance = MagicMock()
|
|
report_instance.as_dict.return_value = {'result': 'quality'}
|
|
ReportMock = MagicMock(return_value=report_instance)
|
|
monkeypatch.setattr(reports, 'Report', ReportMock)
|
|
|
|
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
|
report.add_data_quality_section(run=True)
|
|
|
|
assert report.sections['data_quality'] == {'result': 'quality'}
|
|
report_instance.save_html.assert_not_called()
|
|
|
|
|
|
def test_add_data_quality_section_non_default_target_keeps_conflict_metric(
|
|
monkeypatch, stub_color_options
|
|
):
|
|
summary = object()
|
|
column_metrics = object()
|
|
conflict = object()
|
|
correlations = object()
|
|
|
|
monkeypatch.setattr(reports, 'DatasetSummaryMetric', lambda: summary)
|
|
monkeypatch.setattr(
|
|
reports,
|
|
'generate_column_metrics',
|
|
lambda *args, **kwargs: column_metrics,
|
|
)
|
|
monkeypatch.setattr(reports, 'ConflictTargetMetric', lambda: conflict)
|
|
monkeypatch.setattr(reports, 'DatasetCorrelationsMetric', lambda: correlations)
|
|
|
|
report = reports.Reports(reference_data='ref', current_data='cur', target_name='sales')
|
|
report.add_data_quality_section(columns=['c1'], run=False)
|
|
|
|
assert report.metrics[-4:] == [
|
|
summary,
|
|
column_metrics,
|
|
conflict,
|
|
correlations,
|
|
]
|
|
|
|
|
|
def test_add_data_drift_section_paths(monkeypatch, tmp_path, stub_color_options):
|
|
drift_instances = [object(), object(), object()]
|
|
DataDriftPresetMock = MagicMock(side_effect=drift_instances)
|
|
monkeypatch.setattr(reports, 'DataDriftPreset', DataDriftPresetMock)
|
|
|
|
report_instance = MagicMock()
|
|
report_instance.as_dict.return_value = {'result': 'data_drift'}
|
|
ReportMock = MagicMock(return_value=report_instance)
|
|
monkeypatch.setattr(reports, 'Report', ReportMock)
|
|
|
|
report = reports.Reports(
|
|
reference_data='ref', current_data='cur', target_name='target', base_path=str(tmp_path)
|
|
)
|
|
report.add_data_drift_section(columns=['c1'], run=False)
|
|
assert report.metrics[-1] == drift_instances[0]
|
|
assert 'data_drift' not in report.sections
|
|
|
|
report.add_data_drift_section(columns=['c1'], run=True)
|
|
assert report.sections['data_drift'] == {'result': 'data_drift'}
|
|
ReportMock.assert_called_with(metrics=[drift_instances[2]], options=report.options)
|
|
run_kwargs = report_instance.run.call_args.kwargs
|
|
assert run_kwargs['reference_data'] == 'ref'
|
|
assert run_kwargs['current_data'] == 'cur'
|
|
assert run_kwargs['column_mapping'].target == 'target'
|
|
report_instance.save_html.assert_called_with(os.path.join(str(tmp_path), 'data_drift.html'))
|
|
|
|
|
|
def test_add_data_drift_section_run_without_base_path(monkeypatch, stub_color_options):
|
|
drift_instances = [object(), object(), object()]
|
|
DataDriftPresetMock = MagicMock(side_effect=drift_instances)
|
|
monkeypatch.setattr(reports, 'DataDriftPreset', DataDriftPresetMock)
|
|
|
|
report_instance = MagicMock()
|
|
report_instance.as_dict.return_value = {'result': 'drift'}
|
|
ReportMock = MagicMock(return_value=report_instance)
|
|
monkeypatch.setattr(reports, 'Report', ReportMock)
|
|
|
|
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
|
report.add_data_drift_section(run=True)
|
|
|
|
assert report.sections['data_drift'] == {'result': 'drift'}
|
|
report_instance.save_html.assert_not_called()
|
|
|
|
|
|
def test_add_regression_section(monkeypatch, tmp_path, stub_color_options):
|
|
regression_metrics = [object() for _ in range(7)]
|
|
monkeypatch.setattr(reports, 'RegressionPerformanceMetrics', lambda: regression_metrics[0])
|
|
monkeypatch.setattr(reports, 'RegressionDummyMetric', lambda: regression_metrics[1])
|
|
monkeypatch.setattr(
|
|
reports, 'RegressionPredictedVsActualScatter', lambda: regression_metrics[2]
|
|
)
|
|
monkeypatch.setattr(reports, 'RegressionPredictedVsActualPlot', lambda: regression_metrics[3])
|
|
monkeypatch.setattr(reports, 'RegressionErrorPlot', lambda: regression_metrics[4])
|
|
monkeypatch.setattr(reports, 'RegressionAbsPercentageErrorPlot', lambda: regression_metrics[5])
|
|
monkeypatch.setattr(reports, 'RegressionErrorDistribution', lambda: regression_metrics[6])
|
|
|
|
report_instance = MagicMock()
|
|
report_instance.as_dict.return_value = {'result': 'regression'}
|
|
ReportMock = MagicMock(return_value=report_instance)
|
|
monkeypatch.setattr(reports, 'Report', ReportMock)
|
|
|
|
report = reports.Reports(
|
|
reference_data='ref', current_data='cur', target_name='target', base_path=str(tmp_path)
|
|
)
|
|
|
|
report.add_regression_section(run=False)
|
|
assert report.metrics[-7:] == regression_metrics
|
|
assert 'regression' not in report.sections
|
|
|
|
report.add_regression_section(run=True)
|
|
assert report.sections['regression'] == {'result': 'regression'}
|
|
ReportMock.assert_called_with(metrics=regression_metrics, options=report.options)
|
|
report_instance.run.assert_called_with(
|
|
reference_data='ref',
|
|
current_data='cur',
|
|
column_mapping=report_instance.run.call_args.kwargs['column_mapping'],
|
|
)
|
|
report_instance.save_html.assert_called_with(os.path.join(str(tmp_path), 'regression.html'))
|
|
|
|
|
|
def test_add_regression_section_run_without_base_path(monkeypatch, stub_color_options):
|
|
regression_metrics = [object() for _ in range(7)]
|
|
monkeypatch.setattr(reports, 'RegressionPerformanceMetrics', lambda: regression_metrics[0])
|
|
monkeypatch.setattr(reports, 'RegressionDummyMetric', lambda: regression_metrics[1])
|
|
monkeypatch.setattr(
|
|
reports, 'RegressionPredictedVsActualScatter', lambda: regression_metrics[2]
|
|
)
|
|
monkeypatch.setattr(reports, 'RegressionPredictedVsActualPlot', lambda: regression_metrics[3])
|
|
monkeypatch.setattr(reports, 'RegressionErrorPlot', lambda: regression_metrics[4])
|
|
monkeypatch.setattr(reports, 'RegressionAbsPercentageErrorPlot', lambda: regression_metrics[5])
|
|
monkeypatch.setattr(reports, 'RegressionErrorDistribution', lambda: regression_metrics[6])
|
|
|
|
report_instance = MagicMock()
|
|
report_instance.as_dict.return_value = {'result': 'reg'}
|
|
ReportMock = MagicMock(return_value=report_instance)
|
|
monkeypatch.setattr(reports, 'Report', ReportMock)
|
|
|
|
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
|
report.add_regression_section(run=True)
|
|
|
|
assert report.sections['regression'] == {'result': 'reg'}
|
|
report_instance.save_html.assert_not_called()
|
|
|
|
|
|
def test_set_color_options_appends(monkeypatch):
|
|
calls = []
|
|
|
|
def color_options_mock(**kwargs):
|
|
calls.append(kwargs)
|
|
return kwargs
|
|
|
|
monkeypatch.setattr(reports, 'ColorOptions', color_options_mock)
|
|
|
|
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
|
report.set_color_options(primary_color='#111', secondary_color='#222')
|
|
|
|
options = report.options
|
|
assert options is not None
|
|
assert len(options) == 2
|
|
assert calls[0]['primary_color'] == '#0F4C81'
|
|
assert calls[1]['primary_color'] == '#111'
|
|
assert options[1]['secondary_color'] == '#222'
|
|
|
|
|
|
def test_save_all_sections_html_requires_base_path(stub_color_options):
|
|
report = reports.Reports(reference_data='ref', current_data='cur', target_name='target')
|
|
|
|
with pytest.raises(ValueError):
|
|
report.save_all_sections_html('output/report.html')
|
|
|
|
|
|
def test_save_all_sections_html_requires_template_path(stub_color_options, tmp_path):
|
|
report = reports.Reports(
|
|
reference_data='ref',
|
|
current_data='cur',
|
|
target_name='target',
|
|
base_path=str(tmp_path),
|
|
)
|
|
|
|
with pytest.raises(ValueError, match='template_path is required'):
|
|
report.save_all_sections_html('output/report.html')
|
|
|
|
|
|
def test_save_all_sections_html_writes_output(tmp_path, stub_color_options):
|
|
base_dir = tmp_path / 'templates'
|
|
base_dir.mkdir()
|
|
(base_dir / 'header.html').write_text(
|
|
"<html><body><div id='data_drift'></div><div id='data_quality'></div><div id='regression'></div></body></html>",
|
|
encoding='utf-8',
|
|
)
|
|
(base_dir / 'data_drift.html').write_text('<p>Drift</p>', encoding='utf-8')
|
|
(base_dir / 'data_quality.html').write_text('<p>Quality</p>', encoding='utf-8')
|
|
(base_dir / 'regression.html').write_text('<p>Regression</p>', encoding='utf-8')
|
|
|
|
report = reports.Reports(
|
|
reference_data='ref',
|
|
current_data='cur',
|
|
target_name='target',
|
|
base_path=str(base_dir),
|
|
template_path=str(base_dir),
|
|
)
|
|
output_path = tmp_path / 'reports' / 'combined.html'
|
|
|
|
report.save_all_sections_html(str(output_path))
|
|
|
|
assert output_path.exists()
|
|
content = output_path.read_text(encoding='utf-8')
|
|
assert '<p>Drift</p>' in content
|
|
assert '<p>Quality</p>' in content
|
|
assert '<p>Regression</p>' in content
|
|
|
|
|
|
def test_save_all_sections_html_creates_directory(monkeypatch, tmp_path, stub_color_options):
|
|
base_dir = tmp_path / 'templates'
|
|
base_dir.mkdir()
|
|
(base_dir / 'header.html').write_text(
|
|
"<html><body><div id='data_drift'></div><div id='data_quality'></div><div id='regression'></div></body></html>",
|
|
encoding='utf-8',
|
|
)
|
|
(base_dir / 'data_drift.html').write_text('<p>Drift</p>', encoding='utf-8')
|
|
(base_dir / 'data_quality.html').write_text('<p>Quality</p>', encoding='utf-8')
|
|
(base_dir / 'regression.html').write_text('<p>Regression</p>', encoding='utf-8')
|
|
|
|
make_dirs_called = []
|
|
report = reports.Reports(
|
|
reference_data='ref',
|
|
current_data='cur',
|
|
target_name='target',
|
|
base_path=str(base_dir),
|
|
template_path=str(base_dir),
|
|
)
|
|
output_path = tmp_path / 'nested' / 'report.html'
|
|
output_dir = str(output_path.parent)
|
|
|
|
original_exists = os.path.exists
|
|
original_makedirs = os.makedirs
|
|
|
|
def fake_exists(path):
|
|
if path == output_dir:
|
|
return False
|
|
return original_exists(path)
|
|
|
|
def fake_makedirs(path, exist_ok=False):
|
|
make_dirs_called.append((path, exist_ok))
|
|
return original_makedirs(path, exist_ok=exist_ok)
|
|
|
|
monkeypatch.setattr(os.path, 'exists', fake_exists)
|
|
monkeypatch.setattr(os, 'makedirs', fake_makedirs)
|
|
|
|
report.save_all_sections_html(str(output_path))
|
|
|
|
assert make_dirs_called == [(str(output_path.parent), True)]
|
|
|
|
|
|
def test_save_all_sections_html_no_directory_needed(monkeypatch, tmp_path, stub_color_options):
|
|
base_dir = tmp_path / 'templates'
|
|
base_dir.mkdir()
|
|
(base_dir / 'header.html').write_text(
|
|
"<html><body><div id='data_drift'></div><div id='data_quality'></div><div id='regression'></div></body></html>",
|
|
encoding='utf-8',
|
|
)
|
|
(base_dir / 'data_drift.html').write_text('<p>Drift</p>', encoding='utf-8')
|
|
(base_dir / 'data_quality.html').write_text('<p>Quality</p>', encoding='utf-8')
|
|
(base_dir / 'regression.html').write_text('<p>Regression</p>', encoding='utf-8')
|
|
|
|
mk_calls = []
|
|
|
|
def fake_makedirs(path, exist_ok=False):
|
|
mk_calls.append((path, exist_ok))
|
|
|
|
monkeypatch.setattr(os, 'makedirs', fake_makedirs)
|
|
monkeypatch.chdir(tmp_path)
|
|
|
|
report = reports.Reports(
|
|
reference_data='ref',
|
|
current_data='cur',
|
|
target_name='target',
|
|
base_path=str(base_dir),
|
|
template_path=str(base_dir),
|
|
)
|
|
report.save_all_sections_html('report.html')
|
|
|
|
assert mk_calls == []
|
|
assert (tmp_path / 'report.html').exists()
|