Code import - branch release/SIENTIAPDE-1645
This commit is contained in:
0
model_manager/sientia/__init__.py
Normal file
0
model_manager/sientia/__init__.py
Normal file
3
model_manager/sientia/exceptions.py
Normal file
3
model_manager/sientia/exceptions.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from mlflow.exceptions import MlflowException
|
||||
|
||||
SientiaMlException = MlflowException
|
||||
148
model_manager/sientia/metrics.py
Normal file
148
model_manager/sientia/metrics.py
Normal file
@@ -0,0 +1,148 @@
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
|
||||
|
||||
|
||||
def mse(real_data: pd.Series, predictions: pd.Series) -> float:
|
||||
"""
|
||||
Calculates the mean squared error between the real data and the predictions.
|
||||
"""
|
||||
return round(
|
||||
mean_squared_error(real_data.astype(np.float64), predictions.astype(np.float64)), 2
|
||||
)
|
||||
|
||||
|
||||
def mae(real_data: pd.Series, predictions: pd.Series) -> float:
|
||||
"""
|
||||
Calculates the mean absolute error between the real data and the predictions.
|
||||
"""
|
||||
return round(
|
||||
mean_absolute_error(real_data.astype(np.float64), predictions.astype(np.float64)), 2
|
||||
)
|
||||
|
||||
|
||||
def r2(real_data: pd.Series, predictions: pd.Series) -> float:
|
||||
"""
|
||||
Calculates the R2 score between the real data and the predictions.
|
||||
"""
|
||||
return round(r2_score(real_data.astype(np.float64), predictions.astype(np.float64)), 2)
|
||||
|
||||
|
||||
def silverman_radius(data: np.ndarray) -> float:
|
||||
"""
|
||||
Calculate the Silverman bandwidth (radius) for a given dataset.
|
||||
|
||||
Args:
|
||||
data (np.ndarray): Input data (1D array)
|
||||
|
||||
Returns:
|
||||
float: Silverman bandwidth (radius)
|
||||
"""
|
||||
n = len(data)
|
||||
sigma = np.std(data)
|
||||
iqr = np.percentile(data, 75) - np.percentile(data, 25)
|
||||
radius = 0.9 * min(sigma, iqr / 1.34) * n ** (-1 / 5)
|
||||
return radius
|
||||
|
||||
|
||||
def rce_train(training_set: pd.DataFrame, radius: float | None = None) -> pd.DataFrame:
|
||||
"""
|
||||
Get the Reduced Coulomb Energy (RCE) prototypes.
|
||||
|
||||
Args:
|
||||
training_set (pd.DataFrame): The training set
|
||||
radius (float | None): The radius of the RCE prototypes. If None, computed using Silverman's rule.
|
||||
|
||||
Returns:
|
||||
pd.DataFrame: The RCE prototypes
|
||||
"""
|
||||
train_vectors = training_set.values
|
||||
|
||||
# Vectorized distance computation for the radius calculation
|
||||
diff_vectors = train_vectors[:, np.newaxis] - train_vectors[np.newaxis, :]
|
||||
distances = np.linalg.norm(diff_vectors, axis=-1)
|
||||
|
||||
# Non-parametric radius: Silverman Radius (compute if not provided)
|
||||
effective_radius = radius if radius is not None else silverman_radius(distances.flatten())
|
||||
|
||||
# Initialize prototypes with the first vector
|
||||
prototypes = [train_vectors[0]]
|
||||
|
||||
for vector in train_vectors[1:]:
|
||||
# Vectorized distance check between current vector and all prototypes
|
||||
distances_to_prototypes = np.linalg.norm(prototypes - vector, axis=1)
|
||||
|
||||
# If no prototype is close, add the current vector as a new prototype
|
||||
if np.all(distances_to_prototypes > effective_radius):
|
||||
prototypes.append(vector)
|
||||
|
||||
return pd.DataFrame(prototypes)
|
||||
|
||||
|
||||
def rce_test(test_set: pd.DataFrame, prototypes: pd.DataFrame) -> pd.Series:
|
||||
"""
|
||||
Get the signed Reduced Coulomb Energy (RCE) predictions.
|
||||
|
||||
Args:
|
||||
test_set (pd.DataFrame): The test set
|
||||
prototypes (pd.DataFrame): The RCE prototypes
|
||||
|
||||
Returns:
|
||||
pd.Series: The signed distances to the closest prototype for each test vector
|
||||
"""
|
||||
test_vectors = test_set.values
|
||||
prototype_vectors = prototypes.values
|
||||
|
||||
# Vectorized computation of distances between test vectors and all prototypes
|
||||
diff_vectors = test_vectors[:, np.newaxis] - prototype_vectors[np.newaxis, :]
|
||||
distances = np.linalg.norm(diff_vectors, axis=-1)
|
||||
|
||||
# Find the closest prototype for each test vector
|
||||
min_distances = np.min(distances, axis=1)
|
||||
closest_prototypes = prototype_vectors[np.argmin(distances, axis=1)]
|
||||
|
||||
# Compute the signed distance for each test vector
|
||||
signed_distances = np.sqrt(min_distances**2) * np.sign(
|
||||
np.mean(test_vectors - closest_prototypes, axis=1)
|
||||
)
|
||||
|
||||
return pd.Series(signed_distances)
|
||||
|
||||
|
||||
def rce_drift(reference_data: pd.DataFrame, real_data: pd.DataFrame, column: str) -> pd.Series:
|
||||
"""
|
||||
Detect drift using the Reduced Coulomb Energy (RCE) method.
|
||||
|
||||
Args:
|
||||
reference_data (pd.DataFrame): The reference data
|
||||
real_data (pd.DataFrame): The real data
|
||||
column (str): The target column to be analyzed. 'target' or 'prediction'
|
||||
|
||||
Returns:
|
||||
pd.Series: Normalized drift distances
|
||||
"""
|
||||
common_columns = list(set(reference_data.columns).intersection(real_data.columns))
|
||||
reference_data = reference_data[common_columns]
|
||||
real_data = real_data[common_columns]
|
||||
|
||||
# Get prototypes
|
||||
if column == 'target':
|
||||
prototypes = rce_train(reference_data.drop(columns=['prediction']), 0.1)
|
||||
else:
|
||||
prototypes = rce_train(reference_data.drop(columns=['target']), 0.1)
|
||||
|
||||
# Distances to prototypes
|
||||
if column == 'target':
|
||||
distances_train = rce_test(reference_data.drop(columns=['prediction']), prototypes)
|
||||
distances_test = rce_test(real_data.drop(columns=['prediction']), prototypes)
|
||||
else:
|
||||
distances_train = rce_test(reference_data.drop(columns=['target']), prototypes)
|
||||
distances_test = rce_test(real_data.drop(columns=['target']), prototypes)
|
||||
|
||||
# Find the maximum absolute distance in the training set
|
||||
max_abs_distance = max(abs(distances_train.max()), abs(distances_train.min()))
|
||||
|
||||
# Normalize while preserving sign
|
||||
distances = distances_test / max_abs_distance
|
||||
|
||||
return distances
|
||||
292
model_manager/sientia/reports.py
Normal file
292
model_manager/sientia/reports.py
Normal file
@@ -0,0 +1,292 @@
|
||||
import os
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
from evidently.metric_preset import DataDriftPreset
|
||||
from evidently.metrics import (
|
||||
ColumnSummaryMetric,
|
||||
ConflictTargetMetric,
|
||||
DatasetCorrelationsMetric,
|
||||
DatasetSummaryMetric,
|
||||
RegressionAbsPercentageErrorPlot,
|
||||
RegressionDummyMetric,
|
||||
RegressionErrorDistribution,
|
||||
RegressionErrorPlot,
|
||||
RegressionPerformanceMetrics,
|
||||
RegressionPredictedVsActualPlot,
|
||||
RegressionPredictedVsActualScatter,
|
||||
)
|
||||
from evidently.metrics.base_metric import generate_column_metrics
|
||||
from evidently.options import ColorOptions
|
||||
from evidently.pipeline.column_mapping import ColumnMapping
|
||||
from evidently.report import Report
|
||||
|
||||
COLOR_DISCRETE_SEQUENCE = (
|
||||
'#ed0400',
|
||||
'#0a5f38',
|
||||
'#6c3461',
|
||||
'#71aa34',
|
||||
'#d8dcd6',
|
||||
'#6b8ba4',
|
||||
)
|
||||
|
||||
|
||||
def load_html_from_file(file_path):
|
||||
with open(file_path, encoding='utf-8') as file:
|
||||
return file.read()
|
||||
|
||||
|
||||
def inject_content(main_html, section_id, content):
|
||||
soup = BeautifulSoup(main_html, 'html.parser')
|
||||
section = soup.find(id=section_id)
|
||||
|
||||
# Verifica se a seção foi encontrada E se ela é uma Tag (não uma string)
|
||||
if section and isinstance(section, Tag):
|
||||
section.clear()
|
||||
# Converte o conteúdo para um fragmento de BeautifulSoup e anexa
|
||||
new_content = BeautifulSoup(content, 'html.parser')
|
||||
section.append(new_content)
|
||||
|
||||
return str(soup)
|
||||
|
||||
|
||||
class Reports:
|
||||
"""
|
||||
Report generator using Evidently library.
|
||||
|
||||
Thread-safety: This class is NOT thread-safe. Multiple threads should not
|
||||
call add_*_section() methods on the same instance simultaneously as they
|
||||
modify shared state (self.metrics, self.sections, self.options).
|
||||
|
||||
For multi-threaded environments:
|
||||
- Create separate Reports instances per thread
|
||||
- Or synchronize access using locks
|
||||
- After generation, instances are safe for read-only operations
|
||||
|
||||
I/O Note: This class relies on Evidently's report.save_html() method
|
||||
for file operations. Ensure Evidently properly manages file handles.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
reference_data: Any,
|
||||
current_data: Any,
|
||||
target_name: str,
|
||||
base_path: str | None = None,
|
||||
template_path: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initializes an instance of the AigReport class.
|
||||
|
||||
Args:
|
||||
reference_data: The reference data for the report.
|
||||
current_data: The current data for the report.
|
||||
base_path: The base path for the report.
|
||||
"""
|
||||
self.metrics: list[Any] = []
|
||||
self.options: list[Any] | None = None
|
||||
self.sections: dict[str, Any] = {}
|
||||
self.report: Any = None
|
||||
self.ref_data = reference_data
|
||||
self.cur_data = current_data
|
||||
self.target_name = target_name
|
||||
self.set_color_options(primary_color='#0F4C81', secondary_color='#001E60')
|
||||
self.base_path = base_path
|
||||
self.template_path = template_path
|
||||
|
||||
def add_data_quality_section(self, columns: list[str] | None = None, run: bool = True) -> None:
|
||||
"""
|
||||
Adds a data quality section to the report.
|
||||
|
||||
Args:
|
||||
columns: The list of columns to include in the data quality section. If None, all columns will be included.
|
||||
run: Indicates whether to run the report immediately after adding the section.
|
||||
"""
|
||||
metrics = [
|
||||
DatasetSummaryMetric(),
|
||||
generate_column_metrics(ColumnSummaryMetric, columns=columns, skip_id_column=True),
|
||||
ConflictTargetMetric(),
|
||||
DatasetCorrelationsMetric(),
|
||||
]
|
||||
self.metrics.extend(metrics)
|
||||
if run:
|
||||
mapping = ColumnMapping()
|
||||
mapping.target = self.target_name
|
||||
report = Report(metrics=metrics, options=self.options)
|
||||
report.run(
|
||||
reference_data=self.ref_data,
|
||||
current_data=self.cur_data,
|
||||
column_mapping=mapping,
|
||||
)
|
||||
self.sections['data_quality'] = report.as_dict()
|
||||
if self.base_path:
|
||||
# Note: Relies on Evidently's save_html() to properly manage file I/O
|
||||
report.save_html(os.path.join(self.base_path, 'data_quality.html'))
|
||||
|
||||
def add_data_drift_section(self, columns: list[str] | None = None, run: bool = True) -> None:
|
||||
"""
|
||||
Adds a data drift section to the report.
|
||||
|
||||
Args:
|
||||
columns: The list of columns to include in the data drift section. If None, all columns will be included.
|
||||
run: Indicates whether to run the report immediately after adding the section.
|
||||
"""
|
||||
self.metrics.append(DataDriftPreset(columns=columns))
|
||||
if run:
|
||||
mapping = ColumnMapping()
|
||||
mapping.target = self.target_name
|
||||
report = Report(metrics=[DataDriftPreset(columns=columns)], options=self.options)
|
||||
report.run(
|
||||
reference_data=self.ref_data,
|
||||
current_data=self.cur_data,
|
||||
column_mapping=mapping,
|
||||
)
|
||||
self.sections['data_drift'] = report.as_dict()
|
||||
if self.base_path:
|
||||
# Note: Relies on Evidently's save_html() to properly manage file I/O
|
||||
report.save_html(os.path.join(self.base_path, 'data_drift.html'))
|
||||
|
||||
def add_regression_section(self, run: bool = True) -> None:
|
||||
"""
|
||||
Adds a regression section to the report.
|
||||
|
||||
Args:
|
||||
run: Indicates whether to run the report immediately after adding the section.
|
||||
"""
|
||||
metrics = [
|
||||
RegressionPerformanceMetrics(),
|
||||
RegressionDummyMetric(),
|
||||
RegressionPredictedVsActualScatter(),
|
||||
RegressionPredictedVsActualPlot(),
|
||||
RegressionErrorPlot(),
|
||||
RegressionAbsPercentageErrorPlot(),
|
||||
RegressionErrorDistribution(),
|
||||
]
|
||||
self.metrics.extend(metrics)
|
||||
if run:
|
||||
mapping = ColumnMapping()
|
||||
|
||||
mapping.target = self.target_name
|
||||
mapping.prediction = 'prediction'
|
||||
|
||||
report = Report(metrics=metrics, options=self.options)
|
||||
report.run(
|
||||
reference_data=self.ref_data,
|
||||
current_data=self.cur_data,
|
||||
column_mapping=mapping,
|
||||
)
|
||||
self.sections['regression'] = report.as_dict()
|
||||
if self.base_path:
|
||||
# Note: Relies on Evidently's save_html() to properly manage file I/O
|
||||
report.save_html(os.path.join(self.base_path, 'regression.html'))
|
||||
|
||||
def set_color_options(
|
||||
self,
|
||||
primary_color: str = '#0F4C81',
|
||||
secondary_color: str = '#001E60',
|
||||
current_data_color: str | None = None,
|
||||
reference_data_color: str | None = None,
|
||||
additional_data_color: str = '#0a5f38',
|
||||
color_sequence: Sequence[str] = COLOR_DISCRETE_SEQUENCE,
|
||||
fill_color: str = 'LightGreen',
|
||||
zero_line_color: str = 'green',
|
||||
non_visible_color: str = 'white',
|
||||
underestimation_color: str = '#6574f7',
|
||||
overestimation_color: str = '#ee5540',
|
||||
majority_color: str = '#1acc98',
|
||||
vertical_lines: str = 'green',
|
||||
heatmap: str = 'RdBu_r',
|
||||
) -> None:
|
||||
"""
|
||||
Sets the color options for the report.
|
||||
|
||||
Args:
|
||||
primary_color: The primary color for the report.
|
||||
secondary_color: The secondary color for the report.
|
||||
current_data_color: The color for the current data.
|
||||
reference_data_color: The color for the reference data.
|
||||
additional_data_color: The color for additional data.
|
||||
color_sequence: The color sequence for discrete values.
|
||||
fill_color: The fill color for visualizations.
|
||||
zero_line_color: The color for the zero line.
|
||||
non_visible_color: The color for non-visible elements.
|
||||
underestimation_color: The color for underestimation.
|
||||
overestimation_color: The color for overestimation.
|
||||
majority_color: The color for majority elements.
|
||||
vertical_lines: The color for vertical lines.
|
||||
heatmap: The color map for heatmaps.
|
||||
"""
|
||||
color_scheme = ColorOptions(
|
||||
primary_color=primary_color,
|
||||
secondary_color=secondary_color,
|
||||
current_data_color=current_data_color,
|
||||
reference_data_color=reference_data_color,
|
||||
additional_data_color=additional_data_color,
|
||||
color_sequence=color_sequence,
|
||||
fill_color=fill_color,
|
||||
zero_line_color=zero_line_color,
|
||||
non_visible_color=non_visible_color,
|
||||
underestimation_color=underestimation_color,
|
||||
overestimation_color=overestimation_color,
|
||||
majority_color=majority_color,
|
||||
vertical_lines=vertical_lines,
|
||||
heatmap=heatmap,
|
||||
)
|
||||
|
||||
if self.options is None:
|
||||
self.options = [color_scheme]
|
||||
else:
|
||||
self.options.append(color_scheme)
|
||||
|
||||
def save_all_sections_html(self, report_path):
|
||||
"""
|
||||
Saves the report with all sections as HTML.
|
||||
|
||||
Args:
|
||||
report_path: The path to save the report HTML file.
|
||||
|
||||
Raises:
|
||||
ValueError: If base_path is not set
|
||||
OSError: If directory creation or file writing fails
|
||||
|
||||
Note:
|
||||
This method uses context manager (with open) to ensure file is properly closed.
|
||||
Creates parent directories if they don't exist.
|
||||
"""
|
||||
if not self.base_path:
|
||||
raise ValueError('base_path is required to save all sections HTML')
|
||||
|
||||
if not self.template_path:
|
||||
raise ValueError('template_path is required to save all sections HTML')
|
||||
|
||||
# Ensure output directory exists
|
||||
output_dir = os.path.dirname(report_path)
|
||||
if output_dir and not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
print(f'Output directory: {output_dir}')
|
||||
print(f'Report path: {report_path}')
|
||||
print(f'Base path: {self.base_path}')
|
||||
print(f'Template path: {self.template_path}')
|
||||
|
||||
# Load main HTML template
|
||||
main_html_path = os.path.join(self.template_path, 'header.html')
|
||||
main_html = load_html_from_file(main_html_path)
|
||||
|
||||
# Load content from data_drift.html, data_quality.html, and regression.html
|
||||
data_drift_content = load_html_from_file(os.path.join(self.base_path, 'data_drift.html'))
|
||||
data_quality_content = load_html_from_file(
|
||||
os.path.join(self.base_path, 'data_quality.html')
|
||||
)
|
||||
regression_content = load_html_from_file(os.path.join(self.base_path, 'regression.html'))
|
||||
|
||||
# Inject content into the main HTML template
|
||||
main_html = inject_content(main_html, 'data_drift', data_drift_content)
|
||||
main_html = inject_content(main_html, 'data_quality', data_quality_content)
|
||||
main_html = inject_content(main_html, 'regression', regression_content)
|
||||
|
||||
# Save the final HTML to a new file (report.html)
|
||||
# Context manager ensures file is properly closed even if an error occurs
|
||||
with open(report_path, 'w', encoding='utf-8') as report_file:
|
||||
report_file.write(main_html)
|
||||
Reference in New Issue
Block a user