Code import - branch release/SIENTIAPDE-1645
This commit is contained in:
636
model_manager/utils/repository/data_manager_repository.py
Normal file
636
model_manager/utils/repository/data_manager_repository.py
Normal file
@@ -0,0 +1,636 @@
|
||||
"""
|
||||
Data management repository for the training pipeline.
|
||||
|
||||
This module provides the core data loading and preprocessing logic for the
|
||||
training pipeline, including:
|
||||
- CSV loading from in-memory bytes
|
||||
- datetime parsing and index configuration
|
||||
- optional support filters
|
||||
- train/test split management (when no explicit validation dataset is provided)
|
||||
|
||||
It is intentionally decoupled from any specific model implementation or MLflow
|
||||
integration. Models are trained elsewhere (e.g., via SientiaModel wrappers),
|
||||
and this repository focuses solely on preparing data structures for them.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from os import makedirs, path
|
||||
from shutil import rmtree
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_model.wrappers.sientia_model import SientiaModel
|
||||
|
||||
from model_manager.runtime_paths import PROJECT_BASE_PATH, REPORTS_ROOT
|
||||
from model_manager.sientia.metrics import mae, mse, r2
|
||||
from model_manager.sientia.reports import Reports # type: ignore[import-untyped]
|
||||
from model_manager.utils.models.train_model_params import (
|
||||
FRONTEND_DATE_FORMAT_TO_STRFTIME,
|
||||
TrainModelParams,
|
||||
)
|
||||
from model_manager.utils.models.train_model_result import TrainModelResult
|
||||
|
||||
|
||||
def train_test_split(
|
||||
data: pd.DataFrame,
|
||||
train_size: float,
|
||||
random_state: int | None = None,
|
||||
shuffle: bool = True,
|
||||
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
||||
# 1. Definir a semente (seed) para reprodutibilidade
|
||||
if random_state is not None:
|
||||
np.random.seed(random_state)
|
||||
|
||||
# 2. Gerar índices e embaralhar se necessário
|
||||
indices = np.arange(len(data))
|
||||
|
||||
if shuffle:
|
||||
np.random.shuffle(indices)
|
||||
|
||||
# 3. Calcular o ponto de corte (split point)
|
||||
# Cálculo: N_treino = tamanho_total * proporcao_treino
|
||||
n_train = int(len(data) * train_size)
|
||||
|
||||
# 4. Dividir os índices
|
||||
train_indices = indices[:n_train]
|
||||
test_indices = indices[n_train:]
|
||||
|
||||
# 5. Retornar os dados fatiados
|
||||
return data.iloc[train_indices], data.iloc[test_indices]
|
||||
|
||||
|
||||
def _ensure_date_column_parsed(data: pd.DataFrame, params: TrainModelParams) -> pd.DataFrame:
|
||||
"""
|
||||
Parse params.date_column using the frontend date_format mapping only.
|
||||
|
||||
date_column must exist in ``data`` (callers validate before prepare). No broad
|
||||
pandas inference or alternate timezone formats here—clients must send a supported
|
||||
date_format or rely on the TrainModelParams default.
|
||||
"""
|
||||
if params.date_column not in data.columns:
|
||||
raise ValueError(
|
||||
f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}'
|
||||
)
|
||||
data = data.copy()
|
||||
col = data[params.date_column]
|
||||
if params.date_format not in FRONTEND_DATE_FORMAT_TO_STRFTIME:
|
||||
raise ValueError(
|
||||
f'date_format "{params.date_format}" is not mapped to a strftime pattern '
|
||||
'(must be one of the allowed frontend formats).'
|
||||
)
|
||||
strf = FRONTEND_DATE_FORMAT_TO_STRFTIME[params.date_format]
|
||||
try:
|
||||
parsed = pd.to_datetime(col, format=strf, errors='raise')
|
||||
data[params.date_column] = parsed
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
f'Failed to parse date column "{params.date_column}" with format "{params.date_format}": {e}'
|
||||
) from e
|
||||
return data
|
||||
|
||||
|
||||
class DataManagerRepository(SientiaMonitoring):
|
||||
"""
|
||||
Repository for data preparation in the training pipeline.
|
||||
|
||||
This class encapsulates the core logic for preparing ML training data:
|
||||
loading CSV bytes, applying date/index configuration, support filters, and
|
||||
constructing train/test splits (or using an explicit validation dataset).
|
||||
|
||||
Attributes:
|
||||
logger (Logger): Logger instance for observability and debugging
|
||||
"""
|
||||
|
||||
def __init__(self, logger: Logger):
|
||||
"""
|
||||
Initialize DataManagerRepository with logger.
|
||||
|
||||
Args:
|
||||
logger: Logger instance for observability
|
||||
"""
|
||||
SientiaMonitoring.__init__(
|
||||
self,
|
||||
logger=logger,
|
||||
notification_handler=None,
|
||||
metrics_controller=None,
|
||||
)
|
||||
|
||||
def _drop_rows_with_missing_timestamp(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
params: TrainModelParams,
|
||||
metadata: dict[str, Any] | None,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Remove rows where the configured date_column is missing (NaN/NaT/blank string).
|
||||
|
||||
Empty timestamp cells cannot be placed on a DatetimeIndex and break
|
||||
downstream joins and metrics.
|
||||
"""
|
||||
if params.date_column not in df.columns:
|
||||
return df
|
||||
series = df[params.date_column]
|
||||
mask = series.notna()
|
||||
if series.dtype == object:
|
||||
stripped = series.astype(str).str.strip()
|
||||
mask &= stripped.ne('')
|
||||
mask &= stripped.str.lower().ne('nan')
|
||||
n_drop = int((~mask).sum())
|
||||
if n_drop:
|
||||
self.info(
|
||||
f'Dropping {n_drop} row(s) with missing or blank timestamp column '
|
||||
f'"{params.date_column}"',
|
||||
metadata,
|
||||
)
|
||||
return df.loc[mask].copy()
|
||||
|
||||
def _coerce_non_timestamp_columns_to_numeric(
|
||||
self,
|
||||
df: pd.DataFrame,
|
||||
params: TrainModelParams,
|
||||
metadata: dict[str, Any] | None,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Coerce all non-timestamp columns to numeric dtype.
|
||||
|
||||
The timestamp column defined by params.date_column is excluded from coercion.
|
||||
Non-numeric values are coerced to NaN.
|
||||
"""
|
||||
out = df.copy()
|
||||
for col in out.columns:
|
||||
if col == params.date_column:
|
||||
continue
|
||||
original_na = int(out[col].isna().sum())
|
||||
out[col] = pd.to_numeric(out[col], errors='coerce')
|
||||
new_na = int(out[col].isna().sum())
|
||||
introduced_na = new_na - original_na
|
||||
if introduced_na > 0:
|
||||
self.warning(
|
||||
f'Column "{col}" had {introduced_na} non-numeric value(s) coerced to NaN',
|
||||
metadata,
|
||||
)
|
||||
return out
|
||||
|
||||
def prepare_training_data(
|
||||
self,
|
||||
train_file_bytes: bytes,
|
||||
validation_file_bytes: bytes | None,
|
||||
params: TrainModelParams,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> TrainModelResult:
|
||||
"""
|
||||
Build TrainModelResult from raw CSV bytes for train (and optional validation) data.
|
||||
|
||||
This method orchestrates the data pipeline:
|
||||
1. Load training data from in-memory bytes
|
||||
2. Optionally load validation data from in-memory bytes
|
||||
3. Parse and configure datetime index
|
||||
4. Apply optional support filters
|
||||
5. Split into train/test sets when no explicit validation dataset is provided
|
||||
|
||||
Args:
|
||||
train_file_bytes: Raw bytes of the training CSV.
|
||||
validation_file_bytes: Raw bytes of the validation CSV, or None when
|
||||
validation should be derived via train/test split.
|
||||
params: Training parameters (TrainModelParams).
|
||||
|
||||
Returns:
|
||||
TrainModelResult: Object containing processed data, train/test splits,
|
||||
and scaler dictionary.
|
||||
|
||||
Raises:
|
||||
ValueError: If transformed data is empty.
|
||||
Exception: If data loading or preprocessing fails.
|
||||
"""
|
||||
try:
|
||||
train_df = pd.read_csv(
|
||||
BytesIO(train_file_bytes),
|
||||
sep=params.line_separator,
|
||||
decimal=params.decimal_separator,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise ValueError(
|
||||
'Failed to load training CSV data from MinIO object. '
|
||||
'Check file encoding, line separator and decimal separator.'
|
||||
) from exc
|
||||
|
||||
train_df = self._drop_rows_with_missing_timestamp(train_df, params, metadata)
|
||||
train_df = _ensure_date_column_parsed(train_df, params)
|
||||
train_df = self._configure_datetime_index(train_df, params, metadata)
|
||||
train_df = self._set_timezone_on_index(train_df, metadata)
|
||||
train_df = self._coerce_non_timestamp_columns_to_numeric(train_df, params, metadata)
|
||||
|
||||
if len(train_df) <= 0:
|
||||
raise ValueError('Training data view is empty after transformation')
|
||||
|
||||
# Explicit validation dataset path
|
||||
train_data = pd.DataFrame(train_df[params.variable_columns + [params.target_variable]])
|
||||
if validation_file_bytes is not None:
|
||||
try:
|
||||
val_df = pd.read_csv(
|
||||
BytesIO(validation_file_bytes),
|
||||
sep=params.line_separator,
|
||||
decimal=params.decimal_separator,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise ValueError(
|
||||
'Failed to load validation CSV data from MinIO object. '
|
||||
'Check file encoding, line separator and decimal separator.'
|
||||
) from exc
|
||||
|
||||
val_df = self._drop_rows_with_missing_timestamp(val_df, params, metadata)
|
||||
val_df = _ensure_date_column_parsed(val_df, params)
|
||||
val_df = self._configure_datetime_index(val_df, params, metadata)
|
||||
val_df = self._set_timezone_on_index(val_df, metadata)
|
||||
val_df = self._coerce_non_timestamp_columns_to_numeric(val_df, params, metadata)
|
||||
|
||||
if len(val_df) <= 0:
|
||||
raise ValueError('Validation data view is empty after transformation')
|
||||
|
||||
val_data = pd.DataFrame(val_df[params.variable_columns + [params.target_variable]])
|
||||
else:
|
||||
# Fallback path: derive validation via train/test split from a single dataset.
|
||||
train_data, val_data = train_test_split(
|
||||
train_data,
|
||||
train_size=params.train_size / 100,
|
||||
shuffle=params.shuffle,
|
||||
random_state=params.random_state,
|
||||
)
|
||||
|
||||
self.info(
|
||||
f'Data preprocessed and split successfully - experiment run id: {params.experiment_run_id}',
|
||||
metadata,
|
||||
)
|
||||
|
||||
experiment_name = f'{params.experiment_name}'
|
||||
run_name = f'{experiment_name}_{datetime.now().strftime("%Y%m%d_%H%M%S")}'
|
||||
|
||||
return TrainModelResult(
|
||||
params=params,
|
||||
train_data=train_data,
|
||||
val_data=val_data,
|
||||
run_name=run_name,
|
||||
experiment_name=experiment_name,
|
||||
)
|
||||
|
||||
def _as_series(self, pred: pd.DataFrame | pd.Series) -> pd.Series:
|
||||
if isinstance(pred, pd.Series):
|
||||
return pred
|
||||
# If the wrapper returns a single-column DataFrame, take its first column.
|
||||
if pred.shape[1] == 1:
|
||||
return pred.iloc[:, 0]
|
||||
raise ValueError('y_pred/y_train_pred must be a Series or single-column DataFrame')
|
||||
|
||||
def _extract_model_equation(self, regr: Any, params: TrainModelParams) -> dict:
|
||||
"""
|
||||
Extract the linear regression equation coefficients and create equation metadata.
|
||||
|
||||
This method extracts the coefficients and intercept from the trained model
|
||||
and creates a structured dictionary containing the equation information
|
||||
for serialization as JSON artifact.
|
||||
|
||||
Args:
|
||||
regr: Trained LinearRegressionModel object
|
||||
params: Training parameters containing variable information
|
||||
|
||||
Returns:
|
||||
dict: Equation metadata containing:
|
||||
- target_variable: Name of the target variable
|
||||
- coefficients: Dictionary mapping variable names to coefficients
|
||||
- intercept: Model intercept value
|
||||
- equation_string: Human-readable equation string
|
||||
- latex_equation: LaTeX formatted equation
|
||||
"""
|
||||
coefficients = regr.regr.coef_
|
||||
intercept = regr.regr.intercept_
|
||||
|
||||
# Get feature names - for polynomial models, use poly_feature_names
|
||||
model_kwargs = params.model_kwargs or {}
|
||||
degree = model_kwargs.get('degree', 1)
|
||||
poly_feature_names = model_kwargs.get('poly_feature_names', None)
|
||||
|
||||
if degree > 1 and poly_feature_names:
|
||||
feature_names = poly_feature_names
|
||||
else:
|
||||
feature_names = params.variable_columns
|
||||
|
||||
# Create coefficients dictionary
|
||||
coefficients_dict = {}
|
||||
for i, var in enumerate(feature_names):
|
||||
if i < len(coefficients):
|
||||
coefficients_dict[var] = float(coefficients[i])
|
||||
|
||||
# Create equation string
|
||||
equation_parts = [f'{coef:.6f} * {var}' for var, coef in coefficients_dict.items()]
|
||||
equation_string = f'{params.target_variable} = {intercept:.6f} + ' + ' + '.join(
|
||||
equation_parts
|
||||
)
|
||||
|
||||
# Create LaTeX equation
|
||||
latex_parts = [f'{coef:.6f} \\cdot {var}' for var, coef in coefficients_dict.items()]
|
||||
latex_equation = f'{params.target_variable} = {intercept:.6f} + ' + ' + '.join(latex_parts)
|
||||
|
||||
return {
|
||||
'target_variable': params.target_variable,
|
||||
'coefficients': coefficients_dict,
|
||||
'intercept': float(intercept),
|
||||
'equation_string': equation_string,
|
||||
'latex_equation': latex_equation,
|
||||
'model_type': params.model_name,
|
||||
'degree': degree,
|
||||
'interaction_only': model_kwargs.get('interaction_only', False),
|
||||
'original_features': feature_names,
|
||||
}
|
||||
|
||||
def compute_regression_metrics(
|
||||
self,
|
||||
tmr: TrainModelResult,
|
||||
wrapper: SientiaModel,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> TrainModelResult:
|
||||
"""
|
||||
Compute regression metrics for training results.
|
||||
|
||||
This helper mirrors the previous TrainingRepository.after_train_calculation
|
||||
behavior, assuming that predictions (y_pred/y_train_pred) are already on the
|
||||
correct scale for metric calculation (any scaling is handled inside the
|
||||
model wrapper).
|
||||
|
||||
Args:
|
||||
tmr: Training result containing:
|
||||
- train_data/val_data DataFrames with a target column
|
||||
- y_train_pred/y_pred populated (model predictions for train/val)
|
||||
wrapper: Trained model wrapper (used for linear equation extraction).
|
||||
metadata: Optional workflow metadata for debug logging.
|
||||
|
||||
Return:
|
||||
TrainModelResult: Same object with mse_val, mae_val and r2_val set.
|
||||
"""
|
||||
if tmr.y_pred is None:
|
||||
raise ValueError('y_pred must be set before computing regression metrics')
|
||||
|
||||
params = tmr.params
|
||||
target = params.target_variable
|
||||
|
||||
# True values are expected to come from val_data.
|
||||
y_true_val = tmr.val_data[target]
|
||||
|
||||
y_pred_val = self._as_series(tmr.y_pred).sort_index()
|
||||
y_true_val = y_true_val.sort_index()
|
||||
|
||||
# Align by index to avoid metric calculation errors if ordering differs.
|
||||
common_index = y_true_val.index.intersection(y_pred_val.index)
|
||||
head = min(5, len(y_true_val), len(y_pred_val))
|
||||
self.debug(
|
||||
'compute_regression_metrics index alignment: '
|
||||
f'val_n={len(y_true_val)} pred_n={len(y_pred_val)} common_n={len(common_index)}; '
|
||||
f'val_index_dtype={y_true_val.index.dtype} '
|
||||
f'pred_index_dtype={y_pred_val.index.dtype}; '
|
||||
f'val_index_sample={list(y_true_val.index[:head])} '
|
||||
f'pred_index_sample={list(y_pred_val.index[:head])}',
|
||||
metadata,
|
||||
)
|
||||
|
||||
if len(common_index) == 0:
|
||||
raise ValueError(
|
||||
'No overlapping indices between val_data and y_pred. '
|
||||
f'val_n={len(y_true_val)} pred_n={len(y_pred_val)} '
|
||||
f'val_index_sample={list(y_true_val.index[:head])} '
|
||||
f'pred_index_sample={list(y_pred_val.index[:head])}'
|
||||
)
|
||||
|
||||
y_true_val = y_true_val.loc[common_index]
|
||||
y_pred_val = y_pred_val.loc[common_index]
|
||||
|
||||
# Metrics helpers already round to 2 decimals.
|
||||
tmr.mse_val = mse(y_true_val, y_pred_val)
|
||||
tmr.mae_val = mae(y_true_val, y_pred_val)
|
||||
tmr.r2_val = r2(y_true_val, y_pred_val)
|
||||
|
||||
if params.model_type == 'linear_regression':
|
||||
inner = getattr(wrapper, 'model', None)
|
||||
regr = getattr(inner, 'regr', None) if inner is not None else None
|
||||
if regr is not None and hasattr(regr, 'coef_') and hasattr(regr, 'intercept_'):
|
||||
tmr.equation = self._extract_model_equation(inner, params)
|
||||
|
||||
return tmr
|
||||
|
||||
def _configure_datetime_index(
|
||||
self,
|
||||
data: pd.DataFrame | None,
|
||||
params: TrainModelParams,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Configure datetime index for the DataFrame.
|
||||
|
||||
Guards against None to avoid 'NoneType' object has no attribute 'index' downstream.
|
||||
Uses only params.date_column and assumes it was already parsed exactly once by
|
||||
_ensure_date_column_parsed.
|
||||
Args:
|
||||
data: The DataFrame to configure the datetime index for.
|
||||
params: The training parameters.
|
||||
metadata: The metadata for the training run.
|
||||
|
||||
Returns:
|
||||
The DataFrame with the datetime index configured.
|
||||
"""
|
||||
if data is None:
|
||||
raise ValueError(
|
||||
'Data is None after load_data. '
|
||||
'Check file format, line separator and decimal separator.'
|
||||
)
|
||||
|
||||
if params.date_column not in data.columns:
|
||||
raise ValueError(
|
||||
f'date_column "{params.date_column}" not found in dataset columns: {list(data.columns)}'
|
||||
)
|
||||
|
||||
if not pd.api.types.is_datetime64_any_dtype(data[params.date_column]):
|
||||
raise ValueError(
|
||||
f'date_column "{params.date_column}" must be datetime before index configuration'
|
||||
)
|
||||
|
||||
data = data.set_index(params.date_column)
|
||||
data = data.sort_index()
|
||||
self.info(f'Configured datetime index from column: {params.date_column}', metadata)
|
||||
return data
|
||||
|
||||
def _set_timezone_on_index(
|
||||
self, data: pd.DataFrame, metadata: dict[str, Any] | None = None
|
||||
) -> pd.DataFrame:
|
||||
"""
|
||||
Check if the index has a timezone and if not, set it to UTC timezone.
|
||||
|
||||
Args:
|
||||
data: The DataFrame to set the timezone on.
|
||||
metadata: The metadata for the training run.
|
||||
|
||||
Returns:
|
||||
The DataFrame with the timezone set.
|
||||
"""
|
||||
|
||||
if isinstance(data.index, pd.DatetimeIndex):
|
||||
if data.index.tz is None:
|
||||
data.index = data.index.tz_localize('UTC')
|
||||
else:
|
||||
data.index = data.index.tz_convert('UTC')
|
||||
else:
|
||||
raise ValueError('Index is not a DatetimeIndex')
|
||||
|
||||
return data
|
||||
|
||||
def _get_reports_directory(self) -> str:
|
||||
"""
|
||||
Get the absolute path to the reports directory.
|
||||
|
||||
Returns:
|
||||
str: Absolute path to the runtime reports root.
|
||||
"""
|
||||
return REPORTS_ROOT
|
||||
|
||||
def _create_run_directory(
|
||||
self, base_path: str, run_name: str, metadata: dict[str, Any] | None = None
|
||||
) -> str:
|
||||
"""
|
||||
Creates a directory inside the 'reports' folder with the run name and a timestamp.
|
||||
|
||||
Uses microsecond precision in timestamp to minimize collision probability
|
||||
in high-concurrency scenarios.
|
||||
|
||||
Args:
|
||||
base_path (str): The path to the 'reports' folder.
|
||||
run_name (str): The name of the run.
|
||||
|
||||
Returns:
|
||||
str: The path to the created directory.
|
||||
|
||||
Raises:
|
||||
PermissionError: If there are insufficient permissions to create the directory.
|
||||
OSError: If directory creation fails for any other reason.
|
||||
"""
|
||||
# Use microsecond precision to reduce collision probability
|
||||
run_dir = path.join(base_path, 'temp', f'{run_name}')
|
||||
|
||||
try:
|
||||
makedirs(run_dir, exist_ok=True)
|
||||
return run_dir
|
||||
except PermissionError as e:
|
||||
error_msg = f'Permission denied when creating directory: {run_dir}'
|
||||
self.error(error_msg, metadata)
|
||||
raise PermissionError(error_msg) from e
|
||||
except OSError as e:
|
||||
error_msg = f'Failed to create directory {run_dir}: {str(e)}'
|
||||
self.error(error_msg, metadata)
|
||||
raise OSError(error_msg) from e
|
||||
|
||||
def generate_report(
|
||||
self, data: TrainModelResult, metadata: dict[str, Any] | None = None
|
||||
) -> TrainModelResult:
|
||||
"""
|
||||
Generates a comprehensive report summarizing data quality, data drift, and regression analysis.
|
||||
|
||||
Args:
|
||||
reference_data (pd.DataFrame): The training dataset with predictions added.
|
||||
current_data (pd.DataFrame): The testing dataset with predictions added.
|
||||
data: The training model result containing the datasets, model, and parameters.
|
||||
|
||||
Returns:
|
||||
The updated result object with paths to the generated report and data files.
|
||||
|
||||
Raises:
|
||||
ValueError: If data conversion to float64 fails or DataFrames are invalid.
|
||||
PermissionError: If there are insufficient permissions to write files.
|
||||
OSError: If file writing fails for any other reason.
|
||||
"""
|
||||
|
||||
if data.run_name is None:
|
||||
raise ValueError('run_name is not set, cannot generate report')
|
||||
|
||||
if data.y_train_pred is None or data.y_pred is None:
|
||||
raise ValueError('y_train_pred or y_pred is not set, cannot generate report')
|
||||
|
||||
y_train_pred = data.y_train_pred.rename(columns={data.params.target_variable: 'prediction'})
|
||||
y_val_pred = data.y_pred.rename(columns={data.params.target_variable: 'prediction'})
|
||||
|
||||
# Join the predictions to the data
|
||||
reference_data = y_train_pred[['prediction']].join(data.train_data, how='inner')
|
||||
reference_data_float = reference_data.astype(np.float64)
|
||||
|
||||
current_data = y_val_pred[['prediction']].join(data.val_data, how='inner')
|
||||
current_data_float = current_data.astype(np.float64)
|
||||
|
||||
# Evidently's ConflictTargetMetric expects a literal `target` column name.
|
||||
# Keep the original target column and provide this alias for report metrics.
|
||||
target_col = data.params.target_variable
|
||||
|
||||
reference_data_float['target'] = reference_data_float[target_col]
|
||||
current_data_float['target'] = current_data_float[target_col]
|
||||
|
||||
# Initialize report generator
|
||||
base_path = self._get_reports_directory()
|
||||
data.run_dir = self._create_run_directory(base_path, data.run_name)
|
||||
|
||||
# Template path is the code path of the model_manager package
|
||||
template_path = path.join(PROJECT_BASE_PATH, 'reports')
|
||||
|
||||
report = Reports(
|
||||
reference_data=reference_data_float,
|
||||
current_data=current_data_float,
|
||||
base_path=data.run_dir,
|
||||
template_path=template_path,
|
||||
target_name=data.params.target_variable,
|
||||
)
|
||||
|
||||
# Generate report sections
|
||||
feature_and_target_cols = data.params.variable_columns + [target_col]
|
||||
report.add_data_quality_section(columns=feature_and_target_cols)
|
||||
report.add_data_drift_section(columns=feature_and_target_cols)
|
||||
report.add_regression_section()
|
||||
|
||||
# Save HTML report
|
||||
data.report_path = path.join(data.run_dir, 'report.html')
|
||||
report.save_all_sections_html(data.report_path)
|
||||
|
||||
# Save training / validation CSVs using the same frames as the report (includes
|
||||
# literal `target` alias for Evidently, plus predictions and float-cast features).
|
||||
data.train_data_path = path.join(data.run_dir, 'train_data.csv')
|
||||
reference_data_float.to_csv(data.train_data_path, index=False)
|
||||
|
||||
# Save test data CSV
|
||||
data.test_data_path = path.join(data.run_dir, 'test_data.csv')
|
||||
current_data_float.to_csv(data.test_data_path, index=False)
|
||||
|
||||
# Save equation as JSON
|
||||
if data.equation is not None and data.params.model_type == 'linear_regression':
|
||||
data.equation_path = path.join(data.run_dir, 'model_equation.json')
|
||||
with open(data.equation_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(data.equation, f, indent=2, ensure_ascii=False)
|
||||
|
||||
return data
|
||||
|
||||
def cleanup_run_directory(self, run_dir: str, metadata: dict[str, Any] | None = None) -> None:
|
||||
"""
|
||||
Clean up temporary run directory after model training.
|
||||
|
||||
This activity deletes the temporary directory created during model training
|
||||
and artifact generation. It implements idempotent cleanup to handle cases
|
||||
where the directory may have already been deleted.
|
||||
|
||||
Args:
|
||||
run_dir (str): Path to the run directory to delete
|
||||
"""
|
||||
if not run_dir:
|
||||
self.info('No run directory specified, skipping cleanup')
|
||||
return
|
||||
|
||||
if path.exists(run_dir):
|
||||
rmtree(run_dir)
|
||||
self.info(f'Run directory deleted successfully: {run_dir}')
|
||||
else:
|
||||
self.info(f'Run directory already deleted: {run_dir}')
|
||||
Reference in New Issue
Block a user