gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: log regression metrics as parameters in Training class

Browse Source 
- Added a method to persist computed regression metrics (MSE, MAE, R²) as MLflow parameters during model training, enhancing model evaluation and tracking.
- Updated the Training class to log the equation path if available, improving artifact management.
This commit is contained in:
vitor-aignosi
2026-04-17 10:52:32 -03:00
parent b245801e09
commit 31e95cbdf8
11 changed files with 649 additions and 1167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
model_manager/activities/training.py
View File

@@ -14,7 +14,6 @@ with workflow.unsafe.imports_passed_through():
from typing import Any
import mlflow
import pandas as pd
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
@@ -340,10 +339,30 @@ class Training(SientiaMonitoring):
self.info(f'Storing model for {train_params.model_type}', metadata)
wrapper._input_example = None
wrapper.store_model(name=train_params.model_name)
self._log_regression_metrics_as_params(train_result)
self.info(f'Logging artifacts for {train_params.model_type}', metadata)
mlflow.log_artifact(train_result.report_path)
mlflow.log_artifact(train_result.train_data_path)
mlflow.log_artifact(train_result.test_data_path)
if train_result.equation_path is not None:
mlflow.log_artifact(train_result.equation_path)
def _log_regression_metrics_as_params(self, train_result: TrainModelResult) -> None:
"""
Persist computed regression metrics as MLflow params.
Args:
train_result: Training output containing computed regression metrics.
"""
metric_params = {
'mse_val': train_result.mse_val,
'mae_val': train_result.mae_val,
'r2_val': train_result.r2_val,
}
for key, value in metric_params.items():
if value is not None:
mlflow.log_param(key, value)
@activity.defn(name='cleanup_resources')
def cleanup_resources(self, input_data: dict[str, Any]) -> None:

24
model_manager/sientia/reports.py
View File

@@ -19,8 +19,8 @@ from evidently.metrics import (
)
from evidently.metrics.base_metric import generate_column_metrics
from evidently.options import ColorOptions
from evidently.report import Report
from evidently.pipeline.column_mapping import ColumnMapping
from evidently.report import Report
COLOR_DISCRETE_SEQUENCE = (
'#ed0400',
@@ -36,17 +36,18 @@ def load_html_from_file(file_path):
with open(file_path, encoding='utf-8') as file:
return file.read()
def inject_content(main_html, section_id, content):
soup = BeautifulSoup(main_html, 'html.parser')
section = soup.find(id=section_id)
# Verifica se a seção foi encontrada E se ela é uma Tag (não uma string)
if section and isinstance(section, Tag):
section.clear()
# Converte o conteúdo para um fragmento de BeautifulSoup e anexa
new_content = BeautifulSoup(content, 'html.parser')
section.append(new_content)
return str(soup)
@@ -68,7 +69,12 @@ class Reports:
"""
def __init__(
self, reference_data: Any, current_data: Any, target_name: str, base_path: str | None = None, template_path: str | None = None
self,
reference_data: Any,
current_data: Any,
target_name: str,
base_path: str | None = None,
template_path: str | None = None,
) -> None:
"""
Initializes an instance of the AigReport class.
@@ -151,7 +157,7 @@ class Reports:
mapping.target = self.target_name
mapping.prediction = 'prediction'
report = Report(metrics=metrics, options=self.options)
report.run(
reference_data=self.ref_data,
@@ -247,10 +253,10 @@ class Reports:
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir, exist_ok=True)
print(f"Output directory: {output_dir}")
print(f"Report path: {report_path}")
print(f"Base path: {self.base_path}")
print(f"Template path: {self.template_path}")
print(f'Output directory: {output_dir}')
print(f'Report path: {report_path}')
print(f'Base path: {self.base_path}')
print(f'Template path: {self.template_path}')
# Load main HTML template
main_html_path = os.path.join(self.template_path, 'header.html')

1
model_manager/utils/models/train_model_params.py
View File

@@ -23,6 +23,7 @@ def validate_frontend_date_format(fmt: str | None) -> None:
allowed = ', '.join(sorted(ALLOWED_FRONTEND_DATE_FORMATS))
raise ValueError(f'Invalid date_format "{fmt}". Allowed formats: {allowed}')
# Model name constants
MODEL_LINEAR_REGRESSION = 'Linear Regression'
MODEL_POLYNOMIAL_REGRESSION = 'Polynomial Regression'

41
model_manager/utils/repository/data_manager_repository.py
View File

@@ -20,7 +20,6 @@ from os import makedirs, path
from shutil import rmtree
from typing import Any
from mlflow.entities import experiment
import numpy as np
import pandas as pd
from sientia_do.observability.logger import Logger
@@ -28,7 +27,7 @@ from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_model.wrappers.sientia_model import SientiaModel
from model_manager.runtime_paths import REPORTS_ROOT, PROJECT_BASE_PATH
from model_manager.runtime_paths import PROJECT_BASE_PATH, REPORTS_ROOT
from model_manager.sientia.metrics import mae, mse, r2
from model_manager.sientia.reports import Reports # type: ignore[import-untyped]
from model_manager.utils.models.train_model_params import TrainModelParams
@@ -160,7 +159,7 @@ class DataManagerRepository(SientiaMonitoring):
train_df = _ensure_date_column_parsed(train_df, params)
train_df = self._configure_datetime_index(train_df, params, metadata)
train_df = self._set_timezone_on_index(train_df, metadata)
if len(train_df) <= 0:
raise ValueError('Training data view is empty after transformation')
@@ -201,10 +200,16 @@ class DataManagerRepository(SientiaMonitoring):
metadata,
)
experiment_name = f'train_model_{params.model_type}_{params.model_name}_{params.experiment_run_id}'
experiment_name = f'{params.model_name}'
run_name = f'{experiment_name}_{datetime.now().strftime("%Y%m%d_%H%M%S")}'
return TrainModelResult(params=params, train_data=train_data, val_data=val_data, run_name=run_name, experiment_name=experiment_name)
return TrainModelResult(
params=params,
train_data=train_data,
val_data=val_data,
run_name=run_name,
experiment_name=experiment_name,
)
def _as_series(self, pred: pd.DataFrame | pd.Series) -> pd.Series:
if isinstance(pred, pd.Series):
@@ -422,7 +427,9 @@ class DataManagerRepository(SientiaMonitoring):
)
return data
def _set_timezone_on_index(self, data: pd.DataFrame, metadata: dict[str, Any] | None = None) -> pd.DataFrame:
def _set_timezone_on_index(
self, data: pd.DataFrame, metadata: dict[str, Any] | None = None
) -> pd.DataFrame:
"""
Check if the index has a timezone and if not, set it to UTC timezone.
@@ -441,8 +448,9 @@ class DataManagerRepository(SientiaMonitoring):
data.index = data.index.tz_convert('UTC')
else:
raise ValueError('Index is not a DatetimeIndex')
return data
def _get_reports_directory(self) -> str:
"""
Get the absolute path to the reports directory.
@@ -513,29 +521,20 @@ class DataManagerRepository(SientiaMonitoring):
if data.y_train_pred is None or data.y_pred is None:
raise ValueError('y_train_pred or y_pred is not set, cannot generate report')
y_train_pred = data.y_train_pred.rename(columns={data.params.target_variable: 'prediction'})
y_val_pred = data.y_pred.rename(columns={data.params.target_variable: 'prediction'})
y_train_pred = data.y_train_pred.rename(
columns={data.params.target_variable: 'prediction'}
)
y_val_pred = data.y_pred.rename(
columns={data.params.target_variable: 'prediction'}
)
# Join the predictions to the data
reference_data = y_train_pred[['prediction']].join(
data.train_data, how='inner'
)
reference_data = y_train_pred[['prediction']].join(data.train_data, how='inner')
reference_data_float = reference_data.astype(np.float64)
current_data = y_val_pred[['prediction']].join(
data.val_data, how='inner'
)
current_data = y_val_pred[['prediction']].join(data.val_data, how='inner')
current_data_float = current_data.astype(np.float64)
# Initialize report generator
base_path = self._get_reports_directory()
data.run_dir = self._create_run_directory(base_path, data.run_name)
# Template path is the code path of the model_manager package
template_path = path.join(PROJECT_BASE_PATH, 'reports')