gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1645: Add comprehensive model training observability metrics and Grafana dashboard.

Browse Source
This commit is contained in:
vitor-aignosi
2026-06-17 10:34:56 -03:00
parent 445fe643fe
commit 76f926a8ab
12 changed files with 866 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -254,4 +254,5 @@ openspec
# Training smoke test: track JSON inputs, ignore local sample CSVs
input_dataset.csv
scripts/**/*.csv
!scripts/inputs/
!scripts/inputs/
.claude/

197
dashboards/sientia-dataops-model-manager.json Normal file
View File

@@ -0,0 +1,197 @@
{
"annotations": { "list": [] },
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"panels": [
{
"id": 1,
"title": "Data Preparation Latency (p50 / p95)",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
"targets": [
{
"expr": "histogram_quantile(0.5, rate(sientia_training_data_preparation_lag_bucket[5m]))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, rate(sientia_training_data_preparation_lag_bucket[5m]))",
"legendFormat": "p95",
"refId": "B"
}
],
"fieldConfig": {
"defaults": { "unit": "s", "color": { "mode": "palette-classic" } }
},
"options": { "tooltip": { "mode": "multi" } }
},
{
"id": 2,
"title": "Model Fit Latency (p50 / p95)",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
"targets": [
{
"expr": "histogram_quantile(0.5, rate(sientia_training_model_fit_lag_bucket[5m]))",
"legendFormat": "p50",
"refId": "A"
},
{
"expr": "histogram_quantile(0.95, rate(sientia_training_model_fit_lag_bucket[5m]))",
"legendFormat": "p95",
"refId": "B"
}
],
"fieldConfig": {
"defaults": { "unit": "s", "color": { "mode": "palette-classic" } }
},
"options": { "tooltip": { "mode": "multi" } }
},
{
"id": 3,
"title": "Data Preparation Errors / s",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
"targets": [
{
"expr": "rate(sientia_training_data_preparation_error_count_total[5m])",
"legendFormat": "{{model_name}} / {{model_type}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": { "unit": "ops", "color": { "mode": "palette-classic" } }
}
},
{
"id": 4,
"title": "Model Fit Errors / s",
"type": "timeseries",
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
"targets": [
{
"expr": "rate(sientia_training_model_fit_error_count_total[5m])",
"legendFormat": "{{model_name}} / {{model_type}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": { "unit": "ops", "color": { "mode": "palette-classic" } }
}
},
{
"id": 5,
"title": "Model Quality — MSE",
"type": "stat",
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 16 },
"targets": [
{
"expr": "sientia_training_model_quality_mse",
"legendFormat": "{{model_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": { "unit": "none", "decimals": 4 }
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
},
{
"id": 6,
"title": "Model Quality — MAE",
"type": "stat",
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 16 },
"targets": [
{
"expr": "sientia_training_model_quality_mae",
"legendFormat": "{{model_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": { "unit": "none", "decimals": 4 }
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
},
{
"id": 7,
"title": "Model Quality — R²",
"type": "stat",
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 16 },
"targets": [
{
"expr": "sientia_training_model_quality_r2",
"legendFormat": "{{model_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": { "unit": "percentunit", "decimals": 3, "min": -1, "max": 1 }
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "background" }
},
{
"id": 8,
"title": "Dataset — Train Rows",
"type": "stat",
"gridPos": { "h": 4, "w": 8, "x": 0, "y": 20 },
"targets": [
{
"expr": "sientia_training_dataset_train_rows",
"legendFormat": "{{model_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": { "unit": "short", "decimals": 0 }
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
},
{
"id": 9,
"title": "Dataset — Val Rows",
"type": "stat",
"gridPos": { "h": 4, "w": 8, "x": 8, "y": 20 },
"targets": [
{
"expr": "sientia_training_dataset_val_rows",
"legendFormat": "{{model_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": { "unit": "short", "decimals": 0 }
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
},
{
"id": 10,
"title": "Dataset — Feature Count",
"type": "stat",
"gridPos": { "h": 4, "w": 8, "x": 16, "y": 20 },
"targets": [
{
"expr": "sientia_training_feature_count",
"legendFormat": "{{model_name}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": { "unit": "short", "decimals": 0 }
},
"options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
}
],
"refresh": "30s",
"schemaVersion": 38,
"tags": ["sientia", "model-manager", "training"],
"templating": { "list": [] },
"time": { "from": "now-3h", "to": "now" },
"timepicker": {},
"timezone": "browser",
"title": "Sientia DataOps Model Manager",
"uid": "sientia-dataops-model-manager",
"version": 1
}

44
model_manager/activities/cleanup.py
View File

@@ -22,7 +22,6 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from model_manager.metrics import ACTIVITY_EXECUTION_TOTAL, WORKFLOW_EXECUTION_TOTAL
from model_manager.runtime_paths import REPORTS_TEMP_DIR
RETENTION_HOURS = int(os.getenv('CLEANUP_RETENTION_HOURS', '24'))
@@ -84,7 +83,6 @@ class Cleanup(SientiaMonitoring):
"""
metadata = input_data.get('metadata', {})
temp_path = input_data.get('temp_path', REPORTS_TEMP_DIR)
metrics_status = 'success'
cutoff_time = datetime.now() - timedelta(hours=self.retention_hours)
@@ -164,7 +162,6 @@ class Cleanup(SientiaMonitoring):
)
except Exception as e:
metrics_status = 'error'
error_msg = f'Error in directory cleanup: {str(e)}'
trace = traceback.format_exc()
@@ -178,44 +175,3 @@ class Cleanup(SientiaMonitoring):
)
raise
finally:
self._emit_metrics(
metadata=metadata,
metrics_status=metrics_status,
activity_name='cleanup_temp_directories',
emit_workflow_metric=True,
)
def _emit_metrics(
self,
metadata: dict[str, Any],
metrics_status: str,
activity_name: str,
emit_workflow_metric: bool,
) -> None:
"""
Emit workflow and activity execution metrics.
Args:
metadata: Activity metadata containing pod_id and workflow_name
metrics_status: Execution status ('success' or 'error')
activity_name: Name of the activity being executed
"""
if emit_workflow_metric:
self.emit_metric_sync(
metric_object=WORKFLOW_EXECUTION_TOTAL,
tags={
'pod_id': metadata.get('pod_id'),
'workflow_name': metadata.get('workflow_name'),
'status': metrics_status,
},
)
self.emit_metric_sync(
metric_object=ACTIVITY_EXECUTION_TOTAL,
tags={
'pod_id': metadata.get('pod_id'),
'activity_name': activity_name,
'status': metrics_status,
},
)

188
model_manager/activities/training.py
View File

@@ -10,6 +10,8 @@ from sientia_model.wrappers.sientia_model import SientiaModel
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import os
import time
import traceback
from typing import Any
@@ -23,6 +25,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
from sientia_model.model_repository.plugin_store import PluginStore
from model_manager import metrics as mm_metrics
from model_manager.utils.models.train_model_params import TrainModelParams
from model_manager.utils.models.train_model_result import TrainModelResult
from model_manager.utils.repository.data_manager_repository import DataManagerRepository
@@ -184,12 +187,12 @@ class Training(SientiaMonitoring):
"""
metadata = input_data.get('metadata')
train_params = TrainModelParams.from_dict(input_data['train_params'])
labels = self._get_training_labels(train_params)
self.info('Starting train_model process', metadata)
try:
# Download training file bytes from MinIO
self.info(
f'Downloading training file from MinIO for {train_params.file_name}', metadata
)
@@ -211,11 +214,16 @@ class Training(SientiaMonitoring):
)
self.info(f'Preparing training data for {train_params.file_name}', metadata)
train_result = self.data_manager_repository.prepare_training_data(
train_file_bytes=train_bytes,
validation_file_bytes=val_bytes,
params=train_params,
metadata=metadata,
train_result = self._prepare_data(train_bytes, val_bytes, train_params, metadata)
mm_metrics.SIENTIA_TRAINING_DATASET_TRAIN_ROWS.labels(**labels).set(
len(train_result.train_data)
)
mm_metrics.SIENTIA_TRAINING_DATASET_VAL_ROWS.labels(**labels).set(
len(train_result.val_data)
)
mm_metrics.SIENTIA_TRAINING_FEATURE_COUNT.labels(**labels).set(
len(train_params.variable_columns)
)
self.info(f'Getting model wrapper for {train_params.model_type}', metadata)
@@ -232,15 +240,121 @@ class Training(SientiaMonitoring):
wrapper.logger = self.logger.base_logger
self.info(f'Training model for {train_params.model_type}', metadata)
train_data = train_result.train_data
val_data = train_result.val_data
train_result = self._fit_model(wrapper, train_result, train_params, metadata)
self.debug(
f'train_model prepared data (head 10):\ntrain:\n{train_data.head(10).to_string()}'
f'\nval:\n{val_data.head(10).to_string()}',
metadata,
self.info(f'Computing regression metrics for {train_params.model_type}', metadata)
train_result = self.data_manager_repository.compute_regression_metrics(
train_result,
wrapper,
metadata=metadata,
)
if train_result.mse_val is not None:
mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels(**labels).set(
train_result.mse_val
)
if train_result.mae_val is not None:
mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels(**labels).set(
train_result.mae_val
)
if train_result.r2_val is not None:
mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels(**labels).set(
train_result.r2_val
)
self.info(f'Starting MLflow run for {train_params.model_type}', metadata)
with self.mlflow_repository.start_run(
model_name=train_params.model_name,
run_name=train_result.run_name,
experiment_name=train_result.experiment_name,
tags=None,
metadata=metadata,
) as run_info:
train_result.run_id = run_info.run_id
self._persist_training_artifacts(train_result, train_params, wrapper, metadata)
self.emit_metric_sync(
metric_object=mm_metrics.SIENTIA_TRAINING_MODEL_TRAINED_TOTAL,
tags=labels,
)
return {
'run_name': train_result.run_name,
'experiment_name': train_result.experiment_name,
'run_id': train_result.run_id,
'run_dir': train_result.run_dir,
}
except Exception as e: # noqa: BLE001
error_msg = f'Error training model - error: {str(e)}'
trace = traceback.format_exc()
self.send_notification(
metadata=metadata or {},
notification_id='TRAIN_MODEL_ERROR',
message=error_msg,
block='train_model',
level=NotificationLevel.ERROR,
attachment_content=trace,
)
raise e
def _get_training_labels(self, train_params: TrainModelParams) -> dict:
return {
'pod_id': os.getenv('POD_ID'),
'model_name': train_params.model_name,
'model_type': train_params.model_type,
}
def _prepare_data(
self,
train_bytes: bytes,
val_bytes: bytes | None,
train_params: TrainModelParams,
metadata: dict | None,
) -> TrainModelResult:
labels = self._get_training_labels(train_params)
start_time = time.monotonic()
try:
return self.data_manager_repository.prepare_training_data(
train_file_bytes=train_bytes,
validation_file_bytes=val_bytes,
params=train_params,
metadata=metadata,
)
except Exception:
self.emit_metric_sync(
metric_object=mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL,
tags=labels,
)
raise
finally:
self.observe_lag_sync(
start_time,
mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_LAG,
labels,
)
def _fit_model(
self,
wrapper: Any,
train_result: TrainModelResult,
train_params: TrainModelParams,
metadata: dict | None,
) -> TrainModelResult:
labels = self._get_training_labels(train_params)
train_data = train_result.train_data
val_data = train_result.val_data
self.debug(
f'train_model prepared data (head 10):\ntrain:\n{train_data.head(10).to_string()}'
f'\nval:\n{val_data.head(10).to_string()}',
metadata,
)
start_time = time.monotonic()
try:
wrapper.train(
train_data=train_data,
val_data=val_data,
@@ -251,7 +365,6 @@ class Training(SientiaMonitoring):
f'Generating predictions using the trained wrapper for {train_params.model_type}',
metadata,
)
# Generate predictions using the trained wrapper
transformed_train, _ = wrapper.transform(train_data)
transformed_val, _ = wrapper.transform(val_data)
@@ -275,47 +388,20 @@ class Training(SientiaMonitoring):
train_result.y_train_pred = y_train_pred_df
train_result.y_pred = y_val_pred_df
self.info(f'Computing regression metrics for {train_params.model_type}', metadata)
train_result = self.data_manager_repository.compute_regression_metrics(
train_result,
wrapper,
metadata=metadata,
return train_result
except Exception:
self.emit_metric_sync(
metric_object=mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL,
tags=labels,
)
self.info(f'Starting MLflow run for {train_params.model_type}', metadata)
with self.mlflow_repository.start_run(
model_name=train_params.model_name,
run_name=train_result.run_name,
experiment_name=train_result.experiment_name,
tags=None,
metadata=metadata,
) as run_info:
train_result.run_id = run_info.run_id
self._persist_training_artifacts(train_result, train_params, wrapper, metadata)
return {
'run_name': train_result.run_name,
'experiment_name': train_result.experiment_name,
'run_id': train_result.run_id,
'run_dir': train_result.run_dir,
}
except Exception as e: # noqa: BLE001
error_msg = f'Error training model - error: {str(e)}'
trace = traceback.format_exc()
self.send_notification(
metadata=metadata or {},
notification_id='TRAIN_MODEL_ERROR',
message=error_msg,
block='train_model',
level=NotificationLevel.ERROR,
attachment_content=trace,
raise
finally:
self.observe_lag_sync(
start_time,
mm_metrics.SIENTIA_TRAINING_MODEL_FIT_LAG,
labels,
)
raise e
def _persist_training_artifacts(
self,
train_result: TrainModelResult,

76
model_manager/metrics.py
View File

@@ -16,23 +16,79 @@ Metric Labels:
- pod_id: Kubernetes pod identifier for multi-instance deployments
"""
from prometheus_client import Counter, Gauge
from prometheus_client import Counter, Gauge, Histogram
# Application health metric
APP_UP = Gauge(
'app_up',
'sientia_app_up',
'Indicates if the application is running (1) or shutting down (0)',
['pod_id'],
)
WORKFLOW_EXECUTION_TOTAL = Counter(
'model_manager_workflow_executions_total',
'Total number of workflow executions',
['pod_id', 'workflow_name', 'status'], # status: success, error
_TRAINING_LABELS = ['pod_id', 'model_name', 'model_type']
SIENTIA_TRAINING_MODEL_TRAINED_TOTAL = Counter(
'sientia_training_model_trained_total',
'Number of successfully completed model training runs',
_TRAINING_LABELS,
)
ACTIVITY_EXECUTION_TOTAL = Counter(
'model_manager_activity_executions_total',
'Total number of activity executions',
['pod_id', 'activity_name', 'status'], # status: success, error
SIENTIA_TRAINING_DATA_PREPARATION_LAG = Histogram(
'sientia_training_data_preparation_lag',
'Latency of prepare_training_data() in seconds',
_TRAINING_LABELS,
)
SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL = Counter(
'sientia_training_data_preparation_error_count_total',
'Number of failures in prepare_training_data()',
_TRAINING_LABELS,
)
SIENTIA_TRAINING_MODEL_FIT_LAG = Histogram(
'sientia_training_model_fit_lag',
'Latency of wrapper.train() (model fitting) in seconds',
_TRAINING_LABELS,
)
SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL = Counter(
'sientia_training_model_fit_error_count_total',
'Number of failures in wrapper.train() (model fitting)',
_TRAINING_LABELS,
)
SIENTIA_TRAINING_MODEL_QUALITY_MSE = Gauge(
'sientia_training_model_quality_mse',
'Mean Squared Error of the last successful model training',
_TRAINING_LABELS,
)
SIENTIA_TRAINING_MODEL_QUALITY_MAE = Gauge(
'sientia_training_model_quality_mae',
'Mean Absolute Error of the last successful model training',
_TRAINING_LABELS,
)
SIENTIA_TRAINING_MODEL_QUALITY_R2 = Gauge(
'sientia_training_model_quality_r2',
'R-squared of the last successful model training',
_TRAINING_LABELS,
)
SIENTIA_TRAINING_DATASET_TRAIN_ROWS = Gauge(
'sientia_training_dataset_train_rows',
'Number of rows in the training dataset after preparation',
_TRAINING_LABELS,
)
SIENTIA_TRAINING_DATASET_VAL_ROWS = Gauge(
'sientia_training_dataset_val_rows',
'Number of rows in the validation dataset after preparation',
_TRAINING_LABELS,
)
SIENTIA_TRAINING_FEATURE_COUNT = Gauge(
'sientia_training_feature_count',
'Number of input feature columns used for training',
_TRAINING_LABELS,
)

49
scripts/inputs/sin-approx.json Normal file
View File

@@ -0,0 +1,49 @@
{
"experiment": {
"experiment_run_id": 2000,
"experiment_name": "experiment-sin-approx",
"run_name": "run-sin-approx",
"username": "vitor.santos@aignosi.com.br",
"status": "ORCHESTRATOR_WAITING_PROC"
},
"minio": {
"mc_alias": "open-suse",
"bucket_name": "model-training",
"file_name": "training-sin-approx-dataset-1001.csv",
"local_csv": "data-1780946658143-pivot.csv"
},
"temporal": {
"task_queue": "train_model-basic-queue",
"workflow_name": "train_model",
"execution_timeout_minutes": 5,
"run_timeout_minutes": 5,
"task_timeout_minutes": 5
},
"payload": {
"experiment_run_id": 2000,
"variable_columns": ["SourceTri"],
"target_variable": "TargetSin",
"line_separator": ",",
"decimal_separator": ".",
"train_size": 80,
"shuffle": true,
"random_state": 42,
"model_name": "sin-approx",
"model_type": "linear_regression",
"model_id": 1,
"data_model_kwargs": {},
"model_kwargs": {},
"opt_params": {},
"date_column": "timestamp"
},
"db_only": {
"model_metadata": {
"schemas": {
"components": {
"schemas": {}
}
}
}
}
}

5
scripts/run_training_test.py
View File

@@ -42,6 +42,10 @@ TRAINING_TEST_INPUTS: list[str] = [
'scripts/inputs/xgboost.json',
]
TRAINING_TEST_INPUTS: list[str] = [
'scripts/inputs/sin-approx.json',
]
_REQUIRED_INPUT_KEYS = ('experiment', 'minio', 'temporal', 'payload', 'db_only')
@@ -260,6 +264,7 @@ for exp in experiments:
[
'mc',
'cp',
'--insecure',
str(local_csv),
f'{exp_minio["mc_alias"]}/{exp_minio["bucket_name"]}/{exp_minio["file_name"]}',
],

24
t.py Normal file
View File

@@ -0,0 +1,24 @@
# %%
from pandas import read_csv, to_datetime
df = read_csv('data-1780946658143.csv')
# %%
df.head()
# %%
# normalize timestamp to naive "yyyy-MM-dd HH:mm:ss" (drop the "+00" UTC offset)
df['timestamp'] = to_datetime(df['timestamp'], utc=True).dt.tz_localize(None)
df['timestamp'] = df['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')
# %%
# pivot the dataframe
df = df.pivot(index='timestamp', columns='variable', values='value')
# %%
df.head()
# %%
df.to_csv('data-1780946658143-pivot.csv')
# %%

68
tests/activities/test_cleanup.py
View File

@@ -125,13 +125,11 @@ def test_cleanup_temp_directories_nonexistent_path(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.warning = MagicMock()
cleanup.cleanup_temp_directories({'temp_path': '/nonexistent/path', 'metadata': {}})
cleanup.warning.assert_called_once()
cleanup._emit_metrics.assert_called_once()
@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
@@ -152,8 +150,6 @@ def test_cleanup_temp_directories_success_with_deletions(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
old_dir = os.path.join(temp_dir, f'old_dir_{old_time}')
os.makedirs(old_dir)
@@ -166,7 +162,6 @@ def test_cleanup_temp_directories_success_with_deletions(
assert not os.path.exists(old_dir)
assert os.path.exists(recent_dir)
cleanup._emit_metrics.assert_called_once()
@patch.dict(os.environ, {'CLEANUP_DRY_RUN': 'true'})
@@ -187,8 +182,6 @@ def test_cleanup_temp_directories_dry_run(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
old_dir = os.path.join(temp_dir, f'old_dir_{old_time}')
os.makedirs(old_dir)
@@ -196,7 +189,6 @@ def test_cleanup_temp_directories_dry_run(
cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
assert os.path.exists(old_dir)
cleanup._emit_metrics.assert_called_once()
@patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
@@ -217,7 +209,6 @@ def test_cleanup_temp_directories_delete_error(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.error = MagicMock()
old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
@@ -228,35 +219,9 @@ def test_cleanup_temp_directories_delete_error(
cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
cleanup.error.assert_called_once()
cleanup._emit_metrics.assert_called_once()
# --- Metrics and Utility Tests ---
def test_emit_metrics(
mock_logger,
mock_notification_handler,
mock_metrics_controller,
):
"""Test that _emit_metrics calls the public emit_metric method."""
from model_manager.activities.cleanup import Cleanup
cleanup = Cleanup(
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup.emit_metric_sync = MagicMock()
cleanup._emit_metrics(
metadata={'pod_id': 'p1', 'workflow_name': 'wf1'},
metrics_status='success',
activity_name='test_activity',
emit_workflow_metric=True,
)
assert cleanup.emit_metric_sync.call_count == 2
# --- Utility Tests ---
def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
@@ -276,7 +241,6 @@ def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.debug = MagicMock()
# Create a file and a directory with a non-matching name
@@ -290,7 +254,6 @@ def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
cleanup.debug.assert_called_with(
'Skipping directory without timestamp pattern: a_directory_with_no_timestamp', {}
)
cleanup._emit_metrics.assert_called_once()
def test_cleanup_temp_directories_invalid_timestamp_format(
@@ -310,7 +273,6 @@ def test_cleanup_temp_directories_invalid_timestamp_format(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.error = MagicMock()
# Create a directory with a malformed timestamp that matches the regex but fails parsing
@@ -320,7 +282,6 @@ def test_cleanup_temp_directories_invalid_timestamp_format(
cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
cleanup.error.assert_called_once()
cleanup._emit_metrics.assert_called_once()
def test_cleanup_temp_directories_generic_exception(
@@ -340,7 +301,6 @@ def test_cleanup_temp_directories_generic_exception(
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup._emit_metrics = MagicMock() # type: ignore[method-assign]
cleanup.send_notification = MagicMock()
with patch('os.listdir', side_effect=Exception('Unexpected OS Error')):
@@ -348,29 +308,3 @@ def test_cleanup_temp_directories_generic_exception(
cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
cleanup.send_notification.assert_called_once()
cleanup._emit_metrics.assert_called_once()
def test_emit_metrics_activity_only(
mock_logger,
mock_notification_handler,
mock_metrics_controller,
):
"""Test that _emit_metrics can emit only the activity metric."""
from model_manager.activities.cleanup import Cleanup
cleanup = Cleanup(
logger=mock_logger,
notification_handler=mock_notification_handler,
metrics_controller=mock_metrics_controller,
)
cleanup.emit_metric_sync = MagicMock()
cleanup._emit_metrics(
metadata={'pod_id': 'p1', 'workflow_name': 'wf1'},
metrics_status='success',
activity_name='test_activity',
emit_workflow_metric=False,
)
cleanup.emit_metric_sync.assert_called_once()

261
tests/activities/test_training.py
View File

@@ -330,6 +330,267 @@ def test_train_model_downloads_validation_file_when_set(mock_mlflow, training):
mock_mlflow.log_artifact.assert_called()
def test_prepare_data_observes_lag_on_success(training):
tp = TrainModelParams.from_dict(
{**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
)
train_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr)
training.observe_lag_sync = MagicMock()
training.emit_metric_sync = MagicMock()
from model_manager import metrics as mm_metrics
training._prepare_data(b'csv', None, tp, {})
training.observe_lag_sync.assert_called_once()
call_args = training.observe_lag_sync.call_args
assert call_args.args[1] is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_LAG
training.emit_metric_sync.assert_not_called()
def test_prepare_data_increments_error_counter_and_still_observes_lag_on_failure(training):
tp = TrainModelParams.from_dict(
{**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
)
training.data_manager_repository.prepare_training_data = MagicMock(
side_effect=RuntimeError('prep-fail')
)
training.observe_lag_sync = MagicMock()
training.emit_metric_sync = MagicMock()
from model_manager import metrics as mm_metrics
with pytest.raises(RuntimeError, match='prep-fail'):
training._prepare_data(b'csv', None, tp, {})
training.observe_lag_sync.assert_called_once()
training.emit_metric_sync.assert_called_once()
call_args = training.emit_metric_sync.call_args
assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL
def test_fit_model_observes_lag_on_success(training):
tp = TrainModelParams.from_dict(
{**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
)
train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]})
val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
wrapper = MagicMock()
wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)])
wrapper.predict = MagicMock(
side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)]
)
training.observe_lag_sync = MagicMock()
training.emit_metric_sync = MagicMock()
from model_manager import metrics as mm_metrics
training._fit_model(wrapper, tmr, tp, {})
training.observe_lag_sync.assert_called_once()
call_args = training.observe_lag_sync.call_args
assert call_args.args[1] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_LAG
training.emit_metric_sync.assert_not_called()
def test_fit_model_increments_error_counter_on_failure(training):
tp = TrainModelParams.from_dict(
{**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
)
train_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
wrapper = MagicMock()
wrapper.train = MagicMock(side_effect=RuntimeError('fit-fail'))
training.observe_lag_sync = MagicMock()
training.emit_metric_sync = MagicMock()
from model_manager import metrics as mm_metrics
with pytest.raises(RuntimeError, match='fit-fail'):
training._fit_model(wrapper, tmr, tp, {})
training.observe_lag_sync.assert_called_once()
training.emit_metric_sync.assert_called_once()
call_args = training.emit_metric_sync.call_args
assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL
@patch('model_manager.activities.training.mm_metrics')
@patch('model_manager.activities.training.mlflow')
def test_train_model_sets_quality_gauges_after_compute_metrics(mock_mlflow, mock_mm_metrics, training):
tp = TrainModelParams.from_dict(
{**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
)
train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]})
val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
training.minio_repository.download_file = MagicMock(return_value=b'csv')
training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr)
def _set_metrics(x, _w, **_kw):
x.mse_val = 0.5
x.mae_val = 0.3
x.r2_val = -0.1
return x
training.data_manager_repository.compute_regression_metrics = MagicMock(
side_effect=_set_metrics
)
def _fill_report(x, **_kw):
x.report_path = '/tmp/r.html'
x.train_data_path = '/tmp/tr.csv'
x.test_data_path = '/tmp/te.csv'
x.run_dir = '/tmp/run'
return x
training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report)
wrapper = MagicMock()
wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)])
wrapper.predict = MagicMock(
side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)]
)
wrapper.store_model = MagicMock()
training.plugin_store.get_model = MagicMock(return_value=wrapper)
@contextmanager
def _run_ctx(*_a, **_k):
info = MagicMock()
info.run_id = 'rid'
yield info
training.mlflow_repository.start_run = _run_ctx
training.observe_lag_sync = MagicMock()
training.emit_metric_sync = MagicMock()
training.train_model({'metadata': {}, 'train_params': tp.to_dict()})
mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_called_once_with(0.5)
mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_called_once_with(0.3)
mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_called_once_with(-0.1)
@patch('model_manager.activities.training.mm_metrics')
@patch('model_manager.activities.training.mlflow')
def test_train_model_skips_quality_gauges_when_none(_mock_mlflow, mock_mm_metrics, training):
tp = TrainModelParams.from_dict(
{**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
)
train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]})
val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
training.minio_repository.download_file = MagicMock(return_value=b'csv')
training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr)
training.data_manager_repository.compute_regression_metrics = MagicMock(
side_effect=lambda x, _w, **_kw: x
)
def _fill_report(x, **_kw):
x.report_path = '/tmp/r.html'
x.train_data_path = '/tmp/tr.csv'
x.test_data_path = '/tmp/te.csv'
x.run_dir = '/tmp/run'
return x
training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report)
wrapper = MagicMock()
wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)])
wrapper.predict = MagicMock(
side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)]
)
wrapper.store_model = MagicMock()
training.plugin_store.get_model = MagicMock(return_value=wrapper)
@contextmanager
def _run_ctx(*_a, **_k):
info = MagicMock()
info.run_id = 'rid'
yield info
training.mlflow_repository.start_run = _run_ctx
training.observe_lag_sync = MagicMock()
training.emit_metric_sync = MagicMock()
training.train_model({'metadata': {}, 'train_params': tp.to_dict()})
mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_not_called()
mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_not_called()
mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_not_called()
@patch('model_manager.activities.training.mm_metrics')
@patch('model_manager.activities.training.mlflow')
def test_train_model_increments_trained_total_on_success(_mock_mlflow, mock_mm_metrics, training):
tp = TrainModelParams.from_dict(
{**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
)
train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]})
val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
training.minio_repository.download_file = MagicMock(return_value=b'csv')
training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr)
training.data_manager_repository.compute_regression_metrics = MagicMock(
side_effect=lambda x, _w, **_kw: x
)
def _fill_report(x, **_kw):
x.report_path = '/tmp/r.html'
x.train_data_path = '/tmp/tr.csv'
x.test_data_path = '/tmp/te.csv'
x.run_dir = '/tmp/run'
return x
training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report)
wrapper = MagicMock()
wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)])
wrapper.predict = MagicMock(
side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)]
)
wrapper.store_model = MagicMock()
training.plugin_store.get_model = MagicMock(return_value=wrapper)
@contextmanager
def _run_ctx(*_a, **_k):
info = MagicMock()
info.run_id = 'rid'
yield info
training.mlflow_repository.start_run = _run_ctx
training.observe_lag_sync = MagicMock()
training.emit_metric_sync = MagicMock()
training.train_model({'metadata': {}, 'train_params': tp.to_dict()})
training.emit_metric_sync.assert_called_once_with(
metric_object=mock_mm_metrics.SIENTIA_TRAINING_MODEL_TRAINED_TOTAL,
tags=training._get_training_labels(tp),
)
def test_train_model_does_not_increment_trained_total_on_failure(training):
tp = TrainModelParams.from_dict(
{**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
)
training.minio_repository.download_file = MagicMock(side_effect=RuntimeError('dl-fail'))
training.send_notification = MagicMock()
training.emit_metric_sync = MagicMock()
with pytest.raises(RuntimeError):
training.train_model({'metadata': {}, 'train_params': tp.to_dict()})
training.emit_metric_sync.assert_not_called()
def test_train_model_value_error_when_paths_missing_after_report(training):
"""Raises ValueError when report paths are not populated after generate_report."""
tp = TrainModelParams.from_dict(

124
tests/test_metrics.py
View File

@@ -10,7 +10,7 @@ def test_app_up_metric_exists():
from model_manager.metrics import APP_UP
assert APP_UP is not None
assert APP_UP._name == 'app_up'
assert APP_UP._name == 'sientia_app_up'
assert (
APP_UP._documentation == 'Indicates if the application is running (1) or shutting down (0)'
)
@@ -207,3 +207,125 @@ def test_prometheus_client_gauge_import():
from model_manager.metrics import Gauge
assert Gauge is PrometheusGauge
# ---------------------------------------------------------------------------
# Training metrics — existence, type, and labels
# ---------------------------------------------------------------------------
_TRAINING_LABEL_NAMES = ('pod_id', 'model_name', 'model_type')
def _assert_training_labels(metric):
for label in _TRAINING_LABEL_NAMES:
assert label in metric._labelnames
def test_sientia_training_data_preparation_lag_is_histogram():
from prometheus_client import Histogram
from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_LAG
assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_LAG, Histogram)
assert SIENTIA_TRAINING_DATA_PREPARATION_LAG._name == 'sientia_training_data_preparation_lag'
_assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_LAG)
def test_sientia_training_data_preparation_error_count_total_is_counter():
from prometheus_client import Counter
from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL
assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, Counter)
assert 'sientia_training_data_preparation_error_count' in SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL._name
_assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL)
def test_sientia_training_model_fit_lag_is_histogram():
from prometheus_client import Histogram
from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_LAG
assert isinstance(SIENTIA_TRAINING_MODEL_FIT_LAG, Histogram)
assert SIENTIA_TRAINING_MODEL_FIT_LAG._name == 'sientia_training_model_fit_lag'
_assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_LAG)
def test_sientia_training_model_fit_error_count_total_is_counter():
from prometheus_client import Counter
from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL
assert isinstance(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, Counter)
assert 'sientia_training_model_fit_error_count' in SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL._name
_assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL)
def test_sientia_training_model_quality_mse_is_gauge():
from prometheus_client import Gauge
from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MSE
assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MSE, Gauge)
assert SIENTIA_TRAINING_MODEL_QUALITY_MSE._name == 'sientia_training_model_quality_mse'
_assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MSE)
def test_sientia_training_model_quality_mae_is_gauge():
from prometheus_client import Gauge
from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MAE
assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MAE, Gauge)
assert SIENTIA_TRAINING_MODEL_QUALITY_MAE._name == 'sientia_training_model_quality_mae'
_assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MAE)
def test_sientia_training_model_quality_r2_is_gauge():
from prometheus_client import Gauge
from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_R2
assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_R2, Gauge)
assert SIENTIA_TRAINING_MODEL_QUALITY_R2._name == 'sientia_training_model_quality_r2'
_assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_R2)
def test_sientia_training_dataset_train_rows_is_gauge():
from prometheus_client import Gauge
from model_manager.metrics import SIENTIA_TRAINING_DATASET_TRAIN_ROWS
assert isinstance(SIENTIA_TRAINING_DATASET_TRAIN_ROWS, Gauge)
assert SIENTIA_TRAINING_DATASET_TRAIN_ROWS._name == 'sientia_training_dataset_train_rows'
_assert_training_labels(SIENTIA_TRAINING_DATASET_TRAIN_ROWS)
def test_sientia_training_dataset_val_rows_is_gauge():
from prometheus_client import Gauge
from model_manager.metrics import SIENTIA_TRAINING_DATASET_VAL_ROWS
assert isinstance(SIENTIA_TRAINING_DATASET_VAL_ROWS, Gauge)
assert SIENTIA_TRAINING_DATASET_VAL_ROWS._name == 'sientia_training_dataset_val_rows'
_assert_training_labels(SIENTIA_TRAINING_DATASET_VAL_ROWS)
def test_sientia_training_model_trained_total_is_counter():
from prometheus_client import Counter
from model_manager.metrics import SIENTIA_TRAINING_MODEL_TRAINED_TOTAL
assert isinstance(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, Counter)
assert 'sientia_training_model_trained' in SIENTIA_TRAINING_MODEL_TRAINED_TOTAL._name
_assert_training_labels(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL)
def test_sientia_training_feature_count_is_gauge():
from prometheus_client import Gauge
from model_manager.metrics import SIENTIA_TRAINING_FEATURE_COUNT
assert isinstance(SIENTIA_TRAINING_FEATURE_COUNT, Gauge)
assert SIENTIA_TRAINING_FEATURE_COUNT._name == 'sientia_training_feature_count'
_assert_training_labels(SIENTIA_TRAINING_FEATURE_COUNT)

2
values.yaml
View File

@@ -328,7 +328,7 @@ ssh:
# Configuração para dashboards do Grafana
grafanaDashboard:
# Habilita a criação de ConfigMaps para dashboards
enabled: false
enabled: true
# Namespace onde o Grafana está instalado (ajuste conforme seu ambiente)
namespace: monitoring
# Labels para que o sidecar do Grafana encontre os dashboards