From 76f926a8ab69e455d7175058c8dfaf3df63b7596 Mon Sep 17 00:00:00 2001 From: vitor-aignosi Date: Wed, 17 Jun 2026 10:34:56 -0300 Subject: [PATCH] SIENTIAPDE-1645: Add comprehensive model training observability metrics and Grafana dashboard. --- .gitignore | 3 +- dashboards/sientia-dataops-model-manager.json | 197 +++++++++++++ model_manager/activities/cleanup.py | 44 --- model_manager/activities/training.py | 188 +++++++++---- model_manager/metrics.py | 76 ++++- scripts/inputs/sin-approx.json | 49 ++++ scripts/run_training_test.py | 5 + t.py | 24 ++ tests/activities/test_cleanup.py | 68 +---- tests/activities/test_training.py | 261 ++++++++++++++++++ tests/test_metrics.py | 124 ++++++++- values.yaml | 2 +- 12 files changed, 866 insertions(+), 175 deletions(-) create mode 100644 dashboards/sientia-dataops-model-manager.json create mode 100644 scripts/inputs/sin-approx.json create mode 100644 t.py diff --git a/.gitignore b/.gitignore index 2b0edd4..683505a 100644 --- a/.gitignore +++ b/.gitignore @@ -254,4 +254,5 @@ openspec # Training smoke test: track JSON inputs, ignore local sample CSVs input_dataset.csv scripts/**/*.csv -!scripts/inputs/ \ No newline at end of file +!scripts/inputs/ +.claude/ \ No newline at end of file diff --git a/dashboards/sientia-dataops-model-manager.json b/dashboards/sientia-dataops-model-manager.json new file mode 100644 index 0000000..0b1ab53 --- /dev/null +++ b/dashboards/sientia-dataops-model-manager.json @@ -0,0 +1,197 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "id": 1, + "title": "Data Preparation Latency (p50 / p95)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(sientia_training_data_preparation_lag_bucket[5m]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(sientia_training_data_preparation_lag_bucket[5m]))", + "legendFormat": "p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { "unit": "s", "color": { "mode": "palette-classic" } } + }, + "options": { "tooltip": { "mode": "multi" } } + }, + { + "id": 2, + "title": "Model Fit Latency (p50 / p95)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "histogram_quantile(0.5, rate(sientia_training_model_fit_lag_bucket[5m]))", + "legendFormat": "p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(sientia_training_model_fit_lag_bucket[5m]))", + "legendFormat": "p95", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { "unit": "s", "color": { "mode": "palette-classic" } } + }, + "options": { "tooltip": { "mode": "multi" } } + }, + { + "id": 3, + "title": "Data Preparation Errors / s", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "rate(sientia_training_data_preparation_error_count_total[5m])", + "legendFormat": "{{model_name}} / {{model_type}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "ops", "color": { "mode": "palette-classic" } } + } + }, + { + "id": 4, + "title": "Model Fit Errors / s", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "rate(sientia_training_model_fit_error_count_total[5m])", + "legendFormat": "{{model_name}} / {{model_type}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "ops", "color": { "mode": "palette-classic" } } + } + }, + { + "id": 5, + "title": "Model Quality — MSE", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 16 }, + "targets": [ + { + "expr": "sientia_training_model_quality_mse", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "decimals": 4 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + }, + { + "id": 6, + "title": "Model Quality — MAE", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 16 }, + "targets": [ + { + "expr": "sientia_training_model_quality_mae", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "none", "decimals": 4 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + }, + { + "id": 7, + "title": "Model Quality — R²", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 16 }, + "targets": [ + { + "expr": "sientia_training_model_quality_r2", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "percentunit", "decimals": 3, "min": -1, "max": 1 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "background" } + }, + { + "id": 8, + "title": "Dataset — Train Rows", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 20 }, + "targets": [ + { + "expr": "sientia_training_dataset_train_rows", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "short", "decimals": 0 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + }, + { + "id": 9, + "title": "Dataset — Val Rows", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 20 }, + "targets": [ + { + "expr": "sientia_training_dataset_val_rows", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "short", "decimals": 0 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + }, + { + "id": 10, + "title": "Dataset — Feature Count", + "type": "stat", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 20 }, + "targets": [ + { + "expr": "sientia_training_feature_count", + "legendFormat": "{{model_name}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { "unit": "short", "decimals": 0 } + }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" } + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["sientia", "model-manager", "training"], + "templating": { "list": [] }, + "time": { "from": "now-3h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Sientia DataOps Model Manager", + "uid": "sientia-dataops-model-manager", + "version": 1 +} diff --git a/model_manager/activities/cleanup.py b/model_manager/activities/cleanup.py index 633a774..8b53012 100644 --- a/model_manager/activities/cleanup.py +++ b/model_manager/activities/cleanup.py @@ -22,7 +22,6 @@ with workflow.unsafe.imports_passed_through(): from sientia_do.observability.metrics_controller import MetricsController from sientia_do.observability.sientia_monitoring import SientiaMonitoring - from model_manager.metrics import ACTIVITY_EXECUTION_TOTAL, WORKFLOW_EXECUTION_TOTAL from model_manager.runtime_paths import REPORTS_TEMP_DIR RETENTION_HOURS = int(os.getenv('CLEANUP_RETENTION_HOURS', '24')) @@ -84,7 +83,6 @@ class Cleanup(SientiaMonitoring): """ metadata = input_data.get('metadata', {}) temp_path = input_data.get('temp_path', REPORTS_TEMP_DIR) - metrics_status = 'success' cutoff_time = datetime.now() - timedelta(hours=self.retention_hours) @@ -164,7 +162,6 @@ class Cleanup(SientiaMonitoring): ) except Exception as e: - metrics_status = 'error' error_msg = f'Error in directory cleanup: {str(e)}' trace = traceback.format_exc() @@ -178,44 +175,3 @@ class Cleanup(SientiaMonitoring): ) raise - finally: - self._emit_metrics( - metadata=metadata, - metrics_status=metrics_status, - activity_name='cleanup_temp_directories', - emit_workflow_metric=True, - ) - - def _emit_metrics( - self, - metadata: dict[str, Any], - metrics_status: str, - activity_name: str, - emit_workflow_metric: bool, - ) -> None: - """ - Emit workflow and activity execution metrics. - - Args: - metadata: Activity metadata containing pod_id and workflow_name - metrics_status: Execution status ('success' or 'error') - activity_name: Name of the activity being executed - """ - if emit_workflow_metric: - self.emit_metric_sync( - metric_object=WORKFLOW_EXECUTION_TOTAL, - tags={ - 'pod_id': metadata.get('pod_id'), - 'workflow_name': metadata.get('workflow_name'), - 'status': metrics_status, - }, - ) - - self.emit_metric_sync( - metric_object=ACTIVITY_EXECUTION_TOTAL, - tags={ - 'pod_id': metadata.get('pod_id'), - 'activity_name': activity_name, - 'status': metrics_status, - }, - ) diff --git a/model_manager/activities/training.py b/model_manager/activities/training.py index 80a56ec..2920eeb 100644 --- a/model_manager/activities/training.py +++ b/model_manager/activities/training.py @@ -10,6 +10,8 @@ from sientia_model.wrappers.sientia_model import SientiaModel from temporalio import activity, workflow with workflow.unsafe.imports_passed_through(): + import os + import time import traceback from typing import Any @@ -23,6 +25,7 @@ with workflow.unsafe.imports_passed_through(): from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository from sientia_model.model_repository.plugin_store import PluginStore + from model_manager import metrics as mm_metrics from model_manager.utils.models.train_model_params import TrainModelParams from model_manager.utils.models.train_model_result import TrainModelResult from model_manager.utils.repository.data_manager_repository import DataManagerRepository @@ -184,12 +187,12 @@ class Training(SientiaMonitoring): """ metadata = input_data.get('metadata') train_params = TrainModelParams.from_dict(input_data['train_params']) + labels = self._get_training_labels(train_params) self.info('Starting train_model process', metadata) try: # Download training file bytes from MinIO - self.info( f'Downloading training file from MinIO for {train_params.file_name}', metadata ) @@ -211,11 +214,16 @@ class Training(SientiaMonitoring): ) self.info(f'Preparing training data for {train_params.file_name}', metadata) - train_result = self.data_manager_repository.prepare_training_data( - train_file_bytes=train_bytes, - validation_file_bytes=val_bytes, - params=train_params, - metadata=metadata, + train_result = self._prepare_data(train_bytes, val_bytes, train_params, metadata) + + mm_metrics.SIENTIA_TRAINING_DATASET_TRAIN_ROWS.labels(**labels).set( + len(train_result.train_data) + ) + mm_metrics.SIENTIA_TRAINING_DATASET_VAL_ROWS.labels(**labels).set( + len(train_result.val_data) + ) + mm_metrics.SIENTIA_TRAINING_FEATURE_COUNT.labels(**labels).set( + len(train_params.variable_columns) ) self.info(f'Getting model wrapper for {train_params.model_type}', metadata) @@ -232,15 +240,121 @@ class Training(SientiaMonitoring): wrapper.logger = self.logger.base_logger self.info(f'Training model for {train_params.model_type}', metadata) - train_data = train_result.train_data - val_data = train_result.val_data + train_result = self._fit_model(wrapper, train_result, train_params, metadata) - self.debug( - f'train_model prepared data (head 10):\ntrain:\n{train_data.head(10).to_string()}' - f'\nval:\n{val_data.head(10).to_string()}', - metadata, + self.info(f'Computing regression metrics for {train_params.model_type}', metadata) + train_result = self.data_manager_repository.compute_regression_metrics( + train_result, + wrapper, + metadata=metadata, ) + if train_result.mse_val is not None: + mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels(**labels).set( + train_result.mse_val + ) + if train_result.mae_val is not None: + mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels(**labels).set( + train_result.mae_val + ) + if train_result.r2_val is not None: + mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels(**labels).set( + train_result.r2_val + ) + + self.info(f'Starting MLflow run for {train_params.model_type}', metadata) + with self.mlflow_repository.start_run( + model_name=train_params.model_name, + run_name=train_result.run_name, + experiment_name=train_result.experiment_name, + tags=None, + metadata=metadata, + ) as run_info: + train_result.run_id = run_info.run_id + self._persist_training_artifacts(train_result, train_params, wrapper, metadata) + + self.emit_metric_sync( + metric_object=mm_metrics.SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, + tags=labels, + ) + + return { + 'run_name': train_result.run_name, + 'experiment_name': train_result.experiment_name, + 'run_id': train_result.run_id, + 'run_dir': train_result.run_dir, + } + except Exception as e: # noqa: BLE001 + error_msg = f'Error training model - error: {str(e)}' + + trace = traceback.format_exc() + + self.send_notification( + metadata=metadata or {}, + notification_id='TRAIN_MODEL_ERROR', + message=error_msg, + block='train_model', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + + raise e + + def _get_training_labels(self, train_params: TrainModelParams) -> dict: + return { + 'pod_id': os.getenv('POD_ID'), + 'model_name': train_params.model_name, + 'model_type': train_params.model_type, + } + + def _prepare_data( + self, + train_bytes: bytes, + val_bytes: bytes | None, + train_params: TrainModelParams, + metadata: dict | None, + ) -> TrainModelResult: + labels = self._get_training_labels(train_params) + start_time = time.monotonic() + try: + return self.data_manager_repository.prepare_training_data( + train_file_bytes=train_bytes, + validation_file_bytes=val_bytes, + params=train_params, + metadata=metadata, + ) + except Exception: + self.emit_metric_sync( + metric_object=mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, + tags=labels, + ) + raise + finally: + self.observe_lag_sync( + start_time, + mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_LAG, + labels, + ) + + def _fit_model( + self, + wrapper: Any, + train_result: TrainModelResult, + train_params: TrainModelParams, + metadata: dict | None, + ) -> TrainModelResult: + labels = self._get_training_labels(train_params) + train_data = train_result.train_data + val_data = train_result.val_data + + self.debug( + f'train_model prepared data (head 10):\ntrain:\n{train_data.head(10).to_string()}' + f'\nval:\n{val_data.head(10).to_string()}', + metadata, + ) + + start_time = time.monotonic() + try: wrapper.train( train_data=train_data, val_data=val_data, @@ -251,7 +365,6 @@ class Training(SientiaMonitoring): f'Generating predictions using the trained wrapper for {train_params.model_type}', metadata, ) - # Generate predictions using the trained wrapper transformed_train, _ = wrapper.transform(train_data) transformed_val, _ = wrapper.transform(val_data) @@ -275,47 +388,20 @@ class Training(SientiaMonitoring): train_result.y_train_pred = y_train_pred_df train_result.y_pred = y_val_pred_df - - self.info(f'Computing regression metrics for {train_params.model_type}', metadata) - train_result = self.data_manager_repository.compute_regression_metrics( - train_result, - wrapper, - metadata=metadata, + return train_result + except Exception: + self.emit_metric_sync( + metric_object=mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, + tags=labels, ) - - self.info(f'Starting MLflow run for {train_params.model_type}', metadata) - with self.mlflow_repository.start_run( - model_name=train_params.model_name, - run_name=train_result.run_name, - experiment_name=train_result.experiment_name, - tags=None, - metadata=metadata, - ) as run_info: - train_result.run_id = run_info.run_id - self._persist_training_artifacts(train_result, train_params, wrapper, metadata) - - return { - 'run_name': train_result.run_name, - 'experiment_name': train_result.experiment_name, - 'run_id': train_result.run_id, - 'run_dir': train_result.run_dir, - } - except Exception as e: # noqa: BLE001 - error_msg = f'Error training model - error: {str(e)}' - - trace = traceback.format_exc() - - self.send_notification( - metadata=metadata or {}, - notification_id='TRAIN_MODEL_ERROR', - message=error_msg, - block='train_model', - level=NotificationLevel.ERROR, - attachment_content=trace, + raise + finally: + self.observe_lag_sync( + start_time, + mm_metrics.SIENTIA_TRAINING_MODEL_FIT_LAG, + labels, ) - raise e - def _persist_training_artifacts( self, train_result: TrainModelResult, diff --git a/model_manager/metrics.py b/model_manager/metrics.py index 9533e9b..ad2a1e4 100644 --- a/model_manager/metrics.py +++ b/model_manager/metrics.py @@ -16,23 +16,79 @@ Metric Labels: - pod_id: Kubernetes pod identifier for multi-instance deployments """ -from prometheus_client import Counter, Gauge +from prometheus_client import Counter, Gauge, Histogram # Application health metric APP_UP = Gauge( - 'app_up', + 'sientia_app_up', 'Indicates if the application is running (1) or shutting down (0)', ['pod_id'], ) -WORKFLOW_EXECUTION_TOTAL = Counter( - 'model_manager_workflow_executions_total', - 'Total number of workflow executions', - ['pod_id', 'workflow_name', 'status'], # status: success, error +_TRAINING_LABELS = ['pod_id', 'model_name', 'model_type'] + +SIENTIA_TRAINING_MODEL_TRAINED_TOTAL = Counter( + 'sientia_training_model_trained_total', + 'Number of successfully completed model training runs', + _TRAINING_LABELS, ) -ACTIVITY_EXECUTION_TOTAL = Counter( - 'model_manager_activity_executions_total', - 'Total number of activity executions', - ['pod_id', 'activity_name', 'status'], # status: success, error +SIENTIA_TRAINING_DATA_PREPARATION_LAG = Histogram( + 'sientia_training_data_preparation_lag', + 'Latency of prepare_training_data() in seconds', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL = Counter( + 'sientia_training_data_preparation_error_count_total', + 'Number of failures in prepare_training_data()', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_FIT_LAG = Histogram( + 'sientia_training_model_fit_lag', + 'Latency of wrapper.train() (model fitting) in seconds', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL = Counter( + 'sientia_training_model_fit_error_count_total', + 'Number of failures in wrapper.train() (model fitting)', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_QUALITY_MSE = Gauge( + 'sientia_training_model_quality_mse', + 'Mean Squared Error of the last successful model training', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_QUALITY_MAE = Gauge( + 'sientia_training_model_quality_mae', + 'Mean Absolute Error of the last successful model training', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_MODEL_QUALITY_R2 = Gauge( + 'sientia_training_model_quality_r2', + 'R-squared of the last successful model training', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_DATASET_TRAIN_ROWS = Gauge( + 'sientia_training_dataset_train_rows', + 'Number of rows in the training dataset after preparation', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_DATASET_VAL_ROWS = Gauge( + 'sientia_training_dataset_val_rows', + 'Number of rows in the validation dataset after preparation', + _TRAINING_LABELS, +) + +SIENTIA_TRAINING_FEATURE_COUNT = Gauge( + 'sientia_training_feature_count', + 'Number of input feature columns used for training', + _TRAINING_LABELS, ) diff --git a/scripts/inputs/sin-approx.json b/scripts/inputs/sin-approx.json new file mode 100644 index 0000000..46a2659 --- /dev/null +++ b/scripts/inputs/sin-approx.json @@ -0,0 +1,49 @@ +{ + "experiment": { + "experiment_run_id": 2000, + "experiment_name": "experiment-sin-approx", + "run_name": "run-sin-approx", + "username": "vitor.santos@aignosi.com.br", + "status": "ORCHESTRATOR_WAITING_PROC" + }, + "minio": { + "mc_alias": "open-suse", + "bucket_name": "model-training", + "file_name": "training-sin-approx-dataset-1001.csv", + "local_csv": "data-1780946658143-pivot.csv" + }, + "temporal": { + "task_queue": "train_model-basic-queue", + "workflow_name": "train_model", + "execution_timeout_minutes": 5, + "run_timeout_minutes": 5, + "task_timeout_minutes": 5 + }, + "payload": { + "experiment_run_id": 2000, + "variable_columns": ["SourceTri"], + "target_variable": "TargetSin", + "line_separator": ",", + "decimal_separator": ".", + "train_size": 80, + "shuffle": true, + "random_state": 42, + "model_name": "sin-approx", + "model_type": "linear_regression", + "model_id": 1, + "data_model_kwargs": {}, + "model_kwargs": {}, + "opt_params": {}, + "date_column": "timestamp" + }, + "db_only": { + "model_metadata": { + "schemas": { + "components": { + "schemas": {} + } + } + } + } + } + \ No newline at end of file diff --git a/scripts/run_training_test.py b/scripts/run_training_test.py index 09f7195..71c4c66 100644 --- a/scripts/run_training_test.py +++ b/scripts/run_training_test.py @@ -42,6 +42,10 @@ TRAINING_TEST_INPUTS: list[str] = [ 'scripts/inputs/xgboost.json', ] +TRAINING_TEST_INPUTS: list[str] = [ + 'scripts/inputs/sin-approx.json', +] + _REQUIRED_INPUT_KEYS = ('experiment', 'minio', 'temporal', 'payload', 'db_only') @@ -260,6 +264,7 @@ for exp in experiments: [ 'mc', 'cp', + '--insecure', str(local_csv), f'{exp_minio["mc_alias"]}/{exp_minio["bucket_name"]}/{exp_minio["file_name"]}', ], diff --git a/t.py b/t.py new file mode 100644 index 0000000..5ca0f77 --- /dev/null +++ b/t.py @@ -0,0 +1,24 @@ +# %% +from pandas import read_csv, to_datetime + +df = read_csv('data-1780946658143.csv') + +# %% +df.head() + +# %% +# normalize timestamp to naive "yyyy-MM-dd HH:mm:ss" (drop the "+00" UTC offset) +df['timestamp'] = to_datetime(df['timestamp'], utc=True).dt.tz_localize(None) +df['timestamp'] = df['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S') + +# %% +# pivot the dataframe +df = df.pivot(index='timestamp', columns='variable', values='value') + +# %% +df.head() + + +# %% +df.to_csv('data-1780946658143-pivot.csv') +# %% diff --git a/tests/activities/test_cleanup.py b/tests/activities/test_cleanup.py index 3921117..370f1a0 100644 --- a/tests/activities/test_cleanup.py +++ b/tests/activities/test_cleanup.py @@ -125,13 +125,11 @@ def test_cleanup_temp_directories_nonexistent_path( notification_handler=mock_notification_handler, metrics_controller=mock_metrics_controller, ) - cleanup._emit_metrics = MagicMock() # type: ignore[method-assign] cleanup.warning = MagicMock() cleanup.cleanup_temp_directories({'temp_path': '/nonexistent/path', 'metadata': {}}) cleanup.warning.assert_called_once() - cleanup._emit_metrics.assert_called_once() @patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'}) @@ -152,8 +150,6 @@ def test_cleanup_temp_directories_success_with_deletions( notification_handler=mock_notification_handler, metrics_controller=mock_metrics_controller, ) - cleanup._emit_metrics = MagicMock() # type: ignore[method-assign] - old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000') old_dir = os.path.join(temp_dir, f'old_dir_{old_time}') os.makedirs(old_dir) @@ -166,7 +162,6 @@ def test_cleanup_temp_directories_success_with_deletions( assert not os.path.exists(old_dir) assert os.path.exists(recent_dir) - cleanup._emit_metrics.assert_called_once() @patch.dict(os.environ, {'CLEANUP_DRY_RUN': 'true'}) @@ -187,8 +182,6 @@ def test_cleanup_temp_directories_dry_run( notification_handler=mock_notification_handler, metrics_controller=mock_metrics_controller, ) - cleanup._emit_metrics = MagicMock() # type: ignore[method-assign] - old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000') old_dir = os.path.join(temp_dir, f'old_dir_{old_time}') os.makedirs(old_dir) @@ -196,7 +189,6 @@ def test_cleanup_temp_directories_dry_run( cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) assert os.path.exists(old_dir) - cleanup._emit_metrics.assert_called_once() @patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'}) @@ -217,7 +209,6 @@ def test_cleanup_temp_directories_delete_error( notification_handler=mock_notification_handler, metrics_controller=mock_metrics_controller, ) - cleanup._emit_metrics = MagicMock() # type: ignore[method-assign] cleanup.error = MagicMock() old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000') @@ -228,35 +219,9 @@ def test_cleanup_temp_directories_delete_error( cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) cleanup.error.assert_called_once() - cleanup._emit_metrics.assert_called_once() -# --- Metrics and Utility Tests --- - - -def test_emit_metrics( - mock_logger, - mock_notification_handler, - mock_metrics_controller, -): - """Test that _emit_metrics calls the public emit_metric method.""" - from model_manager.activities.cleanup import Cleanup - - cleanup = Cleanup( - logger=mock_logger, - notification_handler=mock_notification_handler, - metrics_controller=mock_metrics_controller, - ) - cleanup.emit_metric_sync = MagicMock() - - cleanup._emit_metrics( - metadata={'pod_id': 'p1', 'workflow_name': 'wf1'}, - metrics_status='success', - activity_name='test_activity', - emit_workflow_metric=True, - ) - - assert cleanup.emit_metric_sync.call_count == 2 +# --- Utility Tests --- def test_cleanup_temp_directories_with_files_and_unmatched_dirs( @@ -276,7 +241,6 @@ def test_cleanup_temp_directories_with_files_and_unmatched_dirs( notification_handler=mock_notification_handler, metrics_controller=mock_metrics_controller, ) - cleanup._emit_metrics = MagicMock() # type: ignore[method-assign] cleanup.debug = MagicMock() # Create a file and a directory with a non-matching name @@ -290,7 +254,6 @@ def test_cleanup_temp_directories_with_files_and_unmatched_dirs( cleanup.debug.assert_called_with( 'Skipping directory without timestamp pattern: a_directory_with_no_timestamp', {} ) - cleanup._emit_metrics.assert_called_once() def test_cleanup_temp_directories_invalid_timestamp_format( @@ -310,7 +273,6 @@ def test_cleanup_temp_directories_invalid_timestamp_format( notification_handler=mock_notification_handler, metrics_controller=mock_metrics_controller, ) - cleanup._emit_metrics = MagicMock() # type: ignore[method-assign] cleanup.error = MagicMock() # Create a directory with a malformed timestamp that matches the regex but fails parsing @@ -320,7 +282,6 @@ def test_cleanup_temp_directories_invalid_timestamp_format( cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) cleanup.error.assert_called_once() - cleanup._emit_metrics.assert_called_once() def test_cleanup_temp_directories_generic_exception( @@ -340,7 +301,6 @@ def test_cleanup_temp_directories_generic_exception( notification_handler=mock_notification_handler, metrics_controller=mock_metrics_controller, ) - cleanup._emit_metrics = MagicMock() # type: ignore[method-assign] cleanup.send_notification = MagicMock() with patch('os.listdir', side_effect=Exception('Unexpected OS Error')): @@ -348,29 +308,3 @@ def test_cleanup_temp_directories_generic_exception( cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}}) cleanup.send_notification.assert_called_once() - cleanup._emit_metrics.assert_called_once() - - -def test_emit_metrics_activity_only( - mock_logger, - mock_notification_handler, - mock_metrics_controller, -): - """Test that _emit_metrics can emit only the activity metric.""" - from model_manager.activities.cleanup import Cleanup - - cleanup = Cleanup( - logger=mock_logger, - notification_handler=mock_notification_handler, - metrics_controller=mock_metrics_controller, - ) - cleanup.emit_metric_sync = MagicMock() - - cleanup._emit_metrics( - metadata={'pod_id': 'p1', 'workflow_name': 'wf1'}, - metrics_status='success', - activity_name='test_activity', - emit_workflow_metric=False, - ) - - cleanup.emit_metric_sync.assert_called_once() diff --git a/tests/activities/test_training.py b/tests/activities/test_training.py index 3cd2bfc..9c2395e 100644 --- a/tests/activities/test_training.py +++ b/tests/activities/test_training.py @@ -330,6 +330,267 @@ def test_train_model_downloads_validation_file_when_set(mock_mlflow, training): mock_mlflow.log_artifact.assert_called() +def test_prepare_data_observes_lag_on_success(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + from model_manager import metrics as mm_metrics + + training._prepare_data(b'csv', None, tp, {}) + + training.observe_lag_sync.assert_called_once() + call_args = training.observe_lag_sync.call_args + assert call_args.args[1] is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_LAG + training.emit_metric_sync.assert_not_called() + + +def test_prepare_data_increments_error_counter_and_still_observes_lag_on_failure(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + training.data_manager_repository.prepare_training_data = MagicMock( + side_effect=RuntimeError('prep-fail') + ) + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + from model_manager import metrics as mm_metrics + + with pytest.raises(RuntimeError, match='prep-fail'): + training._prepare_data(b'csv', None, tp, {}) + + training.observe_lag_sync.assert_called_once() + training.emit_metric_sync.assert_called_once() + call_args = training.emit_metric_sync.call_args + assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL + + +def test_fit_model_observes_lag_on_success(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + from model_manager import metrics as mm_metrics + + training._fit_model(wrapper, tmr, tp, {}) + + training.observe_lag_sync.assert_called_once() + call_args = training.observe_lag_sync.call_args + assert call_args.args[1] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_LAG + training.emit_metric_sync.assert_not_called() + + +def test_fit_model_increments_error_counter_on_failure(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + wrapper = MagicMock() + wrapper.train = MagicMock(side_effect=RuntimeError('fit-fail')) + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + from model_manager import metrics as mm_metrics + + with pytest.raises(RuntimeError, match='fit-fail'): + training._fit_model(wrapper, tmr, tp, {}) + + training.observe_lag_sync.assert_called_once() + training.emit_metric_sync.assert_called_once() + call_args = training.emit_metric_sync.call_args + assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL + + +@patch('model_manager.activities.training.mm_metrics') +@patch('model_manager.activities.training.mlflow') +def test_train_model_sets_quality_gauges_after_compute_metrics(mock_mlflow, mock_mm_metrics, training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'csv') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + + def _set_metrics(x, _w, **_kw): + x.mse_val = 0.5 + x.mae_val = 0.3 + x.r2_val = -0.1 + return x + + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=_set_metrics + ) + + def _fill_report(x, **_kw): + x.report_path = '/tmp/r.html' + x.train_data_path = '/tmp/tr.csv' + x.test_data_path = '/tmp/te.csv' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_id = 'rid' + yield info + + training.mlflow_repository.start_run = _run_ctx + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_called_once_with(0.5) + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_called_once_with(0.3) + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_called_once_with(-0.1) + + +@patch('model_manager.activities.training.mm_metrics') +@patch('model_manager.activities.training.mlflow') +def test_train_model_skips_quality_gauges_when_none(_mock_mlflow, mock_mm_metrics, training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'csv') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=lambda x, _w, **_kw: x + ) + + def _fill_report(x, **_kw): + x.report_path = '/tmp/r.html' + x.train_data_path = '/tmp/tr.csv' + x.test_data_path = '/tmp/te.csv' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_id = 'rid' + yield info + + training.mlflow_repository.start_run = _run_ctx + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_not_called() + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_not_called() + mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_not_called() + + +@patch('model_manager.activities.training.mm_metrics') +@patch('model_manager.activities.training.mlflow') +def test_train_model_increments_trained_total_on_success(_mock_mlflow, mock_mm_metrics, training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]}) + val_df = pd.DataFrame({'a': [1.0], 't': [1.0]}) + tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df) + + training.minio_repository.download_file = MagicMock(return_value=b'csv') + training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr) + training.data_manager_repository.compute_regression_metrics = MagicMock( + side_effect=lambda x, _w, **_kw: x + ) + + def _fill_report(x, **_kw): + x.report_path = '/tmp/r.html' + x.train_data_path = '/tmp/tr.csv' + x.test_data_path = '/tmp/te.csv' + x.run_dir = '/tmp/run' + return x + + training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report) + wrapper = MagicMock() + wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)]) + wrapper.predict = MagicMock( + side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)] + ) + wrapper.store_model = MagicMock() + training.plugin_store.get_model = MagicMock(return_value=wrapper) + + @contextmanager + def _run_ctx(*_a, **_k): + info = MagicMock() + info.run_id = 'rid' + yield info + + training.mlflow_repository.start_run = _run_ctx + training.observe_lag_sync = MagicMock() + training.emit_metric_sync = MagicMock() + + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + + training.emit_metric_sync.assert_called_once_with( + metric_object=mock_mm_metrics.SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, + tags=training._get_training_labels(tp), + ) + + +def test_train_model_does_not_increment_trained_total_on_failure(training): + tp = TrainModelParams.from_dict( + {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}} + ) + training.minio_repository.download_file = MagicMock(side_effect=RuntimeError('dl-fail')) + training.send_notification = MagicMock() + training.emit_metric_sync = MagicMock() + + with pytest.raises(RuntimeError): + training.train_model({'metadata': {}, 'train_params': tp.to_dict()}) + + training.emit_metric_sync.assert_not_called() + + def test_train_model_value_error_when_paths_missing_after_report(training): """Raises ValueError when report paths are not populated after generate_report.""" tp = TrainModelParams.from_dict( diff --git a/tests/test_metrics.py b/tests/test_metrics.py index edc3ce0..06bc251 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -10,7 +10,7 @@ def test_app_up_metric_exists(): from model_manager.metrics import APP_UP assert APP_UP is not None - assert APP_UP._name == 'app_up' + assert APP_UP._name == 'sientia_app_up' assert ( APP_UP._documentation == 'Indicates if the application is running (1) or shutting down (0)' ) @@ -207,3 +207,125 @@ def test_prometheus_client_gauge_import(): from model_manager.metrics import Gauge assert Gauge is PrometheusGauge + + +# --------------------------------------------------------------------------- +# Training metrics — existence, type, and labels +# --------------------------------------------------------------------------- + +_TRAINING_LABEL_NAMES = ('pod_id', 'model_name', 'model_type') + + +def _assert_training_labels(metric): + for label in _TRAINING_LABEL_NAMES: + assert label in metric._labelnames + + +def test_sientia_training_data_preparation_lag_is_histogram(): + from prometheus_client import Histogram + + from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_LAG + + assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_LAG, Histogram) + assert SIENTIA_TRAINING_DATA_PREPARATION_LAG._name == 'sientia_training_data_preparation_lag' + _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_LAG) + + +def test_sientia_training_data_preparation_error_count_total_is_counter(): + from prometheus_client import Counter + + from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL + + assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, Counter) + assert 'sientia_training_data_preparation_error_count' in SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL._name + _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL) + + +def test_sientia_training_model_fit_lag_is_histogram(): + from prometheus_client import Histogram + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_LAG + + assert isinstance(SIENTIA_TRAINING_MODEL_FIT_LAG, Histogram) + assert SIENTIA_TRAINING_MODEL_FIT_LAG._name == 'sientia_training_model_fit_lag' + _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_LAG) + + +def test_sientia_training_model_fit_error_count_total_is_counter(): + from prometheus_client import Counter + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL + + assert isinstance(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, Counter) + assert 'sientia_training_model_fit_error_count' in SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL._name + _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL) + + +def test_sientia_training_model_quality_mse_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MSE + + assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MSE, Gauge) + assert SIENTIA_TRAINING_MODEL_QUALITY_MSE._name == 'sientia_training_model_quality_mse' + _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MSE) + + +def test_sientia_training_model_quality_mae_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MAE + + assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MAE, Gauge) + assert SIENTIA_TRAINING_MODEL_QUALITY_MAE._name == 'sientia_training_model_quality_mae' + _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MAE) + + +def test_sientia_training_model_quality_r2_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_R2 + + assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_R2, Gauge) + assert SIENTIA_TRAINING_MODEL_QUALITY_R2._name == 'sientia_training_model_quality_r2' + _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_R2) + + +def test_sientia_training_dataset_train_rows_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_DATASET_TRAIN_ROWS + + assert isinstance(SIENTIA_TRAINING_DATASET_TRAIN_ROWS, Gauge) + assert SIENTIA_TRAINING_DATASET_TRAIN_ROWS._name == 'sientia_training_dataset_train_rows' + _assert_training_labels(SIENTIA_TRAINING_DATASET_TRAIN_ROWS) + + +def test_sientia_training_dataset_val_rows_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_DATASET_VAL_ROWS + + assert isinstance(SIENTIA_TRAINING_DATASET_VAL_ROWS, Gauge) + assert SIENTIA_TRAINING_DATASET_VAL_ROWS._name == 'sientia_training_dataset_val_rows' + _assert_training_labels(SIENTIA_TRAINING_DATASET_VAL_ROWS) + + +def test_sientia_training_model_trained_total_is_counter(): + from prometheus_client import Counter + + from model_manager.metrics import SIENTIA_TRAINING_MODEL_TRAINED_TOTAL + + assert isinstance(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, Counter) + assert 'sientia_training_model_trained' in SIENTIA_TRAINING_MODEL_TRAINED_TOTAL._name + _assert_training_labels(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL) + + +def test_sientia_training_feature_count_is_gauge(): + from prometheus_client import Gauge + + from model_manager.metrics import SIENTIA_TRAINING_FEATURE_COUNT + + assert isinstance(SIENTIA_TRAINING_FEATURE_COUNT, Gauge) + assert SIENTIA_TRAINING_FEATURE_COUNT._name == 'sientia_training_feature_count' + _assert_training_labels(SIENTIA_TRAINING_FEATURE_COUNT) diff --git a/values.yaml b/values.yaml index 59fa408..d1fe44f 100644 --- a/values.yaml +++ b/values.yaml @@ -328,7 +328,7 @@ ssh: # Configuração para dashboards do Grafana grafanaDashboard: # Habilita a criação de ConfigMaps para dashboards - enabled: false + enabled: true # Namespace onde o Grafana está instalado (ajuste conforme seu ambiente) namespace: monitoring # Labels para que o sidecar do Grafana encontre os dashboards