SIENTIAPDE-1645: Add comprehensive model training observability metrics and Grafana dashboard.

.gitignore

@@ -255,3 +255,4 @@ openspec
 input_dataset.csv
 scripts/**/*.csv
 !scripts/inputs/
+.claude/

dashboards/sientia-dataops-model-manager.json

@@ -0,0 +1,197 @@
+{
+  "annotations": { "list": [] },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "panels": [
+    {
+      "id": 1,
+      "title": "Data Preparation Latency (p50 / p95)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.5, rate(sientia_training_data_preparation_lag_bucket[5m]))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.95, rate(sientia_training_data_preparation_lag_bucket[5m]))",
+          "legendFormat": "p95",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "s", "color": { "mode": "palette-classic" } }
+      },
+      "options": { "tooltip": { "mode": "multi" } }
+    },
+    {
+      "id": 2,
+      "title": "Model Fit Latency (p50 / p95)",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.5, rate(sientia_training_model_fit_lag_bucket[5m]))",
+          "legendFormat": "p50",
+          "refId": "A"
+        },
+        {
+          "expr": "histogram_quantile(0.95, rate(sientia_training_model_fit_lag_bucket[5m]))",
+          "legendFormat": "p95",
+          "refId": "B"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "s", "color": { "mode": "palette-classic" } }
+      },
+      "options": { "tooltip": { "mode": "multi" } }
+    },
+    {
+      "id": 3,
+      "title": "Data Preparation Errors / s",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "targets": [
+        {
+          "expr": "rate(sientia_training_data_preparation_error_count_total[5m])",
+          "legendFormat": "{{model_name}} / {{model_type}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "ops", "color": { "mode": "palette-classic" } }
+      }
+    },
+    {
+      "id": 4,
+      "title": "Model Fit Errors / s",
+      "type": "timeseries",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "targets": [
+        {
+          "expr": "rate(sientia_training_model_fit_error_count_total[5m])",
+          "legendFormat": "{{model_name}} / {{model_type}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "ops", "color": { "mode": "palette-classic" } }
+      }
+    },
+    {
+      "id": 5,
+      "title": "Model Quality — MSE",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 16 },
+      "targets": [
+        {
+          "expr": "sientia_training_model_quality_mse",
+          "legendFormat": "{{model_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "none", "decimals": 4 }
+      },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
+    },
+    {
+      "id": 6,
+      "title": "Model Quality — MAE",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 16 },
+      "targets": [
+        {
+          "expr": "sientia_training_model_quality_mae",
+          "legendFormat": "{{model_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "none", "decimals": 4 }
+      },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
+    },
+    {
+      "id": 7,
+      "title": "Model Quality — R²",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 16 },
+      "targets": [
+        {
+          "expr": "sientia_training_model_quality_r2",
+          "legendFormat": "{{model_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "percentunit", "decimals": 3, "min": -1, "max": 1 }
+      },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "background" }
+    },
+    {
+      "id": 8,
+      "title": "Dataset — Train Rows",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 20 },
+      "targets": [
+        {
+          "expr": "sientia_training_dataset_train_rows",
+          "legendFormat": "{{model_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "short", "decimals": 0 }
+      },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
+    },
+    {
+      "id": 9,
+      "title": "Dataset — Val Rows",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 20 },
+      "targets": [
+        {
+          "expr": "sientia_training_dataset_val_rows",
+          "legendFormat": "{{model_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "short", "decimals": 0 }
+      },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
+    },
+    {
+      "id": 10,
+      "title": "Dataset — Feature Count",
+      "type": "stat",
+      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 20 },
+      "targets": [
+        {
+          "expr": "sientia_training_feature_count",
+          "legendFormat": "{{model_name}}",
+          "refId": "A"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": { "unit": "short", "decimals": 0 }
+      },
+      "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "auto", "textMode": "auto", "colorMode": "value" }
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "tags": ["sientia", "model-manager", "training"],
+  "templating": { "list": [] },
+  "time": { "from": "now-3h", "to": "now" },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Sientia DataOps Model Manager",
+  "uid": "sientia-dataops-model-manager",
+  "version": 1
+}

model_manager/activities/cleanup.py

@@ -22,7 +22,6 @@ with workflow.unsafe.imports_passed_through():
     from sientia_do.observability.metrics_controller import MetricsController
     from sientia_do.observability.sientia_monitoring import SientiaMonitoring
 
-    from model_manager.metrics import ACTIVITY_EXECUTION_TOTAL, WORKFLOW_EXECUTION_TOTAL
     from model_manager.runtime_paths import REPORTS_TEMP_DIR
 
     RETENTION_HOURS = int(os.getenv('CLEANUP_RETENTION_HOURS', '24'))
@@ -84,7 +83,6 @@ class Cleanup(SientiaMonitoring):
         """
         metadata = input_data.get('metadata', {})
         temp_path = input_data.get('temp_path', REPORTS_TEMP_DIR)
-        metrics_status = 'success'
 
         cutoff_time = datetime.now() - timedelta(hours=self.retention_hours)
 
@@ -164,7 +162,6 @@ class Cleanup(SientiaMonitoring):
             )
 
         except Exception as e:
-            metrics_status = 'error'
             error_msg = f'Error in directory cleanup: {str(e)}'
             trace = traceback.format_exc()
 
@@ -178,44 +175,3 @@ class Cleanup(SientiaMonitoring):
             )
 
             raise
-        finally:
-            self._emit_metrics(
-                metadata=metadata,
-                metrics_status=metrics_status,
-                activity_name='cleanup_temp_directories',
-                emit_workflow_metric=True,
-            )
-
-    def _emit_metrics(
-        self,
-        metadata: dict[str, Any],
-        metrics_status: str,
-        activity_name: str,
-        emit_workflow_metric: bool,
-    ) -> None:
-        """
-        Emit workflow and activity execution metrics.
-
-        Args:
-            metadata: Activity metadata containing pod_id and workflow_name
-            metrics_status: Execution status ('success' or 'error')
-            activity_name: Name of the activity being executed
-        """
-        if emit_workflow_metric:
-            self.emit_metric_sync(
-                metric_object=WORKFLOW_EXECUTION_TOTAL,
-                tags={
-                    'pod_id': metadata.get('pod_id'),
-                    'workflow_name': metadata.get('workflow_name'),
-                    'status': metrics_status,
-                },
-            )
-
-        self.emit_metric_sync(
-            metric_object=ACTIVITY_EXECUTION_TOTAL,
-            tags={
-                'pod_id': metadata.get('pod_id'),
-                'activity_name': activity_name,
-                'status': metrics_status,
-            },
-        )

model_manager/activities/training.py

@@ -10,6 +10,8 @@ from sientia_model.wrappers.sientia_model import SientiaModel
 from temporalio import activity, workflow
 
 with workflow.unsafe.imports_passed_through():
+    import os
+    import time
     import traceback
     from typing import Any
 
@@ -23,6 +25,7 @@ with workflow.unsafe.imports_passed_through():
     from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
     from sientia_model.model_repository.plugin_store import PluginStore
 
+    from model_manager import metrics as mm_metrics
     from model_manager.utils.models.train_model_params import TrainModelParams
     from model_manager.utils.models.train_model_result import TrainModelResult
     from model_manager.utils.repository.data_manager_repository import DataManagerRepository
@@ -184,12 +187,12 @@ class Training(SientiaMonitoring):
         """
         metadata = input_data.get('metadata')
         train_params = TrainModelParams.from_dict(input_data['train_params'])
+        labels = self._get_training_labels(train_params)
 
         self.info('Starting train_model process', metadata)
 
         try:
             # Download training file bytes from MinIO
-
             self.info(
                 f'Downloading training file from MinIO for {train_params.file_name}', metadata
             )
@@ -211,11 +214,16 @@ class Training(SientiaMonitoring):
                 )
 
             self.info(f'Preparing training data for {train_params.file_name}', metadata)
-            train_result = self.data_manager_repository.prepare_training_data(
+            train_result = self._prepare_data(train_bytes, val_bytes, train_params, metadata)
-                train_file_bytes=train_bytes,
+
-                validation_file_bytes=val_bytes,
+            mm_metrics.SIENTIA_TRAINING_DATASET_TRAIN_ROWS.labels(**labels).set(
-                params=train_params,
+                len(train_result.train_data)
-                metadata=metadata,
+            )
+            mm_metrics.SIENTIA_TRAINING_DATASET_VAL_ROWS.labels(**labels).set(
+                len(train_result.val_data)
+            )
+            mm_metrics.SIENTIA_TRAINING_FEATURE_COUNT.labels(**labels).set(
+                len(train_params.variable_columns)
             )
 
             self.info(f'Getting model wrapper for {train_params.model_type}', metadata)
@@ -232,15 +240,121 @@ class Training(SientiaMonitoring):
                 wrapper.logger = self.logger.base_logger
 
             self.info(f'Training model for {train_params.model_type}', metadata)
-            train_data = train_result.train_data
+            train_result = self._fit_model(wrapper, train_result, train_params, metadata)
-            val_data = train_result.val_data
 
-            self.debug(
+            self.info(f'Computing regression metrics for {train_params.model_type}', metadata)
-                f'train_model prepared data (head 10):\ntrain:\n{train_data.head(10).to_string()}'
+            train_result = self.data_manager_repository.compute_regression_metrics(
-                f'\nval:\n{val_data.head(10).to_string()}',
+                train_result,
-                metadata,
+                wrapper,
+                metadata=metadata,
             )
 
+            if train_result.mse_val is not None:
+                mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels(**labels).set(
+                    train_result.mse_val
+                )
+            if train_result.mae_val is not None:
+                mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels(**labels).set(
+                    train_result.mae_val
+                )
+            if train_result.r2_val is not None:
+                mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels(**labels).set(
+                    train_result.r2_val
+                )
+
+            self.info(f'Starting MLflow run for {train_params.model_type}', metadata)
+            with self.mlflow_repository.start_run(
+                model_name=train_params.model_name,
+                run_name=train_result.run_name,
+                experiment_name=train_result.experiment_name,
+                tags=None,
+                metadata=metadata,
+            ) as run_info:
+                train_result.run_id = run_info.run_id
+                self._persist_training_artifacts(train_result, train_params, wrapper, metadata)
+
+            self.emit_metric_sync(
+                metric_object=mm_metrics.SIENTIA_TRAINING_MODEL_TRAINED_TOTAL,
+                tags=labels,
+            )
+
+            return {
+                'run_name': train_result.run_name,
+                'experiment_name': train_result.experiment_name,
+                'run_id': train_result.run_id,
+                'run_dir': train_result.run_dir,
+            }
+        except Exception as e:  # noqa: BLE001
+            error_msg = f'Error training model - error: {str(e)}'
+
+            trace = traceback.format_exc()
+
+            self.send_notification(
+                metadata=metadata or {},
+                notification_id='TRAIN_MODEL_ERROR',
+                message=error_msg,
+                block='train_model',
+                level=NotificationLevel.ERROR,
+                attachment_content=trace,
+            )
+
+            raise e
+
+    def _get_training_labels(self, train_params: TrainModelParams) -> dict:
+        return {
+            'pod_id': os.getenv('POD_ID'),
+            'model_name': train_params.model_name,
+            'model_type': train_params.model_type,
+        }
+
+    def _prepare_data(
+        self,
+        train_bytes: bytes,
+        val_bytes: bytes | None,
+        train_params: TrainModelParams,
+        metadata: dict | None,
+    ) -> TrainModelResult:
+        labels = self._get_training_labels(train_params)
+        start_time = time.monotonic()
+        try:
+            return self.data_manager_repository.prepare_training_data(
+                train_file_bytes=train_bytes,
+                validation_file_bytes=val_bytes,
+                params=train_params,
+                metadata=metadata,
+            )
+        except Exception:
+            self.emit_metric_sync(
+                metric_object=mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL,
+                tags=labels,
+            )
+            raise
+        finally:
+            self.observe_lag_sync(
+                start_time,
+                mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_LAG,
+                labels,
+            )
+
+    def _fit_model(
+        self,
+        wrapper: Any,
+        train_result: TrainModelResult,
+        train_params: TrainModelParams,
+        metadata: dict | None,
+    ) -> TrainModelResult:
+        labels = self._get_training_labels(train_params)
+        train_data = train_result.train_data
+        val_data = train_result.val_data
+
+        self.debug(
+            f'train_model prepared data (head 10):\ntrain:\n{train_data.head(10).to_string()}'
+            f'\nval:\n{val_data.head(10).to_string()}',
+            metadata,
+        )
+
+        start_time = time.monotonic()
+        try:
             wrapper.train(
                 train_data=train_data,
                 val_data=val_data,
@@ -251,7 +365,6 @@ class Training(SientiaMonitoring):
                 f'Generating predictions using the trained wrapper for {train_params.model_type}',
                 metadata,
             )
-            # Generate predictions using the trained wrapper
             transformed_train, _ = wrapper.transform(train_data)
             transformed_val, _ = wrapper.transform(val_data)
 
@@ -275,47 +388,20 @@ class Training(SientiaMonitoring):
 
             train_result.y_train_pred = y_train_pred_df
             train_result.y_pred = y_val_pred_df
-
+            return train_result
-            self.info(f'Computing regression metrics for {train_params.model_type}', metadata)
+        except Exception:
-            train_result = self.data_manager_repository.compute_regression_metrics(
+            self.emit_metric_sync(
-                train_result,
+                metric_object=mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL,
-                wrapper,
+                tags=labels,
-                metadata=metadata,
             )
-
+            raise
-            self.info(f'Starting MLflow run for {train_params.model_type}', metadata)
+        finally:
-            with self.mlflow_repository.start_run(
+            self.observe_lag_sync(
-                model_name=train_params.model_name,
+                start_time,
-                run_name=train_result.run_name,
+                mm_metrics.SIENTIA_TRAINING_MODEL_FIT_LAG,
-                experiment_name=train_result.experiment_name,
+                labels,
-                tags=None,
-                metadata=metadata,
-            ) as run_info:
-                train_result.run_id = run_info.run_id
-                self._persist_training_artifacts(train_result, train_params, wrapper, metadata)
-
-            return {
-                'run_name': train_result.run_name,
-                'experiment_name': train_result.experiment_name,
-                'run_id': train_result.run_id,
-                'run_dir': train_result.run_dir,
-            }
-        except Exception as e:  # noqa: BLE001
-            error_msg = f'Error training model - error: {str(e)}'
-
-            trace = traceback.format_exc()
-
-            self.send_notification(
-                metadata=metadata or {},
-                notification_id='TRAIN_MODEL_ERROR',
-                message=error_msg,
-                block='train_model',
-                level=NotificationLevel.ERROR,
-                attachment_content=trace,
             )
 
-            raise e
-
     def _persist_training_artifacts(
         self,
         train_result: TrainModelResult,

model_manager/metrics.py

@@ -16,23 +16,79 @@ Metric Labels:
 - pod_id: Kubernetes pod identifier for multi-instance deployments
 """
 
-from prometheus_client import Counter, Gauge
+from prometheus_client import Counter, Gauge, Histogram
 
 # Application health metric
 APP_UP = Gauge(
-    'app_up',
+    'sientia_app_up',
     'Indicates if the application is running (1) or shutting down (0)',
     ['pod_id'],
 )
 
-WORKFLOW_EXECUTION_TOTAL = Counter(
+_TRAINING_LABELS = ['pod_id', 'model_name', 'model_type']
-    'model_manager_workflow_executions_total',
+
-    'Total number of workflow executions',
+SIENTIA_TRAINING_MODEL_TRAINED_TOTAL = Counter(
-    ['pod_id', 'workflow_name', 'status'],  # status: success, error
+    'sientia_training_model_trained_total',
+    'Number of successfully completed model training runs',
+    _TRAINING_LABELS,
 )
 
-ACTIVITY_EXECUTION_TOTAL = Counter(
+SIENTIA_TRAINING_DATA_PREPARATION_LAG = Histogram(
-    'model_manager_activity_executions_total',
+    'sientia_training_data_preparation_lag',
-    'Total number of activity executions',
+    'Latency of prepare_training_data() in seconds',
-    ['pod_id', 'activity_name', 'status'],  # status: success, error
+    _TRAINING_LABELS,
+)
+
+SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL = Counter(
+    'sientia_training_data_preparation_error_count_total',
+    'Number of failures in prepare_training_data()',
+    _TRAINING_LABELS,
+)
+
+SIENTIA_TRAINING_MODEL_FIT_LAG = Histogram(
+    'sientia_training_model_fit_lag',
+    'Latency of wrapper.train() (model fitting) in seconds',
+    _TRAINING_LABELS,
+)
+
+SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL = Counter(
+    'sientia_training_model_fit_error_count_total',
+    'Number of failures in wrapper.train() (model fitting)',
+    _TRAINING_LABELS,
+)
+
+SIENTIA_TRAINING_MODEL_QUALITY_MSE = Gauge(
+    'sientia_training_model_quality_mse',
+    'Mean Squared Error of the last successful model training',
+    _TRAINING_LABELS,
+)
+
+SIENTIA_TRAINING_MODEL_QUALITY_MAE = Gauge(
+    'sientia_training_model_quality_mae',
+    'Mean Absolute Error of the last successful model training',
+    _TRAINING_LABELS,
+)
+
+SIENTIA_TRAINING_MODEL_QUALITY_R2 = Gauge(
+    'sientia_training_model_quality_r2',
+    'R-squared of the last successful model training',
+    _TRAINING_LABELS,
+)
+
+SIENTIA_TRAINING_DATASET_TRAIN_ROWS = Gauge(
+    'sientia_training_dataset_train_rows',
+    'Number of rows in the training dataset after preparation',
+    _TRAINING_LABELS,
+)
+
+SIENTIA_TRAINING_DATASET_VAL_ROWS = Gauge(
+    'sientia_training_dataset_val_rows',
+    'Number of rows in the validation dataset after preparation',
+    _TRAINING_LABELS,
+)
+
+SIENTIA_TRAINING_FEATURE_COUNT = Gauge(
+    'sientia_training_feature_count',
+    'Number of input feature columns used for training',
+    _TRAINING_LABELS,
 )

scripts/inputs/sin-approx.json

@@ -0,0 +1,49 @@
+{
+    "experiment": {
+      "experiment_run_id": 2000,
+      "experiment_name": "experiment-sin-approx",
+      "run_name": "run-sin-approx",
+      "username": "vitor.santos@aignosi.com.br",
+      "status": "ORCHESTRATOR_WAITING_PROC"
+    },
+    "minio": {
+      "mc_alias": "open-suse",
+      "bucket_name": "model-training",
+      "file_name": "training-sin-approx-dataset-1001.csv",
+      "local_csv": "data-1780946658143-pivot.csv"
+    },
+    "temporal": {
+      "task_queue": "train_model-basic-queue",
+      "workflow_name": "train_model",
+      "execution_timeout_minutes": 5,
+      "run_timeout_minutes": 5,
+      "task_timeout_minutes": 5
+    },
+    "payload": {
+      "experiment_run_id": 2000,
+      "variable_columns": ["SourceTri"],
+      "target_variable": "TargetSin",
+      "line_separator": ",",
+      "decimal_separator": ".",
+      "train_size": 80,
+      "shuffle": true,
+      "random_state": 42,
+      "model_name": "sin-approx",
+      "model_type": "linear_regression",
+      "model_id": 1,
+      "data_model_kwargs": {},
+      "model_kwargs": {},
+      "opt_params": {},
+      "date_column": "timestamp"
+    },
+    "db_only": {
+      "model_metadata": {
+        "schemas": {
+          "components": {
+            "schemas": {}
+          }
+        }
+      }
+    }
+  }
+  

scripts/run_training_test.py

@@ -42,6 +42,10 @@ TRAINING_TEST_INPUTS: list[str] = [
     'scripts/inputs/xgboost.json',
 ]
 
+TRAINING_TEST_INPUTS: list[str] = [
+    'scripts/inputs/sin-approx.json',
+]
+
 _REQUIRED_INPUT_KEYS = ('experiment', 'minio', 'temporal', 'payload', 'db_only')
 
 
@@ -260,6 +264,7 @@ for exp in experiments:
         [
             'mc',
             'cp',
+            '--insecure',
             str(local_csv),
             f'{exp_minio["mc_alias"]}/{exp_minio["bucket_name"]}/{exp_minio["file_name"]}',
         ],

t.py

@@ -0,0 +1,24 @@
+# %%
+from pandas import read_csv, to_datetime
+
+df = read_csv('data-1780946658143.csv')
+
+# %%
+df.head()
+
+# %%
+# normalize timestamp to naive "yyyy-MM-dd HH:mm:ss" (drop the "+00" UTC offset)
+df['timestamp'] = to_datetime(df['timestamp'], utc=True).dt.tz_localize(None)
+df['timestamp'] = df['timestamp'].dt.strftime('%Y-%m-%d %H:%M:%S')
+
+# %%
+# pivot the dataframe
+df = df.pivot(index='timestamp', columns='variable', values='value')
+
+# %%
+df.head()
+
+
+# %%
+df.to_csv('data-1780946658143-pivot.csv')
+# %%

tests/activities/test_cleanup.py

@@ -125,13 +125,11 @@ def test_cleanup_temp_directories_nonexistent_path(
         notification_handler=mock_notification_handler,
         metrics_controller=mock_metrics_controller,
     )
-    cleanup._emit_metrics = MagicMock()  # type: ignore[method-assign]
     cleanup.warning = MagicMock()
 
     cleanup.cleanup_temp_directories({'temp_path': '/nonexistent/path', 'metadata': {}})
 
     cleanup.warning.assert_called_once()
-    cleanup._emit_metrics.assert_called_once()
 
 
 @patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
@@ -152,8 +150,6 @@ def test_cleanup_temp_directories_success_with_deletions(
         notification_handler=mock_notification_handler,
         metrics_controller=mock_metrics_controller,
     )
-    cleanup._emit_metrics = MagicMock()  # type: ignore[method-assign]
-
     old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
     old_dir = os.path.join(temp_dir, f'old_dir_{old_time}')
     os.makedirs(old_dir)
@@ -166,7 +162,6 @@ def test_cleanup_temp_directories_success_with_deletions(
 
     assert not os.path.exists(old_dir)
     assert os.path.exists(recent_dir)
-    cleanup._emit_metrics.assert_called_once()
 
 
 @patch.dict(os.environ, {'CLEANUP_DRY_RUN': 'true'})
@@ -187,8 +182,6 @@ def test_cleanup_temp_directories_dry_run(
         notification_handler=mock_notification_handler,
         metrics_controller=mock_metrics_controller,
     )
-    cleanup._emit_metrics = MagicMock()  # type: ignore[method-assign]
-
     old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
     old_dir = os.path.join(temp_dir, f'old_dir_{old_time}')
     os.makedirs(old_dir)
@@ -196,7 +189,6 @@ def test_cleanup_temp_directories_dry_run(
     cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
 
     assert os.path.exists(old_dir)
-    cleanup._emit_metrics.assert_called_once()
 
 
 @patch.dict('model_manager.activities.cleanup.os.environ', {'CLEANUP_DRY_RUN': 'false'})
@@ -217,7 +209,6 @@ def test_cleanup_temp_directories_delete_error(
         notification_handler=mock_notification_handler,
         metrics_controller=mock_metrics_controller,
     )
-    cleanup._emit_metrics = MagicMock()  # type: ignore[method-assign]
     cleanup.error = MagicMock()
 
     old_time = (datetime.now() - timedelta(hours=48)).strftime('%Y%m%d_%H%M%S_000000')
@@ -228,35 +219,9 @@ def test_cleanup_temp_directories_delete_error(
         cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
 
     cleanup.error.assert_called_once()
-    cleanup._emit_metrics.assert_called_once()
 
 
-# --- Metrics and Utility Tests ---
+# --- Utility Tests ---
-
-
-def test_emit_metrics(
-    mock_logger,
-    mock_notification_handler,
-    mock_metrics_controller,
-):
-    """Test that _emit_metrics calls the public emit_metric method."""
-    from model_manager.activities.cleanup import Cleanup
-
-    cleanup = Cleanup(
-        logger=mock_logger,
-        notification_handler=mock_notification_handler,
-        metrics_controller=mock_metrics_controller,
-    )
-    cleanup.emit_metric_sync = MagicMock()
-
-    cleanup._emit_metrics(
-        metadata={'pod_id': 'p1', 'workflow_name': 'wf1'},
-        metrics_status='success',
-        activity_name='test_activity',
-        emit_workflow_metric=True,
-    )
-
-    assert cleanup.emit_metric_sync.call_count == 2
 
 
 def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
@@ -276,7 +241,6 @@ def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
         notification_handler=mock_notification_handler,
         metrics_controller=mock_metrics_controller,
     )
-    cleanup._emit_metrics = MagicMock()  # type: ignore[method-assign]
     cleanup.debug = MagicMock()
 
     # Create a file and a directory with a non-matching name
@@ -290,7 +254,6 @@ def test_cleanup_temp_directories_with_files_and_unmatched_dirs(
     cleanup.debug.assert_called_with(
         'Skipping directory without timestamp pattern: a_directory_with_no_timestamp', {}
     )
-    cleanup._emit_metrics.assert_called_once()
 
 
 def test_cleanup_temp_directories_invalid_timestamp_format(
@@ -310,7 +273,6 @@ def test_cleanup_temp_directories_invalid_timestamp_format(
         notification_handler=mock_notification_handler,
         metrics_controller=mock_metrics_controller,
     )
-    cleanup._emit_metrics = MagicMock()  # type: ignore[method-assign]
     cleanup.error = MagicMock()
 
     # Create a directory with a malformed timestamp that matches the regex but fails parsing
@@ -320,7 +282,6 @@ def test_cleanup_temp_directories_invalid_timestamp_format(
     cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
 
     cleanup.error.assert_called_once()
-    cleanup._emit_metrics.assert_called_once()
 
 
 def test_cleanup_temp_directories_generic_exception(
@@ -340,7 +301,6 @@ def test_cleanup_temp_directories_generic_exception(
         notification_handler=mock_notification_handler,
         metrics_controller=mock_metrics_controller,
     )
-    cleanup._emit_metrics = MagicMock()  # type: ignore[method-assign]
     cleanup.send_notification = MagicMock()
 
     with patch('os.listdir', side_effect=Exception('Unexpected OS Error')):
@@ -348,29 +308,3 @@ def test_cleanup_temp_directories_generic_exception(
             cleanup.cleanup_temp_directories({'temp_path': temp_dir, 'metadata': {}})
 
     cleanup.send_notification.assert_called_once()
-    cleanup._emit_metrics.assert_called_once()
-
-
-def test_emit_metrics_activity_only(
-    mock_logger,
-    mock_notification_handler,
-    mock_metrics_controller,
-):
-    """Test that _emit_metrics can emit only the activity metric."""
-    from model_manager.activities.cleanup import Cleanup
-
-    cleanup = Cleanup(
-        logger=mock_logger,
-        notification_handler=mock_notification_handler,
-        metrics_controller=mock_metrics_controller,
-    )
-    cleanup.emit_metric_sync = MagicMock()
-
-    cleanup._emit_metrics(
-        metadata={'pod_id': 'p1', 'workflow_name': 'wf1'},
-        metrics_status='success',
-        activity_name='test_activity',
-        emit_workflow_metric=False,
-    )
-
-    cleanup.emit_metric_sync.assert_called_once()

tests/activities/test_training.py

@@ -330,6 +330,267 @@ def test_train_model_downloads_validation_file_when_set(mock_mlflow, training):
     mock_mlflow.log_artifact.assert_called()
 
 
+def test_prepare_data_observes_lag_on_success(training):
+    tp = TrainModelParams.from_dict(
+        {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
+    )
+    train_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
+    val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
+    tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
+    training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr)
+    training.observe_lag_sync = MagicMock()
+    training.emit_metric_sync = MagicMock()
+
+    from model_manager import metrics as mm_metrics
+
+    training._prepare_data(b'csv', None, tp, {})
+
+    training.observe_lag_sync.assert_called_once()
+    call_args = training.observe_lag_sync.call_args
+    assert call_args.args[1] is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_LAG
+    training.emit_metric_sync.assert_not_called()
+
+
+def test_prepare_data_increments_error_counter_and_still_observes_lag_on_failure(training):
+    tp = TrainModelParams.from_dict(
+        {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
+    )
+    training.data_manager_repository.prepare_training_data = MagicMock(
+        side_effect=RuntimeError('prep-fail')
+    )
+    training.observe_lag_sync = MagicMock()
+    training.emit_metric_sync = MagicMock()
+
+    from model_manager import metrics as mm_metrics
+
+    with pytest.raises(RuntimeError, match='prep-fail'):
+        training._prepare_data(b'csv', None, tp, {})
+
+    training.observe_lag_sync.assert_called_once()
+    training.emit_metric_sync.assert_called_once()
+    call_args = training.emit_metric_sync.call_args
+    assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL
+
+
+def test_fit_model_observes_lag_on_success(training):
+    tp = TrainModelParams.from_dict(
+        {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
+    )
+    train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]})
+    val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
+    tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
+
+    wrapper = MagicMock()
+    wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)])
+    wrapper.predict = MagicMock(
+        side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)]
+    )
+    training.observe_lag_sync = MagicMock()
+    training.emit_metric_sync = MagicMock()
+
+    from model_manager import metrics as mm_metrics
+
+    training._fit_model(wrapper, tmr, tp, {})
+
+    training.observe_lag_sync.assert_called_once()
+    call_args = training.observe_lag_sync.call_args
+    assert call_args.args[1] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_LAG
+    training.emit_metric_sync.assert_not_called()
+
+
+def test_fit_model_increments_error_counter_on_failure(training):
+    tp = TrainModelParams.from_dict(
+        {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
+    )
+    train_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
+    val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
+    tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
+
+    wrapper = MagicMock()
+    wrapper.train = MagicMock(side_effect=RuntimeError('fit-fail'))
+    training.observe_lag_sync = MagicMock()
+    training.emit_metric_sync = MagicMock()
+
+    from model_manager import metrics as mm_metrics
+
+    with pytest.raises(RuntimeError, match='fit-fail'):
+        training._fit_model(wrapper, tmr, tp, {})
+
+    training.observe_lag_sync.assert_called_once()
+    training.emit_metric_sync.assert_called_once()
+    call_args = training.emit_metric_sync.call_args
+    assert call_args.kwargs['metric_object'] is mm_metrics.SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL
+
+
+@patch('model_manager.activities.training.mm_metrics')
+@patch('model_manager.activities.training.mlflow')
+def test_train_model_sets_quality_gauges_after_compute_metrics(mock_mlflow, mock_mm_metrics, training):
+    tp = TrainModelParams.from_dict(
+        {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
+    )
+    train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]})
+    val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
+    tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
+
+    training.minio_repository.download_file = MagicMock(return_value=b'csv')
+    training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr)
+
+    def _set_metrics(x, _w, **_kw):
+        x.mse_val = 0.5
+        x.mae_val = 0.3
+        x.r2_val = -0.1
+        return x
+
+    training.data_manager_repository.compute_regression_metrics = MagicMock(
+        side_effect=_set_metrics
+    )
+
+    def _fill_report(x, **_kw):
+        x.report_path = '/tmp/r.html'
+        x.train_data_path = '/tmp/tr.csv'
+        x.test_data_path = '/tmp/te.csv'
+        x.run_dir = '/tmp/run'
+        return x
+
+    training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report)
+    wrapper = MagicMock()
+    wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)])
+    wrapper.predict = MagicMock(
+        side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)]
+    )
+    wrapper.store_model = MagicMock()
+    training.plugin_store.get_model = MagicMock(return_value=wrapper)
+
+    @contextmanager
+    def _run_ctx(*_a, **_k):
+        info = MagicMock()
+        info.run_id = 'rid'
+        yield info
+
+    training.mlflow_repository.start_run = _run_ctx
+    training.observe_lag_sync = MagicMock()
+    training.emit_metric_sync = MagicMock()
+
+    training.train_model({'metadata': {}, 'train_params': tp.to_dict()})
+
+    mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_called_once_with(0.5)
+    mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_called_once_with(0.3)
+    mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_called_once_with(-0.1)
+
+
+@patch('model_manager.activities.training.mm_metrics')
+@patch('model_manager.activities.training.mlflow')
+def test_train_model_skips_quality_gauges_when_none(_mock_mlflow, mock_mm_metrics, training):
+    tp = TrainModelParams.from_dict(
+        {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
+    )
+    train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]})
+    val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
+    tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
+
+    training.minio_repository.download_file = MagicMock(return_value=b'csv')
+    training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr)
+    training.data_manager_repository.compute_regression_metrics = MagicMock(
+        side_effect=lambda x, _w, **_kw: x
+    )
+
+    def _fill_report(x, **_kw):
+        x.report_path = '/tmp/r.html'
+        x.train_data_path = '/tmp/tr.csv'
+        x.test_data_path = '/tmp/te.csv'
+        x.run_dir = '/tmp/run'
+        return x
+
+    training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report)
+    wrapper = MagicMock()
+    wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)])
+    wrapper.predict = MagicMock(
+        side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)]
+    )
+    wrapper.store_model = MagicMock()
+    training.plugin_store.get_model = MagicMock(return_value=wrapper)
+
+    @contextmanager
+    def _run_ctx(*_a, **_k):
+        info = MagicMock()
+        info.run_id = 'rid'
+        yield info
+
+    training.mlflow_repository.start_run = _run_ctx
+    training.observe_lag_sync = MagicMock()
+    training.emit_metric_sync = MagicMock()
+
+    training.train_model({'metadata': {}, 'train_params': tp.to_dict()})
+
+    mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MSE.labels.return_value.set.assert_not_called()
+    mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_MAE.labels.return_value.set.assert_not_called()
+    mock_mm_metrics.SIENTIA_TRAINING_MODEL_QUALITY_R2.labels.return_value.set.assert_not_called()
+
+
+@patch('model_manager.activities.training.mm_metrics')
+@patch('model_manager.activities.training.mlflow')
+def test_train_model_increments_trained_total_on_success(_mock_mlflow, mock_mm_metrics, training):
+    tp = TrainModelParams.from_dict(
+        {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
+    )
+    train_df = pd.DataFrame({'a': [1.0, 2.0], 't': [1.0, 2.0]})
+    val_df = pd.DataFrame({'a': [1.0], 't': [1.0]})
+    tmr = TrainModelResult(params=tp, train_data=train_df, val_data=val_df)
+
+    training.minio_repository.download_file = MagicMock(return_value=b'csv')
+    training.data_manager_repository.prepare_training_data = MagicMock(return_value=tmr)
+    training.data_manager_repository.compute_regression_metrics = MagicMock(
+        side_effect=lambda x, _w, **_kw: x
+    )
+
+    def _fill_report(x, **_kw):
+        x.report_path = '/tmp/r.html'
+        x.train_data_path = '/tmp/tr.csv'
+        x.test_data_path = '/tmp/te.csv'
+        x.run_dir = '/tmp/run'
+        return x
+
+    training.data_manager_repository.generate_report = MagicMock(side_effect=_fill_report)
+    wrapper = MagicMock()
+    wrapper.transform = MagicMock(side_effect=[(train_df, None), (val_df, None)])
+    wrapper.predict = MagicMock(
+        side_effect=[(pd.DataFrame({'p': [1.0, 2.0]}), None), (pd.DataFrame({'p': [1.0]}), None)]
+    )
+    wrapper.store_model = MagicMock()
+    training.plugin_store.get_model = MagicMock(return_value=wrapper)
+
+    @contextmanager
+    def _run_ctx(*_a, **_k):
+        info = MagicMock()
+        info.run_id = 'rid'
+        yield info
+
+    training.mlflow_repository.start_run = _run_ctx
+    training.observe_lag_sync = MagicMock()
+    training.emit_metric_sync = MagicMock()
+
+    training.train_model({'metadata': {}, 'train_params': tp.to_dict()})
+
+    training.emit_metric_sync.assert_called_once_with(
+        metric_object=mock_mm_metrics.SIENTIA_TRAINING_MODEL_TRAINED_TOTAL,
+        tags=training._get_training_labels(tp),
+    )
+
+
+def test_train_model_does_not_increment_trained_total_on_failure(training):
+    tp = TrainModelParams.from_dict(
+        {**_minimal_params_dict(), 'model_metadata': {'schemas': {'components': {'schemas': {}}}}}
+    )
+    training.minio_repository.download_file = MagicMock(side_effect=RuntimeError('dl-fail'))
+    training.send_notification = MagicMock()
+    training.emit_metric_sync = MagicMock()
+
+    with pytest.raises(RuntimeError):
+        training.train_model({'metadata': {}, 'train_params': tp.to_dict()})
+
+    training.emit_metric_sync.assert_not_called()
+
+
 def test_train_model_value_error_when_paths_missing_after_report(training):
     """Raises ValueError when report paths are not populated after generate_report."""
     tp = TrainModelParams.from_dict(

tests/test_metrics.py

@@ -10,7 +10,7 @@ def test_app_up_metric_exists():
     from model_manager.metrics import APP_UP
 
     assert APP_UP is not None
-    assert APP_UP._name == 'app_up'
+    assert APP_UP._name == 'sientia_app_up'
     assert (
         APP_UP._documentation == 'Indicates if the application is running (1) or shutting down (0)'
     )
@@ -207,3 +207,125 @@ def test_prometheus_client_gauge_import():
     from model_manager.metrics import Gauge
 
     assert Gauge is PrometheusGauge
+
+
+# ---------------------------------------------------------------------------
+# Training metrics — existence, type, and labels
+# ---------------------------------------------------------------------------
+
+_TRAINING_LABEL_NAMES = ('pod_id', 'model_name', 'model_type')
+
+
+def _assert_training_labels(metric):
+    for label in _TRAINING_LABEL_NAMES:
+        assert label in metric._labelnames
+
+
+def test_sientia_training_data_preparation_lag_is_histogram():
+    from prometheus_client import Histogram
+
+    from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_LAG
+
+    assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_LAG, Histogram)
+    assert SIENTIA_TRAINING_DATA_PREPARATION_LAG._name == 'sientia_training_data_preparation_lag'
+    _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_LAG)
+
+
+def test_sientia_training_data_preparation_error_count_total_is_counter():
+    from prometheus_client import Counter
+
+    from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL
+
+    assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, Counter)
+    assert 'sientia_training_data_preparation_error_count' in SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL._name
+    _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL)
+
+
+def test_sientia_training_model_fit_lag_is_histogram():
+    from prometheus_client import Histogram
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_LAG
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_FIT_LAG, Histogram)
+    assert SIENTIA_TRAINING_MODEL_FIT_LAG._name == 'sientia_training_model_fit_lag'
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_LAG)
+
+
+def test_sientia_training_model_fit_error_count_total_is_counter():
+    from prometheus_client import Counter
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, Counter)
+    assert 'sientia_training_model_fit_error_count' in SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL._name
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL)
+
+
+def test_sientia_training_model_quality_mse_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MSE
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MSE, Gauge)
+    assert SIENTIA_TRAINING_MODEL_QUALITY_MSE._name == 'sientia_training_model_quality_mse'
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MSE)
+
+
+def test_sientia_training_model_quality_mae_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MAE
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MAE, Gauge)
+    assert SIENTIA_TRAINING_MODEL_QUALITY_MAE._name == 'sientia_training_model_quality_mae'
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MAE)
+
+
+def test_sientia_training_model_quality_r2_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_R2
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_R2, Gauge)
+    assert SIENTIA_TRAINING_MODEL_QUALITY_R2._name == 'sientia_training_model_quality_r2'
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_R2)
+
+
+def test_sientia_training_dataset_train_rows_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_DATASET_TRAIN_ROWS
+
+    assert isinstance(SIENTIA_TRAINING_DATASET_TRAIN_ROWS, Gauge)
+    assert SIENTIA_TRAINING_DATASET_TRAIN_ROWS._name == 'sientia_training_dataset_train_rows'
+    _assert_training_labels(SIENTIA_TRAINING_DATASET_TRAIN_ROWS)
+
+
+def test_sientia_training_dataset_val_rows_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_DATASET_VAL_ROWS
+
+    assert isinstance(SIENTIA_TRAINING_DATASET_VAL_ROWS, Gauge)
+    assert SIENTIA_TRAINING_DATASET_VAL_ROWS._name == 'sientia_training_dataset_val_rows'
+    _assert_training_labels(SIENTIA_TRAINING_DATASET_VAL_ROWS)
+
+
+def test_sientia_training_model_trained_total_is_counter():
+    from prometheus_client import Counter
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_TRAINED_TOTAL
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, Counter)
+    assert 'sientia_training_model_trained' in SIENTIA_TRAINING_MODEL_TRAINED_TOTAL._name
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL)
+
+
+def test_sientia_training_feature_count_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_FEATURE_COUNT
+
+    assert isinstance(SIENTIA_TRAINING_FEATURE_COUNT, Gauge)
+    assert SIENTIA_TRAINING_FEATURE_COUNT._name == 'sientia_training_feature_count'
+    _assert_training_labels(SIENTIA_TRAINING_FEATURE_COUNT)

values.yaml

@@ -328,7 +328,7 @@ ssh:
 # Configuração para dashboards do Grafana
 grafanaDashboard:
   # Habilita a criação de ConfigMaps para dashboards
-  enabled: false
+  enabled: true
   # Namespace onde o Grafana está instalado (ajuste conforme seu ambiente)
   namespace: monitoring
   # Labels para que o sidecar do Grafana encontre os dashboards