SIENTIAPDE-1645: Add comprehensive model training observability metrics and Grafana dashboard.

tests/test_metrics.py

@@ -10,7 +10,7 @@ def test_app_up_metric_exists():
     from model_manager.metrics import APP_UP
 
     assert APP_UP is not None
-    assert APP_UP._name == 'app_up'
+    assert APP_UP._name == 'sientia_app_up'
     assert (
         APP_UP._documentation == 'Indicates if the application is running (1) or shutting down (0)'
     )
@@ -207,3 +207,125 @@ def test_prometheus_client_gauge_import():
     from model_manager.metrics import Gauge
 
     assert Gauge is PrometheusGauge
+
+
+# ---------------------------------------------------------------------------
+# Training metrics — existence, type, and labels
+# ---------------------------------------------------------------------------
+
+_TRAINING_LABEL_NAMES = ('pod_id', 'model_name', 'model_type')
+
+
+def _assert_training_labels(metric):
+    for label in _TRAINING_LABEL_NAMES:
+        assert label in metric._labelnames
+
+
+def test_sientia_training_data_preparation_lag_is_histogram():
+    from prometheus_client import Histogram
+
+    from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_LAG
+
+    assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_LAG, Histogram)
+    assert SIENTIA_TRAINING_DATA_PREPARATION_LAG._name == 'sientia_training_data_preparation_lag'
+    _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_LAG)
+
+
+def test_sientia_training_data_preparation_error_count_total_is_counter():
+    from prometheus_client import Counter
+
+    from model_manager.metrics import SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL
+
+    assert isinstance(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL, Counter)
+    assert 'sientia_training_data_preparation_error_count' in SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL._name
+    _assert_training_labels(SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL)
+
+
+def test_sientia_training_model_fit_lag_is_histogram():
+    from prometheus_client import Histogram
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_LAG
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_FIT_LAG, Histogram)
+    assert SIENTIA_TRAINING_MODEL_FIT_LAG._name == 'sientia_training_model_fit_lag'
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_LAG)
+
+
+def test_sientia_training_model_fit_error_count_total_is_counter():
+    from prometheus_client import Counter
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL, Counter)
+    assert 'sientia_training_model_fit_error_count' in SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL._name
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL)
+
+
+def test_sientia_training_model_quality_mse_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MSE
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MSE, Gauge)
+    assert SIENTIA_TRAINING_MODEL_QUALITY_MSE._name == 'sientia_training_model_quality_mse'
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MSE)
+
+
+def test_sientia_training_model_quality_mae_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_MAE
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_MAE, Gauge)
+    assert SIENTIA_TRAINING_MODEL_QUALITY_MAE._name == 'sientia_training_model_quality_mae'
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_MAE)
+
+
+def test_sientia_training_model_quality_r2_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_QUALITY_R2
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_QUALITY_R2, Gauge)
+    assert SIENTIA_TRAINING_MODEL_QUALITY_R2._name == 'sientia_training_model_quality_r2'
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_QUALITY_R2)
+
+
+def test_sientia_training_dataset_train_rows_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_DATASET_TRAIN_ROWS
+
+    assert isinstance(SIENTIA_TRAINING_DATASET_TRAIN_ROWS, Gauge)
+    assert SIENTIA_TRAINING_DATASET_TRAIN_ROWS._name == 'sientia_training_dataset_train_rows'
+    _assert_training_labels(SIENTIA_TRAINING_DATASET_TRAIN_ROWS)
+
+
+def test_sientia_training_dataset_val_rows_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_DATASET_VAL_ROWS
+
+    assert isinstance(SIENTIA_TRAINING_DATASET_VAL_ROWS, Gauge)
+    assert SIENTIA_TRAINING_DATASET_VAL_ROWS._name == 'sientia_training_dataset_val_rows'
+    _assert_training_labels(SIENTIA_TRAINING_DATASET_VAL_ROWS)
+
+
+def test_sientia_training_model_trained_total_is_counter():
+    from prometheus_client import Counter
+
+    from model_manager.metrics import SIENTIA_TRAINING_MODEL_TRAINED_TOTAL
+
+    assert isinstance(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL, Counter)
+    assert 'sientia_training_model_trained' in SIENTIA_TRAINING_MODEL_TRAINED_TOTAL._name
+    _assert_training_labels(SIENTIA_TRAINING_MODEL_TRAINED_TOTAL)
+
+
+def test_sientia_training_feature_count_is_gauge():
+    from prometheus_client import Gauge
+
+    from model_manager.metrics import SIENTIA_TRAINING_FEATURE_COUNT
+
+    assert isinstance(SIENTIA_TRAINING_FEATURE_COUNT, Gauge)
+    assert SIENTIA_TRAINING_FEATURE_COUNT._name == 'sientia_training_feature_count'
+    _assert_training_labels(SIENTIA_TRAINING_FEATURE_COUNT)