95 lines
2.7 KiB
Python
95 lines
2.7 KiB
Python
"""
|
|
Model Manager Metrics Module
|
|
|
|
This module defines all Prometheus metrics used by the Sientia DataOps Model Manager system
|
|
for monitoring and observability. The metrics provide insights into system performance,
|
|
training operations, and operational health.
|
|
|
|
The metrics are designed to be scraped by Prometheus and can be visualized in
|
|
Grafana or other monitoring dashboards to provide real-time visibility into
|
|
the system's operation.
|
|
|
|
Key Metric Categories:
|
|
- Application Health: Overall system status and availability
|
|
|
|
Metric Labels:
|
|
- pod_id: Kubernetes pod identifier for multi-instance deployments
|
|
"""
|
|
|
|
from prometheus_client import Counter, Gauge, Histogram
|
|
|
|
# Application health metric
|
|
APP_UP = Gauge(
|
|
'app_up',
|
|
'Indicates if the application is running (1) or shutting down (0)',
|
|
['pod_id'],
|
|
)
|
|
|
|
_TRAINING_LABELS = ['pod_id', 'model_name', 'model_type']
|
|
|
|
SIENTIA_TRAINING_MODEL_TRAINED_TOTAL = Counter(
|
|
'sientia_training_model_trained_total',
|
|
'Number of successfully completed model training runs',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_DATA_PREPARATION_LAG = Histogram(
|
|
'sientia_training_data_preparation_lag',
|
|
'Latency of prepare_training_data() in seconds',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_DATA_PREPARATION_ERROR_COUNT_TOTAL = Counter(
|
|
'sientia_training_data_preparation_error_count_total',
|
|
'Number of failures in prepare_training_data()',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_MODEL_FIT_LAG = Histogram(
|
|
'sientia_training_model_fit_lag',
|
|
'Latency of wrapper.train() (model fitting) in seconds',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_MODEL_FIT_ERROR_COUNT_TOTAL = Counter(
|
|
'sientia_training_model_fit_error_count_total',
|
|
'Number of failures in wrapper.train() (model fitting)',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_MODEL_QUALITY_MSE = Gauge(
|
|
'sientia_training_model_quality_mse',
|
|
'Mean Squared Error of the last successful model training',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_MODEL_QUALITY_MAE = Gauge(
|
|
'sientia_training_model_quality_mae',
|
|
'Mean Absolute Error of the last successful model training',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_MODEL_QUALITY_R2 = Gauge(
|
|
'sientia_training_model_quality_r2',
|
|
'R-squared of the last successful model training',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_DATASET_TRAIN_ROWS = Gauge(
|
|
'sientia_training_dataset_train_rows',
|
|
'Number of rows in the training dataset after preparation',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_DATASET_VAL_ROWS = Gauge(
|
|
'sientia_training_dataset_val_rows',
|
|
'Number of rows in the validation dataset after preparation',
|
|
_TRAINING_LABELS,
|
|
)
|
|
|
|
SIENTIA_TRAINING_FEATURE_COUNT = Gauge(
|
|
'sientia_training_feature_count',
|
|
'Number of input feature columns used for training',
|
|
_TRAINING_LABELS,
|
|
)
|