dfc190c818
This commit includes several changes: - Reorganized imports and class inheritance in activities.py, gates.py and mlflow.py for better readability and maintainability. - Improved error handling and logging in gates.py and mlflow.py. - Added input validation and filtering in gates.py to ensure data quality. - Enhanced prediction formatting and storage policy management in gates.py. - Updated metrics.py to use consistent naming conventions and labels. - Refactored connectors_config.py to use type hints and improve code clarity. - Updated conditional and MLFlow filters for better data quality checks. - Improved model repository logic for retraining and updating models. - Enhanced worker.py to include SDK metrics and improved error handling. - Refactored workflows for better modularity and error handling. - Updated tests to reflect the changes and improve test coverage.
58 lines
1.9 KiB
Python
58 lines
1.9 KiB
Python
"""
|
|
Model Manager Metrics Module
|
|
|
|
This module defines all Prometheus metrics used by the Sientia DataOps Model Manager system
|
|
for monitoring and observability. The metrics provide insights into system performance,
|
|
prediction quality, and operational health.
|
|
|
|
The metrics are designed to be scraped by Prometheus and can be visualized in
|
|
Grafana or other monitoring dashboards to provide real-time visibility into
|
|
the system's operation.
|
|
|
|
Key Metric Categories:
|
|
- Application Health: Overall system status and availability
|
|
- Prediction Operations: Count and performance of prediction operations
|
|
- Data Quality: Confidence levels and validation results
|
|
- Export Operations: Database export performance
|
|
- Response Times: Performance monitoring for various operations
|
|
|
|
Metric Labels:
|
|
- pod_id: Kubernetes pod identifier for multi-instance deployments
|
|
- model_name: Name of the ML model being used
|
|
- pipeline_name: Name of the prediction pipeline
|
|
"""
|
|
|
|
from prometheus_client import Counter, Gauge, Histogram
|
|
|
|
# Application health metric
|
|
APP_UP = Gauge(
|
|
'app_up',
|
|
'Indicates if the application is running (1) or shutting down (0)',
|
|
['pod_id'],
|
|
)
|
|
|
|
# Core labels used across multiple metrics
|
|
CORE_LABELS = ['pod_id', 'model_name', 'pipeline_name']
|
|
|
|
# Prediction operation metrics
|
|
PREDICTIONS_WRITTEN_COUNT = Counter(
|
|
'model_manager_predictions_written_count',
|
|
'Number of predictions written to the database table predictions',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
# Prediction quality metrics
|
|
PREDICTION_CONFIDENCE_MONITOR = Gauge(
|
|
'model_manager_prediction_confidence_monitor',
|
|
'Current confidence of each prediction',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
# Performance monitoring metrics
|
|
PREDICTION_RESPONSE_TIME_MONITOR = Histogram(
|
|
'model_manager_prediction_response_time_monitor',
|
|
'Current response time of each prediction',
|
|
CORE_LABELS,
|
|
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
|
)
|