""" Laborious Metrics Module This module defines all Prometheus metrics used by the Sientia DataOps Laborious system for monitoring and observability. The metrics provide insights into system performance, prediction quality, and operational health. The metrics are designed to be scraped by Prometheus and can be visualized in Grafana or other monitoring dashboards to provide real-time visibility into the system's operation. Key Metric Categories: - Application Health: Overall system status and availability - Prediction Operations: Count and performance of prediction operations - Data Quality: Confidence levels and validation results - Export Operations: Database and OPC export performance - Response Times: Performance monitoring for various operations Metric Labels: - pod_id: Kubernetes pod identifier for multi-instance deployments - runtime: Runtime / environment identifier (matches ``RUNTIME`` env, see ``SientiaMonitoring``) - model_name: Name of the ML model being used - workflow_name: Name of the prediction pipeline - opc_server_id: Identifier for OPC server operations """ from prometheus_client import Counter, Gauge, Histogram from sientia_do.observability.metrics import ( CORE_LABELS as SIENTIA_CORE_LABELS, ) # Application health metric APP_UP = Gauge( 'app_up', 'Indicates if the application is running (1) or shutting down (0)', ['pod_id'], ) # Core labels used across multiple laborious metrics (aligned with ``SientiaMonitoring.labels`` subset) CORE_LABELS = ['pod_id', 'runtime', 'model_name', 'workflow_name'] # Prediction operation metrics PREDICTIONS_WRITTEN_COUNT = Counter( 'laborious_predictions_written_count', 'Number of predictions written to the database table predictions', CORE_LABELS, ) # Prediction quality metrics PREDICTION_CONFIDENCE_MONITOR = Gauge( 'laborious_prediction_confidence_monitor', 'Current confidence of each prediction', CORE_LABELS, ) # Prediction total response time PREDICTION_RESPONSE_TIME_MONITOR = Histogram( 'laborious_prediction_response_time_monitor', 'Current response time of each prediction', CORE_LABELS, buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) # ================== MinIO metrics ================== MINIO_READ_LAG = Histogram( 'laborious_minio_read_lag', 'Lag between the last write to MinIO and the last read from MinIO', [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'], buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) MINIO_WRITE_LAG = Histogram( 'laborious_minio_write_lag', 'Lag between the last write to MinIO and the last read from MinIO', [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'], buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) MINIO_READ_COUNT = Counter( 'laborious_minio_read_count', 'Number of reads from MinIO', [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'], ) MINIO_WRITE_COUNT = Counter( 'laborious_minio_write_count', 'Number of writes to MinIO', [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'], ) MINIO_READ_ERROR_COUNT = Counter( 'laborious_minio_read_error_count', 'Number of errors reading from MinIO', [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'], ) MINIO_WRITE_ERROR_COUNT = Counter( 'laborious_minio_write_error_count', 'Number of errors writing to MinIO', [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'], ) # ================== OPC metrics ================== PREDICTION_OPC_WRITING_COUNT = Counter( 'laborious_prediction_opc_writing_count', 'Number of predictions written to the OPC server', [*CORE_LABELS, 'opc_server_id', 'tag'], ) PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram( 'laborious_prediction_opc_writing_response_time_monitor', 'Current response time of each prediction written to the OPC server', [*CORE_LABELS, 'opc_server_id', 'tag'], buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) OPC_CONNECTIONS_TOTAL = Counter( 'opc_connections_initiated_total', 'Total connection attempts to OPC servers', ['pod_id', 'server_name'], ) OPC_CONNECTIONS_FAILED = Counter( 'opc_connections_failed_total', 'Total failed connection attempts to OPC servers', ['pod_id', 'server_name'], ) OPC_CONNECTION_STATUS = Gauge( 'opc_connection_status', 'Connection status with the OPC server (1=connected, 0=disconnected)', ['pod_id', 'server_name', 'server_url'], ) # ================== Model metrics ================== MODEL_READ_LAG = Histogram( 'laborious_model_read_lag', 'Lag between the start and read of read operations', SIENTIA_CORE_LABELS, buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) MODEL_WRITE_LAG = Histogram( 'laborious_model_write_lag', 'Lag between the start and end of write operations', SIENTIA_CORE_LABELS, buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) MODEL_READ_COUNT = Counter( 'laborious_model_read_count', 'Number of reads from the model', SIENTIA_CORE_LABELS, ) MODEL_WRITE_COUNT = Counter( 'laborious_model_write_count', 'Number of writes to the model', SIENTIA_CORE_LABELS, ) MODEL_READ_ERROR_COUNT = Counter( 'laborious_model_read_error_count', 'Number of errors reading from the model', SIENTIA_CORE_LABELS, ) MODEL_WRITE_ERROR_COUNT = Counter( 'laborious_model_write_error_count', 'Number of errors writing to the model', SIENTIA_CORE_LABELS, ) MODEL_ANALYZE_LAG = Histogram( 'laborious_model_analyze_lag', 'Lag between the start and end of analyze operations', SIENTIA_CORE_LABELS, buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) MODEL_ANALYZE_COUNT = Counter( 'laborious_model_analyze_count', 'Number of analyze operations', SIENTIA_CORE_LABELS, ) MODEL_ANALYZE_ERROR_COUNT = Counter( 'laborious_model_analyze_error_count', 'Number of errors during analyze operations', SIENTIA_CORE_LABELS, ) # ================== PI Web API metrics ================== PI_WEB_API_LABELS = [*CORE_LABELS, 'tag_name'] PI_WEB_API_PREDICTION_WRITTEN_COUNT = Counter( 'laborious_pi_web_api_prediction_written_count', 'Number of predictions written to the PI Web API', PI_WEB_API_LABELS, ) PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT = Counter( 'laborious_pi_web_api_prediction_written_error_count', 'Number of errors writing predictions to the PI Web API', PI_WEB_API_LABELS, )