240 lines
7.4 KiB
Python
240 lines
7.4 KiB
Python
"""
|
|
Laborious Metrics Module
|
|
|
|
This module defines all Prometheus metrics used by the Sientia DataOps Laborious system
|
|
for monitoring and observability. The metrics provide insights into system performance,
|
|
prediction quality, and operational health.
|
|
|
|
The metrics are designed to be scraped by Prometheus and can be visualized in
|
|
Grafana or other monitoring dashboards to provide real-time visibility into
|
|
the system's operation.
|
|
|
|
Key Metric Categories:
|
|
- Application Health: Overall system status and availability
|
|
- Prediction Operations: Count and performance of prediction operations
|
|
- Data Quality: Confidence levels and validation results
|
|
- Export Operations: Database and OPC export performance
|
|
- Response Times: Performance monitoring for various operations
|
|
|
|
Metric Labels:
|
|
- pod_id: Kubernetes pod identifier for multi-instance deployments
|
|
- runtime: Runtime / environment identifier (matches ``RUNTIME`` env, see ``SientiaMonitoring``)
|
|
- model_name: Name of the ML model being used
|
|
- workflow_name: Name of the prediction pipeline
|
|
- opc_server_id: Identifier for OPC server operations
|
|
"""
|
|
|
|
from prometheus_client import Counter, Gauge, Histogram
|
|
from sientia_do.observability.metrics import CORE_LABELS
|
|
|
|
# Application health metric
|
|
APP_UP = Gauge(
|
|
'app_up',
|
|
'Indicates if the application is running (1) or shutting down (0)',
|
|
['pod_id'],
|
|
)
|
|
|
|
# Prediction operation metrics
|
|
PREDICTIONS_WRITTEN_COUNT = Counter(
|
|
'laborious_predictions_written_count',
|
|
'Number of predictions written to the database table predictions',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
# Prediction quality metrics
|
|
PREDICTION_CONFIDENCE_MONITOR = Gauge(
|
|
'laborious_prediction_confidence_monitor',
|
|
'Current confidence of each prediction',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
# Prediction total response time
|
|
PREDICTION_RESPONSE_TIME_MONITOR = Histogram(
|
|
'laborious_prediction_response_time_monitor',
|
|
'Current response time of each prediction',
|
|
CORE_LABELS,
|
|
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
|
)
|
|
|
|
|
|
# ================== OPC metrics ==================
|
|
|
|
|
|
PREDICTION_OPC_WRITING_COUNT = Counter(
|
|
'laborious_prediction_opc_writing_count',
|
|
'Number of predictions written to the OPC server',
|
|
[*CORE_LABELS, 'opc_server_id', 'tag'],
|
|
)
|
|
|
|
PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram(
|
|
'laborious_prediction_opc_writing_response_time_monitor',
|
|
'Current response time of each prediction written to the OPC server',
|
|
[*CORE_LABELS, 'opc_server_id', 'tag'],
|
|
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
|
)
|
|
|
|
OPC_CONNECTIONS_TOTAL = Counter(
|
|
'opc_connections_initiated_total',
|
|
'Total connection attempts to OPC servers',
|
|
['pod_id', 'server_name'],
|
|
)
|
|
OPC_CONNECTIONS_FAILED = Counter(
|
|
'opc_connections_failed_total',
|
|
'Total failed connection attempts to OPC servers',
|
|
['pod_id', 'server_name'],
|
|
)
|
|
OPC_CONNECTION_STATUS = Gauge(
|
|
'opc_connection_status',
|
|
'Connection status with the OPC server (1=connected, 0=disconnected)',
|
|
['pod_id', 'server_name', 'server_url'],
|
|
)
|
|
|
|
_OPC_SESSION_DEBUG_LABELS = ['pod_id', 'server_name', 'runtime', 'opc_server_id', 'session_id']
|
|
|
|
OPC_SESSION_CREATED_TOTAL = Counter(
|
|
'opc_session_created_total',
|
|
'OPC UA sessions established (after successful connect)',
|
|
_OPC_SESSION_DEBUG_LABELS,
|
|
)
|
|
|
|
OPC_SESSION_CLOSED_TOTAL = Counter(
|
|
'opc_session_closed_total',
|
|
'OPC UA client disconnects completed (session tear-down initiated)',
|
|
_OPC_SESSION_DEBUG_LABELS,
|
|
)
|
|
|
|
OPC_SESSION_REVISED_TIMEOUT_MS = Gauge(
|
|
'opc_session_revised_timeout_milliseconds',
|
|
'Server-revised OPC UA session timeout (RevisedSessionTimeout) in ms after connect',
|
|
_OPC_SESSION_DEBUG_LABELS,
|
|
)
|
|
|
|
OPC_WRITE_ATTEMPT_LABELS = [*_OPC_SESSION_DEBUG_LABELS, 'model_id', 'model_name', 'result']
|
|
|
|
OPC_WRITE_ATTEMPTS_TOTAL = Counter(
|
|
'opc_write_attempts_total',
|
|
'OPC UA write attempts with session and outcome (result=OK or exception class name)',
|
|
OPC_WRITE_ATTEMPT_LABELS,
|
|
)
|
|
|
|
OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL = Counter(
|
|
'opc_write_inter_arrival_over_session_timeout_total',
|
|
'Successful writes where seconds since the previous successful write exceeded RevisedSessionTimeout (ms)',
|
|
_OPC_SESSION_DEBUG_LABELS,
|
|
)
|
|
|
|
# ================== Model metrics ==================
|
|
|
|
MODEL_READ_LAG = Histogram(
|
|
'laborious_model_read_lag',
|
|
'Lag between the start and read of read operations',
|
|
CORE_LABELS,
|
|
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
|
)
|
|
|
|
MODEL_WRITE_LAG = Histogram(
|
|
'laborious_model_write_lag',
|
|
'Lag between the start and end of write operations',
|
|
CORE_LABELS,
|
|
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
|
)
|
|
|
|
MODEL_READ_COUNT = Counter(
|
|
'laborious_model_read_count',
|
|
'Number of reads from the model',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
MODEL_WRITE_COUNT = Counter(
|
|
'laborious_model_write_count',
|
|
'Number of writes to the model',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
MODEL_READ_ERROR_COUNT = Counter(
|
|
'laborious_model_read_error_count',
|
|
'Number of errors reading from the model',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
MODEL_WRITE_ERROR_COUNT = Counter(
|
|
'laborious_model_write_error_count',
|
|
'Number of errors writing to the model',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
MODEL_ANALYZE_LAG = Histogram(
|
|
'laborious_model_analyze_lag',
|
|
'Lag between the start and end of analyze operations',
|
|
CORE_LABELS,
|
|
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
|
)
|
|
|
|
MODEL_ANALYZE_COUNT = Counter(
|
|
'laborious_model_analyze_count',
|
|
'Number of analyze operations',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
MODEL_ANALYZE_ERROR_COUNT = Counter(
|
|
'laborious_model_analyze_error_count',
|
|
'Number of errors during analyze operations',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
|
|
# ================== Model import metrics ==================
|
|
#
|
|
# The failing step and the error code travel in `operation_type`, the one label the shared
|
|
# `CORE_LABELS` leaves free. Nothing else about a failure is a label: an error reason is a sentence
|
|
# and an exception is a log line, neither of which belongs in a metric's cardinality.
|
|
|
|
MODEL_IMPORT_STARTED_COUNT = Counter(
|
|
'laborious_model_import_started_count',
|
|
'Number of model imports that claimed their import log row',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
MODEL_IMPORT_COMPLETED_COUNT = Counter(
|
|
'laborious_model_import_completed_count',
|
|
'Number of model imports that reached a registered version and a model document',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
MODEL_IMPORT_ERROR_COUNT = Counter(
|
|
'laborious_model_import_error_count',
|
|
'Number of failed model imports, with the failing step or error code in operation_type',
|
|
CORE_LABELS,
|
|
)
|
|
|
|
MODEL_IMPORT_KDF_DURATION = Histogram(
|
|
'laborious_model_import_kdf_duration_seconds',
|
|
'Time spent deriving the bundle key with Argon2id',
|
|
CORE_LABELS,
|
|
buckets=[0.5, 1.0, 2.0, 3.0, 5.0, 8.0, 13.0, 21.0],
|
|
)
|
|
|
|
MODEL_IMPORT_STEP_DURATION = Histogram(
|
|
'laborious_model_import_step_duration_seconds',
|
|
'Duration of each model import step, named in operation_type',
|
|
CORE_LABELS,
|
|
buckets=[0.1, 0.5, 1.0, 5.0, 10.0, 30.0, 60.0, 300.0],
|
|
)
|
|
|
|
|
|
# ================== PI Web API metrics ==================
|
|
|
|
PI_WEB_API_LABELS = [*CORE_LABELS, 'tag_name']
|
|
|
|
PI_WEB_API_PREDICTION_WRITTEN_COUNT = Counter(
|
|
'laborious_pi_web_api_prediction_written_count',
|
|
'Number of predictions written to the PI Web API',
|
|
PI_WEB_API_LABELS,
|
|
)
|
|
|
|
PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT = Counter(
|
|
'laborious_pi_web_api_prediction_written_error_count',
|
|
'Number of errors writing predictions to the PI Web API',
|
|
PI_WEB_API_LABELS,
|
|
)
|