sientia-dataops-laborious_temporal/laborious/metrics.py

"""
Laborious Metrics Module

This module defines all Prometheus metrics used by the Sientia DataOps Laborious system
for monitoring and observability. The metrics provide insights into system performance,
prediction quality, and operational health.

The metrics are designed to be scraped by Prometheus and can be visualized in
Grafana or other monitoring dashboards to provide real-time visibility into
the system's operation.

Key Metric Categories:
- Application Health: Overall system status and availability
- Prediction Operations: Count and performance of prediction operations
- Data Quality: Confidence levels and validation results
- Export Operations: Database and OPC export performance
- Response Times: Performance monitoring for various operations

Metric Labels:
- pod_id: Kubernetes pod identifier for multi-instance deployments
- model_name: Name of the ML model being used
- workflow_name: Name of the prediction pipeline
- opc_server_id: Identifier for OPC server operations
"""

from prometheus_client import Counter, Gauge, Histogram
from sientia_do.observability.metrics import CORE_LABELS as SIENTIA_CORE_LABELS

# Application health metric
APP_UP = Gauge(
    'app_up',
    'Indicates if the application is running (1) or shutting down (0)',
    ['pod_id'],
)

# Core labels used across multiple metrics
CORE_LABELS = ['pod_id', 'model_name', 'workflow_name']

# Prediction operation metrics
PREDICTIONS_WRITTEN_COUNT = Counter(
    'laborious_predictions_written_count',
    'Number of predictions written to the database table predictions',
    CORE_LABELS,
)

# Prediction quality metrics
PREDICTION_CONFIDENCE_MONITOR = Gauge(
    'laborious_prediction_confidence_monitor',
    'Current confidence of each prediction',
    CORE_LABELS,
)

# Prediction total response time
PREDICTION_RESPONSE_TIME_MONITOR = Histogram(
    'laborious_prediction_response_time_monitor',
    'Current response time of each prediction',
    CORE_LABELS,
    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
)

# ================== MinIO metrics ==================

MINIO_READ_LAG = Histogram(
    'laborious_minio_read_lag',
    'Lag between the last write to MinIO and the last read from MinIO',
    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
)

MINIO_WRITE_LAG = Histogram(
    'laborious_minio_write_lag',
    'Lag between the last write to MinIO and the last read from MinIO',
    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
)

MINIO_READ_COUNT = Counter(
    'laborious_minio_read_count',
    'Number of reads from MinIO',
    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
)

MINIO_WRITE_COUNT = Counter(
    'laborious_minio_write_count',
    'Number of writes to MinIO',
    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
)

MINIO_READ_ERROR_COUNT = Counter(
    'laborious_minio_read_error_count',
    'Number of errors reading from MinIO',
    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
)

MINIO_WRITE_ERROR_COUNT = Counter(
    'laborious_minio_write_error_count',
    'Number of errors writing to MinIO',
    [*SIENTIA_CORE_LABELS, 'bucket_name', 'object_name'],
)

# ================== OPC metrics ==================


PREDICTION_OPC_WRITING_COUNT = Counter(
    'laborious_prediction_opc_writing_count',
    'Number of predictions written to the OPC server',
    [*CORE_LABELS, 'opc_server_id', 'tag'],
)

PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram(
    'laborious_prediction_opc_writing_response_time_monitor',
    'Current response time of each prediction written to the OPC server',
    [*CORE_LABELS, 'opc_server_id', 'tag'],
    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
)

OPC_CONNECTIONS_TOTAL = Counter(
    'opc_connections_initiated_total',
    'Total connection attempts to OPC servers',
    ['pod_id', 'server_name'],
)
OPC_CONNECTIONS_FAILED = Counter(
    'opc_connections_failed_total',
    'Total failed connection attempts to OPC servers',
    ['pod_id', 'server_name'],
)
OPC_CONNECTION_STATUS = Gauge(
    'opc_connection_status',
    'Connection status with the OPC server (1=connected, 0=disconnected)',
    ['pod_id', 'server_name', 'server_url'],
)

# ================== Model metrics ==================

MODEL_READ_LAG = Histogram(
    'laborious_model_read_lag',
    'Lag between the start and read of read operations',
    SIENTIA_CORE_LABELS,
    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
)

MODEL_WRITE_LAG = Histogram(
    'laborious_model_write_lag',
    'Lag between the start and end of write operations',
    SIENTIA_CORE_LABELS,
    buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
)

MODEL_READ_COUNT = Counter(
    'laborious_model_read_count',
    'Number of reads from the model',
    SIENTIA_CORE_LABELS,
)

MODEL_WRITE_COUNT = Counter(
    'laborious_model_write_count',
    'Number of writes to the model',
    SIENTIA_CORE_LABELS,
)

MODEL_READ_ERROR_COUNT = Counter(
    'laborious_model_read_error_count',
    'Number of errors reading from the model',
    SIENTIA_CORE_LABELS,
)

MODEL_WRITE_ERROR_COUNT = Counter(
    'laborious_model_write_error_count',
    'Number of errors writing to the model',
    SIENTIA_CORE_LABELS,
)