sientia-dataops-opc-ingestor/ingestor/metrics.py

"""
Prometheus metrics configuration for the OPC Ingestor application.

This module defines all the metrics used for monitoring and observability
of the OPC Ingestor system. It includes metrics for:

- Application health and performance
- OPC server connections and subscriptions
- Data processing and storage operations
- Resource management and load balancing
- Error tracking and notification systems

All metrics follow Prometheus naming conventions and include appropriate
labels for multi-dimensional analysis and alerting.
"""

from prometheus_client import Counter, Gauge, Histogram

# Metric label definitions for consistent labeling across all metrics
POD_ID_LABEL = ['pod_id']
SERVER_LABELS = ['pod_id', 'server_name', 'server_url']
REDIS_LABELS = ['pod_id', 'operation']
NOTIFICATION_LABELS = ['pod_id', 'level', 'block']

MAIN_LABELS = ['pod_id']

# --- Reliability Metrics ---
TAG_WRITTEN_COUNT = Counter(
    'ingestor_tag_written_count',
    'Number of writing process to the collection',
    [*MAIN_LABELS, 'tag_name', 'collection_name'],
)

# --- General Application Metrics ---
APP_LOOP_COUNT = Counter(
    'app_main_loop_total',
    'Total number of times the application main loop has run',
    POD_ID_LABEL,
)
APP_LOOP_DURATION = Histogram(
    'app_main_loop_duration_seconds',
    'Duration of the application main loop in seconds',
    POD_ID_LABEL,
)
APP_ERRORS_TOTAL = Counter(
    'app_errors_total',
    'Total number of unhandled errors in the main loop',
    POD_ID_LABEL,
)
APP_UP = Gauge(
    'app_up',
    'Indicates if the application is running (1) or shutting down (0)',
    POD_ID_LABEL,
)

# --- Ingestor Manager Metrics ---
ACTIVE_INGESTORS = Gauge(
    'ingestor_active_total',
    'Number of active ingestors reported by Redis',
    POD_ID_LABEL,
)
SLOTS_TOTAL = Gauge(
    'ingestor_slots_total',
    'Total number of slots configured in Redis',
    POD_ID_LABEL,
)
LEASES_TOTAL = Gauge(
    'ingestor_leases_total',
    'Total number of leases (allocated slots) in Redis',
    POD_ID_LABEL,
)
SLOTS_MANAGED = Gauge(
    'ingestor_slots_managed_current',
    'Number of slots currently managed by this ingestor instance',
    POD_ID_LABEL,
)
SLOTS_ACQUIRED = Counter(
    'ingestor_slots_acquired_total',
    'Total number of slots acquired by this instance',
    POD_ID_LABEL,
)
SLOTS_RELEASED = Counter(
    'ingestor_slots_released_total',
    'Total number of slots released by this instance',
    POD_ID_LABEL,
)

# --- OPC Manager Metrics ---
OPC_MANAGERS_ACTIVE = Gauge(
    'ingestor_opc_managers_active',
    'Number of active OPC Managers in this instance',
    POD_ID_LABEL,
)
OPC_SUBSCRIPTION_ERRORS = Counter(
    'ingestor_opc_subscription_errors_total',
    'Errors when trying to subscribe to or unsubscribe from OPC tags',
    ['pod_id', 'server', 'slot'],
)

OPC_CONNECTIONS_TOTAL = Counter(
    'opc_connections_initiated_total',
    'Total connection attempts to OPC servers',
    ['pod_id', 'server_name'],
)
OPC_CONNECTIONS_FAILED = Counter(
    'opc_connections_failed_total',
    'Total failed connection attempts to OPC servers',
    ['pod_id', 'server_name'],
)
OPC_CONNECTION_STATUS = Gauge(
    'opc_connection_status',
    'Connection status with the OPC server (1=connected, 0=disconnected)',
    SERVER_LABELS,
)
OPC_SUBSCRIPTIONS_CREATED = Counter(
    'opc_subscriptions_created_total',
    'Total OPC subscriptions created',
    ['pod_id', 'server_name', 'slot_name'],
)
OPC_TAGS_SUBSCRIBED = Gauge(
    'opc_tags_subscribed_current',
    'Current number of OPC tags subscribed on a server',
    ['pod_id', 'server_name'],
)
OPC_CYCLES_WITHOUT_DATA = Gauge(
    'opc_cycles_without_data',
    'Current number of cycles without receiving data from a server',
    ['pod_id', 'server_name'],
)
OPC_RECONNECTIONS_TOTAL = Counter(
    'opc_reconnections_tried_total',
    'Reconnection attempts to an OPC server after a loss',
    ['pod_id', 'server_name'],
)

# --- Notification Metrics ---
NOTIFICATIONS_SENT = Counter(
    'notifications_sent_total',
    'Total number of notifications sent',
    NOTIFICATION_LABELS,
)

# --- Gateway: Write metrics ---
OPC_WRITE_TOTAL = Counter(
    'opc_write_total',
    'OPC active write attempts',
    ['pod_id', 'server_name', 'model_id', 'result'],  # result: success|error
)
OPC_WRITE_LATENCY = Histogram(
    'opc_write_latency_seconds',
    'OPC active write latency',
    ['pod_id', 'server_name'],
)
OPC_WRITE_STATUS_BY_MODEL = Gauge(
    'opc_write_status_by_model',
    'Last write status per model (1=ok,0=error)',
    ['pod_id', 'model_id', 'status'],
)
OPC_WRITE_VALUE_ONLY_FALLBACK = Counter(
    'opc_write_value_only_fallback_total',
    'Servers that refused the timestamp/quality of a write (BadWriteNotSupported); '
    'their writes go with the value alone',
    ['pod_id', 'server_name'],
)
OPC_WRITE_STATUS_UPDATE_TOTAL = Counter(
    'opc_write_status_update_total',
    'write_status UPDATEs on the prediction rows',
    # result: updated|kept|row_not_found|invalid|error
    ['pod_id', 'result'],
)
# --- Gateway: Active read metrics ---
OPC_ACTIVE_READ_TOTAL = Counter(
    'opc_active_read_total',
    'OPC active read attempts',
    ['pod_id', 'server_name', 'result'],
)
OPC_ACTIVE_READ_LATENCY = Histogram(
    'opc_active_read_latency_seconds',
    'OPC active read latency',
    ['pod_id', 'server_name'],
)
OPC_READ_LAG = Gauge(
    'opc_read_lag_seconds',
    'Lag between OPC source timestamp and machine time',
    ['pod_id', 'server_name'],
)
OPC_TAG_VALUE = Gauge(
    'opc_tag_value',
    'Last value read per tag/model',
    ['pod_id', 'tag_name', 'model_id'],
)
# --- Gateway: connection/error metrics ---
OPC_DISCONNECTIONS_TOTAL = Counter(
    'opc_disconnections_total',
    'OPC disconnections',
    ['pod_id', 'server_name'],
)
OPC_ERROR_BY_TYPE = Counter(
    'opc_error_by_type_total',
    'OPC errors by type',
    ['pod_id', 'server_name', 'error_type'],
)