d93c497914
Enhance Ingestor Class with Asynchronous Metrics and Monitoring Integration - Refactored Ingestor class to inherit from SientiaMonitoring for improved observability. - Updated methods to utilize asynchronous operations for declaring active ingestors and managing slots. - Integrated pod_id label into various metrics for better tracking. - Adjusted unit tests to reflect changes in the Ingestor initialization and asynchronous behavior.
158 lines
4.4 KiB
Python
158 lines
4.4 KiB
Python
"""
|
|
Prometheus metrics configuration for the OPC Ingestor application.
|
|
|
|
This module defines all the metrics used for monitoring and observability
|
|
of the OPC Ingestor system. It includes metrics for:
|
|
|
|
- Application health and performance
|
|
- OPC server connections and subscriptions
|
|
- Data processing and storage operations
|
|
- Resource management and load balancing
|
|
- Error tracking and notification systems
|
|
|
|
All metrics follow Prometheus naming conventions and include appropriate
|
|
labels for multi-dimensional analysis and alerting.
|
|
"""
|
|
|
|
from prometheus_client import Counter, Gauge, Histogram
|
|
|
|
# Metric label definitions for consistent labeling across all metrics
|
|
POD_ID_LABEL = ['pod_id']
|
|
SERVER_LABELS = ['pod_id', 'server_name', 'server_url']
|
|
KAFKA_LABELS = ['pod_id', 'topic']
|
|
REDIS_LABELS = ['pod_id', 'operation']
|
|
NOTIFICATION_LABELS = ['pod_id', 'level', 'block']
|
|
|
|
MAIN_LABELS = ['pod_id']
|
|
|
|
# --- Reliability Metrics ---
|
|
TAG_WRITTEN_COUNT = Counter(
|
|
'ingestor_tag_written_count',
|
|
'Number of writing process to the collection',
|
|
[*MAIN_LABELS, 'tag_name', 'collection_name'],
|
|
)
|
|
|
|
# --- General Application Metrics ---
|
|
APP_LOOP_COUNT = Counter(
|
|
'app_main_loop_total',
|
|
'Total number of times the application main loop has run',
|
|
POD_ID_LABEL,
|
|
)
|
|
APP_LOOP_DURATION = Histogram(
|
|
'app_main_loop_duration_seconds',
|
|
'Duration of the application main loop in seconds',
|
|
POD_ID_LABEL,
|
|
)
|
|
APP_ERRORS_TOTAL = Counter(
|
|
'app_errors_total',
|
|
'Total number of unhandled errors in the main loop',
|
|
POD_ID_LABEL,
|
|
)
|
|
APP_UP = Gauge(
|
|
'app_up',
|
|
'Indicates if the application is running (1) or shutting down (0)',
|
|
POD_ID_LABEL,
|
|
)
|
|
|
|
# --- Ingestor Manager Metrics ---
|
|
ACTIVE_INGESTORS = Gauge(
|
|
'ingestor_active_total',
|
|
'Number of active ingestors reported by Redis',
|
|
POD_ID_LABEL,
|
|
)
|
|
SLOTS_TOTAL = Gauge(
|
|
'ingestor_slots_total',
|
|
'Total number of slots configured in Redis',
|
|
POD_ID_LABEL,
|
|
)
|
|
LEASES_TOTAL = Gauge(
|
|
'ingestor_leases_total',
|
|
'Total number of leases (allocated slots) in Redis',
|
|
POD_ID_LABEL,
|
|
)
|
|
SLOTS_MANAGED = Gauge(
|
|
'ingestor_slots_managed_current',
|
|
'Number of slots currently managed by this ingestor instance',
|
|
POD_ID_LABEL,
|
|
)
|
|
SLOTS_ACQUIRED = Counter(
|
|
'ingestor_slots_acquired_total',
|
|
'Total number of slots acquired by this instance',
|
|
POD_ID_LABEL,
|
|
)
|
|
SLOTS_RELEASED = Counter(
|
|
'ingestor_slots_released_total',
|
|
'Total number of slots released by this instance',
|
|
POD_ID_LABEL,
|
|
)
|
|
OPC_MANAGERS_ACTIVE = Gauge(
|
|
'ingestor_opc_managers_active',
|
|
'Number of active OPC Managers in this instance',
|
|
POD_ID_LABEL,
|
|
)
|
|
OPC_SUBSCRIPTION_ERRORS = Counter(
|
|
'ingestor_opc_subscription_errors_total',
|
|
'Errors when trying to subscribe to OPC tags',
|
|
['pod_id', 'server', 'slot'],
|
|
)
|
|
|
|
# --- OPC Manager Metrics ---
|
|
OPC_CONNECTIONS_TOTAL = Counter(
|
|
'opc_connections_initiated_total',
|
|
'Total connection attempts to OPC servers',
|
|
['pod_id', 'server_name'],
|
|
)
|
|
OPC_CONNECTIONS_FAILED = Counter(
|
|
'opc_connections_failed_total',
|
|
'Total failed connection attempts to OPC servers',
|
|
['pod_id', 'server_name'],
|
|
)
|
|
OPC_CONNECTION_STATUS = Gauge(
|
|
'opc_connection_status',
|
|
'Connection status with the OPC server (1=connected, 0=disconnected)',
|
|
SERVER_LABELS,
|
|
)
|
|
OPC_SUBSCRIPTIONS_CREATED = Counter(
|
|
'opc_subscriptions_created_total',
|
|
'Total OPC subscriptions created',
|
|
['pod_id', 'server_name', 'slot_name'],
|
|
)
|
|
OPC_TAGS_SUBSCRIBED = Gauge(
|
|
'opc_tags_subscribed_current',
|
|
'Current number of OPC tags subscribed on a server',
|
|
['pod_id', 'server_name'],
|
|
)
|
|
OPC_CYCLES_WITHOUT_DATA = Gauge(
|
|
'opc_cycles_without_data',
|
|
'Current number of cycles without receiving data from a server',
|
|
['pod_id', 'server_name'],
|
|
)
|
|
OPC_RECONNECTIONS_TOTAL = Counter(
|
|
'opc_reconnections_tried_total',
|
|
'Reconnection attempts to an OPC server after a loss',
|
|
['pod_id', 'server_name'],
|
|
)
|
|
|
|
# --- Data Manager (Kafka) Metrics ---
|
|
KAFKA_MESSAGES_SENT = Counter(
|
|
'kafka_messages_sent_total', 'Total messages sent to Kafka', KAFKA_LABELS
|
|
)
|
|
KAFKA_MESSAGES_ERRORS = Counter(
|
|
'kafka_messages_errors_total',
|
|
'Total errors sending messages to Kafka',
|
|
KAFKA_LABELS,
|
|
)
|
|
KAFKA_CONNECTION_STATUS = Gauge(
|
|
'kafka_connection_status',
|
|
'Connection status with Kafka (1=connected, 0=disconnected)',
|
|
POD_ID_LABEL,
|
|
)
|
|
|
|
|
|
# --- Notification Metrics ---
|
|
NOTIFICATIONS_SENT = Counter(
|
|
'notifications_sent_total',
|
|
'Total number of notifications sent',
|
|
NOTIFICATION_LABELS,
|
|
)
|