SIENTIAPDE-1083: add prometheus metrics to resource_manager.

ingestor/managers/resource_manager.py

@@ -1,17 +1,59 @@
 import json
 from typing import List
 from redis import Redis
+from time import time
+import ingestor.metrics as metrics
 
 
 class ResourceManager:
-    def __init__(self, host: str, port: int,
-                 lease_ttl: int, heartbeat_ttl: int, pod_id: str,
-                 username: str = None, password: str = None) -> None:
-        self.redis = Redis(host=host, port=port, decode_responses=True,
-                           username=username, password=password)
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        lease_ttl: int,
+        heartbeat_ttl: int,
+        pod_id: str,
+        username: str | None = None,
+        password: str | None = None,
+    ) -> None:
+        self.pod_id = pod_id
+        try:
+            self.redis = Redis(
+                host=host,
+                port=port,
+                decode_responses=True,
+                username=username,
+                password=password,
+            )
+            self.redis.ping()
+            metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(1)
+        except Exception as e:
+            print(f"Failed to connect to Redis: {e}")
+            metrics.REDIS_CONNECTION_STATUS.labels(pod_id=self.pod_id).set(0)
+            raise
+
         self.lease_ttl = lease_ttl
         self.heartbeat_ttl = heartbeat_ttl
-        self.pod_id = pod_id
+
+    def _execute_redis_op(self, operation_name: str, func, *args, **kwargs):
+        """Wrapper to execute Redis operations and record metrics."""
+        start_time = time()
+        try:
+            result = func(*args, **kwargs)
+            metrics.REDIS_OPERATIONS_TOTAL.labels(
+                pod_id=self.pod_id, operation=operation_name
+            ).inc()
+            duration = time() - start_time
+            metrics.REDIS_OPERATIONS_DURATION.labels(
+                pod_id=self.pod_id, operation=operation_name
+            ).observe(duration)
+            return result
+        except Exception as e:
+            metrics.REDIS_OPERATIONS_ERRORS.labels(
+                pod_id=self.pod_id, operation=operation_name
+            ).inc()
+            print(f"Error in Redis operation '{operation_name}': {e}")
+            raise
 
     def get(self, key: str) -> dict:
         """
@@ -19,11 +61,11 @@ class ResourceManager:
         Args:
             key (str): The key to look up in Redis.
         Returns:
-            dict: The value associated with the key, parsed as a dictionary, 
+            dict: The value associated with the key, parsed as a dictionary,
                   or None if the key does not exist or the value is empty.
         """
 
-        history = self.redis.get(key)
+        history = self._execute_redis_op("get", self.redis.get, key)
         return json.loads(history) if history else None
 
     def get_tag_slot(self, id: str) -> dict:
@@ -48,14 +90,19 @@ class ResourceManager:
             None
         """
 
-        self.redis.set(
-            f"heartbeat:ingestor:{self.pod_id}", 1, ex=self.heartbeat_ttl)
+        self._execute_redis_op(
+            "set",
+            self.redis.set,
+            f"heartbeat:ingestor:{self.pod_id}",
+            1,
+            ex=self.heartbeat_ttl,
+        )
 
     def lease_tag(self, tag_id: str) -> bool:
         """
         Attempts to lease a tag by setting a key in Redis with a specified TTL (time-to-live).
-        This method uses the Redis `SET` command with the `NX` option to ensure that the key 
-        is only set if it does not already exist. The key is set with an expiration time 
+        This method uses the Redis `SET` command with the `NX` option to ensure that the key
+        is only set if it does not already exist. The key is set with an expiration time
         defined by `lease_ttl`.
         Args:
             tag_id (str): The unique identifier of the tag to be leased.
@@ -63,8 +110,14 @@ class ResourceManager:
             bool: True if the lease was successfully acquired, False otherwise.
         """
 
-        return self.redis.set(
-            f"lease:opc_tags:{tag_id}", self.pod_id, nx=True, ex=self.lease_ttl)
+        return self._execute_redis_op(
+            "set_nx",
+            self.redis.set,
+            f"lease:opc_tags:{tag_id}",
+            self.pod_id,
+            nx=True,
+            ex=self.lease_ttl,
+        )
 
     def renew_tag_lease(self, tag_id: str) -> bool:
         """
@@ -78,12 +131,14 @@ class ResourceManager:
             bool: True if the lease was successfully renewed, False otherwise.
         """
 
-        current = self.redis.get(
-            f"lease:opc_tags:{tag_id}")
+        current = self._execute_redis_op(
+            "get", self.redis.get, f"lease:opc_tags:{tag_id}"
+        )
         if current == self.pod_id:
-            self.redis.expire(f"lease:opc_tags:{tag_id}", self.lease_ttl)
+            self._execute_redis_op(
+                "expire", self.redis.expire, f"lease:opc_tags:{tag_id}", self.lease_ttl
+            )
             return True
-
         return False
 
     def drop_tag_lease(self, tag_id: str) -> None:
@@ -96,7 +151,7 @@ class ResourceManager:
             None
         """
 
-        self.redis.delete(f"lease:opc_tags:{tag_id}")
+        self._execute_redis_op("delete", self.redis.delete, f"lease:opc_tags:{tag_id}")
 
     def get_all_ingestors(self) -> List[str]:
         """
@@ -107,7 +162,7 @@ class ResourceManager:
             list: A list of active ingestors.
         """
 
-        return self.redis.keys("heartbeat:ingestor:*")
+        return self._execute_redis_op("keys", self.redis.keys, "heartbeat:ingestor:*")
 
     def get_all_slots(self) -> List[str]:
         """
@@ -118,7 +173,7 @@ class ResourceManager:
             int: The number of slots available.
         """
 
-        return self.redis.keys("slot:opc_tags:*")
+        return self._execute_redis_op("keys", self.redis.keys, "slot:opc_tags:*")
 
     def get_all_leases(self) -> List[str]:
         """
@@ -129,4 +184,4 @@ class ResourceManager:
             list: A list of active leases.
         """
 
-        return self.redis.keys("lease:opc_tags:*")
+        return self._execute_redis_op("keys", self.redis.keys, "lease:opc_tags:*")