SIENTIAPDE-1174

Update replica count in values.yaml, add application status metrics in metrics.py, and implement Prometheus server in worker.py for enhanced observability.

scouter/metrics.py

@@ -1,5 +1,11 @@
 from prometheus_client import Gauge, Counter
 
+APP_UP = Gauge(
+    "app_up",
+    "Indicates if the application is running (1) or shutting down (0)",
+    ["pod_id"],
+)
+
 CORE_LABELS = ["pod_id", "model_name", "pipeline_name"]
 
 LABORIOUS_DATA_WRITTEN_COUNT = Counter(

scouter/worker/worker.py

@@ -12,19 +12,26 @@ with workflow.unsafe.imports_passed_through():
     from scouter.workflow.fake_data import FakeData
     from scouter.activities.faker import Faker
     import asyncio
+    from prometheus_client import start_http_server
+    from scouter import metrics
     from scouter.utils.connectors_config import (
         build_postgres_config,
         build_redis_config,
         build_mongodb_config
     )
 
+POD_ID = os.getenv("HOSTNAME", "localhost")
+
 
 async def main():
     host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
 
     logger = get_logger(__name__)
 
-    logger.info('Starting Worker...')
+    logger.info(f"Starting Worker with pod_id: {POD_ID}")
+
+    logger.info("Starting prometheus client...")
+    start_prometheus_server()
 
     logger.info('Starting Notification Handler...')
 
@@ -112,7 +119,20 @@ async def main():
         if activities:
             activities.shutdown()
         # Exit with a non-zero status code to indicate failure to Kubernetes
+        metrics.APP_UP.labels(pod_id=POD_ID).set(0)  # Mark app as DOWN
         sys.exit(1)
 
+
+def start_prometheus_server():
+    try:
+        port = int(os.getenv("HTTP_METRICS_PORT", 9090))
+        start_http_server(port)
+        print(f"Prometheus server started on port {port}.")
+        metrics.APP_UP.labels(pod_id=POD_ID).set(1)  # Mark app as UP
+    except Exception as e:
+        print(f"Failed to start Prometheus server: {e}")
+        os._exit(1)
+
+
 if __name__ == '__main__':
     asyncio.run(main())

values.yaml

@@ -3,7 +3,7 @@
 # Declare variables to be passed into your templates.
 
 # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
-replicaCount: 5
+replicaCount: 3
 
 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
 image: