SIENTIAPDE-1083: add prometheus and app metrics.

ingestor/app.py

@@ -1,32 +1,53 @@
-from threading import Event
-import signal
 import os
+import signal
 import traceback
+from threading import Event
+from time import sleep, time
+
+from prometheus_client import start_http_server
+
+import ingestor.metrics as metrics
 from ingestor.ingestor import Ingestor
 
 exit_signal = Event()
+POD_ID = os.getenv("HOSTNAME", "localhost")
 
 
 def main():
+    start_prometheus_server()
     ingestor = Ingestor()
     ingestor.prepare_ingestor()
     ingestor.logger.info("Ingestor prepared. Starting main loop.")
 
     while not exit_signal.is_set():
+        start_time = time()  # Start loop timer
         try:
             ingestor.loop()
+            metrics.APP_LOOP_COUNT.labels(pod_id=POD_ID).inc()  # Increment loop counter
 
             exit_signal.wait(ingestor.poll_interval)
 
+        except KeyboardInterrupt:  # Handle Ctrl+C gracefully
+            print("KeyboardInterrupt received. Setting exit_signal flag.")
+            exit_signal.set()
         except Exception:
             print("Exception in main loop. Setting exit_signal flag.")
             traceback.print_exc()
+            metrics.APP_ERRORS_TOTAL.labels(pod_id=POD_ID).inc()  # Increment errors
             exit_signal.set()
+        finally:
+            # Record loop duration
+            duration = time() - start_time
+            metrics.APP_LOOP_DURATION.labels(pod_id=POD_ID).observe(duration)
 
     ingestor.shutdown()
+    metrics.APP_UP.labels(pod_id=POD_ID).set(0)  # Mark app as DOWN
 
     ingestor.logger.info("Main loop exit_signaled.")
 
+    # Give Prometheus a chance to scrape one last time before exiting (optional)
+    sleep(5)
+
     os._exit(0)
 
 
@@ -35,6 +56,16 @@ def signal_handler(_signum, _frame):
     exit_signal.set()
 
 
+def start_prometheus_server():
+    try:
+        start_http_server(8000)
+        print("Prometheus server started on port 8000.")
+        metrics.APP_UP.labels(pod_id=POD_ID).set(1)  # Mark app as UP
+    except Exception as e:
+        print(f"Failed to start Prometheus server: {e}")
+        os._exit(1)
+
+
 if __name__ == "__main__":
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)