SIENTIAPDE-1174

Update requirements and environment configurations

- Changed the sientia-dataops-library dependency to a relative path in requirements.txt.
- Updated image tag to 0.3.0 in values.yaml.
- Renamed HTTP_SERVER_PORT to HTTP_METRICS_PORT in app.py and updated its default value to 9090.
- Added TAG_WRITTEN_COUNT metric in metrics.py to track writing processes.
- Incremented TAG_WRITTEN_COUNT in DataManager upon successful data insertion.

ingestor/app.py

@@ -66,7 +66,7 @@ def signal_handler(_signum, _frame):
 
 def start_prometheus_server():
     try:
-        port = int(os.getenv("HTTP_SERVER_PORT", 4840))
+        port = int(os.getenv("HTTP_METRICS_PORT", 9090))
         start_http_server(port)
         print(f"Prometheus server started on port {port}.")
         metrics.APP_UP.labels(pod_id=POD_ID).set(1)  # Mark app as UP

ingestor/managers/data_manager.py

@@ -189,6 +189,12 @@ class DataManager(BaseActivity):
             self.logger.debug(
                 f"Message inserted into MongoDB collection {topic}: {data}")
 
+            metrics.TAG_WRITTEN_COUNT.labels(
+                pod_id=self.pod_id,
+                tag_name=data["name"],
+                collection_name=topic
+            ).inc()
+
         except Exception as e:
             trace = traceback.format_exc()
             self.send_notification(

ingestor/metrics.py

@@ -6,6 +6,15 @@ KAFKA_LABELS = ["pod_id", "topic"]
 REDIS_LABELS = ["pod_id", "operation"]
 NOTIFICATION_LABELS = ["pod_id", "level", "block"]
 
+MAIN_LABELS = ["pod_id"]
+
+# --- Reliability Metrics ---
+TAG_WRITTEN_COUNT = Counter(
+    "ingestor_tag_written_count",
+    "Number of writing process to the collection",
+    [*MAIN_LABELS, "tag_name", "collection_name"],
+)
+
 
 # --- General Application Metrics ---
 APP_LOOP_COUNT = Counter(

requirements.txt

@@ -1,5 +1,6 @@
 asyncua==1.1.5
 redis
-git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.5
+# git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.5
+../sientia-dataops-library
 prometheus_client
 pymongo

values.yaml

@@ -11,7 +11,7 @@ image:
   # This sets the pull policy for images.
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
-  tag: "0.2.7"
+  tag: "0.3.0"
 
 # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
 imagePullSecrets:
@@ -190,7 +190,7 @@ env:
     value: "10"
   - name: LOG_LEVEL
     value: "DEBUG"
-  - name: HTTP_SERVER_PORT
+  - name: HTTP_METRICS_PORT
     value: "9090"