diff --git a/ingestor/app.py b/ingestor/app.py index e2657f7..8fc3672 100644 --- a/ingestor/app.py +++ b/ingestor/app.py @@ -66,7 +66,7 @@ def signal_handler(_signum, _frame): def start_prometheus_server(): try: - port = int(os.getenv("HTTP_SERVER_PORT", 4840)) + port = int(os.getenv("HTTP_METRICS_PORT", 9090)) start_http_server(port) print(f"Prometheus server started on port {port}.") metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP diff --git a/ingestor/managers/data_manager.py b/ingestor/managers/data_manager.py index 68dbe06..903daf0 100644 --- a/ingestor/managers/data_manager.py +++ b/ingestor/managers/data_manager.py @@ -189,6 +189,12 @@ class DataManager(BaseActivity): self.logger.debug( f"Message inserted into MongoDB collection {topic}: {data}") + metrics.TAG_WRITTEN_COUNT.labels( + pod_id=self.pod_id, + tag_name=data["name"], + collection_name=topic + ).inc() + except Exception as e: trace = traceback.format_exc() self.send_notification( diff --git a/ingestor/metrics.py b/ingestor/metrics.py index b2ddc42..8cb2110 100644 --- a/ingestor/metrics.py +++ b/ingestor/metrics.py @@ -6,6 +6,15 @@ KAFKA_LABELS = ["pod_id", "topic"] REDIS_LABELS = ["pod_id", "operation"] NOTIFICATION_LABELS = ["pod_id", "level", "block"] +MAIN_LABELS = ["pod_id"] + +# --- Reliability Metrics --- +TAG_WRITTEN_COUNT = Counter( + "ingestor_tag_written_count", + "Number of writing process to the collection", + [*MAIN_LABELS, "tag_name", "collection_name"], +) + # --- General Application Metrics --- APP_LOOP_COUNT = Counter( diff --git a/requirements.txt b/requirements.txt index a7bba96..efdfaf2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ asyncua==1.1.5 redis -git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.5 +# git+ssh://git@github.com/Aignosi/sientia-dataops-library.git@1.3.5 +../sientia-dataops-library prometheus_client pymongo \ No newline at end of file diff --git a/values.yaml b/values.yaml index 9f3d9ed..1ae0638 100644 --- a/values.yaml +++ b/values.yaml @@ -11,7 +11,7 @@ image: # This sets the pull policy for images. pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. - tag: "0.2.7" + tag: "0.3.0" # This is for the secrets for pulling an image from a private repository more information can be found here: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ imagePullSecrets: @@ -190,7 +190,7 @@ env: value: "10" - name: LOG_LEVEL value: "DEBUG" - - name: HTTP_SERVER_PORT + - name: HTTP_METRICS_PORT value: "9090"