SIENTIAPDE-1174

SIENTIAPDE-1174 Update dependencies, modify image tag, and enhance metrics tracking in various activities for improved observability and performance.

scouter/activities/gates.py

@@ -3,6 +3,8 @@ from temporalio import workflow, activity
 with workflow.unsafe.imports_passed_through():
     from sientia_do.notifications.models import NotificationLevel
     from sientia_do.temporal.activities.base import BaseActivity
+    from sientia_do.notifications.handlers import NotificationHandler
+    from sientia_do.temporal.utils.logger import Logger
     from scouter.utils.quality.filters import null_values_filter, out_of_bounds_filter
     from typing import Any
     import traceback
@@ -16,6 +18,10 @@ quality_gate_filters = {
 
 class Gates(BaseActivity):
 
+    def __init__(self, logger: Logger, notification_handler: NotificationHandler):
+        BaseActivity.__init__(
+            self, logger, notification_handler, set_error_counter=True)
+
     def apply_aggregation(self, group: DataFrame, aggr_function: str,
                           metadata: dict[str, Any]) -> float | None | str:
         """

scouter/activities/mongodb.py

@@ -57,7 +57,8 @@ class MongoDB(BaseActivity):
 
         BaseActivity.__init__(self,
                               logger=logger,
-                              notification_handler=notification_handler)
+                              notification_handler=notification_handler,
+                              set_error_counter=True)
 
     def shutdown(self):
         """

scouter/activities/redis.py

@@ -1,8 +1,8 @@
-import traceback
 from temporalio import workflow, activity
 
 with workflow.unsafe.imports_passed_through():
     from logging import Logger
+    import traceback
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.notifications.models import NotificationLevel
     from sientia_do.temporal.activities.redis_base import Redis as RedisBase
@@ -10,6 +10,7 @@ with workflow.unsafe.imports_passed_through():
     from typing import Any
     from pandas import DataFrame
     from datetime import datetime
+    from scouter import metrics
 
 
 class Redis(RedisBase):
@@ -136,16 +137,29 @@ class Redis(RedisBase):
                 return data_hold
 
         try:
+
+            to_register_metrics = []
             for _, row in data.iterrows():
                 value = row['value']
 
                 data_hold[row['name']] = value
+                to_register_metrics.append(
+                    (row['name'], value))
 
             data_hold['timestamp'] = data['timestamp'].max() if not data.empty else \
                 datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
             self.set(key, data_hold, ttl=retention_time)
 
+            # Register metrics
+            for metric in to_register_metrics:
+                metrics.TAG_CHANGES_MONITOR.labels(
+                    pod_id=metadata['pod_id'],
+                    model_name=input_data['model_name'],
+                    pipeline_name=input_data['schedule_name'],
+                    tag_name=metric[0]
+                ).set(metric[1])
+
             data_hold_df = DataFrame(data_hold, index=[0])
             data_hold_melted = data_hold_df.melt(
                 id_vars='timestamp', var_name='variable', value_name='value')

scouter/metrics.py

@@ -0,0 +1,15 @@
+from prometheus_client import Gauge, Counter
+
+CORE_LABELS = ["pod_id", "model_name", "pipeline_name"]
+
+LABORIOUS_DATA_WRITTEN_COUNT = Counter(
+    "scouter_laborious_data_written_count",
+    "Number of writings to the database table laborious_data",
+    CORE_LABELS,
+)
+
+TAG_CHANGES_MONITOR = Gauge(
+    "scouter_tag_changes_monitor",
+    "Current value change of each tag",
+    [*CORE_LABELS, "tag_name"],
+)

scouter/workflow/scouter.py

@@ -5,6 +5,7 @@ with workflow.unsafe.imports_passed_through():
     from typing import Any
     from datetime import timedelta
     from sientia_do.temporal.utils.policies import retry_policy
+    from os import getenv
 
 
 @workflow.defn(name="scouter")
@@ -37,7 +38,8 @@ class Scouter:
                 'model_id': input_data['model_id'],
                 'model_name': input_data['model_name'],
                 'schedule_name': input_data['schedule_name'],
-                'workflow_name': input_data['workflow_name']
+                'workflow_name': input_data['workflow_name'],
+                'pod_id': getenv("HOSTNAME", "localhost")
             }
         }
 

scouter/workflow/sub_workflows/core_scouter.py

@@ -1,5 +1,7 @@
 from temporalio import workflow
 
+from scouter import metrics
+
 with workflow.unsafe.imports_passed_through():
     from scouter.activities.activities import Activities
     from typing import Any
@@ -86,6 +88,12 @@ class CoreScouter:
             start_to_close_timeout=timedelta(seconds=60)
         )
 
+        metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels(
+            pod_id=metadata['pod_id'],
+            model_name=input_data['model_name'],
+            pipeline_name=input_data['workflow_name']
+        ).inc()
+
         if input_data.get('debug_data_package', False):
             await workflow.execute_activity_method(
                 Activities.store_data_package,