SIENTIAPDE-1199

Enhance logging in Gates, MongoDB, and Redis activities by replacing debug statements with info level logs, improving observability of data processing steps. Update worker configuration to adjust concurrency settings and enable autoscaling for task polling.

scouter/activities/redis.py

@@ -29,6 +29,8 @@ class Redis(RedisBase):
         metadata = input_data['metadata']
         key = f"last_data_timestamp_{input_data['workflow_name']}_{input_data['schedule_name']}"
 
+        self.info(f"Getting last data timestamp for {key}")
+
         try:
             data_hold = self.get(key)
         except Exception as e:
@@ -42,7 +44,7 @@ class Redis(RedisBase):
             )
             raise e
 
-        self.debug(
+        self.info(
             f"Last collected timestamp: {data_hold}",
             metadata=metadata
         )
@@ -60,6 +62,8 @@ class Redis(RedisBase):
         metadata = input_data['metadata']
         key = f"last_data_timestamp_{input_data['workflow_name']}_{input_data['schedule_name']}"
 
+        self.info(f"Putting last data timestamp for {key}")
+
         data = DataFrame(input_data['data'])
 
         if data.empty:
@@ -70,7 +74,7 @@ class Redis(RedisBase):
 
         last_data_timestamp = data['inserted_at'].max()
 
-        self.debug(
+        self.info(
             f"Last collected timestamp to insert: {last_data_timestamp}",
             metadata=metadata
         )
@@ -116,6 +120,8 @@ class Redis(RedisBase):
 
         key = f"held_data_{input_data['workflow_name']}_{input_data['schedule_name']}"
 
+        self.info(f"Getting held data for {key}")
+
         try:
             data_hold = self.get(key)
         except Exception as e:
@@ -137,6 +143,8 @@ class Redis(RedisBase):
                              )
                 return data_hold
 
+        self.info(f"Grouping and holding data for {len(data)} rows")
+
         try:
 
             # Remove possibly removed tags
@@ -195,8 +203,10 @@ class Redis(RedisBase):
             )
             raise e
 
+        self.info(f"Data held and melted has {len(data_hold_melted)} rows")
+
         self.debug(
-            f"Data grouped and held successfully:\n {data_hold_melted.to_string()}",
+            f"Data held and melted:\n {data_hold_melted.to_string()}",
             metadata=metadata
         )