SIENTIAPDE-1478

Update tests.ipynb and API class in scouter module

- Reset execution counts in tests.ipynb for reproducibility.
- Removed unnecessary outputs and adjusted execution counts for clarity.
- Enhanced API class to accept 'end_time' parameter for improved data retrieval flexibility.
- Commented out timestamp formatting and normalization for future adjustments.

scouter/activities/api.py

@@ -110,6 +110,7 @@ class API(SientiaMonitoring):
         endpoint = input_data['endpoint']
         web_ids = input_data['web_ids']
         period = input_data['period']
+        end_time = input_data.get('end_time', '*')
         max_count = input_data.get('max_count', 1)
         api_timeout = input_data['api_timeout']
 
@@ -120,6 +121,7 @@ class API(SientiaMonitoring):
                 endpoint=endpoint,
                 web_ids=web_ids,
                 start_time=period,
+                end_time=end_time,
                 max_count=max_count,
                 metadata=metadata,
                 request_timeout=api_timeout,
@@ -136,13 +138,13 @@ class API(SientiaMonitoring):
             )
             raise e
 
-        latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
+        # latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
 
-        self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata)
+        # self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata)
 
-        # Normalize the package timestamp
-        valid_timestamp_values = latest_values['timestamp'].dropna()
-        latest_values['timestamp'] = valid_timestamp_values.max()
+        # # Normalize the package timestamp
+        # valid_timestamp_values = latest_values['timestamp'].dropna()
+        # latest_values['timestamp'] = valid_timestamp_values.max()
 
         self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata)
 

scouter/worker/prepare_worker.py

@@ -14,12 +14,12 @@ parameters = [
     ('MAX_CONCURRENT_ACTIVITIES', '200'),
     ('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'),
     ('MAX_CACHED_WORKFLOWS', '200'),
-    ('WORKFLOW_POLLER_BEHAVIUR_MINIMUM', '10'),
-    ('WORKFLOW_POLLER_BEHAVIUR_INITIAL', '100'),
-    ('WORKFLOW_POLLER_BEHAVIUR_MAXIMUM', '200'),
-    ('ACTIVITY_POLLER_BEHAVIUR_MINIMUM', '10'),
-    ('ACTIVITY_POLLER_BEHAVIUR_INITIAL', '100'),
-    ('ACTIVITY_POLLER_BEHAVIUR_MAXIMUM', '200'),
+    ('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'),
+    ('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'),
+    ('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'),
+    ('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'),
+    ('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'),
+    ('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'),
 ]
 
 
@@ -62,13 +62,13 @@ def prepare_worker(
         ],
         max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'],
         workflow_task_poller_behavior=PollerBehaviorAutoscaling(
-            minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIUR_MINIMUM'],
-            initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIUR_INITIAL'],
-            maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIUR_MAXIMUM'],
+            minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'],
+            initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'],
+            maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'],
         ),
         activity_task_poller_behavior=PollerBehaviorAutoscaling(
-            minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIUR_MINIMUM'],
-            initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIUR_INITIAL'],
-            maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIUR_MAXIMUM'],
+            minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'],
+            initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'],
+            maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'],
         ),
     )

scouter/worker/worker.py

@@ -39,13 +39,13 @@ MAX_CACHED_WORKFLOWS = int(os.getenv('MAX_CACHED_WORKFLOWS', '200'))
 
 # Temporal docs also recommends an autoscaling policy, with agrresive limits to prioritize latency over throughput.
 
-WORKFLOW_POLLER_BEHAVIUR_MINIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIUR_MINIMUM', '10'))
-WORKFLOW_POLLER_BEHAVIUR_INITIAL = int(os.getenv('WORKFLOW_POLLER_BEHAVIUR_INITIAL', '100'))
-WORKFLOW_POLLER_BEHAVIUR_MAXIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIUR_MAXIMUM', '200'))
+WORKFLOW_POLLER_BEHAVIOUR_MINIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'))
+WORKFLOW_POLLER_BEHAVIOUR_INITIAL = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'))
+WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'))
 
-ACTIVITY_POLLER_BEHAVIUR_MINIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIUR_MINIMUM', '10'))
-ACTIVITY_POLLER_BEHAVIUR_INITIAL = int(os.getenv('ACTIVITY_POLLER_BEHAVIUR_INITIAL', '100'))
-ACTIVITY_POLLER_BEHAVIUR_MAXIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIUR_MAXIMUM', '200'))
+ACTIVITY_POLLER_BEHAVIOUR_MINIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'))
+ACTIVITY_POLLER_BEHAVIOUR_INITIAL = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'))
+ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'))
 
 
 async def main():

scouter/worker/worker_parameters.md

@@ -53,19 +53,19 @@ Temporal supports autoscaling that dynamically adjusts the number of concurrent
 
 These parameters control the autoscaling behavior of the workflow task poller, which retrieves workflow tasks from the Temporal server.
 
-### WORKFLOW_POLLER_BEHAVIUR_MINIMUM
+### WORKFLOW_POLLER_BEHAVIOUR_MINIMUM
 - **Default**: `10`
 - **Description**: Minimum number of concurrent pollers for workflow tasks. The poller count will never go below this value.
 - **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`.
 - **Impact**: Ensures a baseline level of polling activity even during low load periods.
 
-### WORKFLOW_POLLER_BEHAVIUR_INITIAL
+### WORKFLOW_POLLER_BEHAVIOUR_INITIAL
 - **Default**: `100`
 - **Description**: Initial number of concurrent pollers for workflow tasks when the worker starts.
 - **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`.
 - **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources.
 
-### WORKFLOW_POLLER_BEHAVIUR_MAXIMUM
+### WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM
 - **Default**: `200`
 - **Description**: Maximum number of concurrent pollers allowed for workflow tasks. The poller count will not exceed this value even under high load.
 - **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`.
@@ -75,19 +75,19 @@ These parameters control the autoscaling behavior of the workflow task poller, w
 
 These parameters control the autoscaling behavior of the activity task poller, which retrieves activity tasks from the Temporal server.
 
-### ACTIVITY_POLLER_BEHAVIUR_MINIMUM
+### ACTIVITY_POLLER_BEHAVIOUR_MINIMUM
 - **Default**: `10`
 - **Description**: Minimum number of concurrent pollers for activity tasks. The poller count will never go below this value.
 - **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`.
 - **Impact**: Ensures a baseline level of polling activity even during low load periods.
 
-### ACTIVITY_POLLER_BEHAVIUR_INITIAL
+### ACTIVITY_POLLER_BEHAVIOUR_INITIAL
 - **Default**: `100`
 - **Description**: Initial number of concurrent pollers for activity tasks when the worker starts.
 - **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`.
 - **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources.
 
-### ACTIVITY_POLLER_BEHAVIUR_MAXIMUM
+### ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM
 - **Default**: `200`
 - **Description**: Maximum number of concurrent pollers allowed for activity tasks. The poller count will not exceed this value even under high load.
 - **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`.
@@ -103,7 +103,7 @@ To override these parameters, set environment variables using the pattern:
 For example, if your workflow is named `ScouterWorkflow`:
 ```bash
 SCOUTERWORKFLOW_MAX_CONCURRENT_ACTIVITIES=500
-SCOUTERWORKFLOW_WORKFLOW_POLLER_BEHAVIUR_MAXIMUM=300
+SCOUTERWORKFLOW_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM=300
 ```
 
 ## Notes