SIENTIAPDE-1478

Update tests.ipynb and API class in scouter module - Reset execution counts in tests.ipynb for reproducibility. - Removed unnecessary outputs and adjusted execution counts for clarity. - Enhanced API class to accept 'end_time' parameter for improved data retrieval flexibility. - Commented out timestamp formatting and normalization for future adjustments.
2026-01-21 15:38:22 -03:00
parent 9dab8da5b3
commit e64af1aa7c
7 changed files with 2117 additions and 681 deletions
--- a/README.md
+++ b/README.md
@@ -492,16 +492,16 @@ The worker implements aggressive autoscaling policies for workflow and activity
 **Workflow Poller Behavior:**
 | Variable | Description | Default |
 |----------|-------------|---------|
-| `WORKFLOW_POLLER_BEHAVIUR_MINIMUM` | Minimum workflow pollers | `10` |
-| `WORKFLOW_POLLER_BEHAVIUR_INITIAL` | Initial workflow pollers | `100` |
-| `WORKFLOW_POLLER_BEHAVIUR_MAXIMUM` | Maximum workflow pollers | `200` |
+| `WORKFLOW_POLLER_BEHAVIOUR_MINIMUM` | Minimum workflow pollers | `10` |
+| `WORKFLOW_POLLER_BEHAVIOUR_INITIAL` | Initial workflow pollers | `100` |
+| `WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM` | Maximum workflow pollers | `200` |

 **Activity Poller Behavior:**
 | Variable | Description | Default |
 |----------|-------------|---------|
-| `ACTIVITY_POLLER_BEHAVIUR_MINIMUM` | Minimum activity pollers | `10` |
-| `ACTIVITY_POLLER_BEHAVIUR_INITIAL` | Initial activity pollers | `100` |
-| `ACTIVITY_POLLER_BEHAVIUR_MAXIMUM` | Maximum activity pollers | `200` |
+| `ACTIVITY_POLLER_BEHAVIOUR_MINIMUM` | Minimum activity pollers | `10` |
+| `ACTIVITY_POLLER_BEHAVIOUR_INITIAL` | Initial activity pollers | `100` |
+| `ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM` | Maximum activity pollers | `200` |

 ### Workflow Configuration

--- a/scouter/activities/api.py
+++ b/scouter/activities/api.py
@@ -110,6 +110,7 @@ class API(SientiaMonitoring):
        endpoint = input_data['endpoint']
        web_ids = input_data['web_ids']
        period = input_data['period']
+        end_time = input_data.get('end_time', '*')
        max_count = input_data.get('max_count', 1)
        api_timeout = input_data['api_timeout']

@@ -120,6 +121,7 @@ class API(SientiaMonitoring):
                endpoint=endpoint,
                web_ids=web_ids,
                start_time=period,
+                end_time=end_time,
                max_count=max_count,
                metadata=metadata,
                request_timeout=api_timeout,
@@ -136,13 +138,13 @@ class API(SientiaMonitoring):
            )
            raise e

-        latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
+        # latest_values['timestamp'] = latest_values['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)

-        self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata)
+        # self.debug(f'Latest values: {latest_values.to_string()}', metadata=metadata)

-        # Normalize the package timestamp
-        valid_timestamp_values = latest_values['timestamp'].dropna()
-        latest_values['timestamp'] = valid_timestamp_values.max()
+        # # Normalize the package timestamp
+        # valid_timestamp_values = latest_values['timestamp'].dropna()
+        # latest_values['timestamp'] = valid_timestamp_values.max()

        self.info(f'Gathered {len(latest_values)} tag values', metadata=metadata)

--- a/scouter/worker/prepare_worker.py
+++ b/scouter/worker/prepare_worker.py
@@ -14,12 +14,12 @@ parameters = [
    ('MAX_CONCURRENT_ACTIVITIES', '200'),
    ('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'),
    ('MAX_CACHED_WORKFLOWS', '200'),
-    ('WORKFLOW_POLLER_BEHAVIUR_MINIMUM', '10'),
-    ('WORKFLOW_POLLER_BEHAVIUR_INITIAL', '100'),
-    ('WORKFLOW_POLLER_BEHAVIUR_MAXIMUM', '200'),
-    ('ACTIVITY_POLLER_BEHAVIUR_MINIMUM', '10'),
-    ('ACTIVITY_POLLER_BEHAVIUR_INITIAL', '100'),
-    ('ACTIVITY_POLLER_BEHAVIUR_MAXIMUM', '200'),
+    ('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'),
+    ('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'),
+    ('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'),
+    ('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'),
+    ('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'),
+    ('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'),
 ]


@@ -62,13 +62,13 @@ def prepare_worker(
        ],
        max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'],
        workflow_task_poller_behavior=PollerBehaviorAutoscaling(
-            minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIUR_MINIMUM'],
-            initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIUR_INITIAL'],
-            maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIUR_MAXIMUM'],
+            minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'],
+            initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'],
+            maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'],
        ),
        activity_task_poller_behavior=PollerBehaviorAutoscaling(
-            minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIUR_MINIMUM'],
-            initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIUR_INITIAL'],
-            maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIUR_MAXIMUM'],
+            minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'],
+            initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'],
+            maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'],
        ),
    )
--- a/scouter/worker/worker.py
+++ b/scouter/worker/worker.py
@@ -39,13 +39,13 @@ MAX_CACHED_WORKFLOWS = int(os.getenv('MAX_CACHED_WORKFLOWS', '200'))

 # Temporal docs also recommends an autoscaling policy, with agrresive limits to prioritize latency over throughput.

-WORKFLOW_POLLER_BEHAVIUR_MINIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIUR_MINIMUM', '10'))
-WORKFLOW_POLLER_BEHAVIUR_INITIAL = int(os.getenv('WORKFLOW_POLLER_BEHAVIUR_INITIAL', '100'))
-WORKFLOW_POLLER_BEHAVIUR_MAXIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIUR_MAXIMUM', '200'))
+WORKFLOW_POLLER_BEHAVIOUR_MINIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'))
+WORKFLOW_POLLER_BEHAVIOUR_INITIAL = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'))
+WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM = int(os.getenv('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'))

-ACTIVITY_POLLER_BEHAVIUR_MINIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIUR_MINIMUM', '10'))
-ACTIVITY_POLLER_BEHAVIUR_INITIAL = int(os.getenv('ACTIVITY_POLLER_BEHAVIUR_INITIAL', '100'))
-ACTIVITY_POLLER_BEHAVIUR_MAXIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIUR_MAXIMUM', '200'))
+ACTIVITY_POLLER_BEHAVIOUR_MINIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'))
+ACTIVITY_POLLER_BEHAVIOUR_INITIAL = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'))
+ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM = int(os.getenv('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'))


 async def main():
--- a/scouter/worker/worker_parameters.md
+++ b/scouter/worker/worker_parameters.md
@@ -53,19 +53,19 @@ Temporal supports autoscaling that dynamically adjusts the number of concurrent

 These parameters control the autoscaling behavior of the workflow task poller, which retrieves workflow tasks from the Temporal server.

-### WORKFLOW_POLLER_BEHAVIUR_MINIMUM
+### WORKFLOW_POLLER_BEHAVIOUR_MINIMUM
 - **Default**: `10`
 - **Description**: Minimum number of concurrent pollers for workflow tasks. The poller count will never go below this value.
 - **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`.
 - **Impact**: Ensures a baseline level of polling activity even during low load periods.

-### WORKFLOW_POLLER_BEHAVIUR_INITIAL
+### WORKFLOW_POLLER_BEHAVIOUR_INITIAL
 - **Default**: `100`
 - **Description**: Initial number of concurrent pollers for workflow tasks when the worker starts.
 - **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`.
 - **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources.

-### WORKFLOW_POLLER_BEHAVIUR_MAXIMUM
+### WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM
 - **Default**: `200`
 - **Description**: Maximum number of concurrent pollers allowed for workflow tasks. The poller count will not exceed this value even under high load.
 - **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`.
@@ -75,19 +75,19 @@ These parameters control the autoscaling behavior of the workflow task poller, w

 These parameters control the autoscaling behavior of the activity task poller, which retrieves activity tasks from the Temporal server.

-### ACTIVITY_POLLER_BEHAVIUR_MINIMUM
+### ACTIVITY_POLLER_BEHAVIOUR_MINIMUM
 - **Default**: `10`
 - **Description**: Minimum number of concurrent pollers for activity tasks. The poller count will never go below this value.
 - **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`.
 - **Impact**: Ensures a baseline level of polling activity even during low load periods.

-### ACTIVITY_POLLER_BEHAVIUR_INITIAL
+### ACTIVITY_POLLER_BEHAVIOUR_INITIAL
 - **Default**: `100`
 - **Description**: Initial number of concurrent pollers for activity tasks when the worker starts.
 - **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`.
 - **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources.

-### ACTIVITY_POLLER_BEHAVIUR_MAXIMUM
+### ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM
 - **Default**: `200`
 - **Description**: Maximum number of concurrent pollers allowed for activity tasks. The poller count will not exceed this value even under high load.
 - **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`.
@@ -103,7 +103,7 @@ To override these parameters, set environment variables using the pattern:
 For example, if your workflow is named `ScouterWorkflow`:
 ```bash
 SCOUTERWORKFLOW_MAX_CONCURRENT_ACTIVITIES=500
-SCOUTERWORKFLOW_WORKFLOW_POLLER_BEHAVIUR_MAXIMUM=300
+SCOUTERWORKFLOW_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM=300
 ```

 ## Notes
--- a/tests.ipynb
+++ b/tests.ipynb
--- a/values.yaml
+++ b/values.yaml
@@ -242,18 +242,18 @@ env:
  - name: SCOUTER_MAX_CACHED_WORKFLOWS
    value: "200"

-  - name: SCOUTER_WORKFLOW_POLLER_BEHAVIUR_MINIMUM
+  - name: SCOUTER_WORKFLOW_POLLER_BEHAVIOUR_MINIMUM
    value: "10"
-  - name: SCOUTER_WORKFLOW_POLLER_BEHAVIUR_INITIAL
+  - name: SCOUTER_WORKFLOW_POLLER_BEHAVIOUR_INITIAL
    value: "100"
-  - name: SCOUTER_WORKFLOW_POLLER_BEHAVIUR_MAXIMUM
+  - name: SCOUTER_WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM
    value: "200"

-  - name: SCOUTER_ACTIVITY_POLLER_BEHAVIUR_MINIMUM
+  - name: SCOUTER_ACTIVITY_POLLER_BEHAVIOUR_MINIMUM
    value: "10"
-  - name: SCOUTER_ACTIVITY_POLLER_BEHAVIUR_INITIAL
+  - name: SCOUTER_ACTIVITY_POLLER_BEHAVIOUR_INITIAL
    value: "100"
-  - name: SCOUTER_ACTIVITY_POLLER_BEHAVIUR_MAXIMUM
+  - name: SCOUTER_ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM
    value: "200"