diff --git a/scouter/worker/prepare_worker.py b/scouter/worker/prepare_worker.py index c8b68f2..09eb902 100644 --- a/scouter/worker/prepare_worker.py +++ b/scouter/worker/prepare_worker.py @@ -7,6 +7,8 @@ from sientia_do.observability.logger import Logger from temporalio.client import Client from temporalio.worker import PollerBehaviorAutoscaling, Worker +# Worker configuration parameters with default values +# See worker_parameters.md for detailed documentation parameters = [ ('MAX_CONCURRENT_WORKFLOW_TASKS', '200'), ('MAX_CONCURRENT_ACTIVITIES', '200'), diff --git a/scouter/worker/worker_parameters.md b/scouter/worker/worker_parameters.md new file mode 100644 index 0000000..ca2c7cd --- /dev/null +++ b/scouter/worker/worker_parameters.md @@ -0,0 +1,113 @@ +# Worker Parameters Documentation + +This document explains each configuration parameter used in the `prepare_worker.py` file for configuring Temporal workers. + +## Overview + +All parameters can be configured via environment variables using the pattern: `{WORKFLOW_NAME}_{PARAMETER_NAME}`. If not set, default values are used as specified below. + +## Concurrency Parameters + +### MAX_CONCURRENT_WORKFLOW_TASKS +- **Default**: `200` +- **Description**: Maximum number of concurrent workflow tasks that can be processed simultaneously by the worker. This controls how many workflow executions can be actively running at the same time. +- **Usage**: Set via `max_concurrent_workflow_tasks` in the Worker configuration. +- **Impact**: Higher values allow more workflows to run concurrently but consume more resources. Lower values provide better resource control but may limit throughput. + +### MAX_CONCURRENT_ACTIVITIES +- **Default**: `200` +- **Description**: Maximum number of concurrent activity tasks that can be executed simultaneously by the worker. Activities are the actual work units that perform business logic. +- **Usage**: Set via `max_concurrent_activities` in the Worker configuration. +- **Impact**: Controls the parallelism of activity execution. Higher values increase throughput but require more system resources (CPU, memory, network connections). + +### MAX_CONCURRENT_LOCAL_ACTIVITIES +- **Default**: `200` +- **Description**: Maximum number of concurrent local activity tasks that can be executed simultaneously. Local activities run in the same process as the workflow, without requiring a separate activity worker. +- **Usage**: Set via `max_concurrent_local_activities` in the Worker configuration. +- **Impact**: Similar to regular activities, but local activities have lower latency and overhead since they don't require network round-trips. Useful for lightweight operations. + +## Caching Parameters + +### MAX_CACHED_WORKFLOWS +- **Default**: `200` +- **Description**: Maximum number of workflow instances that can be cached in memory by the worker. Cached workflows allow faster resumption of execution without reloading state. +- **Usage**: Set via `max_cached_workflows` in the Worker configuration. +- **Impact**: Higher values improve performance for frequently accessed workflows but consume more memory. Lower values reduce memory usage but may require more frequent state reloads. + +## Understanding Pollers in Temporal + +**Pollers** are components of Temporal Workers that continuously request tasks from the Temporal service's Task Queues via synchronous RPCs. There are separate pollers for workflow tasks and activity tasks. + +### How Pollers Work + +Pollers send requests to the Temporal service to retrieve tasks from Task Queues. When a task is available, the poller retrieves it and the Worker processes it using registered Workflow or Activity handlers. This architecture provides: +- **Load Balancing**: Workers only poll when they have capacity, distributing load across multiple processes +- **Fault Tolerance**: Tasks persist in queues if a Worker fails, allowing recovery +- **Task Routing**: Tasks can be routed to specific Worker processes + +### Autoscaling Poller Behavior + +Temporal supports autoscaling that dynamically adjusts the number of concurrent pollers based on workload. The system scales up during high load and down during low load, maintaining a baseline for responsiveness. Autoscaling is configured with `minimum`, `initial`, and `maximum` parameters that define the scaling bounds. + +## Workflow Poller Behavior (Autoscaling) + +These parameters control the autoscaling behavior of the workflow task poller, which retrieves workflow tasks from the Temporal server. + +### WORKFLOW_POLLER_BEHAVIUR_MINIMUM +- **Default**: `10` +- **Description**: Minimum number of concurrent pollers for workflow tasks. The poller count will never go below this value. +- **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. +- **Impact**: Ensures a baseline level of polling activity even during low load periods. + +### WORKFLOW_POLLER_BEHAVIUR_INITIAL +- **Default**: `100` +- **Description**: Initial number of concurrent pollers for workflow tasks when the worker starts. +- **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. +- **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources. + +### WORKFLOW_POLLER_BEHAVIUR_MAXIMUM +- **Default**: `200` +- **Description**: Maximum number of concurrent pollers allowed for workflow tasks. The poller count will not exceed this value even under high load. +- **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `workflow_task_poller_behavior`. +- **Impact**: Caps the resource consumption for workflow task polling. Prevents excessive polling that could overwhelm the Temporal server or worker. + +## Activity Poller Behavior (Autoscaling) + +These parameters control the autoscaling behavior of the activity task poller, which retrieves activity tasks from the Temporal server. + +### ACTIVITY_POLLER_BEHAVIUR_MINIMUM +- **Default**: `10` +- **Description**: Minimum number of concurrent pollers for activity tasks. The poller count will never go below this value. +- **Usage**: Set via `minimum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. +- **Impact**: Ensures a baseline level of polling activity even during low load periods. + +### ACTIVITY_POLLER_BEHAVIUR_INITIAL +- **Default**: `100` +- **Description**: Initial number of concurrent pollers for activity tasks when the worker starts. +- **Usage**: Set via `initial` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. +- **Impact**: Determines the starting point for poller scaling. Higher values provide faster initial task acquisition but consume more resources. + +### ACTIVITY_POLLER_BEHAVIUR_MAXIMUM +- **Default**: `200` +- **Description**: Maximum number of concurrent pollers allowed for activity tasks. The poller count will not exceed this value even under high load. +- **Usage**: Set via `maximum` in `PollerBehaviorAutoscaling` for `activity_task_poller_behavior`. +- **Impact**: Caps the resource consumption for activity task polling. Prevents excessive polling that could overwhelm the Temporal server or worker. + +## Configuration Example + +To override these parameters, set environment variables using the pattern: +``` +{WORKFLOW_NAME}_{PARAMETER_NAME}={value} +``` + +For example, if your workflow is named `ScouterWorkflow`: +```bash +SCOUTERWORKFLOW_MAX_CONCURRENT_ACTIVITIES=500 +SCOUTERWORKFLOW_WORKFLOW_POLLER_BEHAVIUR_MAXIMUM=300 +``` + +## Notes + +- All parameter values are converted to integers before use. +- The autoscaling poller behavior dynamically adjusts the number of pollers between the minimum and maximum values based on workload. +- These parameters should be tuned based on your specific workload characteristics, available resources, and performance requirements. diff --git a/tests.ipynb b/tests.ipynb index 79f937d..9df4097 100644 --- a/tests.ipynb +++ b/tests.ipynb @@ -168,57 +168,67 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 1, "id": "72af4236", "metadata": {}, - "outputs": [ - { - "ename": "KeyError", - "evalue": "'Items'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mKeyError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[17]\u001b[39m\u001b[32m, line 42\u001b[39m\n\u001b[32m 39\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m tag \u001b[38;5;129;01min\u001b[39;00m TAG_NAMES:\n\u001b[32m 40\u001b[39m response = requests.get(url.replace(\u001b[33m'\u001b[39m\u001b[38;5;132;01m{tag}\u001b[39;00m\u001b[33m'\u001b[39m, tag), headers=headers).json()\n\u001b[32m 41\u001b[39m web_ids[tag] = {\n\u001b[32m---> \u001b[39m\u001b[32m42\u001b[39m \u001b[33m'\u001b[39m\u001b[33mwebid\u001b[39m\u001b[33m'\u001b[39m: \u001b[43mresponse\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mItems\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m]\u001b[49m[\u001b[32m0\u001b[39m][\u001b[33m'\u001b[39m\u001b[33mWebId\u001b[39m\u001b[33m'\u001b[39m],\n\u001b[32m 43\u001b[39m \u001b[33m'\u001b[39m\u001b[33maggr_func\u001b[39m\u001b[33m'\u001b[39m: \u001b[33m'\u001b[39m\u001b[33mlts\u001b[39m\u001b[33m'\u001b[39m,\n\u001b[32m 44\u001b[39m \u001b[33m'\u001b[39m\u001b[33mdata_range\u001b[39m\u001b[33m'\u001b[39m: [-\u001b[32m100000\u001b[39m, \u001b[32m100000\u001b[39m],\n\u001b[32m 45\u001b[39m }\n\u001b[32m 46\u001b[39m sleep(\u001b[32m0.5\u001b[39m)\n", - "\u001b[31mKeyError\u001b[39m: 'Items'" - ] - } - ], + "outputs": [], "source": [ "import requests\n", "from time import sleep\n", "\n", "# Obter web id das seguintes tags:\n", - "TAG_NAMES = [\n", + "TAG_NAMES = [\n", " \"CI-W3A05F1\",\n", - " \"CI-W3W03S1\", \"CI-W3W03I1\", \"CI-W3K01T1\", \"CI-W3W01A3\",\n", - " \"CI-W3W01A2\", \"CI-W3W01A1\", \"CI-J3P01T1A\", \"CI-W3A50T1\", \"CI-W3A55T1\",\n", - " \"CI-W3A55P1\", \"CI-W3V33P1\", \"CI-W3E01F1\", \"CI-W3A50A3\", \"CI-W3A50A2\",\n", - " \"CI-W3A50A1\", \"CI-W3A50P1\", \"CI-W3W01P1\", \"CI-W3A71P1\", \"CI-W3W01P2\",\n", - " \"CI-W3A71P2\", \"CI-W3A71P3\", \"CI-J3J01S1\", \"CI-W3P17S1\", \"CI-J3P03S1\",\n", - " \"CI-W3K01S1\", \"CI-W3K14P1\", \"CI-W3K01T4\", \"CI-W3K01T2\", \n", - "\n", - " \"CI-W3FARCI_FSC\",\n", - " \"CI-W3FARCI_MA\",\n", - " \"CI-W3FARCI_MS\",\n", - " \"CI-W3FARCI_P100\",\n", - " \"CI-W3FARCI_p170\",\n", - "\n", - " \"CI-W3CLK_C3S\",\n", - " \"CI-W3CLK_C3S_EXP\",\n", - " \"CI-W3CLK_C3S_MD_EXP\",\n", - " \"CI-W3CLK_C3S_MD_PETRO\",\n", - " \"CI-W3V04P3\", \"CI-W3V04P1\",\n", - " \"CI-W3W01G1\"\n", + " \"CI-W3W03S1\",\n", + " \"CI-W3W03I1\",\n", + " \"CI-W3K01T1\",\n", + " \"CI-W3W01A3\",\n", + " \"CI-W3W01A2\",\n", + " \"CI-W3W01A1\",\n", + " \"CI-J3P01T1A\",\n", + " \"CI-W3A50T1\",\n", + " \"CI-W3A55T1\",\n", + " \"CI-W3A55P1\",\n", + " \"CI-W3V33P1\",\n", + " \"CI-W3E01F1\",\n", + " \"CI-W3A50A3\",\n", + " \"CI-W3A50A2\",\n", + " \"CI-W3A50A1\",\n", + " \"CI-W3A50P1\",\n", + " \"CI-W3W01P1\",\n", + " \"CI-W3A71P1\",\n", + " \"CI-W3W01P2\",\n", + " \"CI-W3A71P2\",\n", + " \"CI-W3A71P3\",\n", + " \"CI-J3J01S1\",\n", + " \"CI-W3P17S1\",\n", + " \"CI-J3P03S1\",\n", + " \"CI-W3K01S1\",\n", + " \"CI-W3K14P1\",\n", + " \"CI-W3K01T4\",\n", + " \"CI-W3K01T2\",\n", + " \"CI-W3A65_SO3\",\n", + " \"CI-W3A65_CL\",\n", + " \"CI-W3_C3S\",\n", + " \"CI-W3_MS\",\n", + " \"CI-W3_MA\",\n", + " \"CI-W3_PL\",\n", + " \"CI-W3_CAO\",\n", + " \"CI-W3V04P3\",\n", + " \"CI-W3V04P1\",\n", + " \"CI-W3W01G1\",\n", + " \"CI-W3V21F1\",\n", + " \"CI-W3V21P1\",\n", + " \"CI-W3V30F1\",\n", + " \"CI-W3V33P1\"\n", "]\n", - "\n", "url = 'https://pivision.votorantimcimentos.com/piwebapi/dataservers/F1DS-7fYgsRTtUOa7V9NIwSujAUElIQVZD/points?namefilter={tag}'\n", "\n", "headers = {\n", " 'Content-Type': 'application/json',\n", " 'Accept': 'application/json',\n", " 'X-Requested-With': 'piwebapistreams', # Header recomendado pelo PI Web API\n", - " 'Authorization': \"\"\n", + " 'Authorization': \"Basic dmlkX3ZjbmV0XHN2Yy5waW9zaS5wcmQud2ViYXBpOlN2Y1ByRFdlQkBQaQ==\"\n", "}\n", "\n", "web_ids = {}\n", @@ -235,7 +245,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "55793801", "metadata": {}, "outputs": [ @@ -329,31 +339,25 @@ " 'CI-W3K01T2': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAlVQAAAUElIQVZDXENJLVczSzAxVDI',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_FSC': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAgVQAAAUElIQVZDXENJLVczRkFSQ0lfRlND',\n", + " 'CI-W3A65_SO3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAjFQAAAUElIQVZDXENJLVczQTY1X1NPMw',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_MA': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAg1QAAAUElIQVZDXENJLVczRkFSQ0lfTUE',\n", + " 'CI-W3A65_CL': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAf1QAAAUElIQVZDXENJLVczQTY1X0NM',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_MS': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAhVQAAAUElIQVZDXENJLVczRkFSQ0lfTVM',\n", + " 'CI-W3_C3S': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5FMAAAUElIQVZDXENJLVczX0MzUw',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_P100': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAiFQAAAUElIQVZDXENJLVczRkFSQ0lfUDEwMA',\n", + " 'CI-W3_MS': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujABlQAAAUElIQVZDXENJLVczX01T',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3FARCI_p170': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAiVQAAAUElIQVZDXENJLVczRkFSQ0lfUDE3MA',\n", + " 'CI-W3_MA': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA_VMAAAUElIQVZDXENJLVczX01B',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3CLK_C3S': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA4lMAAAUElIQVZDXENJLVczQ0xLX0MzUw',\n", + " 'CI-W3_PL': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAFVQAAAUElIQVZDXENJLVczX1BM',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", - " 'CI-W3CLK_C3S_EXP': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA41MAAAUElIQVZDXENJLVczQ0xLX0MzU19FWFA',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3CLK_C3S_MD_EXP': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5VMAAAUElIQVZDXENJLVczQ0xLX0MzU19NRF9FWFA',\n", - " 'aggr_func': 'lts',\n", - " 'data_range': [-100000, 100000]},\n", - " 'CI-W3CLK_C3S_MD_PETRO': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA5lMAAAUElIQVZDXENJLVczQ0xLX0MzU19NRF9QRVRSTw',\n", + " 'CI-W3_CAO': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujA61MAAAUElIQVZDXENJLVczX0NBTw',\n", " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]},\n", " 'CI-W3V04P3': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAElUAAAUElIQVZDXENJLVczVjA0UDM',\n", @@ -364,15 +368,26 @@ " 'data_range': [-100000, 100000]},\n", " 'CI-W3W01G1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAOlUAAAUElIQVZDXENJLVczVzAxRzE',\n", " 'aggr_func': 'lts',\n", + " 'data_range': [-100000, 100000]},\n", + " 'CI-W3V21F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAQnYEAAUElIQVZDXENJLVczVjIxRjE',\n", + " 'aggr_func': 'lts',\n", + " 'data_range': [-100000, 100000]},\n", + " 'CI-W3V21P1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAQHYEAAUElIQVZDXENJLVczVjIxUDE',\n", + " 'aggr_func': 'lts',\n", + " 'data_range': [-100000, 100000]},\n", + " 'CI-W3V30F1': {'webid': 'F1DP-7fYgsRTtUOa7V9NIwSujAIFUAAAUElIQVZDXENJLVczVjMwRjE',\n", + " 'aggr_func': 'lts',\n", " 'data_range': [-100000, 100000]}}" ] }, - "execution_count": 23, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "import json\n", + "json.dump(web_ids, open('web_ids.json', 'w'), indent=4)\n", "web_ids" ] },