09ee92f100
- Added new ignore rule for Ruff to allow temporary paths in tests. - Introduced MyPy overrides for specific modules to ignore errors. - Refactored `Cleanup` and `ExperimentTracking` classes to remove async keywords from methods, improving consistency in method signatures. - Updated `Training` class methods to handle synchronous operations, enhancing performance and clarity. - Adjusted `requirements.txt` to remove unnecessary Git dependency, streamlining project setup.
107 lines
4.0 KiB
Python
107 lines
4.0 KiB
Python
import os
|
|
import re
|
|
from collections.abc import Sequence
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import Any
|
|
|
|
from sientia_do.observability.logger import Logger
|
|
from temporalio.client import Client
|
|
from temporalio.worker import PollerBehaviorAutoscaling, Worker
|
|
|
|
# Worker configuration parameters with default values.
|
|
parameters = [
|
|
('MAX_CONCURRENT_WORKFLOW_TASKS', '200'),
|
|
('MAX_CONCURRENT_ACTIVITIES', '200'),
|
|
('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'),
|
|
('MAX_CACHED_WORKFLOWS', '200'),
|
|
('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'),
|
|
('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'),
|
|
('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'),
|
|
('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'),
|
|
('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'),
|
|
('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'),
|
|
('ACTIVITY_EXECUTOR_MAX_WORKERS', '32'),
|
|
]
|
|
|
|
|
|
def camel_to_snake(text: str) -> str:
|
|
"""
|
|
Convert a CamelCase or camelCase string into snake_case.
|
|
|
|
Args:
|
|
- text: str, original string in CamelCase or camelCase format
|
|
|
|
Return:
|
|
str: converted string in snake_case format
|
|
"""
|
|
text = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', text)
|
|
text = re.sub('([a-z0-9])([A-Z])', r'\1_\2', text)
|
|
return text.lower()
|
|
|
|
|
|
def prepare_worker(
|
|
main_workflow: type,
|
|
other_workflows: Sequence[type],
|
|
activities: Sequence[Any],
|
|
temporal_client: Client,
|
|
logger: Logger,
|
|
runtime: str | None = None,
|
|
) -> Worker:
|
|
"""
|
|
Build and configure a Temporal worker for the given workflow and activities.
|
|
|
|
Args:
|
|
- main_workflow: type, main workflow class used as worker entry point
|
|
- other_workflows: Sequence[type], additional workflows in the same worker
|
|
- activities: Sequence[Any], activity callables registered in this worker
|
|
- temporal_client: Client, Temporal client used by the worker
|
|
- logger: Logger, logger instance used during worker preparation
|
|
- runtime: str | None, runtime suffix appended to queue name when present
|
|
|
|
Return:
|
|
Worker: fully configured Temporal worker instance ready to run
|
|
"""
|
|
main_workflow_name = main_workflow.__name__.upper()
|
|
queue_name = (
|
|
f'{camel_to_snake(main_workflow.__name__)}-{runtime}-queue'
|
|
if runtime
|
|
else f'{camel_to_snake(main_workflow.__name__)}-queue'
|
|
)
|
|
|
|
local_workflow_parameters: dict[str, int] = {}
|
|
for parameter_name, default_value in parameters:
|
|
local_workflow_parameters[parameter_name] = int(
|
|
os.getenv(f'{main_workflow_name}_{parameter_name}', default_value)
|
|
)
|
|
|
|
logger.info(f'Preparing worker for {main_workflow_name} with queue {queue_name}')
|
|
|
|
activity_executor = ThreadPoolExecutor(
|
|
max_workers=local_workflow_parameters['ACTIVITY_EXECUTOR_MAX_WORKERS'],
|
|
thread_name_prefix=f'{queue_name}-activity',
|
|
)
|
|
|
|
return Worker(
|
|
temporal_client,
|
|
task_queue=queue_name,
|
|
workflows=[main_workflow, *other_workflows],
|
|
activities=[*activities],
|
|
activity_executor=activity_executor,
|
|
max_concurrent_workflow_tasks=local_workflow_parameters['MAX_CONCURRENT_WORKFLOW_TASKS'],
|
|
max_concurrent_activities=local_workflow_parameters['MAX_CONCURRENT_ACTIVITIES'],
|
|
max_concurrent_local_activities=local_workflow_parameters[
|
|
'MAX_CONCURRENT_LOCAL_ACTIVITIES'
|
|
],
|
|
max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'],
|
|
workflow_task_poller_behavior=PollerBehaviorAutoscaling(
|
|
minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'],
|
|
initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'],
|
|
maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'],
|
|
),
|
|
activity_task_poller_behavior=PollerBehaviorAutoscaling(
|
|
minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'],
|
|
initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'],
|
|
maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'],
|
|
),
|
|
)
|