0ae03b246f
- Modified `.env.example` to set local defaults for PostgreSQL, MLflow, and MinIO configurations. - Added MongoDB configuration parameters to the environment setup. - Updated `README.md` to reflect changes in workflow input parameters and task queue naming conventions. - Removed the `ModelServing` class to streamline the codebase, as it was deemed unnecessary. - Adjusted `connectors_config.py` to align with new environment variable names and improve clarity. - Updated tests to reflect changes in configuration handling and removed tests related to the deleted `ModelServing` class.
120 lines
4.5 KiB
Python
120 lines
4.5 KiB
Python
import os
|
|
import re
|
|
from collections.abc import Sequence
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import Any
|
|
|
|
from sientia_do.observability.logger import Logger
|
|
from temporalio.client import Client
|
|
from temporalio.worker import PollerBehaviorAutoscaling, Worker
|
|
|
|
# Worker configuration parameters with default values.
|
|
parameters = [
|
|
('MAX_CONCURRENT_WORKFLOW_TASKS', '200'),
|
|
('MAX_CONCURRENT_ACTIVITIES', '200'),
|
|
('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'),
|
|
('MAX_CACHED_WORKFLOWS', '200'),
|
|
('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'),
|
|
('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'),
|
|
('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'),
|
|
('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'),
|
|
('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'),
|
|
('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'),
|
|
('ACTIVITY_EXECUTOR_MAX_WORKERS', '200'),
|
|
]
|
|
|
|
|
|
def camel_to_snake(text: str) -> str:
|
|
"""
|
|
Convert a CamelCase or camelCase string into snake_case.
|
|
|
|
Args:
|
|
- text: str, original string in CamelCase or camelCase format
|
|
|
|
Return:
|
|
str: converted string in snake_case format
|
|
"""
|
|
text = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', text)
|
|
text = re.sub('([a-z0-9])([A-Z])', r'\1_\2', text)
|
|
return text.lower()
|
|
|
|
|
|
def build_queue_name(workflow_name: str, runtime: str | None = None) -> str:
|
|
"""
|
|
Build Temporal queue name from workflow name and runtime.
|
|
|
|
Args:
|
|
- workflow_name: str, workflow class name in CamelCase format
|
|
- runtime: str | None, runtime suffix for environment-specific queues
|
|
|
|
Return:
|
|
str: queue name in the format <workflow>-<runtime>-queue or <workflow>-queue
|
|
"""
|
|
snake_workflow_name = camel_to_snake(workflow_name)
|
|
if runtime:
|
|
return f'{snake_workflow_name}-{runtime}-queue'
|
|
return f'{snake_workflow_name}-queue'
|
|
|
|
|
|
def prepare_worker(
|
|
main_workflow: type,
|
|
other_workflows: Sequence[type],
|
|
activities: Sequence[Any],
|
|
temporal_client: Client,
|
|
logger: Logger,
|
|
runtime: str | None = None,
|
|
) -> Worker:
|
|
"""
|
|
Build and configure a Temporal worker for the given workflow and activities.
|
|
|
|
Args:
|
|
- main_workflow: type, main workflow class used as worker entry point
|
|
- other_workflows: Sequence[type], additional workflows in the same worker
|
|
- activities: Sequence[Any], activity callables registered in this worker
|
|
- temporal_client: Client, Temporal client used by the worker
|
|
- logger: Logger, logger instance used during worker preparation
|
|
- runtime: str | None, runtime suffix appended to queue name when present
|
|
|
|
Return:
|
|
Worker: fully configured Temporal worker instance ready to run
|
|
"""
|
|
main_workflow_name = main_workflow.__name__.upper()
|
|
queue_name = build_queue_name(main_workflow.__name__, runtime)
|
|
|
|
local_workflow_parameters: dict[str, int] = {}
|
|
for parameter_name, default_value in parameters:
|
|
local_workflow_parameters[parameter_name] = int(
|
|
os.getenv(f'{main_workflow_name}_{parameter_name}', default_value)
|
|
)
|
|
|
|
logger.info(f'Preparing worker for {main_workflow_name} with queue {queue_name}')
|
|
|
|
activity_executor = ThreadPoolExecutor(
|
|
max_workers=local_workflow_parameters['ACTIVITY_EXECUTOR_MAX_WORKERS'],
|
|
thread_name_prefix=f'{queue_name}-activity',
|
|
)
|
|
|
|
return Worker(
|
|
temporal_client,
|
|
task_queue=queue_name,
|
|
workflows=[main_workflow, *other_workflows],
|
|
activities=[*activities],
|
|
activity_executor=activity_executor,
|
|
max_concurrent_workflow_tasks=local_workflow_parameters['MAX_CONCURRENT_WORKFLOW_TASKS'],
|
|
max_concurrent_activities=local_workflow_parameters['MAX_CONCURRENT_ACTIVITIES'],
|
|
max_concurrent_local_activities=local_workflow_parameters[
|
|
'MAX_CONCURRENT_LOCAL_ACTIVITIES'
|
|
],
|
|
max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'],
|
|
workflow_task_poller_behavior=PollerBehaviorAutoscaling(
|
|
minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'],
|
|
initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'],
|
|
maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'],
|
|
),
|
|
activity_task_poller_behavior=PollerBehaviorAutoscaling(
|
|
minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'],
|
|
initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'],
|
|
maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'],
|
|
),
|
|
)
|