feat: enhance configuration and error handling in project setup

- Added new ignore rule for Ruff to allow temporary paths in tests.
- Introduced MyPy overrides for specific modules to ignore errors.
- Refactored `Cleanup` and `ExperimentTracking` classes to remove async keywords from methods, improving consistency in method signatures.
- Updated `Training` class methods to handle synchronous operations, enhancing performance and clarity.
- Adjusted `requirements.txt` to remove unnecessary Git dependency, streamlining project setup.

model_manager/worker/prepare_worker.py

@@ -0,0 +1,106 @@
+import os
+import re
+from collections.abc import Sequence
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+from sientia_do.observability.logger import Logger
+from temporalio.client import Client
+from temporalio.worker import PollerBehaviorAutoscaling, Worker
+
+# Worker configuration parameters with default values.
+parameters = [
+    ('MAX_CONCURRENT_WORKFLOW_TASKS', '200'),
+    ('MAX_CONCURRENT_ACTIVITIES', '200'),
+    ('MAX_CONCURRENT_LOCAL_ACTIVITIES', '200'),
+    ('MAX_CACHED_WORKFLOWS', '200'),
+    ('WORKFLOW_POLLER_BEHAVIOUR_MINIMUM', '10'),
+    ('WORKFLOW_POLLER_BEHAVIOUR_INITIAL', '100'),
+    ('WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM', '200'),
+    ('ACTIVITY_POLLER_BEHAVIOUR_MINIMUM', '10'),
+    ('ACTIVITY_POLLER_BEHAVIOUR_INITIAL', '100'),
+    ('ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM', '200'),
+    ('ACTIVITY_EXECUTOR_MAX_WORKERS', '32'),
+]
+
+
+def camel_to_snake(text: str) -> str:
+    """
+    Convert a CamelCase or camelCase string into snake_case.
+
+    Args:
+     - text: str, original string in CamelCase or camelCase format
+
+    Return:
+     str: converted string in snake_case format
+    """
+    text = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', text)
+    text = re.sub('([a-z0-9])([A-Z])', r'\1_\2', text)
+    return text.lower()
+
+
+def prepare_worker(
+    main_workflow: type,
+    other_workflows: Sequence[type],
+    activities: Sequence[Any],
+    temporal_client: Client,
+    logger: Logger,
+    runtime: str | None = None,
+) -> Worker:
+    """
+    Build and configure a Temporal worker for the given workflow and activities.
+
+    Args:
+     - main_workflow: type, main workflow class used as worker entry point
+     - other_workflows: Sequence[type], additional workflows in the same worker
+     - activities: Sequence[Any], activity callables registered in this worker
+     - temporal_client: Client, Temporal client used by the worker
+     - logger: Logger, logger instance used during worker preparation
+     - runtime: str | None, runtime suffix appended to queue name when present
+
+    Return:
+     Worker: fully configured Temporal worker instance ready to run
+    """
+    main_workflow_name = main_workflow.__name__.upper()
+    queue_name = (
+        f'{camel_to_snake(main_workflow.__name__)}-{runtime}-queue'
+        if runtime
+        else f'{camel_to_snake(main_workflow.__name__)}-queue'
+    )
+
+    local_workflow_parameters: dict[str, int] = {}
+    for parameter_name, default_value in parameters:
+        local_workflow_parameters[parameter_name] = int(
+            os.getenv(f'{main_workflow_name}_{parameter_name}', default_value)
+        )
+
+    logger.info(f'Preparing worker for {main_workflow_name} with queue {queue_name}')
+
+    activity_executor = ThreadPoolExecutor(
+        max_workers=local_workflow_parameters['ACTIVITY_EXECUTOR_MAX_WORKERS'],
+        thread_name_prefix=f'{queue_name}-activity',
+    )
+
+    return Worker(
+        temporal_client,
+        task_queue=queue_name,
+        workflows=[main_workflow, *other_workflows],
+        activities=[*activities],
+        activity_executor=activity_executor,
+        max_concurrent_workflow_tasks=local_workflow_parameters['MAX_CONCURRENT_WORKFLOW_TASKS'],
+        max_concurrent_activities=local_workflow_parameters['MAX_CONCURRENT_ACTIVITIES'],
+        max_concurrent_local_activities=local_workflow_parameters[
+            'MAX_CONCURRENT_LOCAL_ACTIVITIES'
+        ],
+        max_cached_workflows=local_workflow_parameters['MAX_CACHED_WORKFLOWS'],
+        workflow_task_poller_behavior=PollerBehaviorAutoscaling(
+            minimum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MINIMUM'],
+            initial=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_INITIAL'],
+            maximum=local_workflow_parameters['WORKFLOW_POLLER_BEHAVIOUR_MAXIMUM'],
+        ),
+        activity_task_poller_behavior=PollerBehaviorAutoscaling(
+            minimum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MINIMUM'],
+            initial=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_INITIAL'],
+            maximum=local_workflow_parameters['ACTIVITY_POLLER_BEHAVIOUR_MAXIMUM'],
+        ),
+    )

model_manager/worker/worker.py

@@ -30,7 +30,6 @@ Environment Variables:
 
 from temporalio import client, workflow
 from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig
-from temporalio.worker import PollerBehaviorAutoscaling, Worker
 
 with workflow.unsafe.imports_passed_through():
     import asyncio
@@ -40,9 +39,8 @@ with workflow.unsafe.imports_passed_through():
     from prometheus_client import start_http_server
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.observability.logger import Logger as SientiaLogger
-    from sientia_model.model_repository.plugin_store import PluginStore
     from sientia_do.observability.metrics_controller import MetricsController
-    from sientia_do.temporal.worker.prepare_worker import prepare_worker
+    from sientia_model.model_repository.plugin_store import PluginStore
 
     from model_manager import metrics
     from model_manager.activities.activities import Activities
@@ -55,6 +53,7 @@ with workflow.unsafe.imports_passed_through():
         build_postgres_config,
     )
     from model_manager.utils.logger_helper import get_logger
+    from model_manager.worker.prepare_worker import prepare_worker
     from model_manager.workflows.cleanup_files import CleanupFiles
     from model_manager.workflows.train_model import TrainModel
 
@@ -96,7 +95,6 @@ async def main():
         'pod_id': POD_ID,
         'runtime': RUNTIME,
     }
-    
 
     start_prometheus_server(logger, metadata)
     mongo_config = build_mongodb_config()
@@ -110,11 +108,9 @@ async def main():
 
     logger.custom_info(f'MongoDB client initialized at {mongo_config["uri"]}', metadata)
 
-    logger.custom_info(f'Initializing metrics controller', metadata)
+    logger.custom_info('Initializing metrics controller', metadata)
 
-    metrics_controller = MetricsController(
-        logger=logger
-    )
+    metrics_controller = MetricsController(logger=logger)
 
     logger.custom_info(f'Installing runtime {RUNTIME}', metadata)
 
@@ -185,6 +181,7 @@ async def main():
             ],
             temporal_client=temporal_client,
             logger=logger,
+            runtime=RUNTIME,
         ),
         prepare_worker(
             main_workflow=CleanupFiles,
@@ -194,12 +191,11 @@ async def main():
             ],
             temporal_client=temporal_client,
             logger=logger,
+            runtime=RUNTIME,
         ),
     ]
 
-    handlers = [
-        w.run() for w in workers
-    ]
+    handlers = [w.run() for w in workers]
 
     logger.custom_info('Model manager workers initialized', metadata)