SIENTIAPDE-1646

Refactor worker task queue management and update README

- Introduced runtime-scoped task queues for workflows, replacing legacy queue names.
- Updated worker implementation to utilize `sientia_do.temporal.worker.prepare_worker`.
- Added `RUNTIME` environment variable to configure task queue suffixes.
- Enhanced README documentation to reflect changes in task queue structure and worker setup.

laborious/worker/worker.py

@@ -5,12 +5,14 @@ This module provides the main worker implementation for the Sientia DataOps Labo
 It orchestrates Temporal workers, manages task queues, and handles the lifecycle of
 prediction and retraining workflows.
 
-The worker supports multiple task queues:
-- predictions_batch-queue: Handles batch prediction workflows (heavy workload)
-  Includes activities for MLFlow, data quality gates, OPC export, PI Web API export, and PostgreSQL
-- minimal_retrain-queue: Handles model retraining workflows
-- drift-queue: Handles drift detection workflows
-- simple_metrics-queue: Handles simple metrics calculation workflows
+The worker supports multiple runtime-scoped task queues (via ``sientia_do.temporal.worker.prepare_worker``):
+- predictions_batch-{runtime}-queue: Batch prediction workflows (heavy workload)
+- minimal_retrain-{runtime}-queue: Model retraining workflows
+- drift-{runtime}-queue: Drift detection workflows
+- simple_metrics-{runtime}-queue: Simple metrics workflows
+
+``RUNTIME`` must be set; it is passed to every ``prepare_worker`` call. Schedulers must use the
+same queue names (breaking change vs legacy ``drift-queue`` / ``simple_metrics-queue``).
 
 Key Features:
 - Resource-based scaling with WorkerTuner (CPU and memory aware)
@@ -21,6 +23,7 @@ Key Features:
 - Multiple worker instances for different workflow types
 
 Environment Variables:
+- RUNTIME: Required non-empty string; suffix for all task queue names
 - TEMPORAL_HOST: Temporal server address (default: localhost:7233)
 - TEMPORAL_NAMESPACE: Temporal namespace (default: laborious)
 - POD_ID: Kubernetes pod identifier for metrics
@@ -40,6 +43,7 @@ with workflow.unsafe.imports_passed_through():
     from prometheus_client import start_http_server
     from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
     from sientia_do.observability.logger import get_logger
+    from sientia_do.temporal.worker.prepare_worker import prepare_worker
     from sientia_do.utils.connectors_config import (
         build_api_config,
         build_mongodb_config,
@@ -53,7 +57,6 @@ with workflow.unsafe.imports_passed_through():
         build_mlflow_config,
         build_opc_config,
     )
-    from laborious.worker.prepare_worker import prepare_worker
     from laborious.workflows.drift import Drift
     from laborious.workflows.minimal_retrain import MinimalRetrain
     from laborious.workflows.predictions_batch import PredictionsBatch
@@ -100,10 +103,21 @@ async def main():
 
     logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata)
 
-    logger.custom_info('Starting prometheus client...', metadata)
+    runtime = os.getenv('RUNTIME', '').strip()
+    if not runtime:
+        logger.custom_critical(
+            'RUNTIME environment variable is required and must be non-empty',
+            metadata,
+        )
+        metrics.APP_UP.labels(pod_id=POD_ID).set(0)
+        sys.exit(1)
+
+    metadata_runtime = {**metadata, 'runtime': runtime}
+
+    logger.custom_info('Starting prometheus client...', metadata_runtime)
     start_prometheus_server()
 
-    logger.custom_info('Starting Notification Handler...', metadata)
+    logger.custom_info('Starting Notification Handler...', metadata_runtime)
 
     mongo_config = build_mongodb_config()
     notification_handler = NotificationHandler(
@@ -113,7 +127,7 @@ async def main():
         project_name=os.getenv('PROJECT_NAME', 'laborious'),
     )
 
-    logger.custom_info('Starting Activities...', metadata)
+    logger.custom_info('Starting Activities...', metadata_runtime)
 
     activities = Activities(
         postgres_config=build_postgres_config(),
@@ -125,10 +139,13 @@ async def main():
         notification_handler=notification_handler,
     )
 
-    logger.custom_info('Initializing OPC...', metadata)
+    logger.custom_info('Initializing OPC...', metadata_runtime)
     await activities.init_opc()
 
-    logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
+    logger.custom_info(
+        f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...',
+        metadata_runtime,
+    )
 
     new_runtime = Runtime(
         telemetry=TelemetryConfig(
@@ -136,7 +153,7 @@ async def main():
         )
     )
 
-    logger.custom_info(f'Starting Temporal Client at {host}...', metadata)
+    logger.custom_info(f'Starting Temporal Client at {host}...', metadata_runtime)
 
     temporal_client = await client.Client.connect(
         target_host=host,
@@ -144,7 +161,7 @@ async def main():
         runtime=new_runtime,
     )
 
-    logger.custom_info('Starting Workers...', metadata)
+    logger.custom_info(f'Starting Workers (runtime={runtime})...', metadata_runtime)
 
     workers = [
         prepare_worker(
@@ -159,6 +176,7 @@ async def main():
                 activities.export_data_to_postgres,
             ],
             logger=logger,
+            runtime=runtime,
         ),
         prepare_worker(
             temporal_client=temporal_client,
@@ -170,6 +188,7 @@ async def main():
                 activities.export_data_to_postgres,
             ],
             logger=logger,
+            runtime=runtime,
         ),
         prepare_worker(
             temporal_client=temporal_client,
@@ -182,6 +201,7 @@ async def main():
                 activities.export_data_to_postgres,
             ],
             logger=logger,
+            runtime=runtime,
         ),
         prepare_worker(
             temporal_client=temporal_client,
@@ -211,6 +231,7 @@ async def main():
                 activities.write_pi_web_api_data,
             ],
             logger=logger,
+            runtime=runtime,
         ),
     ]
 
@@ -218,7 +239,7 @@ async def main():
     for w in workers:
         handlers.append(w.run())
 
-    logger.custom_info('Workers started successfully', metadata)
+    logger.custom_info('Workers started successfully', metadata_runtime)
 
     exit_code = 0
     try: