fix(worker): pin the runtime versions and refuse to boot when install_runtime swaps an imported package (SIENTIAPDE-2214)

Replaces the re-exec. requirements.txt now pins the union of the basic and xgboost
runtime libraries, so install_runtime has nothing to change in the packages the worker
already imported. If a runtime pin changes and requirements.txt is not updated, the
worker logs which packages changed and exits before starting the workers.

test_main_entrypoint assigned worker.main = AsyncMock() without restoring it, so every
later test of main() ran the mock; it now uses patch.object.

model_manager/worker/worker.py

@@ -21,8 +21,6 @@ Environment Variables:
 - TEMPORAL_NAMESPACE: Temporal namespace (default: model-manager)
 - TEMPORAL_USE_TLS: Enable TLS for Temporal connection (default: false)
 - RUNTIME: Runtime identifier used in queue naming (default: single)
-- SIENTIA_RUNTIME_READY: Internal, set by the worker itself before re-executing after
-  install_runtime. Never set it in the deployment: it skips the runtime install.
 - POD_ID: Kubernetes pod identifier for metrics
 - HTTP_METRICS_PORT: Prometheus metrics server port (default: 9090)
 - HTTP_SDK_METRICS_PORT: Temporal SDK metrics port (default: 9091)
@@ -34,6 +32,7 @@ from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig
 
 with workflow.unsafe.imports_passed_through():
     import asyncio
+    import importlib.metadata
     import os
     import sys
 
@@ -78,6 +77,35 @@ def _get_runtime(runtime: str | None) -> str:
     return normalized_runtime or 'single'
 
 
+def _loaded_distribution_versions() -> dict[str, str]:
+    """Version on disk of every installed distribution whose top-level module is imported."""
+    loaded = {name.partition('.')[0] for name in sys.modules}
+    return {
+        dist: importlib.metadata.version(dist)
+        for module, dists in importlib.metadata.packages_distributions().items()
+        if module in loaded
+        for dist in dists
+    }
+
+
+def _runtime_changed_loaded_packages(before: dict[str, str]) -> dict[str, str]:
+    """
+    Packages that were imported before ``install_runtime`` and whose version changed on disk.
+
+    The process keeps the old code in memory, so a later lazy import mixes two versions
+    (SIENTIAPDE-2214: sklearn 1.9.1 in memory, 1.8.0 on disk, ``_cyutility`` import error).
+    """
+    changed = {}
+    for dist, old in before.items():
+        try:
+            new = importlib.metadata.version(dist)
+        except importlib.metadata.PackageNotFoundError:
+            new = 'removed'
+        if new != old:
+            changed[dist] = f'{old} -> {new}'
+    return changed
+
+
 async def main():
     """
     Main entry point for the Model Manager worker application.
@@ -85,12 +113,11 @@ async def main():
     This function initializes and starts all components of the worker:
     1. Sets up logging and metadata
     2. Starts Prometheus metrics server
-    3. Initializes notification handler
-    4. Installs the PluginStore runtime and re-executes itself once (`SIENTIA_RUNTIME_READY=1`)
-       so every import loads the runtime's package versions
-    5. Creates and configures activities
-    6. Starts Temporal client and workers
-    7. Manages worker lifecycle and graceful shutdown
+    3. Initializes notification handler, then installs the runtime; exits if the install
+       changed the version of a package the worker already imported
+    4. Creates and configures activities
+    5. Starts Temporal client and workers
+    6. Manages worker lifecycle and graceful shutdown
 
     The function runs indefinitely until interrupted or an error occurs.
     On error, it performs cleanup and exits with a non-zero status code.
@@ -129,6 +156,8 @@ async def main():
 
     metrics_controller = MetricsController(logger=logger)
 
+    logger.custom_info(f'Installing runtime {runtime}', metadata)
+
     plugin_store_parameters = build_plugin_store_config()
     plugin_store = PluginStore(
         base_url=plugin_store_parameters['base_url'],
@@ -146,19 +175,18 @@ async def main():
         metrics_controller=metrics_controller,
     )
 
-    if os.environ.get('SIENTIA_RUNTIME_READY') != '1':
-        logger.custom_info(f'Installing runtime {runtime}', metadata)
-        await plugin_store.install_runtime(runtime_name=runtime)
-        os.environ['SIENTIA_RUNTIME_READY'] = '1'
-        logger.custom_info(f'Runtime {runtime} installed, re-executing worker', metadata)
-        # ponytail: fresh interpreter so loaded modules match the runtime on disk (SIENTIAPDE-2214).
-        # The install_runtime success metrics (count/lag) are lost to the exec; no dashboard uses them.
-        os.execv(sys.executable, [sys.executable, *sys.orig_argv[1:]])  # noqa: S606
-    else:
-        logger.custom_info(
-            f'SIENTIA_RUNTIME_READY=1, runtime {runtime} already installed, skipping install',
+    loaded_before_install = _loaded_distribution_versions()
+    await plugin_store.install_runtime(runtime_name=runtime)
+
+    changed = _runtime_changed_loaded_packages(loaded_before_install)
+    if changed:
+        logger.custom_critical(
+            f'Runtime {runtime} changed packages already imported by the worker: {changed}. '
+            'Pin them in requirements.txt to the runtime versions.',
             metadata,
         )
+        metrics.APP_UP.labels(pod_id=POD_ID).set(0)
+        sys.exit(1)
 
     activities = Activities(
         postgres_config=build_postgres_config(),