fix(worker): set app_up only after the runtime version guard (SIENTIAPDE-2214)

The probes only check app_up, which was set to 1 when the metrics server started,
before install_runtime. A pod that would fail the guard could turn Ready first, so a
rollout would remove the old pod. app_up now goes to 1 right after the guard.

Also: copy sys.modules keys before iterating, raise the sientia_model floor to the
runtime's >=0.14.0, pin the same runtime versions in requirements-local.txt, test
that the snapshot is taken before the install and the real importlib.metadata path,
and document the boot failure in README and CLAUDE.md.
This commit is contained in:
PedroHMCosme
2026-10-22 12:17:12 -03:00
parent 0f5e4a5cef
commit dfbdd4b605
6 changed files with 74 additions and 15 deletions

View File

@@ -132,9 +132,8 @@ def test_start_prometheus_server_success(
# Verify HTTP server started
mock_start_http_server.assert_called_once_with(9090)
# Verify APP_UP metric was set to 1
mock_metrics.APP_UP.labels.assert_called_once_with(pod_id='test-pod-123')
mock_app_up.set.assert_called_once_with(1)
# app_up goes to 1 in main(), after the runtime version guard.
mock_app_up.set.assert_not_called()
mock_logger.custom_info.assert_called_once()
@@ -284,7 +283,8 @@ async def test_main_successful_startup(
# Verify cleanup was performed
mock_notification_handler.shutdown.assert_called_once()
mock_activities.shutdown.assert_called_once()
mock_app_up.set.assert_called_with(0)
# Ready after the guard, down again on shutdown.
assert [c.args[0] for c in mock_app_up.set.call_args_list] == [1, 0]
@pytest.mark.asyncio
@@ -941,22 +941,30 @@ async def test_main_runtime_changed_loaded_package_exits(mock_env_vars, mock_log
plugin_store = Mock()
plugin_store.install_runtime = AsyncMock(return_value=None)
before = {'scikit-learn': '1.9.1'}
def snapshot():
# Taken after install_runtime, it would already see 1.8.0 and the guard would pass.
assert plugin_store.install_runtime.await_count == 0
return before
mongo = {'connection_string': 'mongodb://test', 'database_name': 'test_db', 'uri': 'x'}
with (
patch.object(worker_module, 'RUNTIME', 'basic'),
patch.object(worker_module, 'ensure_runtime_directories'),
patch.object(worker_module, 'metrics'),
patch.object(worker_module, 'metrics') as mock_metrics,
patch.object(worker_module, 'start_prometheus_server'),
patch.object(worker_module, 'get_logger', return_value=mock_logger),
patch.object(worker_module, 'build_mongodb_config', return_value=mongo),
patch.object(worker_module, 'NotificationHandler'),
patch.object(worker_module, 'build_plugin_store_config', return_value=MagicMock()),
patch.object(worker_module, 'PluginStore', return_value=plugin_store),
patch.object(worker_module, '_loaded_distribution_versions', side_effect=snapshot),
patch.object(
worker_module,
'_runtime_changed_loaded_packages',
return_value={'scikit-learn': '1.9.1 -> 1.8.0'},
),
) as mock_changed,
patch.object(worker_module, 'Activities') as mock_activities_class,
patch.object(worker_module.sys, 'exit', side_effect=SystemExit(1)) as mock_exit,
):
@@ -965,5 +973,35 @@ async def test_main_runtime_changed_loaded_package_exits(mock_env_vars, mock_log
mock_exit.assert_called_once_with(1)
mock_activities_class.assert_not_called()
plugin_store.install_runtime.assert_awaited_once()
mock_changed.assert_called_once_with(before)
mock_metrics.APP_UP.labels.return_value.set.assert_called_once_with(0)
message = mock_logger.custom_critical.call_args.args[0]
assert "'scikit-learn': '1.9.1 -> 1.8.0'" in message
def test_guard_detects_version_change_on_disk(tmp_path, monkeypatch):
"""The real importlib.metadata path: a package swapped on disk after import is reported."""
import types
from model_manager.worker.worker import (
_loaded_distribution_versions,
_runtime_changed_loaded_packages,
)
info = tmp_path / 'fakepkg_2214-1.0.dist-info'
info.mkdir()
(info / 'METADATA').write_text('Metadata-Version: 2.1\nName: fakepkg-2214\nVersion: 1.0\n')
(info / 'top_level.txt').write_text('fakepkg_2214\n')
monkeypatch.syspath_prepend(str(tmp_path))
# The guard only reads sys.modules keys; setitem removes the entry again on teardown.
monkeypatch.setitem(sys.modules, 'fakepkg_2214', types.ModuleType('fakepkg_2214'))
before = _loaded_distribution_versions()
assert before['fakepkg-2214'] == '1.0'
# What pip does on an upgrade: the old dist-info goes away and a new one takes its place.
new_info = info.rename(tmp_path / 'fakepkg_2214-2.0.dist-info')
(new_info / 'METADATA').write_text('Metadata-Version: 2.1\nName: fakepkg-2214\nVersion: 2.0\n')
assert _runtime_changed_loaded_packages(before) == {'fakepkg-2214': '1.0 -> 2.0'}