fix(worker): set app_up only after the runtime version guard (SIENTIAPDE-2214)
The probes only check app_up, which was set to 1 when the metrics server started, before install_runtime. A pod that would fail the guard could turn Ready first, so a rollout would remove the old pod. app_up now goes to 1 right after the guard. Also: copy sys.modules keys before iterating, raise the sientia_model floor to the runtime's >=0.14.0, pin the same runtime versions in requirements-local.txt, test that the snapshot is taken before the install and the real importlib.metadata path, and document the boot failure in README and CLAUDE.md.
This commit is contained in:
@@ -132,9 +132,8 @@ def test_start_prometheus_server_success(
|
||||
# Verify HTTP server started
|
||||
mock_start_http_server.assert_called_once_with(9090)
|
||||
|
||||
# Verify APP_UP metric was set to 1
|
||||
mock_metrics.APP_UP.labels.assert_called_once_with(pod_id='test-pod-123')
|
||||
mock_app_up.set.assert_called_once_with(1)
|
||||
# app_up goes to 1 in main(), after the runtime version guard.
|
||||
mock_app_up.set.assert_not_called()
|
||||
mock_logger.custom_info.assert_called_once()
|
||||
|
||||
|
||||
@@ -284,7 +283,8 @@ async def test_main_successful_startup(
|
||||
# Verify cleanup was performed
|
||||
mock_notification_handler.shutdown.assert_called_once()
|
||||
mock_activities.shutdown.assert_called_once()
|
||||
mock_app_up.set.assert_called_with(0)
|
||||
# Ready after the guard, down again on shutdown.
|
||||
assert [c.args[0] for c in mock_app_up.set.call_args_list] == [1, 0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -941,22 +941,30 @@ async def test_main_runtime_changed_loaded_package_exits(mock_env_vars, mock_log
|
||||
|
||||
plugin_store = Mock()
|
||||
plugin_store.install_runtime = AsyncMock(return_value=None)
|
||||
before = {'scikit-learn': '1.9.1'}
|
||||
|
||||
def snapshot():
|
||||
# Taken after install_runtime, it would already see 1.8.0 and the guard would pass.
|
||||
assert plugin_store.install_runtime.await_count == 0
|
||||
return before
|
||||
|
||||
mongo = {'connection_string': 'mongodb://test', 'database_name': 'test_db', 'uri': 'x'}
|
||||
with (
|
||||
patch.object(worker_module, 'RUNTIME', 'basic'),
|
||||
patch.object(worker_module, 'ensure_runtime_directories'),
|
||||
patch.object(worker_module, 'metrics'),
|
||||
patch.object(worker_module, 'metrics') as mock_metrics,
|
||||
patch.object(worker_module, 'start_prometheus_server'),
|
||||
patch.object(worker_module, 'get_logger', return_value=mock_logger),
|
||||
patch.object(worker_module, 'build_mongodb_config', return_value=mongo),
|
||||
patch.object(worker_module, 'NotificationHandler'),
|
||||
patch.object(worker_module, 'build_plugin_store_config', return_value=MagicMock()),
|
||||
patch.object(worker_module, 'PluginStore', return_value=plugin_store),
|
||||
patch.object(worker_module, '_loaded_distribution_versions', side_effect=snapshot),
|
||||
patch.object(
|
||||
worker_module,
|
||||
'_runtime_changed_loaded_packages',
|
||||
return_value={'scikit-learn': '1.9.1 -> 1.8.0'},
|
||||
),
|
||||
) as mock_changed,
|
||||
patch.object(worker_module, 'Activities') as mock_activities_class,
|
||||
patch.object(worker_module.sys, 'exit', side_effect=SystemExit(1)) as mock_exit,
|
||||
):
|
||||
@@ -965,5 +973,35 @@ async def test_main_runtime_changed_loaded_package_exits(mock_env_vars, mock_log
|
||||
|
||||
mock_exit.assert_called_once_with(1)
|
||||
mock_activities_class.assert_not_called()
|
||||
plugin_store.install_runtime.assert_awaited_once()
|
||||
mock_changed.assert_called_once_with(before)
|
||||
mock_metrics.APP_UP.labels.return_value.set.assert_called_once_with(0)
|
||||
message = mock_logger.custom_critical.call_args.args[0]
|
||||
assert "'scikit-learn': '1.9.1 -> 1.8.0'" in message
|
||||
|
||||
|
||||
def test_guard_detects_version_change_on_disk(tmp_path, monkeypatch):
|
||||
"""The real importlib.metadata path: a package swapped on disk after import is reported."""
|
||||
import types
|
||||
|
||||
from model_manager.worker.worker import (
|
||||
_loaded_distribution_versions,
|
||||
_runtime_changed_loaded_packages,
|
||||
)
|
||||
|
||||
info = tmp_path / 'fakepkg_2214-1.0.dist-info'
|
||||
info.mkdir()
|
||||
(info / 'METADATA').write_text('Metadata-Version: 2.1\nName: fakepkg-2214\nVersion: 1.0\n')
|
||||
(info / 'top_level.txt').write_text('fakepkg_2214\n')
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
# The guard only reads sys.modules keys; setitem removes the entry again on teardown.
|
||||
monkeypatch.setitem(sys.modules, 'fakepkg_2214', types.ModuleType('fakepkg_2214'))
|
||||
|
||||
before = _loaded_distribution_versions()
|
||||
assert before['fakepkg-2214'] == '1.0'
|
||||
|
||||
# What pip does on an upgrade: the old dist-info goes away and a new one takes its place.
|
||||
new_info = info.rename(tmp_path / 'fakepkg_2214-2.0.dist-info')
|
||||
(new_info / 'METADATA').write_text('Metadata-Version: 2.1\nName: fakepkg-2214\nVersion: 2.0\n')
|
||||
|
||||
assert _runtime_changed_loaded_packages(before) == {'fakepkg-2214': '1.0 -> 2.0'}
|
||||
|
||||
Reference in New Issue
Block a user