Merge branch 'release/SIENTIAPDE-1973' into rc/sqlalchemy-21-mongo-uri-log [skip ci]

This commit is contained in:
2026-10-21 15:24:14 -03:00
20 changed files with 976 additions and 128 deletions

View File

@@ -1,3 +1,5 @@
import asyncio
import inspect
import json
from unittest.mock import AsyncMock, MagicMock, patch
@@ -37,20 +39,99 @@ async def test_handle_invalid_json_drops_message_and_logs_error():
consumer.logger.error.assert_called_once()
CLIENT = 'ingestor.adapters.nats_consumer.Client'
def _fake_nc(connect=None):
nc = MagicMock()
nc.connect = connect or AsyncMock()
nc.subscribe = AsyncMock()
nc.close = AsyncMock()
nc.drain = AsyncMock()
nc.is_closed = False
return nc
async def _hang(*_args, **_kwargs):
await asyncio.sleep(10)
@pytest.mark.asyncio
async def test_start_subscribes_with_subject_and_queue():
consumer = make_consumer(subject='opc.write', queue='gateway-writers')
mock_nc = AsyncMock()
mock_nc.subscribe = AsyncMock()
nc = _fake_nc()
with patch('ingestor.adapters.nats_consumer.nats') as mock_nats:
mock_nats.connect = AsyncMock(return_value=mock_nc)
with patch(CLIENT, return_value=nc):
await consumer.start()
mock_nats.connect.assert_awaited_once_with(servers='nats://localhost:4222')
mock_nc.subscribe.assert_awaited_once_with(
'opc.write', queue='gateway-writers', cb=consumer._handle
)
kwargs = nc.connect.await_args.kwargs
assert kwargs['servers'] == 'nats://localhost:4222'
# Reconnection is unbounded once connected: an outage never closes it for good.
assert kwargs['max_reconnect_attempts'] == -1
nc.subscribe.assert_awaited_once_with('opc.write', queue='gateway-writers', cb=consumer._handle)
assert consumer.closed is False
@pytest.mark.asyncio
async def test_start_gives_up_after_the_connect_timeout():
"""start() runs on the main loop: an unreachable NATS must not block it."""
consumer = make_consumer(connect_timeout_s=0.1)
nc = _fake_nc(connect=AsyncMock(side_effect=_hang))
# Outer guard: fails the test instead of hanging it if start() ignores its timeout.
start = asyncio.wait_for(consumer.start(), timeout=5)
with patch(CLIENT, return_value=nc):
with pytest.raises(TimeoutError):
await start
nc.close.assert_awaited_once()
assert consumer.closed is True
@pytest.mark.asyncio
async def test_start_closes_the_connection_when_subscribe_fails():
consumer = make_consumer()
nc = _fake_nc()
nc.subscribe.side_effect = RuntimeError('permissions violation')
with patch(CLIENT, return_value=nc), pytest.raises(RuntimeError):
await consumer.start()
nc.close.assert_awaited_once()
assert consumer.closed is True
@pytest.mark.asyncio
async def test_handle_unexpected_engine_error_is_logged_not_raised():
consumer = make_consumer()
consumer.write_engine.write.side_effect = RuntimeError('bug')
msg = MagicMock(data=json.dumps({'server_id': '1'}).encode())
await consumer._handle(msg)
assert 'bug' in consumer.logger.error.call_args.args[0]
@pytest.mark.asyncio
async def test_connection_events_are_logged_in_one_line():
consumer = make_consumer()
nc = _fake_nc()
with patch(CLIENT, return_value=nc):
await consumer.start()
callbacks = nc.connect.await_args.kwargs
# nats-py raises InvalidCallbackTypeError on connect for a non-coroutine callback.
for name in ('error_cb', 'disconnected_cb', 'reconnected_cb'):
assert inspect.iscoroutinefunction(callbacks[name]), name
await callbacks['error_cb'](OSError('connection refused'))
await callbacks['disconnected_cb']()
await callbacks['reconnected_cb']()
warnings = [c.args[0] for c in consumer.logger.warning.call_args_list]
assert warnings == [
'NATS error: OSError connection refused',
'Disconnected from NATS; reconnecting',
]
@pytest.mark.asyncio
@@ -63,14 +144,28 @@ async def test_stop_before_start_does_not_raise():
@pytest.mark.asyncio
async def test_stop_after_start_drains_and_unsubscribes():
consumer = make_consumer()
mock_nc = AsyncMock()
mock_sub = AsyncMock()
mock_nc.subscribe = AsyncMock(return_value=mock_sub)
nc = _fake_nc()
sub = AsyncMock()
nc.subscribe = AsyncMock(return_value=sub)
with patch('ingestor.adapters.nats_consumer.nats') as mock_nats:
mock_nats.connect = AsyncMock(return_value=mock_nc)
with patch(CLIENT, return_value=nc):
await consumer.start()
await consumer.stop()
mock_sub.unsubscribe.assert_awaited_once()
mock_nc.drain.assert_awaited_once()
sub.unsubscribe.assert_awaited_once()
nc.drain.assert_awaited_once()
@pytest.mark.asyncio
async def test_stop_on_a_connection_closed_for_good_does_not_raise():
consumer = make_consumer()
nc = _fake_nc()
with patch(CLIENT, return_value=nc):
await consumer.start()
nc.is_closed = True
consumer.sub.unsubscribe = AsyncMock(side_effect=RuntimeError('connection closed'))
await consumer.stop() # no raise
nc.drain.assert_not_awaited()
assert consumer.closed is True

View File

@@ -342,3 +342,23 @@ async def test_connect_server_uri_still_wins_for_application_uri():
# must match the certificate SAN URI (BadCertificateUriInvalid otherwise)
assert fake_client.application_uri == 'urn:sientia:opc-ingestor'
assert fake_client.name == pool.pod_id
@pytest.mark.asyncio
async def test_config_for_returns_a_copy_of_the_live_connection_settings():
pool = make_pool()
cfg = {'url': 'opc.tcp://h:4840', 'server_uri': 'urn:h', 'use_security': False}
with patch('ingestor.connection.pool.Client', return_value=AsyncMock()):
await pool.get(1, cfg)
found = pool.config_for('1')
assert found == cfg
found['url'] = 'mutated'
assert pool.config_for(1)['url'] == 'opc.tcp://h:4840'
await pool.evict('1')
assert pool.config_for('1') is None
def test_config_for_unknown_server_is_none():
assert make_pool().config_for('missing') is None

View File

@@ -0,0 +1,151 @@
from unittest.mock import AsyncMock, MagicMock
import pytest
from ingestor.connection.servers import CONNECTORS, OPC_SERVERS, ServerConfigResolver
CONNECTOR = {
'connector_id': 1,
'type': 'opc_ua',
'status': 'Active',
'name': 'kepserver',
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:kepserver',
'use_security': True,
}
OPC_SERVER = {
'id': '1',
'server_name': 'kepserver',
'url': 'opc.tcp://legacy:49320',
'uri': 'urn:legacy',
'cert_path': '/app/certs/cert.pem',
}
def make_resolver(connector=None, opc_server=None, pooled=None, ttl_s=60.0):
pool = MagicMock()
pool.config_for = MagicMock(return_value=pooled)
mongo = MagicMock()
async def find_one(collection, _filters, _metadata):
return {CONNECTORS: connector, OPC_SERVERS: opc_server}[collection]
mongo.find_one = AsyncMock(side_effect=find_one)
return ServerConfigResolver(pool, mongo, MagicMock(), ttl_s=ttl_s)
@pytest.mark.asyncio
async def test_command_connection_overrides_every_other_source():
resolver = make_resolver(connector=CONNECTOR)
cfg = await resolver.resolve('1', {'url': 'opc.tcp://h:1', 'server_uri': 'urn:h'})
assert cfg == {'url': 'opc.tcp://h:1', 'server_uri': 'urn:h', 'use_security': False}
resolver.mongo.find_one.assert_not_awaited()
@pytest.mark.asyncio
async def test_live_pooled_connection_wins_over_mongo():
"""Writes reuse exactly the settings the read path opened the connection with."""
pooled = {'url': 'opc.tcp://read:1', 'server_uri': 'urn:read', 'use_security': True}
resolver = make_resolver(connector=CONNECTOR, pooled=pooled)
assert await resolver.resolve('1', {}) == pooled
resolver.pool.config_for.assert_called_once_with('1')
resolver.mongo.find_one.assert_not_awaited()
@pytest.mark.asyncio
async def test_tag_database_connector_first():
resolver = make_resolver(connector=CONNECTOR, opc_server=OPC_SERVER)
cfg = await resolver.resolve('1', {})
assert cfg == {
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:kepserver',
'use_security': True,
}
collection, filters, _ = resolver.mongo.find_one.await_args_list[0].args
assert collection == CONNECTORS
# connector_id is an int in the tag database; the command carries a string.
assert filters == {'connector_id': {'$in': ['1', 1]}, 'type': 'opc_ua', 'status': 'Active'}
@pytest.mark.asyncio
async def test_opc_servers_fallback_without_tag_database():
resolver = make_resolver(connector=None, opc_server=OPC_SERVER)
cfg = await resolver.resolve('1', {})
# Same rule as the orchestrator's read slot: a cert_path means security.
assert cfg == {
'url': 'opc.tcp://legacy:49320',
'server_uri': 'urn:legacy',
'use_security': True,
}
collection, filters, _ = resolver.mongo.find_one.await_args_list[1].args
assert collection == OPC_SERVERS
assert filters == {'id': {'$in': ['1', 1]}}
@pytest.mark.asyncio
async def test_opc_server_without_cert_path_is_insecure():
resolver = make_resolver(opc_server={**OPC_SERVER, 'cert_path': ''})
assert (await resolver.resolve('1', {}))['use_security'] is False
@pytest.mark.asyncio
async def test_non_numeric_server_id_is_looked_up_as_is():
resolver = make_resolver(opc_server={**OPC_SERVER, 'id': 'kep-a'})
await resolver.resolve('kep-a', {})
_, filters, _ = resolver.mongo.find_one.await_args_list[0].args
assert filters['connector_id'] == {'$in': ['kep-a']}
@pytest.mark.asyncio
async def test_unknown_server_is_cached_briefly():
resolver = make_resolver(ttl_s=60.0)
resolver.missing_ttl_s = 10.0
resolver.clock = MagicMock()
resolver.clock.elapsed.side_effect = [0.0, 5.0, 11.0]
assert await resolver.resolve('9', {}) is None
assert await resolver.resolve('9', {}) is None # within missing_ttl_s: no lookup
assert await resolver.resolve('9', {}) is None # expired: looked up again
assert resolver.mongo.find_one.await_count == 4 # 2 lookups (connectors + opc-servers)
assert resolver.logger.warning.call_count == 2
@pytest.mark.asyncio
async def test_documents_without_url_are_ignored():
resolver = make_resolver(connector={**CONNECTOR, 'url': ''}, opc_server=OPC_SERVER)
assert (await resolver.resolve('1', {}))['url'] == 'opc.tcp://legacy:49320'
@pytest.mark.asyncio
async def test_lookups_are_cached_for_the_ttl():
resolver = make_resolver(connector=CONNECTOR, ttl_s=60.0)
resolver.clock = MagicMock()
resolver.clock.elapsed.side_effect = [0.0, 30.0, 61.0]
first = await resolver.resolve('1', {})
first['url'] = 'mutated by the caller'
second = await resolver.resolve('1', {}) # within the TTL: cached, and a copy
await resolver.resolve('1', {}) # expired: looked up again
assert second['url'] == 'opc.tcp://kepserver:49320'
assert resolver.mongo.find_one.await_count == 2
@pytest.mark.asyncio
async def test_mongo_errors_propagate_to_the_caller():
resolver = make_resolver()
resolver.mongo.find_one = AsyncMock(side_effect=RuntimeError('mongo down'))
with pytest.raises(RuntimeError, match='mongo down'):
await resolver.resolve('1', {})

View File

@@ -7,8 +7,17 @@ from asyncua import ua
import ingestor.metrics as metrics
from ingestor.engines.write import WriteEngine
SERVER_CFG = {'url': 'opc.tcp://kepserver:49320', 'server_uri': 'urn:kep', 'use_security': True}
def make_resolver(cfg=SERVER_CFG):
resolver = MagicMock()
resolver.resolve = AsyncMock(return_value=cfg)
return resolver
def make_engine(**kwargs):
kwargs.setdefault('servers', make_resolver())
eng = WriteEngine(
pool=AsyncMock(),
sink=AsyncMock(),
@@ -257,3 +266,104 @@ async def test_no_rate_limit_when_min_pause_is_zero():
await eng.write(base_cmd())
await eng.write(base_cmd())
sleep_mock.assert_not_awaited()
# --- SIENTIAPDE-1975: server resolution (GW1) and error reporting (GW4) ---
@pytest.mark.asyncio
async def test_write_uses_the_resolved_server_connection():
"""The C2 command of the Laborious carries only the server_id."""
eng = make_engine()
eng.pool.get.return_value = make_client()
await eng.write(base_cmd())
eng.servers.resolve.assert_awaited_once_with('srv1', base_cmd())
eng.pool.get.assert_awaited_once_with('srv1', SERVER_CFG)
@pytest.mark.asyncio
async def test_write_unknown_server_records_error_without_connecting():
eng = make_engine(servers=make_resolver(cfg=None))
result = await eng.write(base_cmd())
assert result.status == 'error'
assert result.opc_status_code == 'server_not_found'
eng.pool.get.assert_not_awaited()
eng.sink.update_write_status.assert_awaited_once_with(base_cmd(), result)
@pytest.mark.asyncio
async def test_write_server_lookup_failure_records_error():
resolver = make_resolver()
resolver.resolve.side_effect = RuntimeError('mongo down')
eng = make_engine(servers=resolver)
result = await eng.write(base_cmd())
assert result.status == 'error'
assert result.opc_status_code == 'server_lookup'
assert 'mongo down' in result.message
eng.pool.get.assert_not_awaited()
@pytest.mark.asyncio
async def test_write_without_resolver_uses_the_command_connection():
eng = make_engine(servers=None)
eng.pool.get.return_value = make_client()
await eng.write(base_cmd(url='opc.tcp://h:4840', server_uri='urn:h', use_security=True))
eng.pool.get.assert_awaited_once_with(
'srv1', {'url': 'opc.tcp://h:4840', 'use_security': True, 'server_uri': 'urn:h'}
)
@pytest.mark.asyncio
async def test_write_without_resolver_or_url_is_server_not_found():
eng = make_engine(servers=None)
result = await eng.write(base_cmd())
assert result.opc_status_code == 'server_not_found'
eng.pool.get.assert_not_awaited()
@pytest.mark.asyncio
async def test_write_error_logs_the_reason_and_sets_the_model_status():
eng = make_engine()
node = make_node()
node.write_value.side_effect = RuntimeError('BadWriteNotSupported')
eng.pool.get.return_value = make_client(node)
await eng.write(base_cmd())
logged = eng.logger.error.call_args.args[0]
assert 'RuntimeError' in logged and 'BadWriteNotSupported' in logged
assert 'addr=ns=2;i=5' in logged and 'model_id=m1' in logged
status_call = next(
c
for c in eng.emit_metric.await_args_list
if c.kwargs['metric_object'] is metrics.OPC_WRITE_STATUS_BY_MODEL
)
# Same series as a success, so it shows the last outcome.
assert status_call.kwargs['value'] == 0
assert status_call.kwargs['tags'] == {'pod_id': eng.pod_id, 'model_id': 'm1', 'status': 'ok'}
@pytest.mark.asyncio
@pytest.mark.parametrize('timestamp', [None, 'not-a-timestamp'])
async def test_write_bad_timestamp_records_error_without_raising(timestamp):
eng = make_engine(write_timestamp=True)
cmd = base_cmd(timestamp=timestamp)
if timestamp is None:
del cmd['timestamp']
result = await eng.write(cmd)
assert result.status == 'error'
assert result.opc_status_code == 'bad_timestamp'
eng.pool.get.assert_not_awaited()
eng.sink.update_write_status.assert_awaited_once()

View File

@@ -1,89 +1,149 @@
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock
import pytest
from sqlalchemy import create_engine, text
from sqlalchemy.dialects import postgresql
import ingestor.metrics as metrics
from ingestor.engines.write import WriteResult
from ingestor.io.data_sink import DataSink
from ingestor.io.data_sink import DataSink, write_status_statements
CMD = {
'server_id': '1',
'addr': 'ns=2;s=Pred',
'schema': 'sientia_data',
'table_name': 'pred_m1',
'model_id': '1',
'timestamp': '2026-07-13T00:00:00Z',
}
def make_sink():
def _result(status):
return WriteResult(ok=status == 'ok', status=status, opc_status_code='x', message='')
def make_sink(rowcount=1, existing_row=True):
postgres = MagicMock()
connection = MagicMock()
connection.execute.return_value.rowcount = rowcount
connection.execute.return_value.first.return_value = (1,) if existing_row else None
postgres.engine.begin.return_value.__enter__.return_value = connection
sink = DataSink(data_manager=MagicMock(), postgres=postgres)
data_manager = MagicMock()
data_manager.emit_metric = AsyncMock()
sink = DataSink(data_manager=data_manager, postgres=postgres)
return sink, connection
def _outcome_metric(sink):
calls = [
c
for c in sink.data_manager.emit_metric.await_args_list
if c.kwargs['metric_object'] is metrics.OPC_WRITE_STATUS_UPDATE_TOTAL
]
assert len(calls) == 1
return calls[0].kwargs['tags']['result']
@pytest.mark.asyncio
async def test_update_write_status_targets_model_row():
async def test_update_write_status_targets_model_row_with_precedence():
sink, connection = make_sink()
cmd = {
'schema': 'sientia_data',
'table_name': 'pred_m1',
'model_id': '1',
'timestamp': '2026-07-13T00:00:00Z',
}
result = WriteResult(ok=True, status='ok', opc_status_code='Good', message='')
await sink.update_write_status(cmd, result)
outcome = await sink.update_write_status(CMD, _result('ok'))
assert outcome == 'updated'
assert connection.execute.call_count == 1
stmt, params = connection.execute.call_args[0]
sql = str(stmt)
assert 'update' in sql.lower() and 'write_status' in sql.lower()
assert 'sientia_data.pred_m1' in sql
(stmt,) = connection.execute.call_args[0]
compiled = stmt.compile(dialect=postgresql.dialect())
sql = str(compiled)
assert sql.startswith('UPDATE sientia_data.pred_m1 SET write_status=')
assert 'CASE sientia_data.pred_m1.write_status WHEN' in sql
# values must be bound params, never interpolated into the SQL text itself
assert params == {'status': 'ok', 'timestamp': '2026-07-13T00:00:00Z', 'model_id': '1'}
assert 'ok' not in sql
assert '2026-07-13T00:00:00Z' not in sql
assert '2026-07-13T00:00:00Z' not in sql and "'ok'" not in sql
assert compiled.params['write_status'] == 'ok'
assert '2026-07-13T00:00:00Z' in compiled.params.values()
assert _outcome_metric(sink) == 'updated'
@pytest.mark.asyncio
async def test_update_write_status_rejects_bad_schema():
sink, connection = make_sink()
cmd = {
'schema': 'a; DROP TABLE x',
'table_name': 'pred_m1',
'model_id': '1',
'timestamp': '2026-07-13T00:00:00Z',
}
result = WriteResult(ok=False, status='error', opc_status_code='Bad', message='x')
async def test_update_write_status_kept_when_the_row_has_a_worse_outcome():
sink, _ = make_sink(rowcount=0, existing_row=True)
with pytest.raises(ValueError):
await sink.update_write_status(cmd, result)
connection.execute.assert_not_called()
assert await sink.update_write_status(CMD, _result('ok')) == 'kept'
assert _outcome_metric(sink) == 'kept'
@pytest.mark.asyncio
async def test_update_write_status_rejects_trailing_newline_identifier():
sink, connection = make_sink()
cmd = {
'schema': 'sientia_data\n',
'table_name': 'pred_m1',
'model_id': '1',
'timestamp': '2026-07-13T00:00:00Z',
}
result = WriteResult(ok=True, status='ok', opc_status_code='Good', message='')
async def test_update_write_status_row_not_found_is_logged():
"""E.g. the Laborious INSERT failed and it wrote to OPC anyway (SIENTIAPDE-1975 D10)."""
sink, _ = make_sink(rowcount=0, existing_row=False)
with pytest.raises(ValueError):
await sink.update_write_status(cmd, result)
connection.execute.assert_not_called()
assert await sink.update_write_status(CMD, _result('error')) == 'row_not_found'
assert 'no prediction row' in sink.data_manager.logger.warning.call_args.args[0]
assert _outcome_metric(sink) == 'row_not_found'
@pytest.mark.asyncio
async def test_update_write_status_rejects_bad_table_name():
async def test_update_write_status_database_error_is_not_raised():
sink, connection = make_sink()
cmd = {
'schema': 'sientia_data',
'table_name': 'pred_m1; DROP TABLE x',
'model_id': '1',
'timestamp': '2026-07-13T00:00:00Z',
}
result = WriteResult(ok=False, status='error', opc_status_code='Bad', message='x')
connection.execute.side_effect = RuntimeError('column "write_status" does not exist')
with pytest.raises(ValueError):
await sink.update_write_status(cmd, result)
assert await sink.update_write_status(CMD, _result('ok')) == 'error'
assert 'write_status' in sink.data_manager.logger.warning.call_args.args[0]
assert _outcome_metric(sink) == 'error'
@pytest.mark.parametrize(
('field', 'value'),
[
('schema', 'a; DROP TABLE x'),
('schema', 'sientia_data\n'),
('table_name', 'pred_m1; DROP TABLE x'),
('schema', None),
('timestamp', None),
('model_id', None),
],
)
@pytest.mark.asyncio
async def test_update_write_status_rejects_invalid_commands(field, value):
sink, connection = make_sink()
outcome = await sink.update_write_status({**CMD, field: value}, _result('error'))
assert outcome == 'invalid'
connection.execute.assert_not_called()
assert _outcome_metric(sink) == 'invalid'
def _apply(initial, answers):
"""Run the real statements on SQLite for each answer; return the final status."""
engine = create_engine('sqlite://')
with engine.begin() as connection:
connection.execute(
text('CREATE TABLE pred (model_id INTEGER, timestamp TEXT, write_status TEXT)')
)
connection.execute(
text('INSERT INTO pred VALUES (1, :ts, :status)'),
{'ts': '2026-07-13T00:00:00Z', 'status': initial},
)
for answer in answers:
update_stmt, exists_stmt = write_status_statements(
None, 'pred', answer, '2026-07-13T00:00:00Z', 1
)
connection.execute(update_stmt)
assert connection.execute(exists_stmt).first() is not None
return connection.execute(text('SELECT write_status FROM pred')).scalar()
@pytest.mark.parametrize(
('initial', 'answers', 'expected'),
[
('pending', ['ok'], 'ok'),
('pending', ['error', 'ok'], 'error'),
('pending', ['ok', 'error'], 'error'),
('pending', ['session_bad', 'error', 'ok'], 'session_bad'),
('error', ['ok'], 'error'),
(None, ['ok'], 'ok'),
],
)
def test_update_statement_keeps_the_worst_answer(initial, answers, expected):
assert _apply(initial, answers) == expected

View File

@@ -39,6 +39,7 @@ def mocks(monkeypatch):
patched['pool'].release_idle = AsyncMock()
patched['NatsWriteConsumer'].return_value.start = AsyncMock()
patched['NatsWriteConsumer'].return_value.stop = AsyncMock()
patched['NatsWriteConsumer'].return_value.closed = False
patched['GrpcServer'].return_value.start = AsyncMock()
patched['GrpcServer'].return_value.stop = AsyncMock()
return patched
@@ -173,3 +174,47 @@ async def test_idle_sweep_releases_idle_connections(mocks):
await runtime.idle_sweep()
mocks['pool'].release_idle.assert_awaited_once_with(42.0)
def test_write_engine_resolves_servers_from_the_pool_and_mongo(mocks):
"""SIENTIAPDE-1975 GW1: write commands carry only the server_id."""
data_manager = MagicMock()
runtime = gr.GatewayRuntime(
config=GatewayConfig(),
data_manager=data_manager,
pool=mocks['pool'],
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=MagicMock(),
)
_, kwargs = mocks['WriteEngine'].call_args
assert kwargs['servers'] is runtime.servers
assert runtime.servers.pool is mocks['pool']
assert runtime.servers.mongo is data_manager.mongo_repository
@pytest.mark.asyncio
async def test_idle_sweep_restarts_a_closed_nats_connection(mocks):
runtime = make_runtime(pool=mocks['pool'])
nats_mock = mocks['NatsWriteConsumer'].return_value
await runtime.start()
assert nats_mock.start.await_count == 1
nats_mock.closed = True # closed for good (e.g. by the server)
await runtime.idle_sweep()
assert nats_mock.start.await_count == 2
assert runtime._nats_started is True
runtime.logger.warning.assert_called_once()
@pytest.mark.asyncio
async def test_idle_sweep_leaves_a_live_nats_connection_alone(mocks):
runtime = make_runtime(pool=mocks['pool'])
nats_mock = mocks['NatsWriteConsumer'].return_value
await runtime.start()
await runtime.idle_sweep()
assert nats_mock.start.await_count == 1