Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over the shared OpcConnectionPool. The OPC client certificate now always comes from MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the Gitea repository, so a new version branch no longer breaks the reads. - io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway and Orchestrator can be deployed in any order. - managers/server_reader.py: new per-server reader with the OpcManager interface the IngestorManager already drives (connect, create_subscription, subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown). - IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a read DataSink, built without touching the network. Slots from Redis are normalized on lease and refresh; a malformed slot is notified and the last good config is kept. - GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no longer closes it on stop: a Postgres/NATS failure can't stop the reads. - pool: pin/unpin per active subscription so release_idle never evicts a subscribed connection; public evict() for lost servers / changed config. - SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a subscription locally even when the server-side delete fails. - DataSink: model_id only written when the name has one, keeping v1 documents identical to the legacy ones. - config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7). - Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh; README/CLAUDE.md document the MinIO certificate and the slot v2 format. Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against an asyncua server: v1 slot with two topics -> one doc per collection, legacy doc shape, idle sweep keeps the subscription, lost-server shutdown and re-initialization work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
176 lines
5.4 KiB
Python
176 lines
5.4 KiB
Python
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
|
|
from ingestor import gateway_runtime as gr
|
|
from ingestor.config import GatewayConfig
|
|
|
|
_PATCHED_CLASSES = (
|
|
'Postgres',
|
|
'build_postgres_config',
|
|
'DataSink',
|
|
'WriteEngine',
|
|
'NatsWriteConsumer',
|
|
'ActiveReadEngine',
|
|
'GrpcServer',
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def mocks(monkeypatch):
|
|
patched = {}
|
|
for name in _PATCHED_CLASSES:
|
|
m = MagicMock()
|
|
monkeypatch.setattr(gr, name, m)
|
|
patched[name] = m
|
|
|
|
patched['build_postgres_config'].return_value = {
|
|
'host': 'localhost',
|
|
'port': 5432,
|
|
'user': 'u',
|
|
'password': 'p',
|
|
'dbname': 'd',
|
|
'min_connections': 5,
|
|
'max_connections': 20,
|
|
}
|
|
# The pool is owned by IngestorManager (read path) and only borrowed here.
|
|
patched['pool'] = MagicMock()
|
|
patched['pool'].close_all = AsyncMock()
|
|
patched['pool'].release_idle = AsyncMock()
|
|
patched['NatsWriteConsumer'].return_value.start = AsyncMock()
|
|
patched['NatsWriteConsumer'].return_value.stop = AsyncMock()
|
|
patched['GrpcServer'].return_value.start = AsyncMock()
|
|
patched['GrpcServer'].return_value.stop = AsyncMock()
|
|
return patched
|
|
|
|
|
|
def make_runtime(pool=None, **config_overrides):
|
|
if pool is None:
|
|
pool = MagicMock(close_all=AsyncMock(), release_idle=AsyncMock())
|
|
return gr.GatewayRuntime(
|
|
config=GatewayConfig(**config_overrides),
|
|
data_manager=MagicMock(),
|
|
pool=pool,
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=MagicMock(),
|
|
)
|
|
|
|
|
|
def test_builds_object_graph_with_config_flags(mocks):
|
|
runtime = make_runtime(
|
|
pool=mocks['pool'],
|
|
write_timestamp=False,
|
|
write_quality=False,
|
|
write_min_pause_s=0.25,
|
|
grpc_enabled=False,
|
|
)
|
|
|
|
# Writes and active reads share the pool built by the read path.
|
|
assert runtime.pool is mocks['pool']
|
|
mocks['DataSink'].assert_called_once()
|
|
|
|
mocks['WriteEngine'].assert_called_once()
|
|
args, kwargs = mocks['WriteEngine'].call_args
|
|
assert args[0] is mocks['pool']
|
|
assert kwargs['write_timestamp'] is False
|
|
assert kwargs['write_quality'] is False
|
|
assert kwargs['min_pause_s'] == 0.25
|
|
|
|
args, _ = mocks['ActiveReadEngine'].call_args
|
|
assert args[0] is mocks['pool']
|
|
|
|
mocks['GrpcServer'].assert_not_called()
|
|
assert runtime.grpc_server is None
|
|
|
|
|
|
def test_builds_grpc_server_when_enabled(mocks):
|
|
runtime = make_runtime(grpc_enabled=True, grpc_port=12345)
|
|
|
|
mocks['GrpcServer'].assert_called_once()
|
|
assert runtime.grpc_server is mocks['GrpcServer'].return_value
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_starts_nats_consumer(mocks):
|
|
runtime = make_runtime(grpc_enabled=False)
|
|
|
|
await runtime.start()
|
|
|
|
mocks['NatsWriteConsumer'].return_value.start.assert_awaited_once()
|
|
mocks['GrpcServer'].return_value.start.assert_not_called()
|
|
assert runtime._nats_started is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_start_also_starts_grpc_when_enabled(mocks):
|
|
runtime = make_runtime(grpc_enabled=True)
|
|
|
|
await runtime.start()
|
|
|
|
mocks['NatsWriteConsumer'].return_value.start.assert_awaited_once()
|
|
mocks['GrpcServer'].return_value.start.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_nats_start_failure_does_not_raise_and_idle_sweep_retries(mocks):
|
|
runtime = make_runtime()
|
|
runtime.send_notification_async = AsyncMock()
|
|
nats_mock = mocks['NatsWriteConsumer'].return_value
|
|
nats_mock.start = AsyncMock(side_effect=[Exception('boom'), Exception('boom'), None])
|
|
|
|
await runtime.start() # must not raise
|
|
|
|
assert runtime._nats_started is False
|
|
runtime.logger.error.assert_called_once()
|
|
runtime.send_notification_async.assert_awaited_once()
|
|
|
|
await runtime.idle_sweep() # second failure: retry connects, but no re-notify
|
|
|
|
assert runtime._nats_started is False
|
|
runtime.send_notification_async.assert_awaited_once() # still just one notification
|
|
|
|
await runtime.idle_sweep()
|
|
|
|
assert runtime._nats_started is True
|
|
assert nats_mock.start.await_count == 3
|
|
# flag reset on success: a future outage notifies again
|
|
assert runtime._nats_notified is False
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stop_stops_started_components_but_not_the_pool(mocks):
|
|
runtime = make_runtime(pool=mocks['pool'], grpc_enabled=True)
|
|
await runtime.start()
|
|
|
|
await runtime.stop()
|
|
|
|
mocks['GrpcServer'].return_value.stop.assert_awaited_once()
|
|
mocks['NatsWriteConsumer'].return_value.stop.assert_awaited_once()
|
|
# IngestorManager.shutdown() closes it, after its readers unsubscribe.
|
|
mocks['pool'].close_all.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stop_skips_nats_stop_if_never_started(mocks):
|
|
runtime = make_runtime(pool=mocks['pool'])
|
|
runtime.send_notification_async = AsyncMock()
|
|
nats_mock = mocks['NatsWriteConsumer'].return_value
|
|
nats_mock.start = AsyncMock(side_effect=Exception('boom'))
|
|
|
|
await runtime.start() # nats fails to start
|
|
await runtime.stop()
|
|
|
|
nats_mock.stop.assert_not_called()
|
|
mocks['pool'].close_all.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_idle_sweep_releases_idle_connections(mocks):
|
|
runtime = make_runtime(pool=mocks['pool'], idle_conn_max_s=42.0)
|
|
runtime._nats_started = True # isolate from the retry path
|
|
|
|
await runtime.idle_sweep()
|
|
|
|
mocks['pool'].release_idle.assert_awaited_once_with(42.0)
|