Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over the shared OpcConnectionPool. The OPC client certificate now always comes from MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the Gitea repository, so a new version branch no longer breaks the reads. - io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway and Orchestrator can be deployed in any order. - managers/server_reader.py: new per-server reader with the OpcManager interface the IngestorManager already drives (connect, create_subscription, subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown). - IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a read DataSink, built without touching the network. Slots from Redis are normalized on lease and refresh; a malformed slot is notified and the last good config is kept. - GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no longer closes it on stop: a Postgres/NATS failure can't stop the reads. - pool: pin/unpin per active subscription so release_idle never evicts a subscribed connection; public evict() for lost servers / changed config. - SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a subscription locally even when the server-side delete fails. - DataSink: model_id only written when the name has one, keeping v1 documents identical to the legacy ones. - config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7). - Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh; README/CLAUDE.md document the MinIO certificate and the slot v2 format. Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against an asyncua server: v1 slot with two topics -> one doc per collection, legacy doc shape, idle sweep keeps the subscription, lost-server shutdown and re-initialization work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
130 lines
3.9 KiB
Python
130 lines
3.9 KiB
Python
from unittest.mock import AsyncMock, MagicMock, call
|
|
|
|
import pytest
|
|
|
|
import ingestor.metrics as metrics
|
|
from ingestor.managers.server_reader import ServerReader
|
|
|
|
SERVER_CONFIG = {
|
|
'name': 'kepserver',
|
|
'server_id': 1, # the slot may carry it as int; the pool keys by str
|
|
'url': 'opc.tcp://kepserver:49320',
|
|
'server_uri': 'urn:sientia:opc-ingestor',
|
|
'use_security': True,
|
|
'subscription_period_ms': 15000.0,
|
|
}
|
|
|
|
|
|
def make_reader():
|
|
pool = MagicMock(get=AsyncMock(), evict=AsyncMock())
|
|
reader = ServerReader(
|
|
name='kepserver',
|
|
server_config=SERVER_CONFIG,
|
|
pool=pool,
|
|
sink=AsyncMock(),
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=AsyncMock(),
|
|
metadata={},
|
|
)
|
|
reader.engine = MagicMock(
|
|
subscriptions={},
|
|
pod_id='pod',
|
|
create_subscription=AsyncMock(),
|
|
subscribe=AsyncMock(),
|
|
unsubscribe=AsyncMock(),
|
|
check_cycles=AsyncMock(),
|
|
check_opc_listenning=AsyncMock(return_value=False),
|
|
emit_metric=AsyncMock(),
|
|
)
|
|
return reader, pool
|
|
|
|
|
|
def test_server_cfg_keeps_only_connection_fields():
|
|
reader, _ = make_reader()
|
|
|
|
assert reader.server_id == '1'
|
|
assert reader.server_cfg == {
|
|
'url': 'opc.tcp://kepserver:49320',
|
|
'server_uri': 'urn:sientia:opc-ingestor',
|
|
'use_security': True,
|
|
'subscription_period_ms': 15000.0,
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_connect_opens_pooled_connection():
|
|
reader, pool = make_reader()
|
|
|
|
await reader.connect()
|
|
|
|
# use_security=True: the pool fetches the certificate from MinIO (CertProvider)
|
|
pool.get.assert_awaited_once_with('1', reader.server_cfg)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_connect_propagates_failure():
|
|
reader, pool = make_reader()
|
|
pool.get.side_effect = ConnectionError('unreachable')
|
|
|
|
with pytest.raises(ConnectionError):
|
|
await reader.connect()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_subscription_calls_delegate_to_engine_per_slot():
|
|
reader, _ = make_reader()
|
|
nodes = {'ns=2;i=2': {'frequency': '1000', 'names': [{'collection': 'raw_a', 'tag_name': 'a'}]}}
|
|
|
|
await reader.create_subscription('1')
|
|
await reader.subscribe('1', nodes, 10)
|
|
await reader.unsubscribe('1')
|
|
|
|
reader.engine.create_subscription.assert_awaited_once_with('1', reader.server_cfg, '1')
|
|
reader.engine.subscribe.assert_awaited_once_with('1', nodes, 10)
|
|
reader.engine.unsubscribe.assert_awaited_once_with('1')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_supervision_delegates_to_engine():
|
|
reader, _ = make_reader()
|
|
reader.engine.check_opc_listenning.return_value = True
|
|
|
|
await reader.check_cycles()
|
|
lost = await reader.check_opc_listenning()
|
|
|
|
reader.engine.check_cycles.assert_awaited_once()
|
|
reader.engine.check_opc_listenning.assert_awaited_once_with('1')
|
|
assert lost is True
|
|
|
|
|
|
def test_subscriptions_are_the_engine_ones():
|
|
reader, _ = make_reader()
|
|
reader.engine.subscriptions['1'] = MagicMock()
|
|
|
|
assert reader.subscriptions is reader.engine.subscriptions
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_shutdown_unsubscribes_all_slots_evicts_and_zeroes_metrics():
|
|
reader, pool = make_reader()
|
|
reader.engine.subscriptions.update({'1': MagicMock(), '2': MagicMock()})
|
|
|
|
await reader.shutdown()
|
|
|
|
reader.engine.unsubscribe.assert_has_awaits([call('1'), call('2')])
|
|
# Dropping the pooled connection makes the next initialization reconnect.
|
|
pool.evict.assert_awaited_once_with('1')
|
|
reader.engine.emit_metric.assert_any_await(
|
|
metric_object=metrics.OPC_CONNECTION_STATUS,
|
|
method='set',
|
|
value=0,
|
|
tags={'pod_id': 'pod', 'server_name': '1', 'server_url': 'opc.tcp://kepserver:49320'},
|
|
)
|
|
reader.engine.emit_metric.assert_any_await(
|
|
metric_object=metrics.OPC_TAGS_SUBSCRIBED,
|
|
method='set',
|
|
value=0,
|
|
tags={'pod_id': 'pod', 'server_name': '1'},
|
|
)
|