SIENTIAPDE-1974: feat(read): slot v2 reads via SubscriptionEngine, OPC cert from MinIO
Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over the shared OpcConnectionPool. The OPC client certificate now always comes from MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the Gitea repository, so a new version branch no longer breaks the reads. - io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway and Orchestrator can be deployed in any order. - managers/server_reader.py: new per-server reader with the OpcManager interface the IngestorManager already drives (connect, create_subscription, subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown). - IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a read DataSink, built without touching the network. Slots from Redis are normalized on lease and refresh; a malformed slot is notified and the last good config is kept. - GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no longer closes it on stop: a Postgres/NATS failure can't stop the reads. - pool: pin/unpin per active subscription so release_idle never evicts a subscribed connection; public evict() for lost servers / changed config. - SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a subscription locally even when the server-side delete fails. - DataSink: model_id only written when the name has one, keeping v1 documents identical to the legacy ones. - config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7). - Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh; README/CLAUDE.md document the MinIO certificate and the slot v2 format. Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against an asyncua server: v1 slot with two topics -> one doc per collection, legacy doc shape, idle sweep keeps the subscription, lost-server shutdown and re-initialization work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
129
tests/unit/managers/test_server_reader.py
Normal file
129
tests/unit/managers/test_server_reader.py
Normal file
@@ -0,0 +1,129 @@
|
||||
from unittest.mock import AsyncMock, MagicMock, call
|
||||
|
||||
import pytest
|
||||
|
||||
import ingestor.metrics as metrics
|
||||
from ingestor.managers.server_reader import ServerReader
|
||||
|
||||
SERVER_CONFIG = {
|
||||
'name': 'kepserver',
|
||||
'server_id': 1, # the slot may carry it as int; the pool keys by str
|
||||
'url': 'opc.tcp://kepserver:49320',
|
||||
'server_uri': 'urn:sientia:opc-ingestor',
|
||||
'use_security': True,
|
||||
'subscription_period_ms': 15000.0,
|
||||
}
|
||||
|
||||
|
||||
def make_reader():
|
||||
pool = MagicMock(get=AsyncMock(), evict=AsyncMock())
|
||||
reader = ServerReader(
|
||||
name='kepserver',
|
||||
server_config=SERVER_CONFIG,
|
||||
pool=pool,
|
||||
sink=AsyncMock(),
|
||||
logger=MagicMock(),
|
||||
notification_handler=MagicMock(),
|
||||
metrics_controller=AsyncMock(),
|
||||
metadata={},
|
||||
)
|
||||
reader.engine = MagicMock(
|
||||
subscriptions={},
|
||||
pod_id='pod',
|
||||
create_subscription=AsyncMock(),
|
||||
subscribe=AsyncMock(),
|
||||
unsubscribe=AsyncMock(),
|
||||
check_cycles=AsyncMock(),
|
||||
check_opc_listenning=AsyncMock(return_value=False),
|
||||
emit_metric=AsyncMock(),
|
||||
)
|
||||
return reader, pool
|
||||
|
||||
|
||||
def test_server_cfg_keeps_only_connection_fields():
|
||||
reader, _ = make_reader()
|
||||
|
||||
assert reader.server_id == '1'
|
||||
assert reader.server_cfg == {
|
||||
'url': 'opc.tcp://kepserver:49320',
|
||||
'server_uri': 'urn:sientia:opc-ingestor',
|
||||
'use_security': True,
|
||||
'subscription_period_ms': 15000.0,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_opens_pooled_connection():
|
||||
reader, pool = make_reader()
|
||||
|
||||
await reader.connect()
|
||||
|
||||
# use_security=True: the pool fetches the certificate from MinIO (CertProvider)
|
||||
pool.get.assert_awaited_once_with('1', reader.server_cfg)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_propagates_failure():
|
||||
reader, pool = make_reader()
|
||||
pool.get.side_effect = ConnectionError('unreachable')
|
||||
|
||||
with pytest.raises(ConnectionError):
|
||||
await reader.connect()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subscription_calls_delegate_to_engine_per_slot():
|
||||
reader, _ = make_reader()
|
||||
nodes = {'ns=2;i=2': {'frequency': '1000', 'names': [{'collection': 'raw_a', 'tag_name': 'a'}]}}
|
||||
|
||||
await reader.create_subscription('1')
|
||||
await reader.subscribe('1', nodes, 10)
|
||||
await reader.unsubscribe('1')
|
||||
|
||||
reader.engine.create_subscription.assert_awaited_once_with('1', reader.server_cfg, '1')
|
||||
reader.engine.subscribe.assert_awaited_once_with('1', nodes, 10)
|
||||
reader.engine.unsubscribe.assert_awaited_once_with('1')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_supervision_delegates_to_engine():
|
||||
reader, _ = make_reader()
|
||||
reader.engine.check_opc_listenning.return_value = True
|
||||
|
||||
await reader.check_cycles()
|
||||
lost = await reader.check_opc_listenning()
|
||||
|
||||
reader.engine.check_cycles.assert_awaited_once()
|
||||
reader.engine.check_opc_listenning.assert_awaited_once_with('1')
|
||||
assert lost is True
|
||||
|
||||
|
||||
def test_subscriptions_are_the_engine_ones():
|
||||
reader, _ = make_reader()
|
||||
reader.engine.subscriptions['1'] = MagicMock()
|
||||
|
||||
assert reader.subscriptions is reader.engine.subscriptions
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_unsubscribes_all_slots_evicts_and_zeroes_metrics():
|
||||
reader, pool = make_reader()
|
||||
reader.engine.subscriptions.update({'1': MagicMock(), '2': MagicMock()})
|
||||
|
||||
await reader.shutdown()
|
||||
|
||||
reader.engine.unsubscribe.assert_has_awaits([call('1'), call('2')])
|
||||
# Dropping the pooled connection makes the next initialization reconnect.
|
||||
pool.evict.assert_awaited_once_with('1')
|
||||
reader.engine.emit_metric.assert_any_await(
|
||||
metric_object=metrics.OPC_CONNECTION_STATUS,
|
||||
method='set',
|
||||
value=0,
|
||||
tags={'pod_id': 'pod', 'server_name': '1', 'server_url': 'opc.tcp://kepserver:49320'},
|
||||
)
|
||||
reader.engine.emit_metric.assert_any_await(
|
||||
metric_object=metrics.OPC_TAGS_SUBSCRIBED,
|
||||
method='set',
|
||||
value=0,
|
||||
tags={'pod_id': 'pod', 'server_name': '1'},
|
||||
)
|
||||
Reference in New Issue
Block a user