Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over the shared OpcConnectionPool. The OPC client certificate now always comes from MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the Gitea repository, so a new version branch no longer breaks the reads. - io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway and Orchestrator can be deployed in any order. - managers/server_reader.py: new per-server reader with the OpcManager interface the IngestorManager already drives (connect, create_subscription, subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown). - IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a read DataSink, built without touching the network. Slots from Redis are normalized on lease and refresh; a malformed slot is notified and the last good config is kept. - GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no longer closes it on stop: a Postgres/NATS failure can't stop the reads. - pool: pin/unpin per active subscription so release_idle never evicts a subscribed connection; public evict() for lost servers / changed config. - SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a subscription locally even when the server-side delete fails. - DataSink: model_id only written when the name has one, keeping v1 documents identical to the legacy ones. - config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7). - Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh; README/CLAUDE.md document the MinIO certificate and the slot v2 format. Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against an asyncua server: v1 slot with two topics -> one doc per collection, legacy doc shape, idle sweep keeps the subscription, lost-server shutdown and re-initialization work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
164 lines
4.7 KiB
Python
164 lines
4.7 KiB
Python
from ingestor.io.slots import parse_slot
|
|
|
|
RAW = {
|
|
'server1': {
|
|
'server_id': '1',
|
|
'url': 'opc.tcp://h:4840',
|
|
'use_security': True,
|
|
'subscription_period_ms': 1000,
|
|
'tags': {
|
|
'ns=2;i=2': {
|
|
'frequency': '15000',
|
|
'active_read': {'enabled': True, 'cycle_multiple': 3},
|
|
'names': [
|
|
{
|
|
'collection': 'raw_x',
|
|
'tag_name': 'Counter',
|
|
'model_name': 'm1',
|
|
'model_id': '1',
|
|
},
|
|
{
|
|
'collection': 'raw_y',
|
|
'tag_name': 'Contador',
|
|
'model_name': 'm2',
|
|
'model_id': '2',
|
|
},
|
|
],
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
|
|
def test_parse_slot_multi_name():
|
|
slot = parse_slot(RAW)
|
|
tag = slot['server1']['tags']['ns=2;i=2']
|
|
assert len(tag['names']) == 2
|
|
assert tag['active_read']['cycle_multiple'] == 3
|
|
|
|
|
|
def test_parse_slot_defaults_active_read_disabled():
|
|
raw = {
|
|
's': {
|
|
'server_id': '1',
|
|
'url': 'u',
|
|
'tags': {
|
|
'ns=2;i=1': {
|
|
'frequency': '1000',
|
|
'names': [
|
|
{'collection': 'c', 'tag_name': 't', 'model_name': 'm', 'model_id': '1'}
|
|
],
|
|
}
|
|
},
|
|
}
|
|
}
|
|
tag = parse_slot(raw)['s']['tags']['ns=2;i=1']
|
|
assert tag['active_read'] == {'enabled': False}
|
|
|
|
|
|
def test_parse_slot_rejects_tag_without_names():
|
|
raw = {'s': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=1': {'frequency': '1000'}}}}
|
|
import pytest
|
|
|
|
with pytest.raises((KeyError, ValueError)):
|
|
parse_slot(raw)
|
|
|
|
|
|
RAW_V1 = {
|
|
'kepserver': {
|
|
'name': 'kepserver',
|
|
'server_id': '1',
|
|
'url': 'opc.tcp://kepserver:49320',
|
|
'server_uri': 'urn:sientia:opc-ingestor',
|
|
'cert_path': '/app/certs/opc-ingestor-cert.pem',
|
|
'private_key_path': '/app/certs/opc-ingestor-key.pem',
|
|
'server_cert_path': None,
|
|
'subscription_period_ms': 15000.0,
|
|
'tags': {
|
|
'ns=2;s=A.TargetSin': {
|
|
'aggr_func': 'lts',
|
|
'frequency': 30000,
|
|
'server_id': '1',
|
|
'tag_address': 'ns=2;s=A.TargetSin',
|
|
'tag_name': 'TargetSin',
|
|
'topics': ['raw_scouter-sin-approx', 'raw_other-model'],
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
|
|
def test_parse_slot_converts_v1_topics_to_names():
|
|
"""Slot v1 (current Orchestrator) is read as v2: one name per topic, no model_id,
|
|
so the documents stay identical to the legacy ones (SIENTIAPDE-1974 G1)."""
|
|
server = parse_slot(RAW_V1)['kepserver']
|
|
tag = server['tags']['ns=2;s=A.TargetSin']
|
|
|
|
assert tag['names'] == [
|
|
{'collection': 'raw_scouter-sin-approx', 'tag_name': 'TargetSin'},
|
|
{'collection': 'raw_other-model', 'tag_name': 'TargetSin'},
|
|
]
|
|
assert tag['frequency'] == 30000
|
|
assert tag['active_read'] == {'enabled': False}
|
|
|
|
|
|
def test_parse_slot_v1_cert_path_means_security_but_path_is_dropped():
|
|
server = parse_slot(RAW_V1)['kepserver']
|
|
|
|
assert server['use_security'] is True
|
|
assert server['name'] == 'kepserver'
|
|
assert server['server_id'] == '1'
|
|
assert server['subscription_period_ms'] == 15000.0
|
|
# the certificate always comes from MinIO now: no file path survives the parse
|
|
assert 'cert_path' not in server
|
|
assert 'private_key_path' not in server
|
|
|
|
|
|
def test_parse_slot_v1_without_cert_path_is_insecure():
|
|
raw = {
|
|
's': {
|
|
'server_id': '2',
|
|
'url': 'u',
|
|
'cert_path': None,
|
|
'tags': {'ns=2;i=1': {'tag_name': 't', 'frequency': '1000', 'topics': ['raw_c']}},
|
|
}
|
|
}
|
|
|
|
server = parse_slot(raw)['s']
|
|
|
|
assert server['use_security'] is False
|
|
assert server['name'] == 's' # falls back to the slot key
|
|
|
|
|
|
def test_parse_slot_v2_explicit_use_security_wins():
|
|
raw = {
|
|
's': {
|
|
'server_id': '1',
|
|
'url': 'u',
|
|
'use_security': False,
|
|
'cert_path': '/legacy/path',
|
|
'tags': {
|
|
'ns=2;i=1': {
|
|
'frequency': '1000',
|
|
'names': [{'collection': 'c', 'tag_name': 't', 'model_id': '1'}],
|
|
}
|
|
},
|
|
}
|
|
}
|
|
|
|
assert parse_slot(raw)['s']['use_security'] is False
|
|
|
|
|
|
def test_parse_slot_rejects_v1_tag_without_tag_name():
|
|
raw = {
|
|
's': {
|
|
'server_id': '1',
|
|
'url': 'u',
|
|
'tags': {'ns=2;i=1': {'frequency': '1', 'topics': ['c']}},
|
|
}
|
|
}
|
|
import pytest
|
|
|
|
with pytest.raises(ValueError):
|
|
parse_slot(raw)
|