50578fdba8
Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over the shared OpcConnectionPool. The OPC client certificate now always comes from MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the Gitea repository, so a new version branch no longer breaks the reads. - io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway and Orchestrator can be deployed in any order. - managers/server_reader.py: new per-server reader with the OpcManager interface the IngestorManager already drives (connect, create_subscription, subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown). - IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a read DataSink, built without touching the network. Slots from Redis are normalized on lease and refresh; a malformed slot is notified and the last good config is kept. - GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no longer closes it on stop: a Postgres/NATS failure can't stop the reads. - pool: pin/unpin per active subscription so release_idle never evicts a subscribed connection; public evict() for lost servers / changed config. - SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a subscription locally even when the server-side delete fails. - DataSink: model_id only written when the name has one, keeping v1 documents identical to the legacy ones. - config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7). - Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh; README/CLAUDE.md document the MinIO certificate and the slot v2 format. Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against an asyncua server: v1 slot with two topics -> one doc per collection, legacy doc shape, idle sweep keeps the subscription, lost-server shutdown and re-initialization work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
73 lines
2.5 KiB
Python
73 lines
2.5 KiB
Python
from unittest.mock import AsyncMock
|
|
|
|
import pytest
|
|
|
|
from ingestor.connection.certs import CertProvider
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_fetch_returns_local_paths(tmp_path):
|
|
mongo = AsyncMock()
|
|
mongo.find_one.return_value = {
|
|
'client_cert': {'minio_object_name': 'platform/certificates/client_cert_c.pem'},
|
|
'private_key': {'minio_object_name': 'platform/certificates/private_key_k.pem'},
|
|
}
|
|
minio = AsyncMock()
|
|
minio.download_file.side_effect = [b'cert-bytes', b'key-bytes']
|
|
|
|
provider = CertProvider(mongo, minio, cache_dir=str(tmp_path), bucket='sientia-dataops')
|
|
cert, key = await provider.fetch()
|
|
|
|
assert cert.endswith('c.pem') and key.endswith('k.pem')
|
|
assert minio.download_file.await_count == 2
|
|
mongo.find_one.assert_awaited_once_with('platform_certificates', {}, {})
|
|
minio.download_file.assert_any_await(
|
|
'platform/certificates/client_cert_c.pem', 'sientia-dataops', {}
|
|
)
|
|
assert (tmp_path / 'client_cert_c.pem').read_bytes() == b'cert-bytes'
|
|
assert (tmp_path / 'private_key_k.pem').read_bytes() == b'key-bytes'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_lazy_minio_builds_client_once_on_first_download():
|
|
from unittest.mock import MagicMock
|
|
|
|
from ingestor.connection.certs import LazyMinio
|
|
|
|
client = AsyncMock()
|
|
client.download_file.return_value = b'bytes'
|
|
factory = MagicMock(return_value=client)
|
|
minio = LazyMinio(factory)
|
|
|
|
factory.assert_not_called() # no network at construction
|
|
assert await minio.download_file('a.pem', 'bucket', {}) == b'bytes'
|
|
await minio.download_file('b.pem', 'bucket', {})
|
|
|
|
factory.assert_called_once()
|
|
assert client.download_file.await_count == 2
|
|
|
|
|
|
def test_build_cert_provider_is_lazy_and_uses_config(monkeypatch):
|
|
from unittest.mock import MagicMock
|
|
|
|
from ingestor.config import GatewayConfig
|
|
from ingestor.connection import certs
|
|
|
|
minio_cls = MagicMock()
|
|
monkeypatch.setattr(certs, 'MinioRepository', minio_cls)
|
|
mongo = MagicMock()
|
|
|
|
provider = certs.build_cert_provider(
|
|
mongo=mongo,
|
|
config=GatewayConfig(cert_bucket='streamlit-connectors', cert_cache_dir='certs-cache'),
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=MagicMock(),
|
|
)
|
|
|
|
assert provider.mongo is mongo
|
|
assert provider.bucket == 'streamlit-connectors'
|
|
assert provider.cache_dir == 'certs-cache'
|
|
assert isinstance(provider.minio, certs.LazyMinio)
|
|
minio_cls.assert_not_called() # MinIO is only reached when a cert is needed
|