SIENTIAPDE-1974: fix(read): code-review fixes for the slot v2 read path
- config: GatewayConfig.certs_from_env() reads only CERT_BUCKET/CERT_CACHE_DIR; IngestorManager uses it, so a malformed write/gRPC env var (GRPC_PORT, WRITE_MIN_PAUSE_S, IDLE_CONN_MAX_S) can no longer stop the reads. - certs: CertProvider raises CertificateNotConfiguredError with an actionable message when no platform_certificates document exists; downloads once under an asyncio.Lock, caches the paths, writes atomically (tmp + os.replace); invalidate() forces a fresh download. - pool: server_id normalized to str (1 and '1' share a session); warns when a get() asks for other url/server_uri/use_security than the pooled client was opened with (client reused, the slot config wins); invalidates the cert cache when a secured connect fails. - SubscriptionEngine: tracks nodes per subscription and forgets them on unsubscribe (no more cycle alarms / tag count for removed tags), keeping addresses still monitored by another slot. - ServerReader.shutdown(evict) / IngestorManager.remove_server(evict): a server that only left this pod's slots keeps the shared connection for NATS writes/gRPC; lost servers and config changes still evict. - IngestorManager warns once per server when a slot still sets server_cert_path (server certificate pinning is no longer applied). - slots: CONNECTION_FIELDS exported and reused by ServerReader. Accepted, not changed: read metrics label server_name with the server_id (same as the write path, which only knows the id); a malformed slot is notified each lease attempt (alerts are deduplicated by notification_id). Tests: 236 unit tests pass (+18); ruff clean; local e2e against an asyncua server incl. slot moved (connection kept and reused by the write path) and lost-server eviction/re-initialization. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -30,6 +30,8 @@ class OpcConnectionPool(SientiaMonitoring):
|
||||
# callbacks and never calls get(), so without a pin release_idle() would
|
||||
# evict it as idle and silently stop the reads (SIENTIAPDE-1974 G4).
|
||||
self._pins: dict[str, int] = {}
|
||||
# Connection settings each pooled client was opened with (see _check_cfg).
|
||||
self._configs: dict[str, dict] = {}
|
||||
# ponytail: one Lock per server_id, never cleaned up — server set is
|
||||
# small and bounded. Add eviction of stale locks only if it grows.
|
||||
self._locks: dict[str, asyncio.Lock] = {}
|
||||
@@ -45,16 +47,35 @@ class OpcConnectionPool(SientiaMonitoring):
|
||||
"""Return the cached client for `server_id`, connecting one if needed.
|
||||
|
||||
Double-checked under a per-server lock so concurrent callers for the
|
||||
same new server share one connection (no leaked, untracked sessions)."""
|
||||
same new server share one connection (no leaked, untracked sessions).
|
||||
`server_id` is normalized to str, so 1 and '1' share the same session."""
|
||||
server_id = str(server_id)
|
||||
client = self._cached(server_id)
|
||||
if client is not None:
|
||||
self._check_cfg(server_id, server_cfg)
|
||||
return client
|
||||
async with self._locks.setdefault(server_id, asyncio.Lock()):
|
||||
client = self._cached(server_id) # re-check: a racer may have connected
|
||||
if client is not None:
|
||||
self._check_cfg(server_id, server_cfg)
|
||||
return client
|
||||
return await self._open(server_id, server_cfg)
|
||||
|
||||
def _check_cfg(self, server_id: str, server_cfg: dict) -> None:
|
||||
"""Warn when a caller asks for a server with other connection settings than
|
||||
the pooled client was opened with. The client is reused as is: reconnecting
|
||||
on every mismatch would let one malformed write command drop the reads.
|
||||
Config changes from the slot evict the client first (ServerReader.shutdown)."""
|
||||
opened = self._configs.get(server_id, {})
|
||||
diff = [
|
||||
k for k in ('url', 'server_uri', 'use_security') if opened.get(k) != server_cfg.get(k)
|
||||
]
|
||||
if diff:
|
||||
self.logger.warning(
|
||||
f'OPC server {server_id}: reusing the pooled connection, but {diff} differ '
|
||||
f'from the settings it was opened with'
|
||||
)
|
||||
|
||||
def _cached(self, server_id: str) -> Client | None:
|
||||
entry = self._connections.get(server_id)
|
||||
if entry is None:
|
||||
@@ -67,6 +88,10 @@ class OpcConnectionPool(SientiaMonitoring):
|
||||
try:
|
||||
client = await self._connect(server_cfg)
|
||||
except Exception:
|
||||
if server_cfg.get('use_security'):
|
||||
# The cached certificate may be the cause (replaced in MinIO,
|
||||
# rejected by the server): download it again on the next attempt.
|
||||
self.cert_provider.invalidate()
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.OPC_CONNECTIONS_FAILED,
|
||||
tags={'pod_id': self.pod_id, 'server_name': server_id},
|
||||
@@ -74,6 +99,7 @@ class OpcConnectionPool(SientiaMonitoring):
|
||||
raise
|
||||
|
||||
self._connections[server_id] = (client, self.clock.elapsed())
|
||||
self._configs[server_id] = dict(server_cfg)
|
||||
await self._record_connected(server_id, server_cfg['url'])
|
||||
return client
|
||||
|
||||
@@ -102,10 +128,12 @@ class OpcConnectionPool(SientiaMonitoring):
|
||||
|
||||
def pin(self, server_id: str) -> None:
|
||||
"""Mark one more active subscription on `server_id` (never evicted as idle)."""
|
||||
server_id = str(server_id)
|
||||
self._pins[server_id] = self._pins.get(server_id, 0) + 1
|
||||
|
||||
def unpin(self, server_id: str) -> None:
|
||||
"""Drop one active subscription mark from `server_id`."""
|
||||
server_id = str(server_id)
|
||||
count = self._pins.get(server_id, 0) - 1
|
||||
if count > 0:
|
||||
self._pins[server_id] = count
|
||||
@@ -127,6 +155,7 @@ class OpcConnectionPool(SientiaMonitoring):
|
||||
async def evict(self, server_id: str):
|
||||
"""Disconnect `server_id` now (lost server or changed config); the next
|
||||
get() reconnects. No-op when the server has no pooled client."""
|
||||
server_id = str(server_id)
|
||||
if server_id in self._connections:
|
||||
await self._evict(server_id)
|
||||
|
||||
@@ -139,6 +168,7 @@ class OpcConnectionPool(SientiaMonitoring):
|
||||
|
||||
async def _evict(self, server_id: str):
|
||||
client, _ = self._connections.pop(server_id)
|
||||
self._configs.pop(server_id, None)
|
||||
await self._disconnect_with_retries(client, server_id)
|
||||
await self.emit_metric(
|
||||
metric_object=metrics.OPC_DISCONNECTIONS_TOTAL,
|
||||
|
||||
Reference in New Issue
Block a user