50578fdba8
Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over the shared OpcConnectionPool. The OPC client certificate now always comes from MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the Gitea repository, so a new version branch no longer breaks the reads. - io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway and Orchestrator can be deployed in any order. - managers/server_reader.py: new per-server reader with the OpcManager interface the IngestorManager already drives (connect, create_subscription, subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown). - IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a read DataSink, built without touching the network. Slots from Redis are normalized on lease and refresh; a malformed slot is notified and the last good config is kept. - GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no longer closes it on stop: a Postgres/NATS failure can't stop the reads. - pool: pin/unpin per active subscription so release_idle never evicts a subscribed connection; public evict() for lost servers / changed config. - SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a subscription locally even when the server-side delete fails. - DataSink: model_id only written when the name has one, keeping v1 documents identical to the legacy ones. - config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7). - Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh; README/CLAUDE.md document the MinIO certificate and the slot v2 format. Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against an asyncua server: v1 slot with two topics -> one doc per collection, legacy doc shape, idle sweep keeps the subscription, lost-server shutdown and re-initialization work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
102 lines
3.9 KiB
Python
102 lines
3.9 KiB
Python
"""Per-server read path driven by the IngestorManager (SIENTIAPDE-1974).
|
|
|
|
Replaces the legacy OpcManager behind the same interface the IngestorManager
|
|
already uses (connect / create_subscription / subscribe / unsubscribe /
|
|
check_cycles / check_opc_listenning / shutdown), so the slot lease loop is
|
|
unchanged. The difference is underneath:
|
|
|
|
- the connection comes from the shared OpcConnectionPool, so the certificate is
|
|
fetched from MinIO by CertProvider (Basic256Sha256) — never from a file shipped
|
|
in the Gitea repository;
|
|
- readings go through SubscriptionEngine, which fans each value out to every
|
|
`names` entry of the slot v2 (one document per collection / tag name / model).
|
|
"""
|
|
|
|
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
|
from sientia_do.observability.logger import Logger
|
|
from sientia_do.observability.metrics_controller import MetricsController
|
|
|
|
import ingestor.metrics as metrics
|
|
from ingestor.connection.pool import OpcConnectionPool
|
|
from ingestor.engines.subscription import SubscriptionEngine
|
|
from ingestor.io.data_sink import DataSink
|
|
|
|
_CONNECTION_FIELDS = ('url', 'server_uri', 'use_security', 'subscription_period_ms')
|
|
|
|
|
|
class ServerReader:
|
|
"""Reads one OPC server: one SubscriptionEngine (its `non_receive_count` is
|
|
engine-wide, hence one per server) with one subscription per leased slot."""
|
|
|
|
def __init__(
|
|
self,
|
|
name: str,
|
|
server_config: dict,
|
|
pool: OpcConnectionPool,
|
|
sink: DataSink,
|
|
logger: Logger,
|
|
notification_handler: NotificationHandler,
|
|
metrics_controller: MetricsController,
|
|
metadata: dict | None = None,
|
|
):
|
|
self.name = name
|
|
self.server_id = str(server_config['server_id'])
|
|
self.url = server_config['url']
|
|
self.server_cfg = {k: server_config[k] for k in _CONNECTION_FIELDS if k in server_config}
|
|
self.pool = pool
|
|
self.logger = logger
|
|
self.config: dict | None = None
|
|
self.engine = SubscriptionEngine(
|
|
pool, sink, logger, notification_handler, metrics_controller, metadata
|
|
)
|
|
|
|
def __str__(self):
|
|
return f'ServerReader(name={self.name}, server_id={self.server_id}, url={self.url})'
|
|
|
|
@property
|
|
def subscriptions(self) -> dict:
|
|
return self.engine.subscriptions
|
|
|
|
async def connect(self):
|
|
"""Open (or reuse) the pooled connection; raises if the server is unreachable
|
|
or the certificate can't be fetched, like OpcManager.connect did."""
|
|
await self.pool.get(self.server_id, self.server_cfg)
|
|
|
|
async def create_subscription(self, slot: str):
|
|
await self.engine.create_subscription(self.server_id, self.server_cfg, slot)
|
|
|
|
async def subscribe(self, slot: str, nodes: dict, collect_period: int):
|
|
await self.engine.subscribe(slot, nodes, collect_period)
|
|
|
|
async def unsubscribe(self, slot: str):
|
|
await self.engine.unsubscribe(slot)
|
|
|
|
async def check_cycles(self):
|
|
await self.engine.check_cycles()
|
|
|
|
async def check_opc_listenning(self) -> bool:
|
|
return await self.engine.check_opc_listenning(self.server_id)
|
|
|
|
async def shutdown(self):
|
|
"""Drop every subscription and the pooled connection, so the next
|
|
initialization reconnects (lost server or changed server config)."""
|
|
for slot in list(self.engine.subscriptions):
|
|
await self.engine.unsubscribe(slot)
|
|
await self.pool.evict(self.server_id)
|
|
await self.engine.emit_metric(
|
|
metric_object=metrics.OPC_CONNECTION_STATUS,
|
|
method='set',
|
|
value=0,
|
|
tags={
|
|
'pod_id': self.engine.pod_id,
|
|
'server_name': self.server_id,
|
|
'server_url': self.url,
|
|
},
|
|
)
|
|
await self.engine.emit_metric(
|
|
metric_object=metrics.OPC_TAGS_SUBSCRIBED,
|
|
method='set',
|
|
value=0,
|
|
tags={'pod_id': self.engine.pod_id, 'server_name': self.server_id},
|
|
)
|