SIENTIAPDE-1973

SIENTIAPDE-1973: Implement per-connector OPC client certificates and remove legacy read engine.
This update overhauls certificate management, transitioning from a single platform certificate to individual client certificates defined in the slot configuration. The CertProvider now downloads certificates from MinIO based on certificate_id, applying dynamic security policies (mode, policy) from the slot. The legacy READ_ENGINE=legacy option and OpcManager have been entirely removed, standardizing on the ServerReader with strict slot v2 enforcement. Write command resolution (ServerConfigResolver) now consistently uses connectors joined with opc_certificates. The CERT_BUCKET environment variable is no longer used.

ingestor/config.py

@@ -22,9 +22,8 @@ class GatewayConfig:
     nats_queue: str = 'gateway-writers'
     grpc_enabled: bool = False
     grpc_port: int = 50051
-    # Legacy frontend bucket the platform already provisions (SIENTIAPDE-1974 D7);
-    # the design's `sientia-dataops` bucket does not exist in the platform MinIO.
-    cert_bucket: str = 'streamlit-connectors'
+    # The certificate bucket and objects come from the slot / the write lookup
+    # (opc_certificates, SIENTIAPDE-1974); only the local cache dir is configured here.
     cert_cache_dir: str = '/tmp/opc-certs'  # noqa: S108 # nosec B108 - default is spec'd, overridable via env
     write_timestamp: bool = True
     write_quality: bool = True
@@ -36,10 +35,7 @@ class GatewayConfig:
         """Only the certificate settings — what the read path needs. Plain strings,
         no number parsing, so a malformed write/gRPC env var (GRPC_PORT,
         WRITE_MIN_PAUSE_S, ...) can only disable the runtime, never the reads."""
-        return cls(
-            cert_bucket=getenv('CERT_BUCKET', cls.cert_bucket),
-            cert_cache_dir=getenv('CERT_CACHE_DIR', cls.cert_cache_dir),
-        )
+        return cls(cert_cache_dir=getenv('CERT_CACHE_DIR', cls.cert_cache_dir))
 
     @classmethod
     def from_env(cls) -> 'GatewayConfig':
@@ -49,7 +45,6 @@ class GatewayConfig:
             nats_queue=getenv('NATS_QUEUE', cls.nats_queue),
             grpc_enabled=_env_bool('GRPC_ENABLED', cls.grpc_enabled),
             grpc_port=int(getenv('GRPC_PORT', cls.grpc_port)),
-            cert_bucket=getenv('CERT_BUCKET', cls.cert_bucket),
             cert_cache_dir=getenv('CERT_CACHE_DIR', cls.cert_cache_dir),
             write_timestamp=_env_bool('WRITE_TIMESTAMP', cls.write_timestamp),
             write_quality=_env_bool('WRITE_QUALITY', cls.write_quality),

ingestor/connection/certificates.py

@@ -0,0 +1,92 @@
+"""Client certificate of a tag database connector (SIENTIAPDE-1974 D5).
+
+Mirrors the Orchestrator's `tag_database.resolve_certificate`, which puts the
+certificate in the read slot: the write lookup (ServerConfigResolver, D1) resolves a
+server that is not in the pool with the same rule, so a write uses the certificate the
+slot would have. Keep both in sync.
+"""
+
+from typing import Any
+
+# security fields of an OPC UA connector: the source of its OPC security (D4)
+SECURITY_FIELDS = ('mode', 'policy')
+CERTIFICATE_FIELDS = ('certificate_id', 'application_uri')
+CERTIFICATE_OBJECTS = ('cert', 'private_key')
+STORED_OBJECT_FIELDS = ('bucket', 'object_name')
+
+# opc_certificates lookup of a connectors aggregation: the certificate the connector's
+# security.client_certificate_id points to, only when it is active and of kind client.
+CERTIFICATE_LOOKUP = {
+    '$lookup': {
+        'from': 'opc_certificates',
+        'let': {'certificate_id': '$security.client_certificate_id'},
+        'pipeline': [
+            {
+                '$match': {
+                    '$expr': {'$eq': ['$certificate_id', '$$certificate_id']},
+                    'status': 'active',
+                    'kind': 'client',
+                }
+            },
+            {
+                '$project': {
+                    '_id': 0,
+                    'certificate_id': 1,
+                    'application_uri': 1,
+                    'cert.bucket': 1,
+                    'cert.object_name': 1,
+                    'private_key.bucket': 1,
+                    'private_key.object_name': 1,
+                }
+            },
+        ],
+        'as': 'certificates',
+    }
+}
+
+
+def _is_complete(document: Any, fields: tuple[str, ...]) -> bool:
+    return isinstance(document, dict) and all(
+        document.get(field) not in (None, '') for field in fields
+    )
+
+
+def _is_usable_certificate(certificate: Any) -> bool:
+    return _is_complete(certificate, CERTIFICATE_FIELDS) and all(
+        _is_complete(certificate.get(name), STORED_OBJECT_FIELDS) for name in CERTIFICATE_OBJECTS
+    )
+
+
+def client_certificate_id(connector: dict[str, Any]) -> Any:
+    """The connector's security.client_certificate_id (None when it has none)."""
+    security = connector.get('security')
+    return security.get('client_certificate_id') if isinstance(security, dict) else None
+
+
+def has_security(connector: dict[str, Any]) -> bool:
+    """Whether the connector has security.mode and security.policy (D4)."""
+    return _is_complete(connector.get('security'), SECURITY_FIELDS)
+
+
+def resolve_certificate(connector: dict[str, Any]) -> dict[str, Any] | None:
+    """
+    The client certificate of a connector from CERTIFICATE_LOOKUP, or None (D5).
+
+    None means no certificate: the connection is opened without security. A connector
+    whose client_certificate_id is set but has no usable certificate is most likely a
+    registration error; the caller reports it (see client_certificate_id).
+
+    Returns:
+        dict[str, Any] | None: certificate_id, application_uri, and cert and private_key
+            as {bucket, object_name}, as in the slot
+    """
+    for certificate in connector.get('certificates') or []:
+        if _is_usable_certificate(certificate):
+            return {
+                **{field: certificate[field] for field in CERTIFICATE_FIELDS},
+                **{
+                    name: {field: certificate[name][field] for field in STORED_OBJECT_FIELDS}
+                    for name in CERTIFICATE_OBJECTS
+                },
+            }
+    return None

ingestor/connection/certs.py

@@ -7,63 +7,88 @@ from sientia_do.repository.minio_repository import MinioRepository
 
 from ingestor.config import _env_bool
 
-COLLECTION_NAME = 'platform_certificates'
+# The two objects of a client certificate, and the local file each one is saved as
+CERTIFICATE_OBJECTS = (('cert', 'cert'), ('private_key', 'key'))
 
 
 class CertificateNotConfiguredError(RuntimeError):
-    """No usable platform OPC client certificate is registered (Mongo + MinIO)."""
+    """The certificate of a server does not name its MinIO objects (opc_certificates)."""
+
+
+def _objects(certificate: dict) -> tuple[tuple[str, str], ...]:
+    """(bucket, object_name) of the cert and the private key, in CERTIFICATE_OBJECTS order."""
+    try:
+        return tuple(
+            (str(certificate[field]['bucket']), str(certificate[field]['object_name']))
+            for field, _ in CERTIFICATE_OBJECTS
+        )
+    except (KeyError, TypeError) as e:
+        raise CertificateNotConfiguredError(
+            f'OPC client certificate {certificate!r} does not name the MinIO bucket and '
+            f'object of its cert and private_key (opc_certificates)'
+        ) from e
+
+
+def _certificate_id(certificate: dict) -> str:
+    """certificate_id as the name of its cache folder: an integer, never a path."""
+    try:
+        return str(int(certificate['certificate_id']))
+    except (KeyError, TypeError, ValueError) as e:
+        raise CertificateNotConfiguredError(
+            f'OPC client certificate {certificate!r} has no integer certificate_id'
+        ) from e
 
 
 class CertProvider:
-    """Downloads the platform's client cert + private key (Mongo metadata,
-    MinIO bytes) into a writable cache dir — pod root FS is read-only (§12),
-    so `cache_dir` must be an emptyDir/tmpfs mount.
+    """Downloads the client cert + private key of a connection (the MinIO objects its
+    slot server or write lookup names, SIENTIAPDE-1974) into a writable cache dir — pod
+    root FS is read-only (§12), so `cache_dir` must be an emptyDir/tmpfs mount.
 
-    The certificate is single and shared by every connection (reads, writes,
-    gRPC), so it is downloaded once and cached; `invalidate()` forces a fresh
-    download (the pool calls it when a secured connection fails, e.g. after the
-    certificate was replaced in MinIO). The lock serializes downloads so two
-    connections never rewrite the cache files at the same time."""
+    Each certificate (`certificate_id`) has its own folder (`<cache_dir>/<certificate_id>/`),
+    cache entry and lock, so two connections never rewrite the same files at the same
+    time. A cached certificate is reused while it names the same objects; when the slot
+    names other objects for the same id (the certificate was replaced), it is downloaded
+    again. `invalidate(certificate_id)` forces a fresh download (the pool calls it when a
+    secured connection fails, e.g. after the objects were replaced in MinIO)."""
 
-    def __init__(self, mongo, minio, cache_dir: str, bucket: str, metadata: dict | None = None):
-        self.mongo = mongo
+    def __init__(self, minio, cache_dir: str, metadata: dict | None = None):
         self.minio = minio
         self.cache_dir = cache_dir
-        self.bucket = bucket
         self.metadata = metadata or {}
-        self._lock = asyncio.Lock()
-        self._paths: tuple[str, str] | None = None
+        # ponytail: one Lock per certificate, never cleaned up — the set is small.
+        self._locks: dict[str, asyncio.Lock] = {}
+        self._paths: dict[str, tuple[tuple[tuple[str, str], ...], tuple[str, str]]] = {}
 
-    async def fetch(self) -> tuple[str, str]:
-        async with self._lock:
-            if self._paths is None:
-                self._paths = await self._download_all()
-            return self._paths
+    async def fetch(self, certificate: dict) -> tuple[str, str]:
+        """Local paths of the cert and private key of `certificate` (slot format)."""
+        certificate_id = _certificate_id(certificate)
+        objects = _objects(certificate)
+        async with self._locks.setdefault(certificate_id, asyncio.Lock()):
+            cached = self._paths.get(certificate_id)
+            if cached is None or cached[0] != objects:
+                self._paths[certificate_id] = (
+                    objects,
+                    await self._download_all(certificate_id, objects),
+                )
+            return self._paths[certificate_id][1]
 
-    def invalidate(self) -> None:
-        self._paths = None
+    def invalidate(self, certificate_id) -> None:
+        self._paths.pop(str(certificate_id), None)
 
-    async def _download_all(self) -> tuple[str, str]:
-        doc = await self.mongo.find_one(COLLECTION_NAME, {}, self.metadata)
-        try:
-            cert_object = doc['client_cert']['minio_object_name']
-            key_object = doc['private_key']['minio_object_name']
-        except (KeyError, TypeError) as e:
-            raise CertificateNotConfiguredError(
-                f'No OPC client certificate configured: the Mongo collection '
-                f"'{COLLECTION_NAME}' needs a document with client_cert.minio_object_name "
-                f'and private_key.minio_object_name (objects in the MinIO bucket '
-                f"'{self.bucket}'). Register the platform certificate before using "
-                f'secured OPC servers.'
-            ) from e
-        Path(self.cache_dir).mkdir(parents=True, exist_ok=True)
-        cert_path = await self._download(cert_object)
-        key_path = await self._download(key_object)
+    async def _download_all(
+        self, certificate_id: str, objects: tuple[tuple[str, str], ...]
+    ) -> tuple[str, str]:
+        folder = Path(self.cache_dir) / certificate_id
+        folder.mkdir(parents=True, exist_ok=True)
+        (cert_bucket, cert_object), (key_bucket, key_object) = objects
+        cert_path = await self._download(cert_bucket, cert_object, folder / 'cert')
+        key_path = await self._download(key_bucket, key_object, folder / 'key')
         return cert_path, key_path
 
-    async def _download(self, object_name: str) -> str:
-        data = await self.minio.download_file(object_name, self.bucket, self.metadata)
-        local_path = Path(self.cache_dir) / Path(object_name).name
+    async def _download(self, bucket: str, object_name: str, stem: Path) -> str:
+        data = await self.minio.download_file(object_name, bucket, self.metadata)
+        # The extension tells asyncua the format (.pem / .der)
+        local_path = stem.with_suffix(Path(object_name).suffix)
         # Atomic replace: a connection loading the previous file never sees it half-written.
         tmp_path = local_path.with_name(local_path.name + '.tmp')
         tmp_path.write_bytes(data)
@@ -72,43 +97,43 @@ class CertProvider:
 
 
 class LazyMinio:
-    """Builds the MinIO client on first use. MinioRepository's constructor hits the
-    network (bucket check), so building it eagerly would let a MinIO outage at pod
-    start take the whole read path down. Built lazily, only the servers that need a
-    certificate fail — and they are retried on the next loop."""
+    """Builds one MinIO client per bucket on first use. MinioRepository's constructor
+    hits the network (bucket check), so building it eagerly would let a MinIO outage at
+    pod start take the whole read path down. Built lazily, only the servers that need a
+    certificate fail — and they are retried on the next loop. One client per bucket,
+    because the constructor creates its bucket when missing: it is only ever given a
+    bucket a certificate names."""
 
     def __init__(self, factory):
         self._factory = factory
-        self._client = None
+        self._clients: dict[str, MinioRepository] = {}
 
     async def download_file(self, object_name: str, bucket: str, metadata: dict):
-        if self._client is None:
-            self._client = self._factory()
-        return await self._client.download_file(object_name, bucket, metadata)
+        if bucket not in self._clients:
+            self._clients[bucket] = self._factory(bucket)
+        return await self._clients[bucket].download_file(object_name, bucket, metadata)
 
 
 def build_cert_provider(
-    mongo, config, logger, notification_handler, metrics_controller, metadata=None
+    config, logger, notification_handler, metrics_controller, metadata=None
 ) -> CertProvider:
     """CertProvider shared by the read path (IngestorManager) and the write path
-    (GatewayRuntime): one certificate source — MinIO + Mongo — for every connection."""
+    (GatewayRuntime): one certificate cache for every connection."""
 
-    def minio_factory() -> MinioRepository:
+    def minio_factory(bucket: str) -> MinioRepository:
         return MinioRepository(
             endpoint=getenv('MINIO_ENDPOINT', 'localhost:9000'),
             access_key=getenv('MINIO_ACCESS_KEY', 'minioadmin'),
             secret_key=getenv('MINIO_SECRET_KEY', 'minioadmin'),
             secure=_env_bool('MINIO_SECURE', True),
-            bucket=config.cert_bucket,
+            bucket=bucket,
             logger=logger,
             notification_handler=notification_handler,
             metrics_controller=metrics_controller,
         )
 
     return CertProvider(
-        mongo=mongo,
         minio=LazyMinio(minio_factory),
         cache_dir=config.cert_cache_dir,
-        bucket=config.cert_bucket,
         metadata=metadata,
     )

ingestor/connection/pool.py

@@ -9,7 +9,7 @@ from sientia_do.observability.sientia_monitoring import SientiaMonitoring
 import ingestor.metrics as metrics
 from ingestor.connection.certs import CertProvider
 from ingestor.connection.clock import MonotonicClock
-from ingestor.connection.security import apply_security
+from ingestor.connection.security import apply_security, is_secured
 
 
 class OpcConnectionPool(SientiaMonitoring):
@@ -65,11 +65,10 @@ class OpcConnectionPool(SientiaMonitoring):
         """Warn when a caller asks for a server with other connection settings than
         the pooled client was opened with. The client is reused as is: reconnecting
         on every mismatch would let one malformed write command drop the reads.
-        Config changes from the slot evict the client first (ServerReader.shutdown)."""
+        Config changes from the slot (security, certificate) evict the client first:
+        the IngestorManager re-initializes the reader (ServerReader.shutdown)."""
         opened = self._configs.get(server_id, {})
-        diff = [
-            k for k in ('url', 'server_uri', 'use_security') if opened.get(k) != server_cfg.get(k)
-        ]
+        diff = [k for k in ('url', 'security', 'certificate') if opened.get(k) != server_cfg.get(k)]
         if diff:
             self.logger.warning(
                 f'OPC server {server_id}: reusing the pooled connection, but {diff} differ '
@@ -98,10 +97,10 @@ class OpcConnectionPool(SientiaMonitoring):
         try:
             client = await self._connect(server_cfg)
         except Exception:
-            if server_cfg.get('use_security'):
+            if is_secured(server_cfg):
                 # The cached certificate may be the cause (replaced in MinIO,
                 # rejected by the server): download it again on the next attempt.
-                self.cert_provider.invalidate()
+                self.cert_provider.invalidate(server_cfg['certificate']['certificate_id'])
             await self.emit_metric(
                 metric_object=metrics.OPC_CONNECTIONS_FAILED,
                 tags={'pod_id': self.pod_id, 'server_name': server_id},
@@ -123,13 +122,12 @@ class OpcConnectionPool(SientiaMonitoring):
         pod_uri = self.pod_id.replace('-', ':')
         client.product_uri = pod_uri
         client.application_uri = pod_uri
-        server_uri = server_cfg.get('server_uri')
-        if server_uri:
-            # Must match the certificate's SAN URI (KEPServer: BadCertificateUriInvalid).
-            client.application_uri = server_uri
-        if server_cfg.get('use_security'):
-            cert, key = await self.cert_provider.fetch()
-            await apply_security(client, server_cfg.get('server_uri'), cert, key, None)
+        # Security only with a client certificate (SIENTIAPDE-1974 D5); apply_security
+        # sets the certificate's application URI, which must match its SAN URI.
+        if is_secured(server_cfg):
+            certificate = server_cfg['certificate']
+            cert, key = await self.cert_provider.fetch(certificate)
+            await apply_security(client, server_cfg['security'], certificate, cert, key)
         await client.connect()
         return client
 

ingestor/connection/security.py

@@ -1,17 +1,65 @@
-from asyncua.crypto.security_policies import SecurityPolicyBasic256Sha256
+from asyncua import ua
+from asyncua.crypto.security_policies import (
+    SecurityPolicyAes128Sha256RsaOaep,
+    SecurityPolicyAes256Sha256RsaPss,
+    SecurityPolicyBasic128Rsa15,
+    SecurityPolicyBasic256,
+    SecurityPolicyBasic256Sha256,
+)
 
 SESSION_TIMEOUT_MS = 600_000  # 10 min; tune per server (Samarco: long sessions degrade)
 
+# connectors.security.policy / .mode (SIENTIAPDE-1974 D4) to the asyncua classes
+POLICIES = {
+    'Basic256Sha256': SecurityPolicyBasic256Sha256,
+    'Aes128Sha256RsaOaep': SecurityPolicyAes128Sha256RsaOaep,
+    'Aes256Sha256RsaPss': SecurityPolicyAes256Sha256RsaPss,
+    'Basic256': SecurityPolicyBasic256,
+    'Basic128Rsa15': SecurityPolicyBasic128Rsa15,
+}
+MODES = {
+    'Sign': ua.MessageSecurityMode.Sign,
+    'SignAndEncrypt': ua.MessageSecurityMode.SignAndEncrypt,
+}
 
-async def apply_security(client, server_uri, cert_path, private_key_path, server_cert_path):
-    """Apply Basic256Sha256 + certificate auth (1879 §1) and set channel > session
-    lifetime (1879 §3: secure_channel_timeout = session_timeout * 1.25)."""
-    client.application_uri = server_uri
+
+class UnsupportedSecurityError(ValueError):
+    """The server's security mode or policy has no asyncua counterpart."""
+
+
+def is_secured(server_cfg: dict) -> bool:
+    """Whether a connection applies security: only with a client certificate (D5), and
+    not when its mode is 'None'. Without one, it connects without security, and a server
+    that requires a certificate refuses the connection."""
+    security = server_cfg.get('security') or {}
+    return server_cfg.get('certificate') is not None and security.get('mode') != 'None'
+
+
+async def apply_security(client, security, certificate, cert_path, private_key_path):
+    """Apply the server's mode + policy with the connector's client certificate, and set
+    channel > session lifetime (1879 §3: secure_channel_timeout = session_timeout * 1.25).
+
+    The application URI is the certificate's (its SAN URI): the server rejects a
+    different one (KEPServer: BadCertificateUriInvalid).
+
+    Raises:
+        UnsupportedSecurityError: When the mode or policy is unknown.
+    """
+    policy = POLICIES.get(security.get('policy'))
+    mode = MODES.get(security.get('mode'))
+    if policy is None or mode is None:
+        raise UnsupportedSecurityError(
+            f'Unsupported OPC security mode={security.get("mode")!r} '
+            f'policy={security.get("policy")!r} (modes: {", ".join(MODES)}; '
+            f'policies: {", ".join(POLICIES)})'
+        )
+    client.application_uri = certificate['application_uri']
     await client.set_security(
-        SecurityPolicyBasic256Sha256,
+        policy,
         certificate=cert_path,
         private_key=private_key_path,
-        server_certificate=server_cert_path,
+        server_certificate=None,
+        mode=mode,
     )
     client.session_timeout = SESSION_TIMEOUT_MS
     client.secure_channel_timeout = int(SESSION_TIMEOUT_MS * 1.25)

ingestor/connection/servers.py

@@ -1,60 +1,61 @@
 """Connection settings of the OPC server a write command targets (SIENTIAPDE-1975 GW1).
 
 The Laborious only knows the `server_id` of a write tag (pipeline `write_tags`); the
-connection (url, server URI, security) belongs to the gateway, which owns every OPC
-session. It is resolved with the same rule the orchestrator uses to build the read slots
-(SIENTIAPDE-1974): the tag database `connectors` first, the legacy `opc-servers` otherwise.
+connection (url, security, certificate) belongs to the gateway, which owns every OPC
+session. A server already in the pool is written through that connection. Otherwise it
+is resolved from the tag database (`connectors` → `opc_certificates`) with the rule the
+Orchestrator uses to build the read slots (SIENTIAPDE-1974 D1), in the slot format.
 """
 
 from typing import Any
 
 from sientia_do.observability.logger import Logger
 
+from ingestor.connection.certificates import (
+    CERTIFICATE_LOOKUP,
+    client_certificate_id,
+    has_security,
+    resolve_certificate,
+)
 from ingestor.connection.clock import MonotonicClock
 from ingestor.connection.pool import OpcConnectionPool
 
 CONNECTORS = 'connectors'
-OPC_SERVERS = 'opc-servers'
 
 
-def _id_forms(server_id: str) -> list[Any]:
-    """`server_id` as stored: the tag database keeps ints, opc-servers strings or ints."""
-    forms: list[Any] = [server_id]
-    if server_id.isdigit():
-        forms.append(int(server_id))
-    return forms
-
-
-def _connector_cfg(connector: dict[str, Any]) -> dict[str, Any] | None:
-    if not connector.get('url'):
-        return None
-    return {
-        'url': connector['url'],
-        'server_uri': connector.get('server_uri'),
-        'use_security': bool(connector.get('use_security', False)),
-    }
-
-
-def _opc_server_cfg(server: dict[str, Any]) -> dict[str, Any] | None:
-    if not server.get('url'):
-        return None
-    return {
-        'url': server['url'],
-        'server_uri': server.get('uri'),
-        # The certificate comes from MinIO now: a cert_path only says the server uses
-        # security, as in the orchestrator's slot (SIENTIAPDE-1974 RF4).
-        'use_security': bool(server.get('cert_path')),
-    }
+def _connector_pipeline(server_id: str) -> list[dict[str, Any]]:
+    """The active OPC UA connector `server_id` (connector_id is its slug), with its
+    certificate. Only the fields used are projected."""
+    return [
+        {'$match': {'connector_id': server_id, 'type': 'opc_ua', 'status': 'Active'}},
+        {'$limit': 1},
+        CERTIFICATE_LOOKUP,
+        {
+            '$project': {
+                '_id': 0,
+                'connector_id': 1,
+                'url': 1,
+                'subscription_period_ms': 1,
+                'security.mode': 1,
+                'security.policy': 1,
+                'security.client_certificate_id': 1,
+                'certificates': 1,
+            }
+        },
+    ]
 
 
 class ServerConfigResolver:
     """Resolve and cache the connection settings of an OPC server by `server_id`.
 
-    Order: settings carried by the command itself (an explicit override; the C2
-    command of the Laborious carries none), the live pooled connection (what the read
-    path opened, so writes never fight the reads over settings), then MongoDB. Lookups
-    are cached for `ttl_s`; a server that is not found, for `missing_ttl_s`, so a
-    pipeline pointing at a removed server does not query MongoDB on every command."""
+    Order: the live pooled connection (what the read path opened, so writes never fight
+    the reads over settings), then MongoDB. Lookups are cached for `ttl_s`; a server
+    that is not found, for `missing_ttl_s`, so a pipeline pointing at a removed server
+    does not query MongoDB on every command.
+
+    # ponytail: the MongoDB fallback goes away once write commands are routed to the pod
+    # that owns the connection (SIENTIAPDE-1974 D1, new card).
+    """
 
     def __init__(
         self,
@@ -74,19 +75,13 @@ class ServerConfigResolver:
         self.clock = MonotonicClock()
         self._cache: dict[str, tuple[dict[str, Any] | None, float]] = {}
 
-    async def resolve(self, server_id: str, cmd: dict) -> dict[str, Any] | None:
+    async def resolve(self, server_id: str) -> dict[str, Any] | None:
         """Connection settings for `server_id`, or None when no source knows it.
 
         Raises:
             Exception: When MongoDB cannot be read (the caller reports the write as
                 failed).
         """
-        if cmd.get('url'):
-            return {
-                'url': cmd['url'],
-                'server_uri': cmd.get('server_uri'),
-                'use_security': cmd.get('use_security', False),
-            }
         server_id = str(server_id)
         pooled = self.pool.config_for(server_id)
         if pooled is not None:
@@ -102,19 +97,30 @@ class ServerConfigResolver:
         return dict(cfg) if cfg is not None else None
 
     async def _lookup(self, server_id: str) -> dict[str, Any] | None:
-        ids = {'$in': _id_forms(server_id)}
-        connector = await self.mongo.find_one(
-            CONNECTORS,
-            {'connector_id': ids, 'type': 'opc_ua', 'status': 'Active'},
-            self.metadata,
+        connectors = await self.mongo.aggregate(
+            CONNECTORS, _connector_pipeline(server_id), self.metadata
         )
-        cfg = _connector_cfg(connector) if connector else None
-        if cfg is not None:
-            return cfg
-        server = await self.mongo.find_one(OPC_SERVERS, {'id': ids}, self.metadata)
-        cfg = _opc_server_cfg(server) if server else None
-        if cfg is None:
+        connector = connectors[0] if connectors else None
+        if connector is None or not connector.get('url') or not has_security(connector):
             self.logger.warning(
-                f'OPC server {server_id} not found in {CONNECTORS} nor {OPC_SERVERS}'
+                f'OPC server {server_id} not found, not active or incomplete (url, '
+                f'security.mode, security.policy) in {CONNECTORS}'
             )
-        return cfg
+            return None
+        certificate = resolve_certificate(connector)
+        if certificate is None and client_certificate_id(connector) is not None:
+            self.logger.warning(
+                f'OPC server {server_id}: client_certificate_id '
+                f'{client_certificate_id(connector)} has no active client certificate in '
+                f'opc_certificates; connecting without security'
+            )
+        return {
+            'server_id': server_id,
+            'url': connector['url'],
+            'subscription_period_ms': connector.get('subscription_period_ms'),
+            'security': {
+                'mode': connector['security']['mode'],
+                'policy': connector['security']['policy'],
+            },
+            'certificate': certificate,
+        }

ingestor/engines/write.py

@@ -83,7 +83,7 @@ class WriteEngine(SientiaMonitoring):
         self.pool = pool
         self.sink = sink
         # Resolves the server connection by server_id (SIENTIAPDE-1975 GW1). Without it,
-        # only the connection carried by the command itself is used.
+        # no server is known and every write fails as server_not_found.
         self.servers = servers
         self.write_timestamp = write_timestamp
         self.write_quality = write_quality
@@ -146,18 +146,12 @@ class WriteEngine(SientiaMonitoring):
         self._last_write_at = now
 
     async def _server_cfg(self, cmd: dict) -> dict | None:
-        """Connection settings of the target server: resolved by server_id (connection
-        carried by the command, live pooled connection, connectors, opc-servers), or only
-        the ones carried by the command when no resolver is wired."""
-        if self.servers is not None:
-            return await self.servers.resolve(str(cmd['server_id']), cmd)
-        if not cmd.get('url'):
+        """Connection settings of the target server, resolved by server_id (live pooled
+        connection, then connectors → opc_certificates). The command carries only the
+        server_id (C2)."""
+        if self.servers is None:
             return None
-        return {
-            'url': cmd.get('url'),
-            'use_security': cmd.get('use_security', False),
-            'server_uri': cmd.get('server_uri'),
-        }
+        return await self.servers.resolve(str(cmd['server_id']))
 
     def _variant_type(self, data_type: str) -> ua.VariantType:
         # direct index: unknown dtypes already became ConversionError in to_typed_value

ingestor/gateway_runtime.py

@@ -2,8 +2,8 @@
 object graph and wires it into the app's lifecycle.
 
 Reuses the existing ingestor's logger/notification_handler/metrics_controller and
-its already-connected DataManager. The OpcConnectionPool (and its CertProvider,
-MinIO + Mongo) belongs to the read path in IngestorManager (SIENTIAPDE-1974):
+its already-connected DataManager. The OpcConnectionPool (and its CertProvider, which
+downloads each server's client certificate from MinIO) belongs to the read path in IngestorManager (SIENTIAPDE-1974):
 reads start in prepare_ingestor, before this runtime exists, and must keep running
 if anything here fails. The runtime only borrows the pool for writes and gRPC.
 
@@ -63,8 +63,9 @@ class GatewayRuntime(SientiaMonitoring):
         self.sink = DataSink(data_manager, postgres)
 
         # Write commands carry only the server_id (SIENTIAPDE-1975): the connection is
-        # resolved here, from the live pooled connection or the connectors/opc-servers
-        # documents, with the same rule the orchestrator uses for the read slots.
+        # resolved here, from the live pooled connection or the connectors →
+        # opc_certificates documents, with the rule the orchestrator uses for the read
+        # slots (SIENTIAPDE-1974 D1).
         self.servers = ServerConfigResolver(
             self.pool, data_manager.mongo_repository, logger, metadata=self.metadata
         )

ingestor/io/data_sink.py

@@ -52,14 +52,16 @@ class DataSink:
     async def publish_reading(self, tag: str, value, timestamp, names: list[dict]) -> None:
         """Publish one MongoDB document per name entry and record its tag value metric.
 
-        `model_id` is only written when the name carries one: names converted from a
-        slot v1 have none, and their documents must stay identical to the legacy ones."""
+        Every name of the slot v2 carries collection, tag_name and model_id (parse_slot
+        checks it)."""
         for name in names:
-            doc = {'tag': tag, 'name': name['tag_name']}
-            if 'model_id' in name:
-                doc['model_id'] = name['model_id']
-            doc['timestamp'] = timestamp
-            doc['value'] = value
+            doc = {
+                'tag': tag,
+                'name': name['tag_name'],
+                'model_id': name['model_id'],
+                'timestamp': timestamp,
+                'value': value,
+            }
             await self.data_manager.publish(name['collection'], doc)
             await self.emit_metric(
                 metric_object=metrics.OPC_TAG_VALUE,
@@ -68,7 +70,7 @@ class DataSink:
                 tags={
                     'pod_id': self.data_manager.pod_id,
                     'tag_name': name['tag_name'],
-                    'model_id': name.get('model_id', ''),
+                    'model_id': name['model_id'],
                 },
             )
 

ingestor/io/slots.py

@@ -1,28 +1,27 @@
-"""Parse the slot written by the Orchestrator to Redis into the slot v2 structure (contract C1).
+"""Parse the slot v2 the Orchestrator writes to Redis (contract C1, SIENTIAPDE-1974).
 
-Both shapes are accepted so the gateway and the Orchestrator can be deployed in
-any order (SIENTIAPDE-1974):
-
-- v2: each tag carries `names` (one entry per collection / tag name / model);
-- v1 (legacy): each tag carries `tag_name` + `topics`, and the server carries the
-  file-based `cert_path`. v1 is converted to v2: one `names` entry per topic, and
-  `use_security` is true when the server had a certificate path (the certificate
-  itself now always comes from MinIO via CertProvider, never from that path).
+Each server carries its connection (`url`, `subscription_period_ms`), its OPC security
+(`security`: mode + policy) and its client certificate (`certificate`: certificate_id,
+application_uri and the MinIO objects of the cert and the private key; None = connect
+without security, D5). Each tag carries `names`: one entry per collection / tag name /
+model the reading is written to.
 """
 
-SERVER_FIELDS = ('server_id', 'url', 'server_uri', 'use_security', 'subscription_period_ms')
+SERVER_FIELDS = ('server_id', 'url', 'subscription_period_ms', 'security', 'certificate')
 # What a connection/subscription needs (the pool keys clients by server_id itself).
 CONNECTION_FIELDS = tuple(f for f in SERVER_FIELDS if f != 'server_id')
+NAME_FIELDS = ('collection', 'tag_name', 'model_id')
 
 
 def _parse_names(address: str, tag: dict) -> list[dict]:
     names = tag.get('names')
-    if names:
-        return names
-    topics = tag.get('topics')
-    if topics and tag.get('tag_name'):
-        return [{'collection': topic, 'tag_name': tag['tag_name']} for topic in topics]
-    raise ValueError(f'tag {address!r} has no names')
+    if not names:
+        raise ValueError(f'tag {address!r} has no names')
+    for name in names:
+        missing = [field for field in NAME_FIELDS if field not in name]
+        if missing:
+            raise ValueError(f'tag {address!r} has a name without {missing}')
+    return names
 
 
 def _parse_tag(address: str, tag: dict) -> dict:
@@ -34,14 +33,19 @@ def _parse_tag(address: str, tag: dict) -> dict:
 
 
 def _parse_server(name: str, server: dict) -> dict:
-    parsed = {field: server[field] for field in SERVER_FIELDS if field in server}
+    if not isinstance(server.get('security'), dict):
+        raise ValueError(f'server {name!r} has no security')
+    parsed = {field: server.get(field) for field in SERVER_FIELDS}
     parsed['name'] = server.get('name', name)
-    if 'use_security' not in parsed:
-        parsed['use_security'] = bool(server.get('cert_path'))
     parsed['tags'] = {addr: _parse_tag(addr, tag) for addr, tag in server['tags'].items()}
     return parsed
 
 
 def parse_slot(raw: dict) -> dict:
-    """Normalize a raw slot (v1 or v2) into `{server_name: {..., tags: {addr: tag}}}`."""
+    """Normalize a raw slot v2 into `{server_name: {..., tags: {addr: tag}}}`.
+
+    Raises:
+        KeyError | TypeError | ValueError: When the slot is not a valid slot v2 (e.g. a
+            slot v1 with `topics`).
+    """
     return {name: _parse_server(name, server) for name, server in raw.items()}

ingestor/managers/ingestor_manager.py

@@ -1,6 +1,5 @@
 import traceback
 from copy import deepcopy
-from os import getenv
 
 from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
 from sientia_do.notifications.models import NotificationLevel
@@ -15,16 +14,9 @@ from ingestor.connection.pool import OpcConnectionPool
 from ingestor.io.data_sink import DataSink
 from ingestor.io.slots import parse_slot
 from ingestor.managers.data_manager import DataManager
-from ingestor.managers.opc_manager import OpcManager
 from ingestor.managers.resource_manager import ResourceManager
 from ingestor.managers.server_reader import ServerReader
 
-# READ_ENGINE (SIENTIAPDE-1974): 'v2' = ServerReader/SubscriptionEngine over the shared
-# pool, certificate from MinIO (default). 'legacy' = plan B: the untouched rc3
-# OpcManager, slot v1 only, certificate files from the Gitea clone (cert_path). Kept
-# for validation until the end of 1974; remove once unused and the tests pass without it.
-READ_ENGINES = ('v2', 'legacy')
-
 
 class IngestorManager(SientiaMonitoring):
     """
@@ -85,7 +77,6 @@ class IngestorManager(SientiaMonitoring):
         notification_handler: NotificationHandler,
         metrics_controller: MetricsController,
         gateway_config: GatewayConfig | None = None,
-        read_engine: str | None = None,
     ):
         redis_host: str = redis_data['host']
         redis_port: int = int(redis_data['port'])
@@ -108,12 +99,12 @@ class IngestorManager(SientiaMonitoring):
             metrics_controller=metrics_controller,
         )
         self.opc_managers: dict = {}
-        # Read path (SIENTIAPDE-1974): pooled connections with the certificate from
-        # MinIO. Nothing here touches the network at construction (MinIO is lazy), so
-        # a MinIO/Postgres/NATS outage can't stop the reads before they start.
+        # Read path (SIENTIAPDE-1974): pooled connections with each server's client
+        # certificate (named in the slot) from MinIO. Nothing here touches the network at
+        # construction (MinIO is lazy), so a MinIO/Postgres/NATS outage can't stop the
+        # reads before they start.
         config = gateway_config or GatewayConfig.certs_from_env()
         self.cert_provider = build_cert_provider(
-            mongo=self.data_manager.mongo_repository,
             config=config,
             logger=logger,
             notification_handler=notification_handler,
@@ -124,10 +115,6 @@ class IngestorManager(SientiaMonitoring):
             self.cert_provider, logger, notification_handler, metrics_controller
         )
         self.sink = DataSink(self.data_manager, None)
-        self._server_cert_warned: set[str] = set()
-        self.read_engine = self._resolve_read_engine(
-            read_engine or getenv('READ_ENGINE') or 'v2', logger
-        )
         self.resource_manager = ResourceManager(
             host=redis_host,
             port=redis_port,
@@ -147,32 +134,7 @@ class IngestorManager(SientiaMonitoring):
 
         self.metadata = metadata
 
-    @staticmethod
-    def _resolve_read_engine(value: str, logger: Logger) -> str:
-        engine = value.strip().lower()
-        if engine not in READ_ENGINES:
-            logger.warning(f'Invalid READ_ENGINE {value!r}, using v2 ({"|".join(READ_ENGINES)})')
-            engine = 'v2'
-        logger.info(f'OPC read engine: {engine}')
-        return engine
-
-    def _build_reader(self, server_config: dict) -> ServerReader | OpcManager:
-        if self.read_engine == 'legacy':
-            # Plan B: the rc3 OpcManager, unchanged (own connection, cert files).
-            return OpcManager(
-                name=server_config['name'],
-                url=server_config['url'],
-                subscription_period_ms=server_config['subscription_period_ms'],
-                data_manager=self.data_manager,
-                logger=self.logger,
-                server_uri=server_config['server_uri'],
-                notification_handler=self.notification_handler,
-                metadata=self.metadata,
-                cert_path=server_config.get('cert_path'),
-                private_key_path=server_config.get('private_key_path'),
-                server_cert_path=server_config.get('server_cert_path'),
-                metrics_controller=self.metrics_controller,
-            )
+    def _build_reader(self, server_config: dict) -> ServerReader:
         return ServerReader(
             name=server_config['name'],
             server_config=server_config,
@@ -184,24 +146,21 @@ class IngestorManager(SientiaMonitoring):
             metadata=self.metadata,
         )
 
-    async def initialize_opc_from_config(
-        self, server_config: dict
-    ) -> ServerReader | OpcManager | None:
+    async def initialize_opc_from_config(self, server_config: dict) -> ServerReader | None:
         """
         Initializes the reader of one OPC UA server from its slot configuration.
 
-        With READ_ENGINE=v2 (default) creates a ServerReader and opens its pooled
-        connection; with `use_security`, the certificate comes from MinIO. With
-        READ_ENGINE=legacy (plan B) creates the rc3 OpcManager, which connects on its
-        own with the certificate files of the slot v1 (`cert_path`).
+        Creates a ServerReader and opens its pooled connection. With a `certificate`, the
+        connection uses the slot's security mode and policy, with the certificate
+        downloaded from MinIO; without one, it connects without security (D5).
 
         Args:
-            server_config (dict): The server configuration of the slot, without the
-                'tags' key. v2: 'name', 'server_id', 'url', 'server_uri',
-                'use_security', 'subscription_period_ms'. legacy: the slot v1 fields.
+            server_config (dict): The server configuration of the slot v2, without the
+                'tags' key: 'name', 'server_id', 'url', 'subscription_period_ms',
+                'security' and 'certificate'.
 
         Returns:
-            ServerReader | OpcManager | None: The connected reader, or None if the
+            ServerReader | None: The connected reader, or None if the
             connection fails (logged and notified, so the other servers keep working).
         """
 
@@ -254,11 +213,7 @@ class IngestorManager(SientiaMonitoring):
                 the shared connection stays for writes/gRPC until the idle sweep.
         """
         if server in self.opc_managers:
-            if self.read_engine == 'legacy':
-                # OpcManager owns its connection: shutdown always disconnects it.
-                await self.opc_managers[server].shutdown()
-            else:
-                await self.opc_managers[server].shutdown(evict=evict)
+            await self.opc_managers[server].shutdown(evict=evict)
             del self.opc_managers[server]
         for slot, _config in self.managed_tags.items():
             self.managed_tags[slot].pop(server, None)
@@ -567,14 +522,11 @@ class IngestorManager(SientiaMonitoring):
             if slot in reader.subscriptions:
                 await self._unsubscribe_reader(server, reader, slot)
 
-    async def _unsubscribe_reader(
-        self, server: str, reader: ServerReader | OpcManager, slot: str
-    ) -> None:
+    async def _unsubscribe_reader(self, server: str, reader: ServerReader, slot: str) -> None:
         """
         Unsubscribe one reader from a slot without letting a failure stop the main loop.
 
-        A failure (e.g. the legacy OpcManager, whose delete() raises when the server
-        session is gone) increments OPC_SUBSCRIPTION_ERRORS, sends
+        A failure (e.g. delete() raising when the server session is gone) increments OPC_SUBSCRIPTION_ERRORS, sends
         OPC_UNSUBSCRIBE_ERROR_<slot>:<server> and is logged. The slot is dropped from the
         reader's books, as the v2 engine already does, so it can be subscribed again.
 
@@ -584,7 +536,7 @@ class IngestorManager(SientiaMonitoring):
 
         Args:
             server (str): The reader's key in opc_managers
-            reader (ServerReader | OpcManager): The OPC reader
+            reader (ServerReader): The OPC reader
             slot (str): The slot to unsubscribe
         """
         try:
@@ -664,22 +616,17 @@ class IngestorManager(SientiaMonitoring):
 
     def parse_slot_config(self, slot: str, raw: dict) -> dict | None:
         """
-        Normalizes the raw slot read from Redis to the slot v2 structure.
+        Normalizes the raw slot read from Redis to the slot v2 structure (contract C1).
 
-        Accepts slot v1 (`topics`) and v2 (`names`), so the gateway and the
-        Orchestrator can be deployed in any order (SIENTIAPDE-1974). With
-        READ_ENGINE=legacy the raw slot v1 is kept as is (what OpcManager reads), and
-        a slot v2 is rejected.
+        Only the slot v2 is accepted (SIENTIAPDE-1974): the gateway and the Orchestrator
+        are deployed together, and a slot v1 (`topics`, no `security`) is rejected.
 
         Returns:
             dict | None: The parsed slot, or None when the config is malformed
             (logged and notified).
         """
         try:
-            if self.read_engine == 'legacy':
-                self._validate_slot_v1(raw)
-                return raw
-            parsed = parse_slot(raw)
+            return parse_slot(raw)
         except (AttributeError, KeyError, TypeError, ValueError) as e:
             self.send_notification(
                 metadata=self.metadata,
@@ -690,32 +637,6 @@ class IngestorManager(SientiaMonitoring):
             )
             self.logger.error(f'Slot {slot} has an invalid configuration: {e!r}')
             return None
-        self._warn_server_cert_path(raw)
-        return parsed
-
-    @staticmethod
-    def _validate_slot_v1(raw: dict) -> None:
-        """READ_ENGINE=legacy: OpcManager only understands slot v1 (`tag_name` +
-        `topics` per tag); a v2 slot (`names`) would fail in its datachange callback."""
-        for name, server in raw.items():
-            for address, tag in server['tags'].items():
-                if 'topics' not in tag or 'tag_name' not in tag or 'frequency' not in tag:
-                    raise ValueError(
-                        f'READ_ENGINE=legacy needs a slot v1 (tag_name + topics); '
-                        f'server {name!r}, tag {address!r} is not'
-                    )
-
-    def _warn_server_cert_path(self, raw: dict) -> None:
-        """Logs once per server when a slot still pins a server certificate
-        (`server_cert_path`): connections now come from the pool with the platform
-        certificate from MinIO, and that file-based pin is no longer applied."""
-        for name, server in raw.items():
-            if server.get('server_cert_path') and name not in self._server_cert_warned:
-                self._server_cert_warned.add(name)
-                self.logger.warning(
-                    f'OPC server {name}: server_cert_path is set in the slot but is no '
-                    f'longer applied (server certificate validation is disabled)'
-                )
 
     async def drop_slot_leases(self, ids: list[str]) -> None:
         """

ingestor/managers/opc_manager.py

@@ -1,588 +0,0 @@
-import asyncio
-import json
-import traceback
-from pathlib import Path
-
-from asyncua import Client
-from asyncua.crypto.security_policies import SecurityPolicyBasic256
-from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
-from sientia_do.notifications.models import NotificationLevel
-from sientia_do.observability.logger import Logger
-from sientia_do.observability.metrics_controller import MetricsController
-from sientia_do.observability.sientia_monitoring import SientiaMonitoring
-from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, OPC_TIMEZONE
-
-import ingestor.metrics as metrics
-from ingestor.managers.data_manager import DataManager
-
-
-class OpcManager(SientiaMonitoring):
-    """
-    Manages OPC UA server connections and tag subscriptions.
-
-    The OpcManager is responsible for:
-    - Establishing and maintaining secure connections to OPC UA servers
-    - Managing tag subscriptions and data collection
-    - Handling server reconnection and error recovery
-    - Processing OPC data and forwarding it to the data manager
-    - Monitoring connection health and performance metrics
-
-    The manager supports both secure and unsecured connections, with optional
-    certificate-based authentication for enhanced security.
-
-    Args:
-        name (str): Unique identifier for the OPC server
-        url (str): OPC UA server endpoint URL
-        data_manager (DataManager): Manager for data persistence and export
-        logger (Logger): Logger instance for application logging
-        server_uri (str): OPC UA server application URI
-        notification_handler (NotificationHandler): Handler for sending notifications
-        metadata (dict): Application metadata for notifications and tracking
-        cert_path (str, optional): Path to client certificate file for secure connections
-        private_key_path (str, optional): Path to client private key file
-        server_cert_path (str, optional): Path to server certificate file for validation
-
-    Attributes:
-        url (str): OPC UA server endpoint URL
-        name (str): Unique identifier for the OPC server
-        server_uri (str): OPC UA server application URI
-        data_queue (dict): Queue for buffering OPC data before processing
-        non_receive_count (int): Counter for cycles without data reception
-        client (Client): OPC UA client instance
-        cert_path (str): Path to client certificate file
-        private_key_path (str): Path to client private key file
-        server_cert_path (str): Path to server certificate file
-        nodes (dict): Dictionary of OPC node references
-        subscriptions (dict): Active OPC subscriptions
-        data_manager (DataManager): Manager for data persistence and export
-        metadata (dict): Application metadata
-    """
-
-    def __init__(
-        self,
-        name: str,
-        url: str,
-        subscription_period_ms: int,
-        data_manager: DataManager,
-        logger: Logger,
-        server_uri: str,
-        notification_handler: NotificationHandler,
-        metrics_controller: MetricsController,
-        metadata: dict,
-        cert_path: str | None = None,
-        private_key_path: str | None = None,
-        server_cert_path: str | None = None,
-    ):
-        self.url = url
-        self.name = name
-        self.server_uri = server_uri
-        self.data_queue: dict = {}
-        self.non_receive_count = 0
-        self.client: Client | None = None
-        self.subscription_period_ms = subscription_period_ms
-        self.cert_path = cert_path
-        self.private_key_path = private_key_path
-        self.server_cert_path = server_cert_path
-        self.nodes: dict = {}
-        self.subscriptions: dict = {}
-        self.data_manager = data_manager
-        self.metadata = metadata
-
-        SientiaMonitoring.__init__(
-            self,
-            logger=logger,
-            notification_handler=notification_handler,
-            metrics_controller=metrics_controller,
-        )
-
-        metrics.OPC_CONNECTION_STATUS.labels(
-            pod_id=self.pod_id, server_name=self.name, server_url=self.url
-        ).set(0)
-        metrics.OPC_TAGS_SUBSCRIBED.labels(pod_id=self.pod_id, server_name=self.name).set(0)
-
-    def __str__(self):
-        """
-        String representation of the OPC Manager.
-
-        Returns:
-            str: Human-readable representation showing server details and current state.
-        """
-        return (
-            f'OpcManager(name={self.name}, url={self.url}, server_uri={self.server_uri})\n'
-            f'nodes={self.nodes}, subscriptions={self.subscriptions}'
-        )
-
-    async def shutdown(self):
-        """
-        Comprehensive cleanup method for graceful shutdown.
-
-        This method ensures proper cleanup of all OPC UA resources:
-        - Closes active subscriptions
-        - Disconnects from the OPC server
-        - Releases allocated resources
-
-        Should be called before the application terminates to prevent resource leaks
-        and ensure clean disconnection from OPC servers.
-        """
-        try:
-            await self.disconnect()
-        except Exception as e:
-            self.logger.error(f'Error during cleanup: {e}')
-
-    async def set_security(self):
-        """
-        Configures the security settings for the OPC UA client.
-
-        This method sets up the security policy, certificates, and timeouts
-        required for establishing a secure connection with the OPC UA server.
-        It implements Basic256 security policy with certificate-based authentication.
-
-        Raises:
-            ValueError: If either the certificate path or private key path is not provided.
-
-        Security Settings:
-            - Security Policy: Basic256
-            - Secure Channel Timeout: 10,000,000 ms
-            - Session Timeout: 10,000,000 ms
-
-        The method configures:
-            - Client application URI
-            - Certificate-based authentication
-            - Server certificate validation (if provided)
-            - Connection timeouts for stability
-        """
-
-        if not all([self.cert_path, self.private_key_path]):
-            raise ValueError(
-                'Certificate and private key paths must be provided for secure connection.'
-            )
-        cert = str(Path(self.cert_path)) if self.cert_path else None
-        private_key = str(Path(self.private_key_path)) if self.private_key_path else None
-        server_cert = str(Path(self.server_cert_path)) if self.server_cert_path else None
-        assert cert is not None and private_key is not None
-
-        if self.client:
-            self.client.application_uri = self.server_uri
-            self.logger.info('Setting security...')
-            await self.client.set_security(
-                SecurityPolicyBasic256,
-                certificate=cert,
-                private_key=private_key,
-                server_certificate=server_cert,
-            )
-            self.client.secure_channel_timeout = 10000000
-            self.client.session_timeout = 10000000
-
-    async def connect(self):
-        """
-        Establishes a connection to the OPC server.
-
-        This method initializes the OPC client using the provided URL and
-        sets up security if a certificate path is specified. It then
-        attempts to connect to the server and logs the connection status.
-
-        The connection process includes:
-        1. Client initialization with server URL
-        2. Security configuration (if certificates are provided)
-        3. Connection establishment
-        4. Metrics recording for monitoring
-
-        Raises:
-            Exception: If the connection to the OPC server fails.
-
-        Metrics:
-            - OPC_CONNECTIONS_TOTAL: Incremented on connection attempt
-            - OPC_CONNECTION_STATUS: Set to 1 on successful connection
-        """
-
-        await self.emit_metric(
-            metric_object=metrics.OPC_CONNECTIONS_TOTAL,
-            tags={
-                'pod_id': self.pod_id,
-                'server_name': self.name,
-            },
-        )
-        try:
-            self.client = Client(self.url, timeout=10, watchdog_intervall=3600000)
-            assert self.client is not None  # Informa ao mypy que client não é None
-
-            self.client.name = self.pod_id
-            self.client.description = self.pod_id
-            pod_uri = self.pod_id.replace('-', ':')
-            self.client.application_uri = pod_uri
-            self.client.product_uri = pod_uri
-
-            if self.cert_path:
-                await self.set_security()
-            self.logger.info(f'Starting connection to {self.name}...')
-            await self.client.connect()
-            await self.emit_metric(
-                metric_object=metrics.OPC_CONNECTION_STATUS,
-                method='set',
-                value=1,
-                tags={
-                    'pod_id': self.pod_id,
-                    'server_name': self.name,
-                    'server_url': self.url,
-                },
-            )
-            self.logger.info(f'Connection to {self.name} successful.')
-        except Exception:
-            await self.disconnect()
-
-            raise
-
-    async def create_subscription(self, name: str):
-        """
-        Creates a subscription with the specified monitoring period.
-
-        This method establishes a subscription to monitor data changes or events
-        from the OPC UA server. If the client is not connected, an exception is raised.
-
-        Args:
-            name (str): The name identifier for the subscription
-            period (int, optional): The monitoring period in milliseconds. Defaults to 500 ms.
-
-        Raises:
-            ValueError: If the client is not connected.
-
-        Side Effects:
-            - Sets the `self.period` attribute to the specified or default period.
-            - Creates a subscription and assigns it to `self.subscriptions[name]`.
-            - Logs the creation of the subscription.
-            - Increments subscription creation metrics.
-        """
-
-        if not self.client:
-            raise ValueError('Client not connected. Call connect first.')
-        try:
-            self.subscriptions[name] = await self.client.create_subscription(
-                self.subscription_period_ms, self
-            )
-            self.logger.info(f'Subscription {name} created on {self.name}.')
-            await self.emit_metric(
-                metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED,
-                method='inc',
-                value=1,
-                tags={
-                    'pod_id': self.pod_id,
-                    'server_name': self.name,
-                    'slot_name': name,
-                },
-            )
-        except Exception as e:
-            self.logger.error(f'Failed to create subscription {name} on {self.name}: {e}')
-            raise
-
-    async def subscribe(self, subscription: str, nodes: dict, collect_period: int):
-        """
-        Subscribes to a set of OPC UA nodes for data change notifications.
-
-        This method adds the specified nodes to the subscription and configures
-        their data collection rules based on the provided collection period and
-        node-specific frequency.
-
-        Args:
-            subscription (str): The name of the subscription to use
-            nodes (dict): A dictionary where keys are node identifiers and values are
-                configurations for each node. Each configuration must include a 'frequency'
-                key indicating the frequency of data collection in Hz.
-            collect_period (int): The data collection period in seconds.
-
-        Raises:
-            ValueError: If the subscription has not been created by calling
-                `create_subscription` prior to this method.
-
-        Side Effects:
-            - Updates internal node tracking and cycle rules
-            - Establishes data change monitoring for specified nodes
-            - Updates metrics for subscribed tags count
-        """
-
-        if not self.subscriptions.get(subscription):
-            raise ValueError('Subscription not created. Call create_subscription first.')
-
-        self.logger.info(f'Subscribing to {subscription} on {self.name}...')
-        self.logger.info(f'Subscribing to nodes: {nodes}')
-        assert self.client is not None  # Informa ao mypy que client não é None
-        addr_nodes = [self.client.get_node(n) for n in nodes]
-        self.logger.debug(f'Addr nodes: {addr_nodes}')
-        self.nodes.update(nodes)
-        self.logger.debug(f'Nodes: {self.nodes}')
-
-        self.collect_period = collect_period
-
-        for node, config in self.nodes.items():
-            self.nodes[node]['cycle_rule'] = {
-                'cycle_increment': collect_period * 1000 / float(config['frequency']),
-                'cycle_count': 0,
-            }
-
-        await self.subscriptions[subscription].subscribe_data_change(addr_nodes)
-
-        await self.emit_metric(
-            metric_object=metrics.OPC_TAGS_SUBSCRIBED,
-            method='set',
-            value=len(self.nodes),
-            tags={
-                'pod_id': self.pod_id,
-                'server_name': self.name,
-            },
-        )
-
-    async def unsubscribe(self, subscription: str):
-        """
-        Unsubscribes from a given subscription.
-
-        This method removes the specified subscription and cleans up associated
-        resources. It handles cases where the subscription doesn't exist gracefully.
-
-        Args:
-            subscription (str): The name of the subscription to unsubscribe from.
-
-        Logs:
-            - A warning if the specified subscription does not exist.
-            - An info message upon successful unsubscription.
-
-        Behavior:
-            - If the subscription exists, it is deleted and removed from the
-              subscriptions dictionary.
-            - If the subscription does not exist, no action is taken.
-        """
-
-        if not self.subscriptions.get(subscription):
-            self.logger.warning(f"Subscription '{subscription}' not found. Cannot unsubscribe.")
-            return
-        await self.subscriptions[subscription].delete()
-        del self.subscriptions[subscription]
-        self.logger.info(f'Unsubscribed from {subscription}.')
-
-    async def disconnection_fallback(self) -> list:
-        """
-        Tries 5 times to disconnect from the OPC UA server, with a delay of 100ms x try.
-        """
-
-        assert self.client is not None
-        error_stack = []
-        for i in range(5):
-            try:
-                self.logger.info(f'Disconnecting from OPC UA server, attempt {i + 1} of 5')
-                await self.client.disconnect()
-                return []
-            except Exception as e:
-                self.logger.error(
-                    f'Failed to disconnect from OPC UA serve  in attempt {i + 1} of 5: {e}'
-                )
-                error_stack.append(
-                    {
-                        'attempt': i + 1,
-                        'error': str(e),
-                        'traceback': traceback.format_exc(),
-                    }
-                )
-                await asyncio.sleep(0.1 * i)
-        return error_stack
-
-    async def disconnect(self):
-        """
-        Disconnects from the OPC UA server.
-
-        This method handles the disconnection process by deleting all subscriptions
-        and disconnecting the client from the OPC UA server. It logs the disconnection
-        process and handles any exceptions that may occur during cleanup.
-
-        Raises:
-            Exception: If an error occurs while deleting the subscription or disconnecting
-                       from the OPC UA server, it logs the error details.
-
-        Side Effects:
-            - Deletes all active subscriptions
-            - Disconnects the OPC client
-            - Updates connection status metrics
-            - Clears internal client reference
-        """
-
-        self.logger.warning('Disconnecting from OPC server')
-        if self.client is None:
-            self.logger.warning('Client already disconnected.')
-            return
-        try:
-            for sub in self.subscriptions:
-                await self.subscriptions[sub].delete()
-            self.logger.warning('Deleted all subscriptions.')
-        except Exception as sub_error:
-            self.logger.error(f'Failed to clean up subscription: {sub_error}')
-
-        errors = await self.disconnection_fallback()
-
-        if errors:
-            await self.send_notification_async(
-                metadata=self.metadata,
-                notification_id=f'OPC_DISCONNECTION_ERROR_{self.name}',
-                message=f'Failed to disconnect from OPC UA server {self.name} after 5 attempts',
-                block='opc_manager',
-                level=NotificationLevel.ERROR,
-                attachment_content=json.dumps(errors, indent=4),
-            )
-        else:
-            self.logger.warning('Disconnected from OPC UA server.')
-
-        del self.client
-        self.client = None
-        await self.emit_metric(
-            metric_object=metrics.OPC_CONNECTION_STATUS,
-            method='set',
-            value=0,
-            tags={
-                'pod_id': self.pod_id,
-                'server_name': self.name,
-                'server_url': self.url,
-            },
-        )
-        await self.emit_metric(
-            metric_object=metrics.OPC_TAGS_SUBSCRIBED,
-            method='set',
-            value=0,
-            tags={
-                'pod_id': self.pod_id,
-                'server_name': self.name,
-            },
-        )
-
-    async def datachange_notification(self, node, _val, data):
-        """
-        Handles data change notifications for monitored OPC UA nodes.
-
-        This method is triggered when a monitored node's value changes. It processes
-        the notification, updates internal state, and publishes the data to the
-        appropriate topics.
-
-        Args:
-            node (NodeId): The OPC UA node that triggered the data change notification.
-            _val (Any): The new value of the node (unused in this implementation).
-            data (DataChangeNotification): The data change notification object containing
-                details about the change.
-
-        Behavior:
-            - Extracts the value and source timestamp from the monitored item.
-            - Resets the cycle count for the node's cycle rule.
-            - Resets the non-receive count.
-            - Constructs a data dictionary containing the tag, tag name, timestamp, and value.
-            - Publishes the data to all topics associated with the node.
-        """
-
-        # get data value
-        monitored_item = data.monitored_item
-        value = monitored_item.Value.Value.Value
-        # source_timestamp
-        source_timestamp = monitored_item.Value.SourceTimestamp.replace(tzinfo=OPC_TIMEZONE)
-        tag = str(node)
-
-        self.logger.debug(
-            f'Data change notification received for tag:'
-            f'{tag} after {self.nodes[tag]["cycle_rule"]["cycle_count"]} cycles'
-        )
-
-        data = {
-            'tag': tag,
-            'name': self.nodes[str(node)]['tag_name'],
-            'timestamp': source_timestamp.strftime(DATETIME_FORMAT_WITH_TZ),
-            'value': value,
-        }
-
-        for topic in self.nodes[tag]['topics']:
-            await self.data_manager.publish(topic, data)
-
-        self.nodes[tag]['cycle_rule']['cycle_count'] = 0
-        self.non_receive_count = 0
-        await self.emit_metric(
-            metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
-            method='set',
-            value=0,
-            tags={
-                'pod_id': self.pod_id,
-                'server_name': self.name,
-            },
-        )
-
-    async def check_cycles(self):
-        """
-        Checks the cycle counts for all monitored nodes and sends
-        notifications if thresholds are exceeded.
-
-        This method iterates through all monitored nodes and updates their cycle counts based on
-        configured increments. If a node's cycle count exceeds a threshold (5 cycles), it triggers
-        a warning notification.
-
-        Side Effects:
-            - Updates cycle counts for all monitored nodes
-            - Sends warning notifications for nodes exceeding cycle thresholds
-        """
-        for node, config in self.nodes.items():
-            self.nodes[node]['cycle_rule']['cycle_count'] += config['cycle_rule']['cycle_increment']
-            if self.nodes[node]['cycle_rule']['cycle_count'] >= 5:
-                name = config['tag_name']
-                cycles = self.nodes[node]['cycle_rule']['cycle_count']
-                await self.send_notification_async(
-                    metadata=self.metadata,
-                    notification_id=f'TAG_{node}:{name}_LISTENNING_STOPPED',
-                    message=f'{cycles} cycles without receive from {node}:{name}',
-                    block='opc_manager',
-                    level=NotificationLevel.WARNING,
-                )
-
-    async def check_opc_listenning(self) -> bool:
-        """
-        Checks the OPC connection and triggers notifications if the connection is lost.
-
-        This method monitors the data reception health by tracking cycles without
-        data. It sends notifications at different thresholds and can trigger
-        reconnection attempts.
-
-        Returns:
-            bool: True if the connection is lost and reconnection should be attempted,
-                  False otherwise.
-
-        Side Effects:
-            - Increments non-receive count
-            - Updates metrics for cycles without data
-            - Sends warning notifications at 5 cycles
-            - Sends error notifications and triggers reconnection at 15 cycles
-        """
-
-        self.non_receive_count += 1
-        await self.emit_metric(
-            metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
-            method='set',
-            value=self.non_receive_count,
-            tags={
-                'pod_id': self.pod_id,
-                'server_name': self.name,
-            },
-        )
-        if self.non_receive_count >= 5:
-            await self.send_notification_async(
-                metadata=self.metadata,
-                notification_id=f'OPC_LISTENNING_STOPPED__{self.name}',
-                message=f'{self.non_receive_count} cycles without '
-                f'receive from OPC {self.name}. Tags: {json.dumps(self.nodes)}',
-                block='opc_manager',
-                level=NotificationLevel.ERROR,
-            )
-        if self.non_receive_count >= 15:
-            await self.emit_metric(
-                metric_object=metrics.OPC_RECONNECTIONS_TOTAL,
-                tags={
-                    'pod_id': self.pod_id,
-                    'server_name': self.name,
-                },
-            )
-            await self.send_notification_async(
-                metadata=self.metadata,
-                notification_id=f'OPC_CONNECTION_RETRY__{self.name}',
-                message=f'Retrying to connect to server {self.name}',
-                block='opc_manager',
-                level=NotificationLevel.ERROR,
-            )
-            return True
-        return False

ingestor/managers/server_reader.py

@@ -1,13 +1,11 @@
 """Per-server read path driven by the IngestorManager (SIENTIAPDE-1974).
 
-Replaces the legacy OpcManager behind the same interface the IngestorManager
-already uses (connect / create_subscription / subscribe / unsubscribe /
-check_cycles / check_opc_listenning / shutdown), so the slot lease loop is
-unchanged. The difference is underneath:
+Exposes the interface the IngestorManager drives (connect / create_subscription /
+subscribe / unsubscribe / check_cycles / check_opc_listenning / shutdown):
 
-- the connection comes from the shared OpcConnectionPool, so the certificate is
-  fetched from MinIO by CertProvider (Basic256Sha256) — never from a file shipped
-  in the Gitea repository;
+- the connection comes from the shared OpcConnectionPool, with the slot's security
+  mode and policy and the client certificate the slot names, which CertProvider
+  downloads from MinIO (no certificate: no security);
 - readings go through SubscriptionEngine, which fans each value out to every
   `names` entry of the slot v2 (one document per collection / tag name / model).
 """
@@ -58,7 +56,7 @@ class ServerReader:
 
     async def connect(self):
         """Open (or reuse) the pooled connection; raises if the server is unreachable
-        or the certificate can't be fetched, like OpcManager.connect did."""
+        or the certificate can't be fetched."""
         await self.pool.get(self.server_id, self.server_cfg)
 
     async def create_subscription(self, slot: str):