gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1973

Browse Source 
SIENTIAPDE-1973: Implement per-connector OPC client certificates and remove legacy read engine.
This update overhauls certificate management, transitioning from a single platform certificate to individual client certificates defined in the slot configuration. The CertProvider now downloads certificates from MinIO based on certificate_id, applying dynamic security policies (mode, policy) from the slot. The legacy READ_ENGINE=legacy option and OpcManager have been entirely removed, standardizing on the ServerReader with strict slot v2 enforcement. Write command resolution (ServerConfigResolver) now consistently uses connectors joined with opc_certificates. The CERT_BUCKET environment variable is no longer used.
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-29 12:39:21 -03:00
parent d68e842e6e
commit ecb055eefb
32 changed files with 1639 additions and 2284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
ingestor/managers/ingestor_manager.py
View File

@@ -1,6 +1,5 @@
import traceback
from copy import deepcopy
from os import getenv
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
@@ -15,16 +14,9 @@ from ingestor.connection.pool import OpcConnectionPool
from ingestor.io.data_sink import DataSink
from ingestor.io.slots import parse_slot
from ingestor.managers.data_manager import DataManager
from ingestor.managers.opc_manager import OpcManager
from ingestor.managers.resource_manager import ResourceManager
from ingestor.managers.server_reader import ServerReader
# READ_ENGINE (SIENTIAPDE-1974): 'v2' = ServerReader/SubscriptionEngine over the shared
# pool, certificate from MinIO (default). 'legacy' = plan B: the untouched rc3
# OpcManager, slot v1 only, certificate files from the Gitea clone (cert_path). Kept
# for validation until the end of 1974; remove once unused and the tests pass without it.
READ_ENGINES = ('v2', 'legacy')
class IngestorManager(SientiaMonitoring):
"""
@@ -85,7 +77,6 @@ class IngestorManager(SientiaMonitoring):
notification_handler: NotificationHandler,
metrics_controller: MetricsController,
gateway_config: GatewayConfig | None = None,
read_engine: str | None = None,
):
redis_host: str = redis_data['host']
redis_port: int = int(redis_data['port'])
@@ -108,12 +99,12 @@ class IngestorManager(SientiaMonitoring):
metrics_controller=metrics_controller,
)
self.opc_managers: dict = {}
# Read path (SIENTIAPDE-1974): pooled connections with the certificate from
# MinIO. Nothing here touches the network at construction (MinIO is lazy), so
# a MinIO/Postgres/NATS outage can't stop the reads before they start.
# Read path (SIENTIAPDE-1974): pooled connections with each server's client
# certificate (named in the slot) from MinIO. Nothing here touches the network at
# construction (MinIO is lazy), so a MinIO/Postgres/NATS outage can't stop the
# reads before they start.
config = gateway_config or GatewayConfig.certs_from_env()
self.cert_provider = build_cert_provider(
mongo=self.data_manager.mongo_repository,
config=config,
logger=logger,
notification_handler=notification_handler,
@@ -124,10 +115,6 @@ class IngestorManager(SientiaMonitoring):
self.cert_provider, logger, notification_handler, metrics_controller
)
self.sink = DataSink(self.data_manager, None)
self._server_cert_warned: set[str] = set()
self.read_engine = self._resolve_read_engine(
read_engine or getenv('READ_ENGINE') or 'v2', logger
)
self.resource_manager = ResourceManager(
host=redis_host,
port=redis_port,
@@ -147,32 +134,7 @@ class IngestorManager(SientiaMonitoring):
self.metadata = metadata
@staticmethod
def _resolve_read_engine(value: str, logger: Logger) -> str:
engine = value.strip().lower()
if engine not in READ_ENGINES:
logger.warning(f'Invalid READ_ENGINE {value!r}, using v2 ({"|".join(READ_ENGINES)})')
engine = 'v2'
logger.info(f'OPC read engine: {engine}')
return engine
def _build_reader(self, server_config: dict) -> ServerReader | OpcManager:
if self.read_engine == 'legacy':
# Plan B: the rc3 OpcManager, unchanged (own connection, cert files).
return OpcManager(
name=server_config['name'],
url=server_config['url'],
subscription_period_ms=server_config['subscription_period_ms'],
data_manager=self.data_manager,
logger=self.logger,
server_uri=server_config['server_uri'],
notification_handler=self.notification_handler,
metadata=self.metadata,
cert_path=server_config.get('cert_path'),
private_key_path=server_config.get('private_key_path'),
server_cert_path=server_config.get('server_cert_path'),
metrics_controller=self.metrics_controller,
)
def _build_reader(self, server_config: dict) -> ServerReader:
return ServerReader(
name=server_config['name'],
server_config=server_config,
@@ -184,24 +146,21 @@ class IngestorManager(SientiaMonitoring):
metadata=self.metadata,
)
async def initialize_opc_from_config(
self, server_config: dict
) -> ServerReader | OpcManager | None:
async def initialize_opc_from_config(self, server_config: dict) -> ServerReader | None:
"""
Initializes the reader of one OPC UA server from its slot configuration.
With READ_ENGINE=v2 (default) creates a ServerReader and opens its pooled
connection; with `use_security`, the certificate comes from MinIO. With
READ_ENGINE=legacy (plan B) creates the rc3 OpcManager, which connects on its
own with the certificate files of the slot v1 (`cert_path`).
Creates a ServerReader and opens its pooled connection. With a `certificate`, the
connection uses the slot's security mode and policy, with the certificate
downloaded from MinIO; without one, it connects without security (D5).
Args:
server_config (dict): The server configuration of the slot, without the
'tags' key. v2: 'name', 'server_id', 'url', 'server_uri',
'use_security', 'subscription_period_ms'. legacy: the slot v1 fields.
server_config (dict): The server configuration of the slot v2, without the
'tags' key: 'name', 'server_id', 'url', 'subscription_period_ms',
'security' and 'certificate'.
Returns:
ServerReader | OpcManager | None: The connected reader, or None if the
ServerReader | None: The connected reader, or None if the
connection fails (logged and notified, so the other servers keep working).
"""
@@ -254,11 +213,7 @@ class IngestorManager(SientiaMonitoring):
the shared connection stays for writes/gRPC until the idle sweep.
"""
if server in self.opc_managers:
if self.read_engine == 'legacy':
# OpcManager owns its connection: shutdown always disconnects it.
await self.opc_managers[server].shutdown()
else:
await self.opc_managers[server].shutdown(evict=evict)
await self.opc_managers[server].shutdown(evict=evict)
del self.opc_managers[server]
for slot, _config in self.managed_tags.items():
self.managed_tags[slot].pop(server, None)
@@ -567,14 +522,11 @@ class IngestorManager(SientiaMonitoring):
if slot in reader.subscriptions:
await self._unsubscribe_reader(server, reader, slot)
async def _unsubscribe_reader(
self, server: str, reader: ServerReader | OpcManager, slot: str
) -> None:
async def _unsubscribe_reader(self, server: str, reader: ServerReader, slot: str) -> None:
"""
Unsubscribe one reader from a slot without letting a failure stop the main loop.
A failure (e.g. the legacy OpcManager, whose delete() raises when the server
session is gone) increments OPC_SUBSCRIPTION_ERRORS, sends
A failure (e.g. delete() raising when the server session is gone) increments OPC_SUBSCRIPTION_ERRORS, sends
OPC_UNSUBSCRIBE_ERROR_<slot>:<server> and is logged. The slot is dropped from the
reader's books, as the v2 engine already does, so it can be subscribed again.
@@ -584,7 +536,7 @@ class IngestorManager(SientiaMonitoring):
Args:
server (str): The reader's key in opc_managers
reader (ServerReader | OpcManager): The OPC reader
reader (ServerReader): The OPC reader
slot (str): The slot to unsubscribe
"""
try:
@@ -664,22 +616,17 @@ class IngestorManager(SientiaMonitoring):
def parse_slot_config(self, slot: str, raw: dict) -> dict | None:
"""
Normalizes the raw slot read from Redis to the slot v2 structure.
Normalizes the raw slot read from Redis to the slot v2 structure (contract C1).
Accepts slot v1 (`topics`) and v2 (`names`), so the gateway and the
Orchestrator can be deployed in any order (SIENTIAPDE-1974). With
READ_ENGINE=legacy the raw slot v1 is kept as is (what OpcManager reads), and
a slot v2 is rejected.
Only the slot v2 is accepted (SIENTIAPDE-1974): the gateway and the Orchestrator
are deployed together, and a slot v1 (`topics`, no `security`) is rejected.
Returns:
dict | None: The parsed slot, or None when the config is malformed
(logged and notified).
"""
try:
if self.read_engine == 'legacy':
self._validate_slot_v1(raw)
return raw
parsed = parse_slot(raw)
return parse_slot(raw)
except (AttributeError, KeyError, TypeError, ValueError) as e:
self.send_notification(
metadata=self.metadata,
@@ -690,32 +637,6 @@ class IngestorManager(SientiaMonitoring):
)
self.logger.error(f'Slot {slot} has an invalid configuration: {e!r}')
return None
self._warn_server_cert_path(raw)
return parsed
@staticmethod
def _validate_slot_v1(raw: dict) -> None:
"""READ_ENGINE=legacy: OpcManager only understands slot v1 (`tag_name` +
`topics` per tag); a v2 slot (`names`) would fail in its datachange callback."""
for name, server in raw.items():
for address, tag in server['tags'].items():
if 'topics' not in tag or 'tag_name' not in tag or 'frequency' not in tag:
raise ValueError(
f'READ_ENGINE=legacy needs a slot v1 (tag_name + topics); '
f'server {name!r}, tag {address!r} is not'
)
def _warn_server_cert_path(self, raw: dict) -> None:
"""Logs once per server when a slot still pins a server certificate
(`server_cert_path`): connections now come from the pool with the platform
certificate from MinIO, and that file-based pin is no longer applied."""
for name, server in raw.items():
if server.get('server_cert_path') and name not in self._server_cert_warned:
self._server_cert_warned.add(name)
self.logger.warning(
f'OPC server {name}: server_cert_path is set in the slot but is no '
f'longer applied (server certificate validation is disabled)'
)
async def drop_slot_leases(self, ids: list[str]) -> None:
"""