gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1974

Browse Source 
SIENTIAPDE-1974-connectors-pi-web-api: Implement OPC UA client certificate handling and update connector security model. Connectors now define security using security.mode and security.policy. Client certificate details (MinIO paths) are fetched from the opc_certificates collection via a MongoDB $lookup and included in the OPC slot. A warning is issued if a connector's client_certificate_id does not resolve to an active client certificate. Documentation (README, CLAUDE.md) and tests have been updated.
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-29 12:21:11 -03:00
parent 5b47726a64
commit 0b928a0ea3
19 changed files with 595 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
orchestrator/activities/formatters.py
View File

@@ -43,23 +43,35 @@ topic_separator = '\n ========== \n'
# Schedule fields left out of config_hash: updated_at is compared on its own
CONFIG_HASH_EXCLUDED_FIELDS = ('updated_at', 'config_hash')
# Read tags left out of the slots, by gather_read_tags issue kind
# Read tags left out of the slots, by gather_read_tags issue kind:
# (kind, notification_id, message, level)
SLOT_ISSUES = (
(
'tags',
'ORCHESTRATOR_TAG_NOT_FOUND_DURING_SLOT_CONFIGURATION',
'Tags {} not found among the active, complete Read tags of connector_tags',
NotificationLevel.ERROR,
),
(
'connectors',
'ORCHESTRATOR_CONNECTOR_NOT_FOUND_DURING_SLOT_CONFIGURATION',
'Connectors {} not found, not active or incomplete in connectors',
NotificationLevel.ERROR,
),
(
'invalid',
'ORCHESTRATOR_INVALID_READ_TAG_DURING_SLOT_CONFIGURATION',
'Read tags {} have no integer tag_id, no integer frequency or an active_read without an '
'integer cycle_multiple; or their pipeline lacks schedule_name, model_id or model.name',
NotificationLevel.ERROR,
),
(
'certificates',
'ORCHESTRATOR_CERTIFICATE_NOT_FOUND_DURING_SLOT_CONFIGURATION',
'Connectors {} have a security.client_certificate_id without an active client '
'certificate in opc_certificates: they go into the slot without a certificate, so '
'without security',
NotificationLevel.WARNING,
),
)
@@ -71,16 +83,19 @@ WRITE_TAG_ISSUES = (
'ORCHESTRATOR_WRITE_TAG_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION',
'Write tags {} not found among the active, complete Write and Prediction Confidence '
'tags of connector_tags',
NotificationLevel.ERROR,
),
(
'connectors',
'ORCHESTRATOR_WRITE_TAG_CONNECTOR_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION',
'Connectors {} of write tags not found, not active or incomplete in connectors',
NotificationLevel.ERROR,
),
(
'invalid',
'ORCHESTRATOR_INVALID_WRITE_TAG_DURING_SCHEDULE_CONFIGURATION',
'Write tags {} have no integer tag_id or a data_type the OPC Gateway does not accept',
NotificationLevel.ERROR,
),
)
@@ -330,14 +345,17 @@ class Formatters(SientiaMonitoring):
The division is by address: all the names of an address stay in the same slot
(SIENTIAPDE-1974, O5). Read tags that cannot be resolved are left out and
notified once per run: a read tag without tag_id or a usable frequency, a tag_id
missing from connector_tags, or a connector missing or inactive.
missing from connector_tags, or a connector missing or inactive. A connector whose
client_certificate_id has no active client certificate goes into the slot without
a certificate, with a warning (D5).
Args:
- input_data (dict[str, Any]): The input data containing
the schedules to process.
- pipelines (list[dict[str, Any]]): The schedules to process.
- connectors (list[dict[str, Any]], optional): Active OPC UA connectors of
the tag database, the only source of the OPC servers.
the tag database, the only source of the OPC servers, each with the
certificates of its client_certificate_id (connectors_query).
- connector_tags (list[dict[str, Any]], optional): The connector_tags the
pipelines reference; only the active Read tags are used.
- active_ingestors (list[str]): The active ingestors to divide into slots.
@@ -374,7 +392,7 @@ class Formatters(SientiaMonitoring):
def _notify_issues(
self,
issues: dict[str, set[str]],
kinds: tuple[tuple[str, str, str], ...],
kinds: tuple[tuple[str, str, str, str], ...],
metadata: dict[str, Any],
) -> None:
"""
@@ -382,18 +400,19 @@ class Formatters(SientiaMonitoring):
Args:
issues (dict[str, set[str]]): What was left out, by kind
kinds (tuple[tuple[str, str, str], ...]): (kind, notification_id, message with
one {} for the joined values), e.g. SLOT_ISSUES or WRITE_TAG_ISSUES
kinds (tuple[tuple[str, str, str, str], ...]): (kind, notification_id, message
with one {} for the joined values, level), e.g. SLOT_ISSUES or
WRITE_TAG_ISSUES
metadata (dict[str, Any]): Metadata for the notifications
"""
for kind, notification_id, message in kinds:
for kind, notification_id, message, level in kinds:
if issues.get(kind):
self.send_notification(
metadata=metadata,
notification_id=notification_id,
message=message.format(', '.join(sorted(issues[kind]))),
block='orchestrator',
level=NotificationLevel.ERROR,
level=level,
)
@activity.defn(name='format_schedule_config')

54
orchestrator/utils/opc_slots.py
View File

@@ -2,18 +2,35 @@
from typing import Any, cast
from orchestrator.utils.tag_database import is_complete, resolve_tag_names, tag_id_of
from orchestrator.utils.tag_database import (
client_certificate_id,
is_complete,
resolve_certificate,
resolve_tag_names,
tag_id_of,
)
def _connector_server(connector: dict[str, Any]) -> dict[str, Any]:
"""Slot server fields of a tag database connector (connectors collection)."""
def _connector_server(connector: dict[str, Any], issues: dict[str, set[str]]) -> dict[str, Any]:
"""
Slot server fields of a tag database connector (connectors collection).
A connector whose client_certificate_id has no active client certificate still goes
into the slot, without a certificate, and is recorded in issues['certificates'] (D5).
"""
certificate = resolve_certificate(connector)
if certificate is None and client_certificate_id(connector) is not None:
issues['certificates'].add(str(connector['connector_id']))
return {
'server_id': str(connector['connector_id']),
'name': connector['name'],
'url': connector['url'],
'server_uri': connector['server_uri'],
'use_security': bool(connector.get('use_security', False)),
'subscription_period_ms': connector.get('subscription_period_ms'),
'security': {
'mode': connector['security']['mode'],
'policy': connector['security']['policy'],
},
'certificate': certificate,
}
@@ -35,7 +52,8 @@ def _locate_read_tag(
Server and address of a read tag, or None when they cannot be resolved.
The address and connector come from connector_tags and the server from connectors
(RF3). What cannot be resolved is recorded in issues ('tags' or 'connectors').
(RF3). What cannot be resolved is recorded in issues ('tags' or 'connectors'), and a
server without its registered certificate in issues['certificates'].
"""
connector_tag = tags_by_id.get(tag_id)
if connector_tag is None:
@@ -46,7 +64,7 @@ def _locate_read_tag(
if connector is None:
issues['connectors'].add(connector_id)
return None
return _connector_server(connector), connector_tag['address']
return _connector_server(connector, issues), connector_tag['address']
def _cycle_multiple(active_read: Any) -> int | None:
@@ -195,9 +213,10 @@ def gather_read_tags(
Returns:
tuple[list[dict[str, Any]], dict[str, set[str]]]:
- One entry per (server, address), sorted by server_id then address, with:
- server (dict): server_id, name, url, server_uri, use_security,
subscription_period_ms (None = derive from the frequencies) and key
(the server's unique slot key)
- server (dict): server_id, name, url, subscription_period_ms (None =
derive from the frequencies), security ({mode, policy}), certificate
(resolve_certificate; None = connect without security) and key (the
server's unique slot key)
- address (str): The OPC node address
- frequency (int): The lowest frequency among the pipelines (RF6)
- active_read (dict): {'enabled': False} unless a pipeline enables it (D3)
@@ -206,12 +225,14 @@ def gather_read_tags(
- What was left out: 'tags' (tag_id), 'connectors' (connector_id) and
'invalid' (schedule_name/read tag without an int tag_id, a usable frequency or
cycle_multiple, or the schedule_name of a pipeline without schedule_name,
model_id or model.name)
model_id or model.name); and 'certificates': connector_ids that went into the
slot without the certificate their client_certificate_id points to
"""
issues: dict[str, set[str]] = {
'tags': set(),
'connectors': set(),
'invalid': set(),
'certificates': set(),
}
servers: dict[str, dict[str, Any]] = {}
entries: dict[tuple[str, str], dict[str, Any]] = {}
@@ -238,15 +259,16 @@ def gather_read_tags(
def build_tag_config(entries: list[dict[str, Any]]) -> dict[str, Any]:
"""
Build one slot in the slot v2 format (contract C1) from address entries. The slot
carries no certificate: the gateway takes it from MinIO (SIENTIAPDE-1974, O4).
Build one slot in the slot v2 format (contract C1) from address entries. Each server
carries its security and the MinIO objects of its client certificate, which the
gateway downloads (SIENTIAPDE-1974); certificate None means no security (D5).
Args:
entries (list[dict[str, Any]]): Address entries from gather_read_tags for this slot
Returns:
dict[str, Any]: The slot keyed by server key, each server with server_id, name,
url, server_uri, use_security, subscription_period_ms and tags. Each tag (keyed
url, subscription_period_ms, security, certificate and tags. Each tag (keyed
by address) has frequency, active_read and names. A connector without a
subscription_period_ms gets half its lowest tag frequency in this slot.
"""
@@ -259,9 +281,9 @@ def build_tag_config(entries: list[dict[str, Any]]) -> dict[str, Any]:
'server_id': server['server_id'],
'name': server['name'],
'url': server['url'],
'server_uri': server['server_uri'],
'use_security': server['use_security'],
'subscription_period_ms': server['subscription_period_ms'],
'security': server['security'],
'certificate': server['certificate'],
'tags': {},
}

72
orchestrator/utils/tag_database.py
View File

@@ -11,7 +11,17 @@ from typing import Any
CONNECTOR_TAG_FIELDS = ('tag_id', 'connector_id', 'address', 'tag_name')
CONNECTOR_FIELDS = ('connector_id', 'name', 'url', 'server_uri')
CONNECTOR_FIELDS = ('connector_id', 'name', 'url')
# security fields of an OPC UA connector: the source of its OPC security (D4)
SECURITY_FIELDS = ('mode', 'policy')
# Fields of a client certificate the slot carries, and of each object it names in MinIO
CERTIFICATE_FIELDS = ('certificate_id', 'application_uri')
CERTIFICATE_OBJECTS = ('cert', 'private_key')
STORED_OBJECT_FIELDS = ('bucket', 'object_name')
# connector_tags categories each use accepts
@@ -146,8 +156,8 @@ def index_connectors(connectors: list[dict[str, Any]]) -> dict[str, dict[str, An
"""
Index the tag database's connectors by connector_id (as text).
Documents that are not active or lack connector_id, name, url or server_uri are
left out.
Documents that are not active or lack connector_id, name, url, security.mode or
security.policy are left out.
Args:
connectors (list[dict[str, Any]]): connectors documents (SIENTIAPDE-1569)
@@ -158,10 +168,64 @@ def index_connectors(connectors: list[dict[str, Any]]) -> dict[str, dict[str, An
return {
str(connector['connector_id']): connector
for connector in connectors
if is_complete(connector, CONNECTOR_FIELDS) and _is_active(connector)
if is_complete(connector, CONNECTOR_FIELDS)
and _has_security(connector)
and _is_active(connector)
}
def _has_security(connector: dict[str, Any]) -> bool:
security = connector.get('security')
return isinstance(security, dict) and is_complete(security, SECURITY_FIELDS)
def client_certificate_id(connector: dict[str, Any]) -> Any:
"""The connector's security.client_certificate_id (None when it has none)."""
security = connector.get('security')
return security.get('client_certificate_id') if isinstance(security, dict) else None
def _is_usable_certificate(certificate: Any) -> bool:
return (
isinstance(certificate, dict)
and is_complete(certificate, CERTIFICATE_FIELDS)
and all(
isinstance(certificate.get(name), dict)
and is_complete(certificate[name], STORED_OBJECT_FIELDS)
for name in CERTIFICATE_OBJECTS
)
)
def resolve_certificate(connector: dict[str, Any]) -> dict[str, Any] | None:
"""
The client certificate a connector goes into the slot with, or None (D5).
connectors_query's $lookup puts in `certificates` the opc_certificates document the
connector's security.client_certificate_id points to, only when it is active and of
kind client. None means no certificate: the gateway connects without security. A
connector with a client_certificate_id and no certificate is most likely a
registration error, reported by the caller (see client_certificate_id).
Args:
connector (dict[str, Any]): A connectors document from connectors_query
Returns:
dict[str, Any] | None: certificate_id, application_uri, and cert and private_key
as {bucket, object_name}; None without a complete certificate
"""
for certificate in connector.get('certificates') or []:
if _is_usable_certificate(certificate):
return {
**{field: certificate[field] for field in CERTIFICATE_FIELDS},
**{
name: {field: certificate[name][field] for field in STORED_OBJECT_FIELDS}
for name in CERTIFICATE_OBJECTS
},
}
return None
def resolve_tag_names(
read_tags: list[dict[str, Any]], tags_by_id: dict[int, dict[str, Any]]
) -> list[dict[str, Any]]:

40
orchestrator/workflows/orchestrator.py
View File

@@ -15,6 +15,7 @@ with workflow.unsafe.imports_passed_through():
# MongoDB aggregation stages
MATCH = '$match'
PROJECT = '$project'
LOOKUP = '$lookup'
# Tag database (SIENTIAPDE-1569) read by the slot v2 and the schedules (SIENTIAPDE-1974).
# The workflow input may override these aggregations (connectors_query /
@@ -22,20 +23,53 @@ PROJECT = '$project'
# connector_tags_query. Every category and status of the referenced tags is loaded: the
# slot uses the active Read tags, predictions_batch the active Write / Prediction
# Confidence ones (status and category are filtered in the code). Only the fields used
# are projected: other attributes may hold BSON types a Temporal payload cannot carry.
# are projected: other attributes may hold BSON types a Temporal payload cannot carry
# (so the certificate dates are left out).
# Each connector brings, in `certificates`, the opc_certificates document its
# security.client_certificate_id points to, only when it is active and of kind client
# (D5); resolve_certificate turns it into the slot's certificate.
DEFAULT_CONNECTORS_QUERY = {
'collection': 'connectors',
'aggregation': [
{MATCH: {'type': 'opc_ua', 'status': 'Active'}},
{
LOOKUP: {
'from': 'opc_certificates',
'let': {'certificate_id': '$security.client_certificate_id'},
'pipeline': [
{
MATCH: {
'$expr': {'$eq': ['$certificate_id', '$$certificate_id']},
'status': 'active',
'kind': 'client',
}
},
{
PROJECT: {
'_id': 0,
'certificate_id': 1,
'application_uri': 1,
'cert.bucket': 1,
'cert.object_name': 1,
'private_key.bucket': 1,
'private_key.object_name': 1,
}
},
],
'as': 'certificates',
}
},
{
PROJECT: {
'_id': 0,
'connector_id': 1,
'name': 1,
'url': 1,
'server_uri': 1,
'use_security': 1,
'subscription_period_ms': 1,
'security.mode': 1,
'security.policy': 1,
'security.client_certificate_id': 1,
'certificates': 1,
}
},
],