gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1974

Browse Source 
SIENTIAPDE-1974-connectors-pi-web-api: Implement OPC UA client certificate handling and update connector security model. Connectors now define security using security.mode and security.policy. Client certificate details (MinIO paths) are fetched from the opc_certificates collection via a MongoDB $lookup and included in the OPC slot. A warning is issued if a connector's client_certificate_id does not resolve to an active client certificate. Documentation (README, CLAUDE.md) and tests have been updated.
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-29 12:21:11 -03:00
parent 5b47726a64
commit 0b928a0ea3
19 changed files with 595 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
tests/orchestrator/activities/test_formatters.py
View File

@@ -7,6 +7,12 @@ from sientia_do.notifications.models import NotificationLevel
from orchestrator.activities.formatters import Formatters, schedule_config_hash
from orchestrator.utils.pipeline_configs import predictions_batch
from tests.orchestrator.utils.tag_database_fixtures import (
CERTIFICATE,
NO_SECURITY,
SECURE,
secure_connector_fields,
)
@fixture
@@ -237,7 +243,7 @@ slot_connectors = [
'connector_id': 'kepserver',
'name': 'kepserver',
'url': 'opc.tcp://kep:49320',
'server_uri': 'urn:kep',
'security': NO_SECURITY,
'status': 'Active',
},
]
@@ -293,8 +299,8 @@ def test_process_slots(formatters):
'server_id': 'kepserver',
'name': 'kepserver',
'url': 'opc.tcp://kep:49320',
'server_uri': 'urn:kep',
'use_security': False,
'security': NO_SECURITY,
'certificate': None,
}
assert result == {
'1': {
@@ -392,10 +398,9 @@ def test_process_slots_with_tag_database(formatters):
'type': 'opc_ua',
'name': 'plant',
'url': 'opc.tcp://plant:4840',
'server_uri': 'urn:plant',
'use_security': True,
'subscription_period_ms': 1500,
'status': 'Active',
**secure_connector_fields(),
}
],
'connector_tags': [
@@ -414,9 +419,9 @@ def test_process_slots_with_tag_database(formatters):
'server_id': '3',
'name': 'plant',
'url': 'opc.tcp://plant:4840',
'server_uri': 'urn:plant',
'use_security': True,
'subscription_period_ms': 1500,
'security': SECURE,
'certificate': CERTIFICATE,
'tags': {
'ns=3;i=1001': {
'frequency': 1000,
@@ -491,6 +496,42 @@ def test_process_slots_notifies_unresolved_tags_once(formatters):
assert formatters.send_notification.call_count == 3
def test_process_slots_warns_about_a_missing_certificate(formatters):
"""A client_certificate_id without an active client certificate: the server goes into
the slot without a certificate, and a warning names the connector (D5)"""
connectors = [
{**slot_connectors[0], **secure_connector_fields(certificate_id=9, certificates=[])},
{
**slot_connectors[0],
'connector_id': 'plant',
'name': 'plant',
**secure_connector_fields(),
},
]
connector_tags = [*slot_connector_tags, _read_tag_doc(7, 'ns=3;i=7', 'P', 'plant')]
pipelines = [
_scouter_pipeline('sin', 'm1', [{'tag_id': 1, 'frequency': 1000}]),
_scouter_pipeline('cos', 'm2', [{'tag_id': 7, 'frequency': 1000}]),
]
result = formatters.process_slots(
_slot_input(pipelines, [], connectors=connectors, connector_tags=connector_tags)
)
assert result['1']['kepserver']['certificate'] is None
assert result['1']['kepserver']['security'] == SECURE
assert result['1']['plant']['certificate'] == CERTIFICATE
formatters.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='ORCHESTRATOR_CERTIFICATE_NOT_FOUND_DURING_SLOT_CONFIGURATION',
message='Connectors kepserver have a security.client_certificate_id without an active '
'client certificate in opc_certificates: they go into the slot without a certificate, '
'so without security',
block='orchestrator',
level=NotificationLevel.WARNING,
)
def test_process_slots_survives_malformed_tag_database(formatters):
"""An incomplete connector or tag leaves only its own read tags out of the slot"""
pipelines = [
@@ -504,7 +545,7 @@ def test_process_slots_survives_malformed_tag_database(formatters):
],
),
]
connectors = [*slot_connectors, {'connector_id': 3, 'name': 'plant', 'server_uri': 'urn:plant'}]
connectors = [*slot_connectors, {'connector_id': 3, 'name': 'plant', 'security': NO_SECURITY}]
connector_tags = [
*slot_connector_tags,
{'tag_id': 7, 'connector_id': 3, 'address': 'ns=3;i=1', 'tag_name': 'P'},
@@ -554,7 +595,7 @@ def test_process_schedules_names_scouter_tags_like_the_slot(formatters):
'connector_id': 3,
'name': 'plant',
'url': 'opc.tcp://plant:4840',
'server_uri': 'urn:plant',
'security': NO_SECURITY,
}
]
@@ -1470,7 +1511,7 @@ PLANT_CONNECTOR = {
'connector_id': 'plant',
'name': 'plant',
'url': 'opc.tcp://plant:4840',
'server_uri': 'urn:plant',
'security': NO_SECURITY,
'status': 'Active',
}

31
tests/orchestrator/utils/tag_database_fixtures.py
View File

@@ -17,6 +17,35 @@ TAGS_BY_ID = {
}
# Connector security and client certificates (SIENTIAPDE-1974)
NO_SECURITY = {'mode': 'None', 'policy': 'None'}
SECURE = {'mode': 'SignAndEncrypt', 'policy': 'Basic256Sha256'}
# A client certificate as connectors_query's $lookup brings it (certificates[])
def certificate_doc(certificate_id=1, name='sientia-platform', **extra):
return {
'certificate_id': certificate_id,
'application_uri': 'urn:sientia:opc-ingestor',
'cert': {'bucket': 'opc-certificates', 'object_name': f'client/{name}/cert.pem'},
'private_key': {'bucket': 'opc-certificates', 'object_name': f'client/{name}/key.pem'},
**extra,
}
# The slot's certificate for certificate_doc(): the same fields
CERTIFICATE = certificate_doc()
def secure_connector_fields(certificate_id=1, certificates=None):
"""security and certificates of a connector with a client certificate."""
return {
'security': {**SECURE, 'client_certificate_id': certificate_id},
'certificates': [certificate_doc(certificate_id)] if certificates is None else certificates,
}
# Write tags (predictions_batch write_tags through the tag database)
OPC_CONNECTORS = {
@@ -24,7 +53,7 @@ OPC_CONNECTORS = {
'connector_id': 'plant',
'name': 'plant',
'url': 'opc.tcp://plant:4840',
'server_uri': 'urn:plant',
'security': NO_SECURITY,
},
}

104
tests/orchestrator/utils/test_opc_slots.py
View File

@@ -1,12 +1,21 @@
import pytest
from orchestrator.utils.opc_slots import (
build_tag_config,
gather_read_tags,
)
from tests.orchestrator.utils.tag_database_fixtures import (
CERTIFICATE,
NO_SECURITY,
SECURE,
TAGS_BY_ID,
certificate_doc,
scouter_pipeline_doc,
secure_connector_fields,
)
NO_ISSUES = {'tags': set(), 'connectors': set(), 'invalid': set(), 'certificates': set()}
def _name(schedule_name, tag_name, model_id):
return {
@@ -26,7 +35,7 @@ def _connector(connector_id, name, **extra):
'connector_id': connector_id,
'name': name,
'url': f'opc.tcp://{name}:4840',
'server_uri': f'urn:{name}',
'security': NO_SECURITY,
**extra,
}
@@ -41,8 +50,8 @@ def _db_tag(tag_id, connector_id, address, tag_name):
connectors = {
'1': _connector(1, 'kepserver', use_security=False),
'3': _connector(3, 'plant', use_security=True, subscription_period_ms=1500),
'1': _connector(1, 'kepserver'),
'3': _connector(3, 'plant', subscription_period_ms=1500, **secure_connector_fields()),
}
@@ -77,7 +86,7 @@ def test_gather_read_tags_multi_name():
assert entries[0]['address'] == 'ns=2;s=A'
assert entries[0]['frequency'] == 5000
assert entries[0]['names'] == [_name('cos', 'Contador', 'm2'), _name('sin', 'Counter', 'm1')]
assert issues == {'tags': set(), 'connectors': set(), 'invalid': set()}
assert issues == NO_ISSUES
def test_gather_read_tags_same_name_in_two_pipelines():
@@ -125,9 +134,9 @@ def test_gather_read_tags_with_tag_database():
'server_id': '3',
'name': 'plant',
'url': 'opc.tcp://plant:4840',
'server_uri': 'urn:plant',
'use_security': True,
'subscription_period_ms': 1500,
'security': SECURE,
'certificate': CERTIFICATE,
'key': 'plant',
},
'address': 'ns=3;i=1001',
@@ -148,7 +157,7 @@ def test_gather_read_tags_unknown_tag_id():
entries, issues = _gather([scouter_pipeline_doc('sin', 'm1', [_read(404)])])
assert entries == []
assert issues == {'tags': {'404'}, 'connectors': set(), 'invalid': set()}
assert issues == {**NO_ISSUES, 'tags': {'404'}}
def test_gather_read_tags_inactive_connector():
@@ -156,7 +165,7 @@ def test_gather_read_tags_inactive_connector():
entries, issues = _gather([scouter_pipeline_doc('sin', 'm1', [_read(8)])])
assert entries == []
assert issues == {'tags': set(), 'connectors': {'5'}, 'invalid': set()}
assert issues == {**NO_ISSUES, 'connectors': {'5'}}
def test_gather_read_tags_without_tag_id():
@@ -179,11 +188,7 @@ def test_gather_read_tags_without_tag_id():
entries, issues = _gather(pipelines)
assert [e['address'] for e in entries] == ['ns=2;s=A']
assert issues == {
'tags': set(),
'connectors': set(),
'invalid': {'sin/b', 'sin/c', 'sin/d', 'sin/11'},
}
assert issues == {**NO_ISSUES, 'invalid': {'sin/b', 'sin/c', 'sin/d', 'sin/11'}}
def test_gather_read_tags_server_name_clash():
@@ -241,11 +246,11 @@ def test_gather_read_tags_without_read_tags():
pipeline = scouter_pipeline_doc('sin', 'm1', [])
del pipeline['read_tags']
assert _gather([pipeline]) == ([], {'tags': set(), 'connectors': set(), 'invalid': set()})
assert _gather([pipeline]) == ([], NO_ISSUES)
def test_build_tag_config():
"""Slot v2 (contract C1): no certificate paths, subscription period per server"""
"""Slot v2 (contract C1): security, certificate and subscription period per server"""
pipelines = [
scouter_pipeline_doc('sin', 'm1', [_read(10, 1000), _read(11, '300'), _read(7, 100)])
]
@@ -258,8 +263,9 @@ def test_build_tag_config():
'server_id': '1',
'name': 'kepserver',
'url': 'opc.tcp://kepserver:4840',
'server_uri': 'urn:kepserver',
'use_security': False,
# No client certificate: no security (D5)
'security': NO_SECURITY,
'certificate': None,
# No subscription_period_ms on the connector: half its lowest tag frequency
'subscription_period_ms': 150,
'tags': {
@@ -279,9 +285,9 @@ def test_build_tag_config():
'server_id': '3',
'name': 'plant',
'url': 'opc.tcp://plant:4840',
'server_uri': 'urn:plant',
'use_security': True,
'subscription_period_ms': 1500,
'security': SECURE,
'certificate': CERTIFICATE,
'tags': {
'ns=3;i=1001': {
'frequency': 100,
@@ -382,3 +388,63 @@ def test_gather_read_tags_skips_incomplete_pipelines():
assert [e['address'] for e in entries] == ['ns=2;s=B']
assert issues['invalid'] == {'no-model', 'no-name', 'no-id', 'None'}
def _plant_with(**fields):
return {'1': connectors['1'], '3': {**connectors['3'], **fields}}
@pytest.mark.parametrize(
'fields',
[
# $lookup found nothing: missing id, inactive or not a client certificate
pytest.param(secure_connector_fields(certificates=[]), id='not-found'),
pytest.param(
secure_connector_fields(certificates=[certificate_doc(application_uri='')]),
id='no-application-uri',
),
pytest.param(
secure_connector_fields(
certificates=[{**certificate_doc(), 'private_key': {'bucket': 'b'}}]
),
id='key-without-object',
),
],
)
def test_gather_read_tags_registered_certificate_missing(fields):
"""An id without a usable certificate: in the slot without a certificate, with a warning (D5)"""
entries, issues = _gather(
[scouter_pipeline_doc('sin', 'm1', [_read(7)])], _plant_with(**fields)
)
assert entries[0]['server']['certificate'] is None
assert entries[0]['server']['security'] == SECURE
assert issues == {**NO_ISSUES, 'certificates': {'3'}}
@pytest.mark.parametrize('certificate_id', [None, 'absent'])
def test_gather_read_tags_without_certificate_id(certificate_id):
"""No client_certificate_id: no certificate and nothing to report (D5)"""
security = {**NO_SECURITY, 'client_certificate_id': certificate_id}
if certificate_id == 'absent':
security = NO_SECURITY
entries, issues = _gather(
[scouter_pipeline_doc('sin', 'm1', [_read(7)])],
_plant_with(security=security, certificates=[]),
)
assert entries[0]['server']['certificate'] is None
assert issues == NO_ISSUES
def test_gather_read_tags_reports_a_certificate_once_per_connector():
entries, issues = _gather(
[
scouter_pipeline_doc('sin', 'm1', [_read(7)]),
scouter_pipeline_doc('cos', 'm2', [_read(7)]),
],
_plant_with(**secure_connector_fields(certificates=[])),
)
assert len(entries) == 1
assert issues['certificates'] == {'3'}

67
tests/orchestrator/utils/test_tag_database.py
View File

@@ -1,19 +1,27 @@
import pytest
from orchestrator.utils.tag_database import (
WRITE_CATEGORIES,
client_certificate_id,
filter_by_tag_ids,
index_connector_tags,
index_connectors,
index_tag_documents,
referenced_tag_ids,
resolve_certificate,
resolve_tag_names,
resolve_write_tags,
tag_id_of,
)
from tests.orchestrator.utils.tag_database_fixtures import (
CERTIFICATE,
NO_SECURITY,
OPC_CONNECTORS,
TAGS_BY_ID,
batch_pipeline_doc,
certificate_doc,
scouter_pipeline_doc,
secure_connector_fields,
write_tag_doc,
)
@@ -92,9 +100,23 @@ def test_index_skips_incomplete_documents():
{'tag_id': 9, 'connector_id': 3, 'address': '', 'tag_name': 'no address'},
]
connector_list = [
{'connector_id': 3, 'name': 'plant', 'url': 'opc.tcp://p:4840', 'server_uri': 'urn:p'},
{'connector_id': 4, 'name': 'no url', 'server_uri': 'urn:x'},
{'name': 'no id', 'url': 'opc.tcp://x:4840', 'server_uri': 'urn:x'},
{'connector_id': 3, 'name': 'plant', 'url': 'opc.tcp://p:4840', 'security': NO_SECURITY},
{'connector_id': 4, 'name': 'no url', 'security': NO_SECURITY},
{'name': 'no id', 'url': 'opc.tcp://x:4840', 'security': NO_SECURITY},
# security.mode and security.policy are the source of the OPC security (D4)
{'connector_id': 5, 'name': 'no security', 'url': 'opc.tcp://x:4840'},
{
'connector_id': 6,
'name': 'no mode',
'url': 'opc.tcp://x:4840',
'security': {'policy': 'None'},
},
{
'connector_id': 7,
'name': 'no policy',
'url': 'opc.tcp://x:4840',
'security': {'mode': 'None'},
},
]
assert list(index_connector_tags(tags)) == [7]
@@ -216,3 +238,42 @@ def test_resolve_write_tags_issues():
"{'tag_id': None}",
'ns=2;s=raw',
}
def test_resolve_certificate():
"""Only the fields the gateway needs: dates and other fields stay out of the slot"""
connector = {
**OPC_CONNECTORS['plant'],
**secure_connector_fields(
certificates=[certificate_doc(thumbprint_sha1='ab', not_after=None)]
),
}
assert resolve_certificate(connector) == CERTIFICATE
assert client_certificate_id(connector) == 1
@pytest.mark.parametrize(
'certificates',
[
pytest.param(None, id='no-lookup'),
pytest.param([], id='not-found'),
pytest.param([certificate_doc(application_uri=None)], id='no-application-uri'),
pytest.param([{**certificate_doc(), 'cert': None}], id='no-cert'),
pytest.param(
[{**certificate_doc(), 'private_key': {'object_name': 'k'}}], id='key-no-bucket'
),
pytest.param(['not a document'], id='not-a-document'),
],
)
def test_resolve_certificate_without_a_usable_certificate(certificates):
connector = {**OPC_CONNECTORS['plant'], **secure_connector_fields()}
connector['certificates'] = certificates
assert resolve_certificate(connector) is None
def test_client_certificate_id_without_one():
assert client_certificate_id(OPC_CONNECTORS['plant']) is None
assert client_certificate_id({'connector_id': 'x'}) is None
assert client_certificate_id({'security': None}) is None