gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1973

Browse Source 
SIENTIAPDE-1973: Implement per-connector OPC client certificates and remove legacy read engine.
This update overhauls certificate management, transitioning from a single platform certificate to individual client certificates defined in the slot configuration. The CertProvider now downloads certificates from MinIO based on certificate_id, applying dynamic security policies (mode, policy) from the slot. The legacy READ_ENGINE=legacy option and OpcManager have been entirely removed, standardizing on the ServerReader with strict slot v2 enforcement. Write command resolution (ServerConfigResolver) now consistently uses connectors joined with opc_certificates. The CERT_BUCKET environment variable is no longer used.
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-29 12:39:21 -03:00
parent d68e842e6e
commit ecb055eefb
32 changed files with 1639 additions and 2284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

244
tests/unit/managers/test_ingestor_manager.py
View File

@@ -9,7 +9,6 @@ from ingestor.config import GatewayConfig
from ingestor.connection.pool import OpcConnectionPool
from ingestor.io.slots import parse_slot
from ingestor.managers.ingestor_manager import IngestorManager
from ingestor.managers.opc_manager import OpcManager
from ingestor.managers.server_reader import ServerReader
metadata = {
@@ -22,17 +21,27 @@ metadata = {
}
def raw_slot_v1(tag_name='Counter', address='ns=2;i=2'):
"""A slot as the current (v1) Orchestrator writes it to Redis."""
SECURITY = {'mode': 'SignAndEncrypt', 'policy': 'Basic256Sha256'}
CERTIFICATE = {
'certificate_id': 1,
'application_uri': 'urn:sientia:opc-ingestor',
'cert': {'bucket': 'opc-certificates', 'object_name': 'client/p/cert.pem'},
'private_key': {'bucket': 'opc-certificates', 'object_name': 'client/p/key.pem'},
}
def raw_slot(tag_name='Counter', address='ns=2;i=2'):
"""A slot v2 as the Orchestrator writes it to Redis (contract C1)."""
name = {'collection': 'raw_a', 'tag_name': tag_name, 'model_id': '1', 'model_name': 'm'}
return {
'server1': {
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'server_uri': 'urn:sientia:opc-ingestor',
'cert_path': '/app/certs/opc-ingestor-cert.pem',
'subscription_period_ms': 500.0,
'tags': {address: {'tag_name': tag_name, 'frequency': '1000', 'topics': ['raw_a']}},
'security': SECURITY,
'certificate': CERTIFICATE,
'tags': {address: {'frequency': '1000', 'names': [name]}},
}
}
@@ -124,7 +133,6 @@ def test___init__(
# Read path (SIENTIAPDE-1974): pool + read sink built here, nothing connected yet.
assert isinstance(ingestor.pool, OpcConnectionPool)
assert ingestor.pool.cert_provider is ingestor.cert_provider
assert ingestor.cert_provider.mongo is data_manager_mock.return_value.mongo_repository
assert ingestor.sink.data_manager is data_manager_mock.return_value
assert ingestor.sink.postgres is None
@@ -143,10 +151,9 @@ def test___init___cert_provider_uses_gateway_config(resource_manager_mock, data_
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=MagicMock(),
gateway_config=GatewayConfig(cert_bucket='my-bucket', cert_cache_dir='certs-cache'),
gateway_config=GatewayConfig(cert_cache_dir='certs-cache'),
)
assert ingestor.cert_provider.bucket == 'my-bucket'
assert ingestor.cert_provider.cache_dir == 'certs-cache'
@@ -157,9 +164,9 @@ async def test_initialize_opc_from_config(server_reader, ingestor_manager):
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 1000,
'security': SECURITY,
'certificate': CERTIFICATE,
}
server_reader.return_value = MagicMock(connect=AsyncMock())
@@ -191,9 +198,9 @@ async def test_initialize_opc_from_config_exception(
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 1000,
'security': SECURITY,
'certificate': CERTIFICATE,
}
ingestor_manager.logger.error = MagicMock()
@@ -483,13 +490,13 @@ async def test_get_number_of_slots_empty(metrics, ingestor_manager):
@mark.asyncio
async def test_get_slot_leases_1_success(ingestor_manager):
ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True)
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value=raw_slot_v1())
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value=raw_slot())
ingestor_manager.number_of_slots = 1
result = await ingestor_manager.get_slot_leases()
# Stored already normalized to slot v2 (v1 converted by parse_slot).
assert result == {'1': parse_slot(raw_slot_v1())}
assert result == {'1': parse_slot(raw_slot())}
assert ingestor_manager.managed_tags == result
@@ -497,22 +504,22 @@ async def test_get_slot_leases_1_success(ingestor_manager):
async def test_get_slot_leases_2_success(ingestor_manager):
ingestor_manager.resource_manager.lease_tag = AsyncMock(side_effect=[True, True])
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
side_effect=[raw_slot_v1('Counter'), raw_slot_v1('Ramp', 'ns=2;i=3')]
side_effect=[raw_slot('Counter'), raw_slot('Ramp', 'ns=2;i=3')]
)
ingestor_manager.number_of_slots = 2
result = await ingestor_manager.get_slot_leases(max_slots=2)
assert result == {
'1': parse_slot(raw_slot_v1('Counter')),
'2': parse_slot(raw_slot_v1('Ramp', 'ns=2;i=3')),
'1': parse_slot(raw_slot('Counter')),
'2': parse_slot(raw_slot('Ramp', 'ns=2;i=3')),
}
@mark.asyncio
async def test_get_slot_leases_skips_malformed_slot(ingestor_manager):
ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True)
# tag without names/topics/frequency
# tag without names/frequency
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
return_value={'server1': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=2': {}}}}
)
@@ -624,8 +631,8 @@ async def test_unsubscribe_slot_reader_failure_does_not_raise(ingestor_manager,
@mark.asyncio
async def test_unsubscribe_slot_with_real_readers(ingestor_manager):
"""Contract: both real readers keep `subscriptions` keyed by slot, and
unsubscribe_slot closes the slot's subscription on each of them"""
"""Contract: the real reader keeps `subscriptions` keyed by slot, and
unsubscribe_slot closes the slot's subscription"""
v2_subscription = AsyncMock()
client = MagicMock(create_subscription=AsyncMock(return_value=v2_subscription))
pool = MagicMock(get=AsyncMock(return_value=client), evict=AsyncMock())
@@ -646,35 +653,13 @@ async def test_unsubscribe_slot_with_real_readers(ingestor_manager):
server_reader.engine.emit_metric = AsyncMock()
await server_reader.create_subscription('slot1')
with patch('ingestor.managers.opc_manager.metrics'):
legacy_reader = OpcManager(
name='legacy',
url='opc.tcp://legacy:4840',
data_manager=AsyncMock(),
subscription_period_ms=1000,
logger=MagicMock(),
server_uri='opc.tcp://legacy:4840',
notification_handler=MagicMock(),
metadata={},
metrics_controller=AsyncMock(),
)
legacy_reader.emit_metric = AsyncMock()
legacy_subscription = AsyncMock()
legacy_reader.client = MagicMock(
create_subscription=AsyncMock(return_value=legacy_subscription)
)
await legacy_reader.create_subscription('slot1')
assert 'slot1' in server_reader.subscriptions
assert 'slot1' in legacy_reader.subscriptions
ingestor_manager.opc_managers = {'kepserver': server_reader, 'legacy': legacy_reader}
ingestor_manager.opc_managers = {'kepserver': server_reader}
await ingestor_manager.unsubscribe_slot('slot1')
v2_subscription.delete.assert_awaited_once()
legacy_subscription.delete.assert_awaited_once()
assert 'slot1' not in server_reader.subscriptions
assert 'slot1' not in legacy_reader.subscriptions
pool.pin.assert_called_once_with('1')
pool.unpin.assert_called_once_with('1')
@@ -683,13 +668,13 @@ async def test_unsubscribe_slot_with_real_readers(ingestor_manager):
async def test_update_slot_config_unsubscribes_removed_slot(ingestor_manager, reader_double):
"""A slot deleted from Redis is unsubscribed right away, not left to the main loop"""
ingestor_manager.managed_tags = {
'slot1': parse_slot(raw_slot_v1('Same')),
'slot2': parse_slot(raw_slot_v1('Same')),
'slot1': parse_slot(raw_slot('Same')),
'slot2': parse_slot(raw_slot('Same')),
}
reader = reader_double('slot1', 'slot2')
ingestor_manager.opc_managers = {'test_server': reader}
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
side_effect=lambda slot: raw_slot_v1('Same') if slot == 'slot1' else None
side_effect=lambda slot: raw_slot('Same') if slot == 'slot1' else None
)
ingestor_manager.resource_manager.renew_tag_lease = AsyncMock()
@@ -703,20 +688,20 @@ async def test_update_slot_config_unsubscribes_removed_slot(ingestor_manager, re
@mark.asyncio
async def test_update_slot_config(ingestor_manager):
ingestor_manager.managed_tags = {
'slot1': parse_slot(raw_slot_v1('Old')),
'slot2': parse_slot(raw_slot_v1('Same')),
'slot3': parse_slot(raw_slot_v1('Old')),
'slot1': parse_slot(raw_slot('Old')),
'slot2': parse_slot(raw_slot('Same')),
'slot3': parse_slot(raw_slot('Old')),
}
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
side_effect=[raw_slot_v1('Updated'), raw_slot_v1('Same'), None]
side_effect=[raw_slot('Updated'), raw_slot('Same'), None]
)
ingestor_manager.resource_manager.renew_tag_lease = AsyncMock()
await ingestor_manager.update_slot_config()
assert ingestor_manager.managed_tags['slot1'] == parse_slot(raw_slot_v1('Updated'))
assert ingestor_manager.managed_tags['slot2'] == parse_slot(raw_slot_v1('Same'))
assert ingestor_manager.managed_tags['slot1'] == parse_slot(raw_slot('Updated'))
assert ingestor_manager.managed_tags['slot2'] == parse_slot(raw_slot('Same'))
assert 'slot3' not in ingestor_manager.managed_tags
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot1')
@@ -783,8 +768,8 @@ async def test_manage_server_create_subscription_failure(ingestor_manager):
@mark.asyncio
async def test_manage_server_subscribe_and_unsubscribe_failure(ingestor_manager):
"""subscribe() fails and so does the cleanup unsubscribe (legacy reader, session
gone): manage_server reports the failure and nothing escapes to the main loop"""
"""subscribe() fails and so does the cleanup unsubscribe (session gone):
manage_server reports the failure and nothing escapes to the main loop"""
reader = AsyncMock()
reader.subscriptions = {}
reader.subscribe.side_effect = Exception('Subscription error')
@@ -886,7 +871,7 @@ async def test_subscribe_to_tags(ingestor_manager):
@mark.asyncio
async def test_update_slot_config_keeps_last_good_config_when_malformed(ingestor_manager):
good = parse_slot(raw_slot_v1('Counter'))
good = parse_slot(raw_slot('Counter'))
ingestor_manager.managed_tags = {'slot1': good}
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
return_value={'server1': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=2': {}}}}
@@ -922,7 +907,7 @@ def test___init___ignores_malformed_write_env(
"""A malformed write/gRPC-only env var must not stop the read path (review #1)."""
monkeypatch.setenv('GRPC_PORT', '')
monkeypatch.setenv('WRITE_MIN_PAUSE_S', '0,5')
monkeypatch.setenv('CERT_BUCKET', 'streamlit-connectors')
monkeypatch.setenv('CERT_CACHE_DIR', 'certs-cache')
ingestor = IngestorManager(
redis_data={'host': 'localhost', 'port': 6379},
@@ -937,147 +922,4 @@ def test___init___ignores_malformed_write_env(
metrics_controller=MagicMock(),
)
assert ingestor.cert_provider.bucket == 'streamlit-connectors'
def test_parse_slot_config_warns_once_about_server_cert_path(ingestor_manager):
raw = raw_slot_v1()
raw['server1']['server_cert_path'] = '/app/certs/kepserver.der'
ingestor_manager.parse_slot_config('1', raw)
ingestor_manager.parse_slot_config('1', raw) # every loop refreshes the slot
warnings = [
c for c in ingestor_manager.logger.warning.call_args_list if 'server_cert_path' in str(c)
]
assert len(warnings) == 1
# --- READ_ENGINE=legacy: plan B with the untouched rc3 OpcManager -------------------
def make_manager(read_engine=None):
with (
patch('ingestor.managers.ingestor_manager.DataManager'),
patch('ingestor.managers.ingestor_manager.ResourceManager'),
):
manager = IngestorManager(
redis_data={'host': 'localhost', 'port': 6379},
lease_ttl=60,
heartbeat_ttl=60,
poll_interval=5,
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
logger=MagicMock(),
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=MagicMock(),
read_engine=read_engine,
)
manager.send_notification = MagicMock()
manager.emit_metric = AsyncMock()
return manager
def test_read_engine_defaults_to_v2(monkeypatch):
monkeypatch.delenv('READ_ENGINE', raising=False)
assert make_manager().read_engine == 'v2'
def test_read_engine_from_env(monkeypatch):
monkeypatch.setenv('READ_ENGINE', 'Legacy ')
assert make_manager().read_engine == 'legacy'
def test_invalid_read_engine_falls_back_to_v2(monkeypatch):
monkeypatch.setenv('READ_ENGINE', 'v3')
manager = make_manager()
assert manager.read_engine == 'v2'
assert any('Invalid READ_ENGINE' in str(c) for c in manager.logger.warning.call_args_list)
@mark.asyncio
@patch('ingestor.managers.ingestor_manager.ServerReader')
@patch('ingestor.managers.ingestor_manager.OpcManager')
async def test_legacy_initialize_builds_rc3_opc_manager(opc_manager, server_reader):
manager = make_manager('legacy')
server_config = {
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'subscription_period_ms': 1000,
'server_uri': 'urn:sientia:opc-ingestor',
'cert_path': '/app/certs/opc-ingestor-cert.pem',
'private_key_path': '/app/certs/opc-ingestor-key.pem',
'server_cert_path': None,
}
opc_manager.return_value = MagicMock(connect=AsyncMock())
result = await manager.initialize_opc_from_config(server_config)
server_reader.assert_not_called()
opc_manager.assert_called_once_with(
name='server1',
url='opc.tcp://localhost:4840',
subscription_period_ms=1000,
data_manager=manager.data_manager,
logger=manager.logger,
server_uri='urn:sientia:opc-ingestor',
notification_handler=manager.notification_handler,
metadata=metadata['metadata'],
cert_path='/app/certs/opc-ingestor-cert.pem',
private_key_path='/app/certs/opc-ingestor-key.pem',
server_cert_path=None,
metrics_controller=manager.metrics_controller,
)
assert result is opc_manager.return_value
result.connect.assert_awaited_once()
def test_legacy_keeps_raw_slot_v1():
manager = make_manager('legacy')
raw = raw_slot_v1()
# OpcManager reads tag_name/topics and the file cert paths straight from the slot
assert manager.parse_slot_config('1', raw) is raw
def test_legacy_rejects_slot_v2():
manager = make_manager('legacy')
raw_v2 = {
'server1': {
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'use_security': True,
'subscription_period_ms': 500,
'tags': {
'ns=2;i=2': {
'frequency': '1000',
'names': [{'collection': 'raw_a', 'tag_name': 'Counter', 'model_id': '1'}],
}
},
}
}
assert manager.parse_slot_config('1', raw_v2) is None
_, kwargs = manager.send_notification.call_args
assert kwargs['notification_id'] == 'OPC_SLOT_CONFIG_ERROR_1'
assert 'READ_ENGINE=legacy' in kwargs['message']
@mark.asyncio
async def test_legacy_remove_server_calls_plain_shutdown():
manager = make_manager('legacy')
opc_manager = MagicMock(shutdown=AsyncMock())
manager.opc_managers = {'server1': opc_manager}
manager.managed_tags = {'slot1': {'server1': {}}}
await manager.remove_server('server1', evict=False)
opc_manager.shutdown.assert_awaited_once_with()
assert manager.opc_managers == {}
assert ingestor.cert_provider.cache_dir == 'certs-cache'