gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

SIENTIAPDE-1973

Browse Source 
SIENTIAPDE-1973: Implement per-connector OPC client certificates and remove legacy read engine.
This update overhauls certificate management, transitioning from a single platform certificate to individual client certificates defined in the slot configuration. The CertProvider now downloads certificates from MinIO based on certificate_id, applying dynamic security policies (mode, policy) from the slot. The legacy READ_ENGINE=legacy option and OpcManager have been entirely removed, standardizing on the ServerReader with strict slot v2 enforcement. Write command resolution (ServerConfigResolver) now consistently uses connectors joined with opc_certificates. The CERT_BUCKET environment variable is no longer used.
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-29 12:39:21 -03:00
parent d68e842e6e
commit ecb055eefb
32 changed files with 1639 additions and 2284 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
tests/unit/connection/test_certificates.py Normal file
View File

@@ -0,0 +1,67 @@
import pytest
from ingestor.connection.certificates import (
client_certificate_id,
has_security,
resolve_certificate,
)
LOOKED_UP = {
'certificate_id': 1,
'application_uri': 'urn:sientia:opc-ingestor',
'cert': {'bucket': 'opc-certificates', 'object_name': 'client/p/cert.pem'},
'private_key': {'bucket': 'opc-certificates', 'object_name': 'client/p/key.pem'},
}
def _connector(certificates, certificate_id=1):
return {
'connector_id': 'kepserver',
'url': 'opc.tcp://kep:49320',
'security': {
'mode': 'SignAndEncrypt',
'policy': 'Basic256Sha256',
'client_certificate_id': certificate_id,
},
'certificates': certificates,
}
def test_resolve_certificate_keeps_only_the_slot_fields():
looked_up = {
**LOOKED_UP,
'thumbprint_sha1': 'ab',
'cert': {**LOOKED_UP['cert'], 'sha256': None},
}
assert resolve_certificate(_connector([looked_up])) == LOOKED_UP
@pytest.mark.parametrize(
'certificates',
[
pytest.param(None, id='no-lookup'),
pytest.param([], id='not-found'),
pytest.param([{**LOOKED_UP, 'application_uri': ''}], id='no-application-uri'),
pytest.param([{**LOOKED_UP, 'cert': None}], id='no-cert'),
pytest.param([{**LOOKED_UP, 'private_key': {'object_name': 'k'}}], id='key-no-bucket'),
pytest.param(['not a document'], id='not-a-document'),
],
)
def test_resolve_certificate_without_a_usable_certificate(certificates):
assert resolve_certificate(_connector(certificates)) is None
def test_client_certificate_id():
assert client_certificate_id(_connector([])) == 1
assert client_certificate_id(_connector([], certificate_id=None)) is None
assert client_certificate_id({'connector_id': 'x'}) is None
assert client_certificate_id({'security': None}) is None
def test_has_security():
assert has_security(_connector([]))
assert has_security({'security': {'mode': 'None', 'policy': 'None'}})
assert not has_security({'security': {'mode': 'Sign'}})
assert not has_security({'security': None})
assert not has_security({})

221
tests/unit/connection/test_certs.py
View File

@@ -1,136 +1,107 @@
from unittest.mock import AsyncMock
import asyncio
from unittest.mock import AsyncMock, MagicMock
import pytest
from ingestor.connection.certs import CertProvider
from ingestor.connection.certs import CertificateNotConfiguredError, CertProvider, LazyMinio
@pytest.mark.asyncio
async def test_fetch_returns_local_paths(tmp_path):
mongo = AsyncMock()
mongo.find_one.return_value = {
'client_cert': {'minio_object_name': 'platform/certificates/client_cert_c.pem'},
'private_key': {'minio_object_name': 'platform/certificates/private_key_k.pem'},
def certificate(certificate_id=1, name='sientia-platform', ext='pem', bucket='opc-certificates'):
"""A client certificate as the slot v2 carries it."""
return {
'certificate_id': certificate_id,
'application_uri': 'urn:sientia:opc-ingestor',
'cert': {'bucket': bucket, 'object_name': f'client/{name}/cert.{ext}'},
'private_key': {'bucket': bucket, 'object_name': f'client/{name}/key.{ext}'},
}
minio = AsyncMock()
minio.download_file.side_effect = [b'cert-bytes', b'key-bytes']
provider = CertProvider(mongo, minio, cache_dir=str(tmp_path), bucket='sientia-dataops')
cert, key = await provider.fetch()
assert cert.endswith('c.pem') and key.endswith('k.pem')
assert minio.download_file.await_count == 2
mongo.find_one.assert_awaited_once_with('platform_certificates', {}, {})
minio.download_file.assert_any_await(
'platform/certificates/client_cert_c.pem', 'sientia-dataops', {}
)
assert (tmp_path / 'client_cert_c.pem').read_bytes() == b'cert-bytes'
assert (tmp_path / 'private_key_k.pem').read_bytes() == b'key-bytes'
@pytest.mark.asyncio
async def test_lazy_minio_builds_client_once_on_first_download():
from unittest.mock import MagicMock
from ingestor.connection.certs import LazyMinio
client = AsyncMock()
client.download_file.return_value = b'bytes'
factory = MagicMock(return_value=client)
minio = LazyMinio(factory)
factory.assert_not_called() # no network at construction
assert await minio.download_file('a.pem', 'bucket', {}) == b'bytes'
await minio.download_file('b.pem', 'bucket', {})
factory.assert_called_once()
assert client.download_file.await_count == 2
def test_build_cert_provider_is_lazy_and_uses_config(monkeypatch):
from unittest.mock import MagicMock
from ingestor.config import GatewayConfig
from ingestor.connection import certs
minio_cls = MagicMock()
monkeypatch.setattr(certs, 'MinioRepository', minio_cls)
mongo = MagicMock()
provider = certs.build_cert_provider(
mongo=mongo,
config=GatewayConfig(cert_bucket='streamlit-connectors', cert_cache_dir='certs-cache'),
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=MagicMock(),
)
assert provider.mongo is mongo
assert provider.bucket == 'streamlit-connectors'
assert provider.cache_dir == 'certs-cache'
assert isinstance(provider.minio, certs.LazyMinio)
minio_cls.assert_not_called() # MinIO is only reached when a cert is needed
def _provider(tmp_path, doc=None, data=b'pem'):
from unittest.mock import MagicMock
mongo = MagicMock()
mongo.find_one = AsyncMock(
return_value=doc
if doc is not None
else {
'client_cert': {'minio_object_name': 'platform_certificates/c.pem'},
'private_key': {'minio_object_name': 'platform_certificates/k.pem'},
}
)
def _provider(tmp_path, data=b'pem'):
minio = MagicMock()
minio.download_file = AsyncMock(return_value=data)
return CertProvider(mongo=mongo, minio=minio, cache_dir=str(tmp_path), bucket='b'), minio
return CertProvider(minio=minio, cache_dir=str(tmp_path)), minio
@pytest.mark.asyncio
async def test_fetch_downloads_the_objects_the_certificate_names(tmp_path):
minio = AsyncMock()
minio.download_file.side_effect = [b'cert-bytes', b'key-bytes']
provider = CertProvider(minio, cache_dir=str(tmp_path))
cert, key = await provider.fetch(certificate(7))
assert (cert, key) == (str(tmp_path / '7' / 'cert.pem'), str(tmp_path / '7' / 'key.pem'))
assert [c.args for c in minio.download_file.await_args_list] == [
('client/sientia-platform/cert.pem', 'opc-certificates', {}),
('client/sientia-platform/key.pem', 'opc-certificates', {}),
]
assert (tmp_path / '7' / 'cert.pem').read_bytes() == b'cert-bytes'
assert (tmp_path / '7' / 'key.pem').read_bytes() == b'key-bytes'
@pytest.mark.asyncio
async def test_local_files_keep_the_object_extension(tmp_path):
"""asyncua picks the format (.pem / .der) by the extension"""
provider, _minio = _provider(tmp_path)
cert, key = await provider.fetch(certificate(ext='der'))
assert cert.endswith('cert.der') and key.endswith('key.der')
@pytest.mark.asyncio
@pytest.mark.parametrize(
'doc',
'broken',
[
None,
{},
{'client_cert': {'minio_object_name': 'c.pem'}},
{'client_cert': None, 'private_key': {}},
pytest.param({'cert': None}, id='no-cert'),
pytest.param({'private_key': {'bucket': 'b'}}, id='key-without-object'),
pytest.param({'certificate_id': None}, id='no-id'),
pytest.param({'certificate_id': '../etc'}, id='id-not-an-integer'),
],
)
async def test_fetch_without_platform_certificate_raises_clear_error(tmp_path, doc):
from ingestor.connection.certs import CertificateNotConfiguredError
async def test_fetch_of_an_incomplete_certificate_raises_clear_error(tmp_path, broken):
provider, minio = _provider(tmp_path)
provider.mongo.find_one.return_value = doc
with pytest.raises(CertificateNotConfiguredError, match='platform_certificates'):
await provider.fetch()
with pytest.raises(CertificateNotConfiguredError, match='opc_certificates|certificate_id'):
await provider.fetch({**certificate(), **broken})
minio.download_file.assert_not_awaited()
@pytest.mark.asyncio
async def test_fetch_is_cached_until_invalidated(tmp_path):
async def test_fetch_is_cached_per_certificate_until_invalidated(tmp_path):
provider, minio = _provider(tmp_path)
first = await provider.fetch()
second = await provider.fetch()
assert first == second
first = await provider.fetch(certificate(1))
assert await provider.fetch(certificate(1)) == first
assert minio.download_file.await_count == 2 # cert + key, once
provider.invalidate()
await provider.fetch()
other = await provider.fetch(certificate(2, name='other'))
assert other != first
assert minio.download_file.await_count == 4
provider.invalidate(1)
await provider.fetch(certificate(1))
await provider.fetch(certificate(2, name='other')) # still cached
assert minio.download_file.await_count == 6
@pytest.mark.asyncio
async def test_a_replaced_certificate_is_downloaded_again(tmp_path):
"""Same certificate_id, other objects in the slot: the cache does not serve the old files"""
provider, minio = _provider(tmp_path)
await provider.fetch(certificate(1, name='old'))
await provider.fetch(certificate(1, name='new'))
assert minio.download_file.await_count == 4
assert minio.download_file.await_args_list[-1].args[0] == 'client/new/key.pem'
@pytest.mark.asyncio
async def test_concurrent_fetches_download_once(tmp_path):
import asyncio
provider, minio = _provider(tmp_path)
results = await asyncio.gather(*(provider.fetch() for _ in range(5)))
results = await asyncio.gather(*(provider.fetch(certificate()) for _ in range(5)))
assert len(set(results)) == 1
assert minio.download_file.await_count == 2
@@ -138,11 +109,51 @@ async def test_concurrent_fetches_download_once(tmp_path):
@pytest.mark.asyncio
async def test_download_replaces_file_atomically(tmp_path):
provider, minio = _provider(tmp_path, data=b'new')
(tmp_path / 'c.pem').write_bytes(b'old')
provider, _minio = _provider(tmp_path, data=b'new')
(tmp_path / '1').mkdir()
(tmp_path / '1' / 'cert.pem').write_bytes(b'old')
cert_path, _ = await provider.fetch()
cert_path, _ = await provider.fetch(certificate(1))
assert (tmp_path / 'c.pem').read_bytes() == b'new'
assert cert_path == str(tmp_path / 'c.pem')
assert not list(tmp_path.glob('*.tmp'))
assert (tmp_path / '1' / 'cert.pem').read_bytes() == b'new'
assert cert_path == str(tmp_path / '1' / 'cert.pem')
assert not list(tmp_path.glob('**/*.tmp'))
@pytest.mark.asyncio
async def test_lazy_minio_builds_one_client_per_bucket_on_first_download():
clients = {'a': AsyncMock(), 'b': AsyncMock()}
for client in clients.values():
client.download_file.return_value = b'bytes'
factory = MagicMock(side_effect=lambda bucket: clients[bucket])
minio = LazyMinio(factory)
factory.assert_not_called() # no network at construction
assert await minio.download_file('x.pem', 'a', {}) == b'bytes'
await minio.download_file('y.pem', 'a', {})
await minio.download_file('z.pem', 'b', {})
assert [c.args for c in factory.call_args_list] == [('a',), ('b',)]
assert clients['a'].download_file.await_count == 2
clients['b'].download_file.assert_awaited_once_with('z.pem', 'b', {})
def test_build_cert_provider_is_lazy_and_uses_config(monkeypatch):
from ingestor.config import GatewayConfig
from ingestor.connection import certs
minio_cls = MagicMock()
monkeypatch.setattr(certs, 'MinioRepository', minio_cls)
provider = certs.build_cert_provider(
config=GatewayConfig(cert_cache_dir='certs-cache'),
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=MagicMock(),
)
assert provider.cache_dir == 'certs-cache'
assert isinstance(provider.minio, certs.LazyMinio)
minio_cls.assert_not_called() # MinIO is only reached when a cert is needed
provider.minio._factory('opc-certificates')
assert minio_cls.call_args.kwargs['bucket'] == 'opc-certificates'

125
tests/unit/connection/test_pool.py
View File

@@ -6,6 +6,15 @@ import pytest
import ingestor.metrics as metrics
from ingestor.connection.pool import OpcConnectionPool
SECURITY = {'mode': 'SignAndEncrypt', 'policy': 'Basic256Sha256'}
CERTIFICATE = {
'certificate_id': 7,
'application_uri': 'urn:sientia:opc-ingestor',
'cert': {'bucket': 'opc-certificates', 'object_name': 'client/p/cert.pem'},
'private_key': {'bucket': 'opc-certificates', 'object_name': 'client/p/key.pem'},
}
SECURED = {'url': 'opc.tcp://h:4840', 'security': SECURITY, 'certificate': CERTIFICATE}
def make_pool(cert_provider=None):
pool = OpcConnectionPool(
@@ -23,8 +32,8 @@ async def test_get_reuses_connection_per_server():
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
c1 = await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
c2 = await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
c1 = await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
c2 = await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
assert c1 is c2
assert fake_client.connect.await_count == 1
@@ -37,7 +46,7 @@ async def test_get_failure_records_metric_and_raises():
fake_client.connect.side_effect = RuntimeError('boom')
with patch('ingestor.connection.pool.Client', return_value=fake_client):
with pytest.raises(RuntimeError):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
pool.emit_metric.assert_any_await(
metric_object=metrics.OPC_CONNECTIONS_FAILED,
@@ -48,13 +57,13 @@ async def test_get_failure_records_metric_and_raises():
# a later successful get must reconnect (failed attempt was not cached)
fake_client2 = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client2):
client = await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
client = await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
assert client is fake_client2
assert fake_client2.connect.await_count == 1
@pytest.mark.asyncio
async def test_get_applies_security_when_requested():
async def test_get_applies_the_certificate_and_security_of_the_server():
cert_provider = AsyncMock()
cert_provider.fetch.return_value = ('cert.pem', 'key.pem')
pool = make_pool(cert_provider=cert_provider)
@@ -63,13 +72,32 @@ async def test_get_applies_security_when_requested():
patch('ingestor.connection.pool.Client', return_value=fake_client),
patch('ingestor.connection.pool.apply_security', new=AsyncMock()) as mock_apply,
):
await pool.get(
'srv',
{'url': 'opc.tcp://h:4840', 'server_uri': 'urn:test', 'use_security': True},
)
await pool.get('srv', SECURED)
cert_provider.fetch.assert_awaited_once()
mock_apply.assert_awaited_once_with(fake_client, 'urn:test', 'cert.pem', 'key.pem', None)
cert_provider.fetch.assert_awaited_once_with(CERTIFICATE)
mock_apply.assert_awaited_once_with(fake_client, SECURITY, CERTIFICATE, 'cert.pem', 'key.pem')
@pytest.mark.asyncio
@pytest.mark.parametrize(
'cfg',
[
pytest.param({**SECURED, 'certificate': None}, id='no-certificate'),
pytest.param({**SECURED, 'security': {'mode': 'None', 'policy': 'None'}}, id='mode-none'),
],
)
async def test_get_without_certificate_connects_without_security(cfg):
"""No client certificate (or mode None): no security, no MinIO download (D5)"""
cert_provider = AsyncMock()
pool = make_pool(cert_provider=cert_provider)
with (
patch('ingestor.connection.pool.Client', return_value=AsyncMock()),
patch('ingestor.connection.pool.apply_security', new=AsyncMock()) as mock_apply,
):
await pool.get('srv', cfg)
cert_provider.fetch.assert_not_awaited()
mock_apply.assert_not_awaited()
@pytest.mark.asyncio
@@ -77,7 +105,7 @@ async def test_release_idle_evicts_and_disconnects():
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
client, _ = pool._connections['srv']
pool._connections['srv'] = (client, -1000.0) # force idle
@@ -97,7 +125,7 @@ async def test_release_idle_keeps_recently_used():
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
await pool.release_idle(max_idle_s=3600)
@@ -110,8 +138,8 @@ async def test_close_all_disconnects_everything():
pool = make_pool()
fake_a, fake_b = AsyncMock(), AsyncMock()
with patch('ingestor.connection.pool.Client', side_effect=[fake_a, fake_b]):
await pool.get('a', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('b', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('a', {'url': 'opc.tcp://h:4840', 'certificate': None})
await pool.get('b', {'url': 'opc.tcp://h:4840', 'certificate': None})
await pool.close_all()
@@ -126,7 +154,7 @@ async def test_release_idle_disconnect_retries_then_gives_up():
fake_client = AsyncMock()
fake_client.disconnect.side_effect = RuntimeError('nope')
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
client, _ = pool._connections['srv']
pool._connections['srv'] = (client, -1000.0)
@@ -142,7 +170,7 @@ async def test_success_path_emits_connection_metrics():
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
pool.emit_metric.assert_any_await(
metric_object=metrics.OPC_CONNECTIONS_TOTAL,
@@ -170,7 +198,7 @@ async def test_concurrent_get_same_server_connects_once():
clients[0].connect.side_effect = slow_connect
cfg = {'url': 'opc.tcp://h:4840', 'use_security': False}
cfg = {'url': 'opc.tcp://h:4840', 'certificate': None}
with patch('ingestor.connection.pool.Client', side_effect=clients) as ctor:
t1 = asyncio.create_task(pool.get('srv', cfg))
await started.wait() # t1 is mid-connect, holding the per-server lock
@@ -191,7 +219,7 @@ async def test_release_idle_keeps_pinned_connection():
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
pool.pin('srv')
await pool.release_idle(max_idle_s=0)
@@ -205,7 +233,7 @@ async def test_release_idle_evicts_after_last_unpin():
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
pool.pin('srv')
pool.pin('srv') # two slots subscribed on the same server
@@ -233,9 +261,9 @@ async def test_evict_disconnects_now_and_next_get_reconnects():
pool = make_pool()
first, second = AsyncMock(), AsyncMock()
with patch('ingestor.connection.pool.Client', side_effect=[first, second]):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
await pool.evict('srv')
client = await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
client = await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
first.disconnect.assert_awaited_once()
assert client is second
@@ -255,8 +283,8 @@ async def test_get_normalizes_server_id_to_str():
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
c1 = await pool.get(1, {'url': 'opc.tcp://h:4840', 'use_security': False})
c2 = await pool.get('1', {'url': 'opc.tcp://h:4840', 'use_security': False})
c1 = await pool.get(1, {'url': 'opc.tcp://h:4840', 'certificate': None})
c2 = await pool.get('1', {'url': 'opc.tcp://h:4840', 'certificate': None})
assert c1 is c2
assert fake_client.connect.await_count == 1
@@ -267,8 +295,8 @@ async def test_get_with_different_cfg_reuses_client_and_warns():
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
client = await pool.get('srv', {'url': 'opc.tcp://10.0.0.5:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
client = await pool.get('srv', {'url': 'opc.tcp://10.0.0.5:4840', 'certificate': None})
assert client is fake_client
assert fake_client.connect.await_count == 1
@@ -276,6 +304,21 @@ async def test_get_with_different_cfg_reuses_client_and_warns():
assert 'url' in pool.logger.warning.call_args.args[0]
@pytest.mark.asyncio
async def test_get_with_another_certificate_warns():
cert_provider = AsyncMock()
cert_provider.fetch.return_value = ('cert.pem', 'key.pem')
pool = make_pool(cert_provider=cert_provider)
with (
patch('ingestor.connection.pool.Client', return_value=AsyncMock()),
patch('ingestor.connection.pool.apply_security', new=AsyncMock()),
):
await pool.get('srv', SECURED)
await pool.get('srv', {**SECURED, 'certificate': {**CERTIFICATE, 'certificate_id': 8}})
assert 'certificate' in pool.logger.warning.call_args.args[0]
@pytest.mark.asyncio
async def test_secured_connect_failure_invalidates_cert_cache():
from unittest.mock import MagicMock
@@ -290,9 +333,9 @@ async def test_secured_connect_failure_invalidates_cert_cache():
patch('ingestor.connection.pool.apply_security', AsyncMock()),
):
with pytest.raises(RuntimeError):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': True})
await pool.get('srv', SECURED)
cert_provider.invalidate.assert_called_once()
cert_provider.invalidate.assert_called_once_with(7)
@pytest.mark.asyncio
@@ -305,7 +348,7 @@ async def test_insecure_connect_failure_keeps_cert_cache():
fake_client.connect.side_effect = RuntimeError('unreachable')
with patch('ingestor.connection.pool.Client', return_value=fake_client):
with pytest.raises(RuntimeError):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
cert_provider.invalidate.assert_not_called()
@@ -317,7 +360,7 @@ async def test_connect_identifies_the_pod_on_the_server():
pool = make_pool()
fake_client = MagicMock(connect=AsyncMock())
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'certificate': None})
assert fake_client.name == pool.pod_id
assert fake_client.description == pool.pod_id
@@ -326,20 +369,16 @@ async def test_connect_identifies_the_pod_on_the_server():
@pytest.mark.asyncio
async def test_connect_server_uri_still_wins_for_application_uri():
pool = make_pool()
fake_client = MagicMock(connect=AsyncMock())
async def test_connect_with_certificate_uses_its_application_uri():
"""apply_security sets the certificate's URI: it must match its SAN URI
(BadCertificateUriInvalid otherwise)"""
cert_provider = AsyncMock()
cert_provider.fetch.return_value = ('cert.pem', 'key.pem')
pool = make_pool(cert_provider=cert_provider)
fake_client = MagicMock(connect=AsyncMock(), set_security=AsyncMock())
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get(
'srv',
{
'url': 'opc.tcp://h:4840',
'use_security': False,
'server_uri': 'urn:sientia:opc-ingestor',
},
)
await pool.get('srv', SECURED)
# must match the certificate SAN URI (BadCertificateUriInvalid otherwise)
assert fake_client.application_uri == 'urn:sientia:opc-ingestor'
assert fake_client.name == pool.pod_id
@@ -347,7 +386,7 @@ async def test_connect_server_uri_still_wins_for_application_uri():
@pytest.mark.asyncio
async def test_config_for_returns_a_copy_of_the_live_connection_settings():
pool = make_pool()
cfg = {'url': 'opc.tcp://h:4840', 'server_uri': 'urn:h', 'use_security': False}
cfg = {'url': 'opc.tcp://h:4840', 'security': SECURITY, 'certificate': None}
with patch('ingestor.connection.pool.Client', return_value=AsyncMock()):
await pool.get(1, cfg)

88
tests/unit/connection/test_security.py
View File

@@ -1,18 +1,92 @@
from unittest.mock import AsyncMock, MagicMock
import pytest
from asyncua import ua
from asyncua.crypto.security_policies import (
SecurityPolicyAes128Sha256RsaOaep,
SecurityPolicyBasic256Sha256,
)
from ingestor.connection.security import SESSION_TIMEOUT_MS, apply_security
from ingestor.connection.security import (
SESSION_TIMEOUT_MS,
UnsupportedSecurityError,
apply_security,
is_secured,
)
CERTIFICATE = {'certificate_id': 1, 'application_uri': 'urn:sientia:opc-ingestor'}
def _client():
client = MagicMock()
client.set_security = AsyncMock()
return client
@pytest.mark.asyncio
async def test_apply_security_sets_sha256_and_channel_lifetime():
client = MagicMock()
client.set_security = AsyncMock()
await apply_security(client, 'urn:x', 'c.pem', 'k.pem', None)
async def test_apply_security_uses_mode_and_policy_and_channel_lifetime():
client = _client()
await apply_security(
client, {'mode': 'SignAndEncrypt', 'policy': 'Basic256Sha256'}, CERTIFICATE, 'c', 'k'
)
assert client.application_uri == 'urn:sientia:opc-ingestor'
assert client.session_timeout == SESSION_TIMEOUT_MS
assert client.secure_channel_timeout == int(SESSION_TIMEOUT_MS * 1.25)
args, kwargs = client.set_security.call_args
from asyncua.crypto.security_policies import SecurityPolicyBasic256Sha256
assert args[0] is SecurityPolicyBasic256Sha256
assert kwargs == {
'certificate': 'c',
'private_key': 'k',
'server_certificate': None,
'mode': ua.MessageSecurityMode.SignAndEncrypt,
}
@pytest.mark.asyncio
async def test_apply_security_sign_with_another_policy():
client = _client()
await apply_security(
client, {'mode': 'Sign', 'policy': 'Aes128Sha256RsaOaep'}, CERTIFICATE, 'c', 'k'
)
args, kwargs = client.set_security.call_args
assert args[0] is SecurityPolicyAes128Sha256RsaOaep
assert kwargs['mode'] == ua.MessageSecurityMode.Sign
@pytest.mark.asyncio
@pytest.mark.parametrize(
'security',
[
{'mode': 'Encrypt', 'policy': 'Basic256Sha256'},
{'mode': 'Sign', 'policy': 'Basic999'},
{'mode': 'Sign', 'policy': 'None'},
{},
],
)
async def test_apply_security_rejects_unknown_mode_or_policy(security):
client = _client()
with pytest.raises(UnsupportedSecurityError):
await apply_security(client, security, CERTIFICATE, 'c', 'k')
client.set_security.assert_not_awaited()
@pytest.mark.parametrize(
('server_cfg', 'secured'),
[
({'security': {'mode': 'SignAndEncrypt'}, 'certificate': CERTIFICATE}, True),
({'security': {'mode': 'Sign'}, 'certificate': CERTIFICATE}, True),
# No certificate: no security (D5), whatever the mode
({'security': {'mode': 'SignAndEncrypt'}, 'certificate': None}, False),
({'security': {'mode': 'SignAndEncrypt'}}, False),
# mode None: no security even with a certificate
({'security': {'mode': 'None'}, 'certificate': CERTIFICATE}, False),
({}, False),
],
)
def test_is_secured(server_cfg, secured):
assert is_secured(server_cfg) is secured

170
tests/unit/connection/test_servers.py
View File

@@ -2,108 +2,113 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from ingestor.connection.servers import CONNECTORS, OPC_SERVERS, ServerConfigResolver
from ingestor.connection.servers import CONNECTORS, ServerConfigResolver
LOOKED_UP = {
'certificate_id': 1,
'application_uri': 'urn:sientia:opc-ingestor',
'cert': {'bucket': 'opc-certificates', 'object_name': 'client/p/cert.pem'},
'private_key': {'bucket': 'opc-certificates', 'object_name': 'client/p/key.pem'},
}
# A connector as the aggregation returns it ($lookup on opc_certificates)
CONNECTOR = {
'connector_id': 1,
'type': 'opc_ua',
'status': 'Active',
'name': 'kepserver',
'connector_id': 'kepserver',
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:kepserver',
'use_security': True,
'subscription_period_ms': 500,
'security': {
'mode': 'SignAndEncrypt',
'policy': 'Basic256Sha256',
'client_certificate_id': 1,
},
'certificates': [LOOKED_UP],
}
OPC_SERVER = {
'id': '1',
'server_name': 'kepserver',
'url': 'opc.tcp://legacy:49320',
'uri': 'urn:legacy',
'cert_path': '/app/certs/cert.pem',
# The same server in the slot v2 format
CONFIG = {
'server_id': 'kepserver',
'url': 'opc.tcp://kepserver:49320',
'subscription_period_ms': 500,
'security': {'mode': 'SignAndEncrypt', 'policy': 'Basic256Sha256'},
'certificate': LOOKED_UP,
}
def make_resolver(connector=None, opc_server=None, pooled=None, ttl_s=60.0):
def make_resolver(connector=None, pooled=None, ttl_s=60.0):
pool = MagicMock()
pool.config_for = MagicMock(return_value=pooled)
mongo = MagicMock()
async def find_one(collection, _filters, _metadata):
return {CONNECTORS: connector, OPC_SERVERS: opc_server}[collection]
mongo.find_one = AsyncMock(side_effect=find_one)
mongo.aggregate = AsyncMock(return_value=[connector] if connector is not None else [])
return ServerConfigResolver(pool, mongo, MagicMock(), ttl_s=ttl_s)
@pytest.mark.asyncio
async def test_command_connection_overrides_every_other_source():
resolver = make_resolver(connector=CONNECTOR)
cfg = await resolver.resolve('1', {'url': 'opc.tcp://h:1', 'server_uri': 'urn:h'})
assert cfg == {'url': 'opc.tcp://h:1', 'server_uri': 'urn:h', 'use_security': False}
resolver.mongo.find_one.assert_not_awaited()
@pytest.mark.asyncio
async def test_live_pooled_connection_wins_over_mongo():
"""Writes reuse exactly the settings the read path opened the connection with."""
pooled = {'url': 'opc.tcp://read:1', 'server_uri': 'urn:read', 'use_security': True}
pooled = {'url': 'opc.tcp://read:1', 'security': CONFIG['security'], 'certificate': None}
resolver = make_resolver(connector=CONNECTOR, pooled=pooled)
assert await resolver.resolve('1', {}) == pooled
resolver.pool.config_for.assert_called_once_with('1')
resolver.mongo.find_one.assert_not_awaited()
assert await resolver.resolve('kepserver') == pooled
resolver.pool.config_for.assert_called_once_with('kepserver')
resolver.mongo.aggregate.assert_not_awaited()
@pytest.mark.asyncio
async def test_tag_database_connector_first():
resolver = make_resolver(connector=CONNECTOR, opc_server=OPC_SERVER)
async def test_connector_with_its_certificate_in_the_slot_format():
resolver = make_resolver(connector=CONNECTOR)
cfg = await resolver.resolve('1', {})
assert cfg == {
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:kepserver',
'use_security': True,
}
collection, filters, _ = resolver.mongo.find_one.await_args_list[0].args
assert await resolver.resolve('kepserver') == CONFIG
collection, pipeline, _ = resolver.mongo.aggregate.await_args.args
assert collection == CONNECTORS
# connector_id is an int in the tag database; the command carries a string.
assert filters == {'connector_id': {'$in': ['1', 1]}, 'type': 'opc_ua', 'status': 'Active'}
@pytest.mark.asyncio
async def test_opc_servers_fallback_without_tag_database():
resolver = make_resolver(connector=None, opc_server=OPC_SERVER)
cfg = await resolver.resolve('1', {})
# Same rule as the orchestrator's read slot: a cert_path means security.
assert cfg == {
'url': 'opc.tcp://legacy:49320',
'server_uri': 'urn:legacy',
'use_security': True,
assert pipeline[0] == {
'$match': {'connector_id': 'kepserver', 'type': 'opc_ua', 'status': 'Active'}
}
collection, filters, _ = resolver.mongo.find_one.await_args_list[1].args
assert collection == OPC_SERVERS
assert filters == {'id': {'$in': ['1', 1]}}
# The same rule as the orchestrator: only an active client certificate (D5)
[lookup] = [stage['$lookup'] for stage in pipeline if '$lookup' in stage]
assert lookup['from'] == 'opc_certificates'
assert lookup['pipeline'][0]['$match']['status'] == 'active'
assert lookup['pipeline'][0]['$match']['kind'] == 'client'
resolver.logger.warning.assert_not_called()
@pytest.mark.asyncio
async def test_opc_server_without_cert_path_is_insecure():
resolver = make_resolver(opc_server={**OPC_SERVER, 'cert_path': ''})
async def test_connector_without_certificate_id_connects_without_security():
connector = {
**CONNECTOR,
'security': {'mode': 'None', 'policy': 'None', 'client_certificate_id': None},
'certificates': [],
}
resolver = make_resolver(connector=connector)
assert (await resolver.resolve('1', {}))['use_security'] is False
cfg = await resolver.resolve('kepserver')
assert cfg['certificate'] is None
assert cfg['security'] == {'mode': 'None', 'policy': 'None'}
resolver.logger.warning.assert_not_called()
@pytest.mark.asyncio
async def test_non_numeric_server_id_is_looked_up_as_is():
resolver = make_resolver(opc_server={**OPC_SERVER, 'id': 'kep-a'})
async def test_certificate_id_without_an_active_client_certificate_warns():
resolver = make_resolver(connector={**CONNECTOR, 'certificates': []})
await resolver.resolve('kep-a', {})
cfg = await resolver.resolve('kepserver')
_, filters, _ = resolver.mongo.find_one.await_args_list[0].args
assert filters['connector_id'] == {'$in': ['kep-a']}
assert cfg == {**CONFIG, 'certificate': None}
assert 'client_certificate_id 1' in resolver.logger.warning.call_args.args[0]
@pytest.mark.asyncio
@pytest.mark.parametrize(
'connector',
[
pytest.param({**CONNECTOR, 'url': ''}, id='no-url'),
pytest.param({**CONNECTOR, 'security': {'mode': 'Sign'}}, id='no-policy'),
pytest.param({k: v for k, v in CONNECTOR.items() if k != 'security'}, id='no-security'),
],
)
async def test_incomplete_connectors_are_not_found(connector):
resolver = make_resolver(connector=connector)
assert await resolver.resolve('kepserver') is None
resolver.logger.warning.assert_called_once()
@pytest.mark.asyncio
@@ -113,39 +118,32 @@ async def test_unknown_server_is_cached_briefly():
resolver.clock = MagicMock()
resolver.clock.elapsed.side_effect = [0.0, 5.0, 11.0]
assert await resolver.resolve('9', {}) is None
assert await resolver.resolve('9', {}) is None # within missing_ttl_s: no lookup
assert await resolver.resolve('9', {}) is None # expired: looked up again
assert resolver.mongo.find_one.await_count == 4 # 2 lookups (connectors + opc-servers)
assert await resolver.resolve('9') is None
assert await resolver.resolve('9') is None # within missing_ttl_s: no lookup
assert await resolver.resolve('9') is None # expired: looked up again
assert resolver.mongo.aggregate.await_count == 2
assert resolver.logger.warning.call_count == 2
@pytest.mark.asyncio
async def test_documents_without_url_are_ignored():
resolver = make_resolver(connector={**CONNECTOR, 'url': ''}, opc_server=OPC_SERVER)
assert (await resolver.resolve('1', {}))['url'] == 'opc.tcp://legacy:49320'
@pytest.mark.asyncio
async def test_lookups_are_cached_for_the_ttl():
resolver = make_resolver(connector=CONNECTOR, ttl_s=60.0)
resolver.clock = MagicMock()
resolver.clock.elapsed.side_effect = [0.0, 30.0, 61.0]
first = await resolver.resolve('1', {})
first = await resolver.resolve('kepserver')
first['url'] = 'mutated by the caller'
second = await resolver.resolve('1', {}) # within the TTL: cached, and a copy
await resolver.resolve('1', {}) # expired: looked up again
second = await resolver.resolve('kepserver') # within the TTL: cached, and a copy
await resolver.resolve('kepserver') # expired: looked up again
assert second['url'] == 'opc.tcp://kepserver:49320'
assert resolver.mongo.find_one.await_count == 2
assert resolver.mongo.aggregate.await_count == 2
@pytest.mark.asyncio
async def test_mongo_errors_propagate_to_the_caller():
resolver = make_resolver()
resolver.mongo.find_one = AsyncMock(side_effect=RuntimeError('mongo down'))
resolver.mongo.aggregate = AsyncMock(side_effect=RuntimeError('mongo down'))
with pytest.raises(RuntimeError, match='mongo down'):
await resolver.resolve('1', {})
await resolver.resolve('kepserver')

25
tests/unit/engines/test_write.py
View File

@@ -7,7 +7,12 @@ from asyncua import ua
import ingestor.metrics as metrics
from ingestor.engines.write import WriteEngine
SERVER_CFG = {'url': 'opc.tcp://kepserver:49320', 'server_uri': 'urn:kep', 'use_security': True}
SERVER_CFG = {
'server_id': 'srv1',
'url': 'opc.tcp://kepserver:49320',
'security': {'mode': 'SignAndEncrypt', 'policy': 'Basic256Sha256'},
'certificate': None,
}
def make_resolver(cfg=SERVER_CFG):
@@ -279,7 +284,7 @@ async def test_write_uses_the_resolved_server_connection():
await eng.write(base_cmd())
eng.servers.resolve.assert_awaited_once_with('srv1', base_cmd())
eng.servers.resolve.assert_awaited_once_with('srv1')
eng.pool.get.assert_awaited_once_with('srv1', SERVER_CFG)
@@ -310,22 +315,22 @@ async def test_write_server_lookup_failure_records_error():
@pytest.mark.asyncio
async def test_write_without_resolver_uses_the_command_connection():
eng = make_engine(servers=None)
async def test_write_ignores_a_connection_carried_by_the_command():
"""The server is always resolved by server_id (SIENTIAPDE-1974): connection fields
in the command are not an override"""
eng = make_engine()
eng.pool.get.return_value = make_client()
await eng.write(base_cmd(url='opc.tcp://h:4840', server_uri='urn:h', use_security=True))
await eng.write(base_cmd(url='opc.tcp://h:4840'))
eng.pool.get.assert_awaited_once_with(
'srv1', {'url': 'opc.tcp://h:4840', 'use_security': True, 'server_uri': 'urn:h'}
)
eng.pool.get.assert_awaited_once_with('srv1', SERVER_CFG)
@pytest.mark.asyncio
async def test_write_without_resolver_or_url_is_server_not_found():
async def test_write_without_resolver_is_server_not_found():
eng = make_engine(servers=None)
result = await eng.write(base_cmd())
result = await eng.write(base_cmd(url='opc.tcp://h:4840'))
assert result.opc_status_code == 'server_not_found'
eng.pool.get.assert_not_awaited()

23
tests/unit/io/test_data_sink_publish.py
View File

@@ -67,26 +67,3 @@ async def test_publish_reading_emits_tag_value_metric_per_name():
value=42,
tags={'pod_id': dm.pod_id, 'tag_name': 'Counter', 'model_id': '1'},
)
@pytest.mark.asyncio
async def test_publish_reading_without_model_id_keeps_legacy_doc():
"""Names converted from a slot v1 carry no model_id: the document must be the
same as the legacy OpcManager one (SIENTIAPDE-1974 NF2)."""
sink, dm = make_sink()
names = [{'collection': 'raw_x', 'tag_name': 'Counter'}]
await sink.publish_reading(
tag='ns=2;i=2', value=42, timestamp='2026-07-13 00:00:00+0000', names=names
)
collection, doc = dm.publish.await_args_list[0].args
assert collection == 'raw_x'
assert doc == {
'tag': 'ns=2;i=2',
'name': 'Counter',
'timestamp': '2026-07-13 00:00:00+0000',
'value': 42,
}
_, kwargs = sink.emit_metric.await_args
assert kwargs['tags']['model_id'] == ''

196
tests/unit/io/test_slots.py
View File

@@ -1,11 +1,24 @@
from ingestor.io.slots import parse_slot
import pytest
from ingestor.io.slots import CONNECTION_FIELDS, parse_slot
SECURITY = {'mode': 'SignAndEncrypt', 'policy': 'Basic256Sha256'}
CERTIFICATE = {
'certificate_id': 1,
'application_uri': 'urn:sientia:opc-ingestor',
'cert': {'bucket': 'opc-certificates', 'object_name': 'client/p/cert.pem'},
'private_key': {'bucket': 'opc-certificates', 'object_name': 'client/p/key.pem'},
}
NAME = {'collection': 'c', 'tag_name': 't', 'model_name': 'm', 'model_id': '1'}
RAW = {
'server1': {
'server_id': '1',
'name': 'server1',
'url': 'opc.tcp://h:4840',
'use_security': True,
'subscription_period_ms': 1000,
'security': SECURITY,
'certificate': CERTIFICATE,
'tags': {
'ns=2;i=2': {
'frequency': '15000',
@@ -30,6 +43,17 @@ RAW = {
}
def _server(**fields):
server = {
'server_id': '1',
'url': 'u',
'security': {'mode': 'None', 'policy': 'None'},
'certificate': None,
'tags': {'ns=2;i=1': {'frequency': '1000', 'names': [NAME]}},
}
return {'s': {**server, **fields}}
def test_parse_slot_multi_name():
slot = parse_slot(RAW)
tag = slot['server1']['tags']['ns=2;i=2']
@@ -37,127 +61,71 @@ def test_parse_slot_multi_name():
assert tag['active_read']['cycle_multiple'] == 3
def test_parse_slot_defaults_active_read_disabled():
raw = {
's': {
'server_id': '1',
'url': 'u',
'tags': {
'ns=2;i=1': {
'frequency': '1000',
'names': [
{'collection': 'c', 'tag_name': 't', 'model_name': 'm', 'model_id': '1'}
],
}
},
}
def test_parse_slot_keeps_security_and_certificate():
server = parse_slot(RAW)['server1']
assert server['security'] == SECURITY
assert server['certificate'] == CERTIFICATE
assert {field: server[field] for field in CONNECTION_FIELDS} == {
'url': 'opc.tcp://h:4840',
'subscription_period_ms': 1000,
'security': SECURITY,
'certificate': CERTIFICATE,
}
tag = parse_slot(raw)['s']['tags']['ns=2;i=1']
assert tag['active_read'] == {'enabled': False}
def test_parse_slot_rejects_tag_without_names():
raw = {'s': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=1': {'frequency': '1000'}}}}
import pytest
with pytest.raises((KeyError, ValueError)):
parse_slot(raw)
RAW_V1 = {
'kepserver': {
'name': 'kepserver',
'server_id': '1',
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:sientia:opc-ingestor',
'cert_path': '/app/certs/opc-ingestor-cert.pem',
'private_key_path': '/app/certs/opc-ingestor-key.pem',
'server_cert_path': None,
'subscription_period_ms': 15000.0,
'tags': {
'ns=2;s=A.TargetSin': {
'aggr_func': 'lts',
'frequency': 30000,
'server_id': '1',
'tag_address': 'ns=2;s=A.TargetSin',
'tag_name': 'TargetSin',
'topics': ['raw_scouter-sin-approx', 'raw_other-model'],
}
},
}
}
def test_parse_slot_converts_v1_topics_to_names():
"""Slot v1 (current Orchestrator) is read as v2: one name per topic, no model_id,
so the documents stay identical to the legacy ones (SIENTIAPDE-1974 G1)."""
server = parse_slot(RAW_V1)['kepserver']
tag = server['tags']['ns=2;s=A.TargetSin']
assert tag['names'] == [
{'collection': 'raw_scouter-sin-approx', 'tag_name': 'TargetSin'},
{'collection': 'raw_other-model', 'tag_name': 'TargetSin'},
]
assert tag['frequency'] == 30000
assert tag['active_read'] == {'enabled': False}
def test_parse_slot_v1_cert_path_means_security_but_path_is_dropped():
server = parse_slot(RAW_V1)['kepserver']
assert server['use_security'] is True
assert server['name'] == 'kepserver'
assert server['server_id'] == '1'
assert server['subscription_period_ms'] == 15000.0
# the certificate always comes from MinIO now: no file path survives the parse
assert 'cert_path' not in server
assert 'private_key_path' not in server
def test_parse_slot_v1_without_cert_path_is_insecure():
raw = {
's': {
'server_id': '2',
'url': 'u',
'cert_path': None,
'tags': {'ns=2;i=1': {'tag_name': 't', 'frequency': '1000', 'topics': ['raw_c']}},
}
}
def test_parse_slot_without_certificate_key_means_no_certificate():
raw = _server()
del raw['s']['certificate']
server = parse_slot(raw)['s']
assert server['use_security'] is False
assert server['certificate'] is None
assert server['name'] == 's' # falls back to the slot key
def test_parse_slot_v2_explicit_use_security_wins():
raw = {
's': {
'server_id': '1',
'url': 'u',
'use_security': False,
'cert_path': '/legacy/path',
'tags': {
'ns=2;i=1': {
'frequency': '1000',
'names': [{'collection': 'c', 'tag_name': 't', 'model_id': '1'}],
def test_parse_slot_defaults_active_read_disabled():
tag = parse_slot(_server())['s']['tags']['ns=2;i=1']
assert tag['active_read'] == {'enabled': False}
@pytest.mark.parametrize(
'raw',
[
pytest.param(_server(tags={'ns=2;i=1': {'frequency': '1000'}}), id='tag-without-names'),
pytest.param(
_server(tags={'ns=2;i=1': {'frequency': '1', 'names': [{'collection': 'c'}]}}),
id='name-without-tag-name',
),
pytest.param(
_server(
tags={
'ns=2;i=1': {
'frequency': '1',
'names': [{'collection': 'c', 'tag_name': 't'}],
}
}
),
id='name-without-model-id',
),
pytest.param(_server(security=None), id='no-security'),
# Slot v1 (an Orchestrator before SIENTIAPDE-1974): no longer accepted
pytest.param(
{
'kepserver': {
'server_id': '1',
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:sientia:opc-ingestor',
'cert_path': '/app/certs/opc-ingestor-cert.pem',
'tags': {
'ns=2;s=A': {'tag_name': 'A', 'frequency': 30000, 'topics': ['raw_x']}
},
}
},
}
}
assert parse_slot(raw)['s']['use_security'] is False
def test_parse_slot_rejects_v1_tag_without_tag_name():
raw = {
's': {
'server_id': '1',
'url': 'u',
'tags': {'ns=2;i=1': {'frequency': '1', 'topics': ['c']}},
}
}
import pytest
with pytest.raises(ValueError):
id='slot-v1',
),
],
)
def test_parse_slot_rejects_invalid_slots(raw):
with pytest.raises((KeyError, ValueError)):
parse_slot(raw)

244
tests/unit/managers/test_ingestor_manager.py
View File

@@ -9,7 +9,6 @@ from ingestor.config import GatewayConfig
from ingestor.connection.pool import OpcConnectionPool
from ingestor.io.slots import parse_slot
from ingestor.managers.ingestor_manager import IngestorManager
from ingestor.managers.opc_manager import OpcManager
from ingestor.managers.server_reader import ServerReader
metadata = {
@@ -22,17 +21,27 @@ metadata = {
}
def raw_slot_v1(tag_name='Counter', address='ns=2;i=2'):
"""A slot as the current (v1) Orchestrator writes it to Redis."""
SECURITY = {'mode': 'SignAndEncrypt', 'policy': 'Basic256Sha256'}
CERTIFICATE = {
'certificate_id': 1,
'application_uri': 'urn:sientia:opc-ingestor',
'cert': {'bucket': 'opc-certificates', 'object_name': 'client/p/cert.pem'},
'private_key': {'bucket': 'opc-certificates', 'object_name': 'client/p/key.pem'},
}
def raw_slot(tag_name='Counter', address='ns=2;i=2'):
"""A slot v2 as the Orchestrator writes it to Redis (contract C1)."""
name = {'collection': 'raw_a', 'tag_name': tag_name, 'model_id': '1', 'model_name': 'm'}
return {
'server1': {
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'server_uri': 'urn:sientia:opc-ingestor',
'cert_path': '/app/certs/opc-ingestor-cert.pem',
'subscription_period_ms': 500.0,
'tags': {address: {'tag_name': tag_name, 'frequency': '1000', 'topics': ['raw_a']}},
'security': SECURITY,
'certificate': CERTIFICATE,
'tags': {address: {'frequency': '1000', 'names': [name]}},
}
}
@@ -124,7 +133,6 @@ def test___init__(
# Read path (SIENTIAPDE-1974): pool + read sink built here, nothing connected yet.
assert isinstance(ingestor.pool, OpcConnectionPool)
assert ingestor.pool.cert_provider is ingestor.cert_provider
assert ingestor.cert_provider.mongo is data_manager_mock.return_value.mongo_repository
assert ingestor.sink.data_manager is data_manager_mock.return_value
assert ingestor.sink.postgres is None
@@ -143,10 +151,9 @@ def test___init___cert_provider_uses_gateway_config(resource_manager_mock, data_
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=MagicMock(),
gateway_config=GatewayConfig(cert_bucket='my-bucket', cert_cache_dir='certs-cache'),
gateway_config=GatewayConfig(cert_cache_dir='certs-cache'),
)
assert ingestor.cert_provider.bucket == 'my-bucket'
assert ingestor.cert_provider.cache_dir == 'certs-cache'
@@ -157,9 +164,9 @@ async def test_initialize_opc_from_config(server_reader, ingestor_manager):
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 1000,
'security': SECURITY,
'certificate': CERTIFICATE,
}
server_reader.return_value = MagicMock(connect=AsyncMock())
@@ -191,9 +198,9 @@ async def test_initialize_opc_from_config_exception(
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 1000,
'security': SECURITY,
'certificate': CERTIFICATE,
}
ingestor_manager.logger.error = MagicMock()
@@ -483,13 +490,13 @@ async def test_get_number_of_slots_empty(metrics, ingestor_manager):
@mark.asyncio
async def test_get_slot_leases_1_success(ingestor_manager):
ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True)
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value=raw_slot_v1())
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value=raw_slot())
ingestor_manager.number_of_slots = 1
result = await ingestor_manager.get_slot_leases()
# Stored already normalized to slot v2 (v1 converted by parse_slot).
assert result == {'1': parse_slot(raw_slot_v1())}
assert result == {'1': parse_slot(raw_slot())}
assert ingestor_manager.managed_tags == result
@@ -497,22 +504,22 @@ async def test_get_slot_leases_1_success(ingestor_manager):
async def test_get_slot_leases_2_success(ingestor_manager):
ingestor_manager.resource_manager.lease_tag = AsyncMock(side_effect=[True, True])
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
side_effect=[raw_slot_v1('Counter'), raw_slot_v1('Ramp', 'ns=2;i=3')]
side_effect=[raw_slot('Counter'), raw_slot('Ramp', 'ns=2;i=3')]
)
ingestor_manager.number_of_slots = 2
result = await ingestor_manager.get_slot_leases(max_slots=2)
assert result == {
'1': parse_slot(raw_slot_v1('Counter')),
'2': parse_slot(raw_slot_v1('Ramp', 'ns=2;i=3')),
'1': parse_slot(raw_slot('Counter')),
'2': parse_slot(raw_slot('Ramp', 'ns=2;i=3')),
}
@mark.asyncio
async def test_get_slot_leases_skips_malformed_slot(ingestor_manager):
ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True)
# tag without names/topics/frequency
# tag without names/frequency
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
return_value={'server1': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=2': {}}}}
)
@@ -624,8 +631,8 @@ async def test_unsubscribe_slot_reader_failure_does_not_raise(ingestor_manager,
@mark.asyncio
async def test_unsubscribe_slot_with_real_readers(ingestor_manager):
"""Contract: both real readers keep `subscriptions` keyed by slot, and
unsubscribe_slot closes the slot's subscription on each of them"""
"""Contract: the real reader keeps `subscriptions` keyed by slot, and
unsubscribe_slot closes the slot's subscription"""
v2_subscription = AsyncMock()
client = MagicMock(create_subscription=AsyncMock(return_value=v2_subscription))
pool = MagicMock(get=AsyncMock(return_value=client), evict=AsyncMock())
@@ -646,35 +653,13 @@ async def test_unsubscribe_slot_with_real_readers(ingestor_manager):
server_reader.engine.emit_metric = AsyncMock()
await server_reader.create_subscription('slot1')
with patch('ingestor.managers.opc_manager.metrics'):
legacy_reader = OpcManager(
name='legacy',
url='opc.tcp://legacy:4840',
data_manager=AsyncMock(),
subscription_period_ms=1000,
logger=MagicMock(),
server_uri='opc.tcp://legacy:4840',
notification_handler=MagicMock(),
metadata={},
metrics_controller=AsyncMock(),
)
legacy_reader.emit_metric = AsyncMock()
legacy_subscription = AsyncMock()
legacy_reader.client = MagicMock(
create_subscription=AsyncMock(return_value=legacy_subscription)
)
await legacy_reader.create_subscription('slot1')
assert 'slot1' in server_reader.subscriptions
assert 'slot1' in legacy_reader.subscriptions
ingestor_manager.opc_managers = {'kepserver': server_reader, 'legacy': legacy_reader}
ingestor_manager.opc_managers = {'kepserver': server_reader}
await ingestor_manager.unsubscribe_slot('slot1')
v2_subscription.delete.assert_awaited_once()
legacy_subscription.delete.assert_awaited_once()
assert 'slot1' not in server_reader.subscriptions
assert 'slot1' not in legacy_reader.subscriptions
pool.pin.assert_called_once_with('1')
pool.unpin.assert_called_once_with('1')
@@ -683,13 +668,13 @@ async def test_unsubscribe_slot_with_real_readers(ingestor_manager):
async def test_update_slot_config_unsubscribes_removed_slot(ingestor_manager, reader_double):
"""A slot deleted from Redis is unsubscribed right away, not left to the main loop"""
ingestor_manager.managed_tags = {
'slot1': parse_slot(raw_slot_v1('Same')),
'slot2': parse_slot(raw_slot_v1('Same')),
'slot1': parse_slot(raw_slot('Same')),
'slot2': parse_slot(raw_slot('Same')),
}
reader = reader_double('slot1', 'slot2')
ingestor_manager.opc_managers = {'test_server': reader}
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
side_effect=lambda slot: raw_slot_v1('Same') if slot == 'slot1' else None
side_effect=lambda slot: raw_slot('Same') if slot == 'slot1' else None
)
ingestor_manager.resource_manager.renew_tag_lease = AsyncMock()
@@ -703,20 +688,20 @@ async def test_update_slot_config_unsubscribes_removed_slot(ingestor_manager, re
@mark.asyncio
async def test_update_slot_config(ingestor_manager):
ingestor_manager.managed_tags = {
'slot1': parse_slot(raw_slot_v1('Old')),
'slot2': parse_slot(raw_slot_v1('Same')),
'slot3': parse_slot(raw_slot_v1('Old')),
'slot1': parse_slot(raw_slot('Old')),
'slot2': parse_slot(raw_slot('Same')),
'slot3': parse_slot(raw_slot('Old')),
}
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
side_effect=[raw_slot_v1('Updated'), raw_slot_v1('Same'), None]
side_effect=[raw_slot('Updated'), raw_slot('Same'), None]
)
ingestor_manager.resource_manager.renew_tag_lease = AsyncMock()
await ingestor_manager.update_slot_config()
assert ingestor_manager.managed_tags['slot1'] == parse_slot(raw_slot_v1('Updated'))
assert ingestor_manager.managed_tags['slot2'] == parse_slot(raw_slot_v1('Same'))
assert ingestor_manager.managed_tags['slot1'] == parse_slot(raw_slot('Updated'))
assert ingestor_manager.managed_tags['slot2'] == parse_slot(raw_slot('Same'))
assert 'slot3' not in ingestor_manager.managed_tags
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot1')
@@ -783,8 +768,8 @@ async def test_manage_server_create_subscription_failure(ingestor_manager):
@mark.asyncio
async def test_manage_server_subscribe_and_unsubscribe_failure(ingestor_manager):
"""subscribe() fails and so does the cleanup unsubscribe (legacy reader, session
gone): manage_server reports the failure and nothing escapes to the main loop"""
"""subscribe() fails and so does the cleanup unsubscribe (session gone):
manage_server reports the failure and nothing escapes to the main loop"""
reader = AsyncMock()
reader.subscriptions = {}
reader.subscribe.side_effect = Exception('Subscription error')
@@ -886,7 +871,7 @@ async def test_subscribe_to_tags(ingestor_manager):
@mark.asyncio
async def test_update_slot_config_keeps_last_good_config_when_malformed(ingestor_manager):
good = parse_slot(raw_slot_v1('Counter'))
good = parse_slot(raw_slot('Counter'))
ingestor_manager.managed_tags = {'slot1': good}
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
return_value={'server1': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=2': {}}}}
@@ -922,7 +907,7 @@ def test___init___ignores_malformed_write_env(
"""A malformed write/gRPC-only env var must not stop the read path (review #1)."""
monkeypatch.setenv('GRPC_PORT', '')
monkeypatch.setenv('WRITE_MIN_PAUSE_S', '0,5')
monkeypatch.setenv('CERT_BUCKET', 'streamlit-connectors')
monkeypatch.setenv('CERT_CACHE_DIR', 'certs-cache')
ingestor = IngestorManager(
redis_data={'host': 'localhost', 'port': 6379},
@@ -937,147 +922,4 @@ def test___init___ignores_malformed_write_env(
metrics_controller=MagicMock(),
)
assert ingestor.cert_provider.bucket == 'streamlit-connectors'
def test_parse_slot_config_warns_once_about_server_cert_path(ingestor_manager):
raw = raw_slot_v1()
raw['server1']['server_cert_path'] = '/app/certs/kepserver.der'
ingestor_manager.parse_slot_config('1', raw)
ingestor_manager.parse_slot_config('1', raw) # every loop refreshes the slot
warnings = [
c for c in ingestor_manager.logger.warning.call_args_list if 'server_cert_path' in str(c)
]
assert len(warnings) == 1
# --- READ_ENGINE=legacy: plan B with the untouched rc3 OpcManager -------------------
def make_manager(read_engine=None):
with (
patch('ingestor.managers.ingestor_manager.DataManager'),
patch('ingestor.managers.ingestor_manager.ResourceManager'),
):
manager = IngestorManager(
redis_data={'host': 'localhost', 'port': 6379},
lease_ttl=60,
heartbeat_ttl=60,
poll_interval=5,
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
logger=MagicMock(),
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=MagicMock(),
read_engine=read_engine,
)
manager.send_notification = MagicMock()
manager.emit_metric = AsyncMock()
return manager
def test_read_engine_defaults_to_v2(monkeypatch):
monkeypatch.delenv('READ_ENGINE', raising=False)
assert make_manager().read_engine == 'v2'
def test_read_engine_from_env(monkeypatch):
monkeypatch.setenv('READ_ENGINE', 'Legacy ')
assert make_manager().read_engine == 'legacy'
def test_invalid_read_engine_falls_back_to_v2(monkeypatch):
monkeypatch.setenv('READ_ENGINE', 'v3')
manager = make_manager()
assert manager.read_engine == 'v2'
assert any('Invalid READ_ENGINE' in str(c) for c in manager.logger.warning.call_args_list)
@mark.asyncio
@patch('ingestor.managers.ingestor_manager.ServerReader')
@patch('ingestor.managers.ingestor_manager.OpcManager')
async def test_legacy_initialize_builds_rc3_opc_manager(opc_manager, server_reader):
manager = make_manager('legacy')
server_config = {
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'subscription_period_ms': 1000,
'server_uri': 'urn:sientia:opc-ingestor',
'cert_path': '/app/certs/opc-ingestor-cert.pem',
'private_key_path': '/app/certs/opc-ingestor-key.pem',
'server_cert_path': None,
}
opc_manager.return_value = MagicMock(connect=AsyncMock())
result = await manager.initialize_opc_from_config(server_config)
server_reader.assert_not_called()
opc_manager.assert_called_once_with(
name='server1',
url='opc.tcp://localhost:4840',
subscription_period_ms=1000,
data_manager=manager.data_manager,
logger=manager.logger,
server_uri='urn:sientia:opc-ingestor',
notification_handler=manager.notification_handler,
metadata=metadata['metadata'],
cert_path='/app/certs/opc-ingestor-cert.pem',
private_key_path='/app/certs/opc-ingestor-key.pem',
server_cert_path=None,
metrics_controller=manager.metrics_controller,
)
assert result is opc_manager.return_value
result.connect.assert_awaited_once()
def test_legacy_keeps_raw_slot_v1():
manager = make_manager('legacy')
raw = raw_slot_v1()
# OpcManager reads tag_name/topics and the file cert paths straight from the slot
assert manager.parse_slot_config('1', raw) is raw
def test_legacy_rejects_slot_v2():
manager = make_manager('legacy')
raw_v2 = {
'server1': {
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'use_security': True,
'subscription_period_ms': 500,
'tags': {
'ns=2;i=2': {
'frequency': '1000',
'names': [{'collection': 'raw_a', 'tag_name': 'Counter', 'model_id': '1'}],
}
},
}
}
assert manager.parse_slot_config('1', raw_v2) is None
_, kwargs = manager.send_notification.call_args
assert kwargs['notification_id'] == 'OPC_SLOT_CONFIG_ERROR_1'
assert 'READ_ENGINE=legacy' in kwargs['message']
@mark.asyncio
async def test_legacy_remove_server_calls_plain_shutdown():
manager = make_manager('legacy')
opc_manager = MagicMock(shutdown=AsyncMock())
manager.opc_managers = {'server1': opc_manager}
manager.managed_tags = {'slot1': {'server1': {}}}
await manager.remove_server('server1', evict=False)
opc_manager.shutdown.assert_awaited_once_with()
assert manager.opc_managers == {}
assert ingestor.cert_provider.cache_dir == 'certs-cache'

627
tests/unit/managers/test_opc_manager.py
View File

@@ -1,627 +0,0 @@
import json
from datetime import datetime
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
import pytest
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
from ingestor.managers.opc_manager import OpcManager
tags = {
'ns=3;i=1001': {
'aggregation_function': 'LTS',
'frequency': 1000,
'max_value': 100,
'min_value': 0,
'tag_name': 'Counter',
},
'ns=3;i=1003': {
'aggregation_function': 'AVG',
'frequency': 1000,
'max_value': 100,
'min_value': 0,
'tag_name': 'Random',
},
'ns=3;i=1004': {
'aggregation_function': 'MDN',
'frequency': 1000,
'max_value': 100,
'min_value': 0,
'tag_name': 'Sawtooth',
},
}
metadata = {
'metadata': {
'model_id': 'test_model',
'model_name': 'test_model',
'workflow_name': 'test_workflow',
'schema_name': 'test_schedule',
},
}
@fixture
@patch('ingestor.managers.opc_manager.metrics')
def raw_opc_manager(mock_metrics):
opc_manager = OpcManager(
name='TestConnector',
url='opc.tcp://localhost:4840',
data_manager=AsyncMock(),
subscription_period_ms=1000,
logger=MagicMock(),
server_uri='opc.tcp://localhost:4840',
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=AsyncMock(),
)
opc_manager.emit_metric = AsyncMock()
opc_manager.send_notification_async = AsyncMock()
opc_manager.send_notification = MagicMock()
return opc_manager
@fixture
def opc_manager(raw_opc_manager):
raw_opc_manager.client = AsyncMock()
raw_opc_manager.cert_path = 'cert.pem'
raw_opc_manager.private_key_path = 'private_key.pem'
raw_opc_manager.server_cert_path = 'server_cert.pem'
return raw_opc_manager
@fixture
def opc_manager_subscribed(opc_manager):
opc_manager.subscriptions['sub1'] = AsyncMock()
return opc_manager
def test___str__(opc_manager):
assert (
str(opc_manager)
== 'OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}'
)
@mark.asyncio
async def test_shutdown_success(opc_manager):
opc_manager.disconnect = AsyncMock()
await opc_manager.shutdown()
opc_manager.disconnect.assert_called_once()
@mark.asyncio
async def test_shutdown_error(opc_manager):
opc_manager.disconnect = AsyncMock(side_effect=Exception('Test error'))
await opc_manager.shutdown()
opc_manager.logger.error.assert_called_once_with('Error during cleanup: Test error')
@mark.asyncio
async def test_set_security_success(opc_manager):
await opc_manager.set_security()
assert opc_manager.client.application_uri == opc_manager.server_uri
opc_manager.client.set_security.assert_called_once_with(
SecurityPolicyBasic256,
certificate=opc_manager.cert_path,
private_key=opc_manager.private_key_path,
server_certificate=opc_manager.server_cert_path,
)
assert opc_manager.client.secure_channel_timeout == 10000000
assert opc_manager.client.session_timeout == 10000000
@mark.asyncio
async def test_set_security_no_cert(opc_manager):
opc_manager.cert_path = None
opc_manager.private_key_path = None
try:
await opc_manager.set_security()
except ValueError as e:
assert str(e) == 'Certificate and private key paths must be provided for secure connection.'
else:
raise AssertionError('ValueError not raised')
assert opc_manager.client.set_security.call_count == 0
@mark.asyncio
@patch('ingestor.managers.opc_manager.metrics')
@patch('ingestor.managers.opc_manager.Client')
async def test_connect_no_security(client, mock_metrics, raw_opc_manager):
raw_opc_manager.set_security = AsyncMock()
client.return_value = AsyncMock()
await raw_opc_manager.connect()
client.assert_called_once_with(raw_opc_manager.url, timeout=10, watchdog_intervall=3600000)
raw_opc_manager.client.connect.assert_called_once()
raw_opc_manager.set_security.assert_not_called()
raw_opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=mock_metrics.OPC_CONNECTIONS_TOTAL,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
},
),
call(
metric_object=mock_metrics.OPC_CONNECTION_STATUS,
method='set',
value=1,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
'server_url': raw_opc_manager.url,
},
),
],
any_order=True,
)
@mark.asyncio
@patch('ingestor.managers.opc_manager.Client')
async def test_connect_with_security(client, raw_opc_manager):
raw_opc_manager.cert_path = 'cert.pem'
raw_opc_manager.private_key_path = 'private_key.pem'
raw_opc_manager.server_cert_path = 'server_cert.pem'
raw_opc_manager.set_security = AsyncMock()
client.return_value = AsyncMock()
await raw_opc_manager.connect()
client.assert_called_once_with(raw_opc_manager.url, timeout=10, watchdog_intervall=3600000)
raw_opc_manager.client.connect.assert_called_once()
raw_opc_manager.set_security.assert_called_once()
@mark.asyncio
@patch('ingestor.managers.opc_manager.Client')
@patch('ingestor.managers.opc_manager.metrics')
async def test_connect_exception_handling_and_metrics(
mock_metrics_module, mock_opc_client_class, raw_opc_manager
):
mock_client_instance = mock_opc_client_class.return_value
simulated_error_message = 'Erro de conexão simulado'
mock_client_instance.connect.side_effect = Exception(simulated_error_message)
raw_opc_manager.disconnect = AsyncMock()
opc_manager_instance = raw_opc_manager
opc_manager_instance.cert_path = None
with pytest.raises(Exception, match=simulated_error_message):
await opc_manager_instance.connect()
opc_manager_instance.disconnect.assert_called_once()
@mark.asyncio
async def test_create_subscription_no_client(raw_opc_manager):
try:
await raw_opc_manager.create_subscription('sub1')
except ValueError as e:
assert str(e) == 'Client not connected. Call connect first.'
else:
raise AssertionError('ValueError not raised')
@mark.asyncio
async def test_create_subscription_success_has_period(opc_manager):
await opc_manager.create_subscription('sub1')
opc_manager.client.create_subscription.assert_called_once_with(
opc_manager.subscription_period_ms, opc_manager
)
assert opc_manager.subscriptions['sub1'] is not None
@mark.asyncio
async def test_create_subscription_success_no_period(opc_manager):
await opc_manager.create_subscription('sub1')
opc_manager.client.create_subscription.assert_called_once_with(
opc_manager.subscription_period_ms, opc_manager
)
assert opc_manager.subscriptions['sub1'] is not None
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_create_subscription_with_metrics(metrics, opc_manager):
await opc_manager.create_subscription('sub1')
opc_manager.emit_metric.assert_called_once_with(
metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED,
method='inc',
value=1,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
'slot_name': 'sub1',
},
)
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_create_subscription_exception_during_client_call(
mock_metrics_module, raw_opc_manager
):
opc_manager_instance = raw_opc_manager
opc_manager_instance.client = AsyncMock()
subscription_name = 'test_sub_client_error'
simulated_error_message = 'Falha ao criar subscrição no cliente OPC'
opc_manager_instance.client.create_subscription.side_effect = Exception(simulated_error_message)
with pytest.raises(Exception, match=simulated_error_message):
await opc_manager_instance.create_subscription(subscription_name)
opc_manager_instance.client.create_subscription.assert_called_once_with(
opc_manager_instance.subscription_period_ms, opc_manager_instance
)
opc_manager_instance.logger.error.assert_called_once_with(
f'Failed to create subscription {subscription_name} on {opc_manager_instance.name}: {simulated_error_message}'
)
mock_metrics_module.OPC_SUBSCRIPTIONS_CREATED.labels.assert_not_called()
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_subscribe_no_subscription(metrics, opc_manager):
try:
await opc_manager.subscribe('sub1', tags, 1000)
except ValueError as e:
assert str(e) == 'Subscription not created. Call create_subscription first.'
else:
raise AssertionError('ValueError not raised')
metrics.OPC_TAGS_SUBSCRIBED.labels.assert_not_called()
@mark.asyncio
async def test_subscribe_success(opc_manager_subscribed):
opc_manager_subscribed.client.get_node = MagicMock()
opc_manager_subscribed.nodes = {'ns=3;i=1001': 'data'}
await opc_manager_subscribed.subscribe('sub1', tags, 1000)
assert opc_manager_subscribed.nodes == tags
opc_manager_subscribed.subscriptions['sub1'].subscribe_data_change.assert_called_once_with(
[opc_manager_subscribed.client.get_node(n) for n in tags]
)
@mark.asyncio
async def test_unsubscribe_no_subscription(opc_manager):
await opc_manager.unsubscribe('sub1')
opc_manager.logger.warning.assert_called_once_with(
"Subscription 'sub1' not found. Cannot unsubscribe."
)
assert opc_manager.subscriptions.get('sub1') is None
@mark.asyncio
async def test_unsubscribe_success(opc_manager_subscribed):
await opc_manager_subscribed.unsubscribe('sub1')
assert opc_manager_subscribed.subscriptions.get('sub1') is None
@mark.asyncio
async def test_disconnection_fallback_success(opc_manager):
opc_manager.client = AsyncMock()
opc_manager.client.disconnect.return_value = True
result = await opc_manager.disconnection_fallback()
assert result == []
@mark.asyncio
async def test_disconnection_fallback_fail(opc_manager):
opc_manager.client = AsyncMock()
opc_manager.client.disconnect.side_effect = Exception('Test error')
result = await opc_manager.disconnection_fallback()
assert result == [
{'attempt': 1, 'error': 'Test error', 'traceback': ANY},
{'attempt': 2, 'error': 'Test error', 'traceback': ANY},
{'attempt': 3, 'error': 'Test error', 'traceback': ANY},
{'attempt': 4, 'error': 'Test error', 'traceback': ANY},
{'attempt': 5, 'error': 'Test error', 'traceback': ANY},
]
assert opc_manager.client.disconnect.call_count == 5
@mark.asyncio
async def test_disconnect_success(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock()
await opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
assert opc_manager_subscribed.client is None
@mark.asyncio
async def test_disconnect_no_client(opc_manager_subscribed):
opc_manager_subscribed.client = None
assert await opc_manager_subscribed.disconnect() is None
opc_manager_subscribed.logger.warning.assert_has_calls(
[
call('Client already disconnected.'),
]
)
@mark.asyncio
async def test_disconnect_error_unsubscribe(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock(disconnect=AsyncMock())
opc_manager_subscribed.subscriptions['sub1'] = MagicMock(
delete=AsyncMock(side_effect=Exception('Test error'))
)
await opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
opc_manager_subscribed.client = None
opc_manager_subscribed.logger.error.assert_called_once_with(
'Failed to clean up subscription: Test error'
)
@mark.asyncio
async def test_disconnect_error(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock()
opc_manager_subscribed.client.disconnect = MagicMock(side_effect=Exception('Test error'))
await opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
opc_manager_subscribed.client = None
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_opc_manager):
mock_metrics_module.OPC_CONNECTION_STATUS.reset_mock()
mock_metrics_module.OPC_TAGS_SUBSCRIBED.reset_mock()
raw_opc_manager.client = MagicMock()
mock_sub1 = MagicMock()
mock_sub2 = MagicMock()
raw_opc_manager.subscriptions = {'sub1': mock_sub1, 'sub2': mock_sub2}
await raw_opc_manager.disconnect()
raw_opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=mock_metrics_module.OPC_CONNECTION_STATUS,
method='set',
value=0,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
'server_url': raw_opc_manager.url,
},
),
call(
metric_object=mock_metrics_module.OPC_TAGS_SUBSCRIBED,
method='set',
value=0,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
},
),
],
any_order=True,
)
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_datachange_notification(metrics, opc_manager_subscribed):
data = MagicMock(
monitored_item=MagicMock(
Value=MagicMock(
Value=MagicMock(Value=42),
SourceTimestamp=datetime.strptime('2021-01-01T00:00:00', '%Y-%m-%dT%H:%M:%S'),
)
)
)
opc_manager_subscribed.nodes = {
'ns=3;i=1001': {
'tag_name': 'Counter',
'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 2},
'topics': ['topic1', 'topic2'],
}
}
await opc_manager_subscribed.datachange_notification('ns=3;i=1001', None, data)
opc_manager_subscribed.data_manager.publish.assert_any_call(
'topic1',
{
'tag': 'ns=3;i=1001',
'name': 'Counter',
'timestamp': '2021-01-01 00:00:00-0300',
'value': 42,
},
)
opc_manager_subscribed.data_manager.publish.assert_any_call(
'topic2',
{
'tag': 'ns=3;i=1001',
'name': 'Counter',
'timestamp': '2021-01-01 00:00:00-0300',
'value': 42,
},
)
assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0
opc_manager_subscribed.emit_metric.assert_called_once_with(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=0,
tags={
'pod_id': opc_manager_subscribed.pod_id,
'server_name': opc_manager_subscribed.name,
},
)
@mark.asyncio
async def test_check_cycles_no_notification(opc_manager):
# Setup: node with cycle_count just below threshold
opc_manager.nodes = {
'ns=3;i=1001': {
'tag_name': 'Counter',
'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 3.0},
}
}
await opc_manager.check_cycles()
# After one increment, cycle_count = 4.0, still below threshold
assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(4.0)
opc_manager.send_notification_async.assert_not_called()
@mark.asyncio
async def test_check_cycles_triggers_notification(opc_manager):
# Setup: node with cycle_count just below threshold, increment will cross threshold
opc_manager.nodes = {
'ns=3;i=1001': {
'tag_name': 'Counter',
'cycle_rule': {'cycle_increment': 2.5, 'cycle_count': 3.0},
}
}
opc_manager.notification_handler.build_and_send_notification = MagicMock()
await opc_manager.check_cycles()
# After increment, cycle_count = 5.5, should trigger notification
assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(5.5)
opc_manager.send_notification_async.assert_called_once_with(
notification_id='TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED',
message='5.5 cycles without receive from ns=3;i=1001:Counter',
block='opc_manager',
level=NotificationLevel.WARNING,
metadata=metadata['metadata'],
)
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_check_opc_listenning_no_notification(metrics, opc_manager):
opc_manager.non_receive_count = 3
result = await opc_manager.check_opc_listenning()
assert opc_manager.non_receive_count == 4
opc_manager.send_notification_async.assert_not_called()
assert result is False
opc_manager.emit_metric.assert_called_once_with(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=opc_manager.non_receive_count,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
},
)
@mark.asyncio
async def test_check_opc_listenning_warning_notification(opc_manager):
opc_manager.non_receive_count = 4
result = await opc_manager.check_opc_listenning()
assert opc_manager.non_receive_count == 5
opc_manager.send_notification_async.assert_called_once_with(
notification_id=f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
message=f'5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}',
block='opc_manager',
level=NotificationLevel.ERROR,
metadata=metadata['metadata'],
)
assert result is False
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager):
opc_manager.non_receive_count = 14
result = await opc_manager.check_opc_listenning()
assert opc_manager.non_receive_count == 15
# Should be called twice: once for 5, once for 15
assert opc_manager.send_notification_async.call_count == 2
calls = opc_manager.send_notification_async.call_args_list
# First call: 5 cycles warning
assert calls[0].kwargs == {
'notification_id': f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
'message': f'15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}',
'block': 'opc_manager',
'level': NotificationLevel.ERROR,
'metadata': metadata['metadata'],
}
# Second call: 15 cycles retry
assert calls[1].kwargs == {
'notification_id': f'OPC_CONNECTION_RETRY__{opc_manager.name}',
'message': f'Retrying to connect to server {opc_manager.name}',
'block': 'opc_manager',
'level': NotificationLevel.ERROR,
'metadata': metadata['metadata'],
}
assert result is True
opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=opc_manager.non_receive_count,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
},
),
]
)
opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=metrics.OPC_RECONNECTIONS_TOTAL,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
},
),
]
)

17
tests/unit/managers/test_server_reader.py
View File

@@ -9,9 +9,14 @@ SERVER_CONFIG = {
'name': 'kepserver',
'server_id': 1, # the slot may carry it as int; the pool keys by str
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 15000.0,
'security': {'mode': 'SignAndEncrypt', 'policy': 'Basic256Sha256'},
'certificate': {
'certificate_id': 1,
'application_uri': 'urn:sientia:opc-ingestor',
'cert': {'bucket': 'opc-certificates', 'object_name': 'client/p/cert.pem'},
'private_key': {'bucket': 'opc-certificates', 'object_name': 'client/p/key.pem'},
},
}
@@ -45,10 +50,8 @@ def test_server_cfg_keeps_only_connection_fields():
assert reader.server_id == '1'
assert reader.server_cfg == {
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 15000.0,
key: SERVER_CONFIG[key]
for key in ('url', 'subscription_period_ms', 'security', 'certificate')
}
@@ -58,7 +61,7 @@ async def test_connect_opens_pooled_connection():
await reader.connect()
# use_security=True: the pool fetches the certificate from MinIO (CertProvider)
# With a certificate, the pool fetches it from MinIO (CertProvider)
pool.get.assert_awaited_once_with('1', reader.server_cfg)

4
tests/unit/test_config.py
View File

@@ -8,7 +8,6 @@ _ENV_VARS = (
'NATS_QUEUE',
'GRPC_ENABLED',
'GRPC_PORT',
'CERT_BUCKET',
'CERT_CACHE_DIR',
'WRITE_TIMESTAMP',
'WRITE_QUALITY',
@@ -31,7 +30,6 @@ def test_from_env_defaults():
assert cfg.nats_queue == 'gateway-writers'
assert cfg.grpc_enabled is False
assert cfg.grpc_port == 50051
assert cfg.cert_bucket == 'streamlit-connectors'
assert cfg.cert_cache_dir == '/tmp/opc-certs' # noqa: S108
assert cfg.write_timestamp is True
assert cfg.write_quality is True
@@ -76,12 +74,10 @@ def test_certs_from_env_ignores_malformed_numeric_vars(monkeypatch):
monkeypatch.setenv('GRPC_PORT', '')
monkeypatch.setenv('IDLE_CONN_MAX_S', 'abc')
monkeypatch.setenv('CERT_BUCKET', 'my-bucket')
monkeypatch.setenv('CERT_CACHE_DIR', 'certs-cache')
cfg = GatewayConfig.certs_from_env()
assert cfg.cert_bucket == 'my-bucket'
assert cfg.cert_cache_dir == 'certs-cache'
with pytest.raises(ValueError):
GatewayConfig.from_env() # the full config still fails, only the runtime uses it