gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

SIENTIAPDE-1974: feat(read): slot v2 reads via SubscriptionEngine, OPC cert from MinIO

Browse Source 
Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over
the shared OpcConnectionPool. The OPC client certificate now always comes from
MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the
Gitea repository, so a new version branch no longer breaks the reads.

- io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy
  v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway
  and Orchestrator can be deployed in any order.
- managers/server_reader.py: new per-server reader with the OpcManager
  interface the IngestorManager already drives (connect, create_subscription,
  subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown).
- IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a
  read DataSink, built without touching the network. Slots from Redis are
  normalized on lease and refresh; a malformed slot is notified and the last
  good config is kept.
- GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no
  longer closes it on stop: a Postgres/NATS failure can't stop the reads.
- pool: pin/unpin per active subscription so release_idle never evicts a
  subscribed connection; public evict() for lost servers / changed config.
- SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a
  subscription locally even when the server-side delete fails.
- DataSink: model_id only written when the name has one, keeping v1 documents
  identical to the legacy ones.
- config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7).
- Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh;
  README/CLAUDE.md document the MinIO certificate and the slot v2 format.

Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against
an asyncua server: v1 slot with two topics -> one doc per collection, legacy
doc shape, idle sweep keeps the subscription, lost-server shutdown and
re-initialization work.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Eduardo Rios
2026-10-06 13:27:18 -03:00
parent 4c85a9c5c2
commit 50578fdba8
25 changed files with 957 additions and 1534 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
tests/unit/connection/test_certs.py
View File

@@ -26,3 +26,47 @@ async def test_fetch_returns_local_paths(tmp_path):
)
assert (tmp_path / 'client_cert_c.pem').read_bytes() == b'cert-bytes'
assert (tmp_path / 'private_key_k.pem').read_bytes() == b'key-bytes'
@pytest.mark.asyncio
async def test_lazy_minio_builds_client_once_on_first_download():
from unittest.mock import MagicMock
from ingestor.connection.certs import LazyMinio
client = AsyncMock()
client.download_file.return_value = b'bytes'
factory = MagicMock(return_value=client)
minio = LazyMinio(factory)
factory.assert_not_called() # no network at construction
assert await minio.download_file('a.pem', 'bucket', {}) == b'bytes'
await minio.download_file('b.pem', 'bucket', {})
factory.assert_called_once()
assert client.download_file.await_count == 2
def test_build_cert_provider_is_lazy_and_uses_config(monkeypatch):
from unittest.mock import MagicMock
from ingestor.config import GatewayConfig
from ingestor.connection import certs
minio_cls = MagicMock()
monkeypatch.setattr(certs, 'MinioRepository', minio_cls)
mongo = MagicMock()
provider = certs.build_cert_provider(
mongo=mongo,
config=GatewayConfig(cert_bucket='streamlit-connectors', cert_cache_dir='certs-cache'),
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=MagicMock(),
)
assert provider.mongo is mongo
assert provider.bucket == 'streamlit-connectors'
assert provider.cache_dir == 'certs-cache'
assert isinstance(provider.minio, certs.LazyMinio)
minio_cls.assert_not_called() # MinIO is only reached when a cert is needed

66
tests/unit/connection/test_pool.py
View File

@@ -182,3 +182,69 @@ async def test_concurrent_get_same_server_connects_once():
assert c1 is c2 is clients[0]
assert ctor.call_count == 1
assert clients[0].connect.await_count == 1
@pytest.mark.asyncio
async def test_release_idle_keeps_pinned_connection():
"""A subscribed connection only gets callbacks (never get()), so it looks idle:
the pin keeps release_idle from dropping it (SIENTIAPDE-1974 G4)."""
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
pool.pin('srv')
await pool.release_idle(max_idle_s=0)
assert 'srv' in pool._connections
fake_client.disconnect.assert_not_awaited()
@pytest.mark.asyncio
async def test_release_idle_evicts_after_last_unpin():
pool = make_pool()
fake_client = AsyncMock()
with patch('ingestor.connection.pool.Client', return_value=fake_client):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
pool.pin('srv')
pool.pin('srv') # two slots subscribed on the same server
pool.unpin('srv')
await pool.release_idle(max_idle_s=0)
assert 'srv' in pool._connections # one subscription still active
pool.unpin('srv')
await pool.release_idle(max_idle_s=0)
assert 'srv' not in pool._connections
fake_client.disconnect.assert_awaited_once()
def test_unpin_never_goes_negative():
pool = make_pool()
pool.unpin('srv')
pool.pin('srv')
assert pool._pins == {'srv': 1}
@pytest.mark.asyncio
async def test_evict_disconnects_now_and_next_get_reconnects():
pool = make_pool()
first, second = AsyncMock(), AsyncMock()
with patch('ingestor.connection.pool.Client', side_effect=[first, second]):
await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
await pool.evict('srv')
client = await pool.get('srv', {'url': 'opc.tcp://h:4840', 'use_security': False})
first.disconnect.assert_awaited_once()
assert client is second
@pytest.mark.asyncio
async def test_evict_unknown_server_is_noop():
pool = make_pool()
await pool.evict('nope')
pool.emit_metric.assert_not_awaited()

54
tests/unit/engines/test_subscription.py
View File

@@ -201,3 +201,57 @@ async def test_check_opc_listenning_triggers_reconnect_at_15():
assert result is True
assert eng.send_notification_async.call_count == 2
@pytest.mark.asyncio
async def test_create_subscription_pins_pool_connection():
eng = make_engine()
eng.pool = MagicMock(get=AsyncMock(return_value=AsyncMock()))
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
eng.pool.pin.assert_called_once_with('1')
@pytest.mark.asyncio
async def test_create_subscription_failure_does_not_pin():
eng = make_engine()
client = AsyncMock()
client.create_subscription.side_effect = RuntimeError('boom')
eng.pool = MagicMock(get=AsyncMock(return_value=client))
with pytest.raises(RuntimeError):
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
eng.pool.pin.assert_not_called()
@pytest.mark.asyncio
async def test_unsubscribe_unpins_pool_connection():
eng = make_engine()
eng.pool = MagicMock(get=AsyncMock(return_value=AsyncMock()))
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
await eng.unsubscribe('slot1')
eng.pool.unpin.assert_called_once_with('1')
assert 'slot1' not in eng.subscriptions
@pytest.mark.asyncio
async def test_unsubscribe_tolerates_dead_connection():
"""A lost server's subscription can't be deleted on the server side; it must
still be dropped locally (and unpinned) so the slot can be re-subscribed."""
eng = make_engine()
sub = AsyncMock()
sub.delete.side_effect = ConnectionError('closed')
client = AsyncMock()
client.create_subscription.return_value = sub
eng.pool = MagicMock(get=AsyncMock(return_value=client))
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
await eng.unsubscribe('slot1') # must not raise
assert 'slot1' not in eng.subscriptions
eng.pool.unpin.assert_called_once_with('1')
eng.logger.warning.assert_called_once()

23
tests/unit/io/test_data_sink_publish.py
View File

@@ -67,3 +67,26 @@ async def test_publish_reading_emits_tag_value_metric_per_name():
value=42,
tags={'pod_id': dm.pod_id, 'tag_name': 'Counter', 'model_id': '1'},
)
@pytest.mark.asyncio
async def test_publish_reading_without_model_id_keeps_legacy_doc():
"""Names converted from a slot v1 carry no model_id: the document must be the
same as the legacy OpcManager one (SIENTIAPDE-1974 NF2)."""
sink, dm = make_sink()
names = [{'collection': 'raw_x', 'tag_name': 'Counter'}]
await sink.publish_reading(
tag='ns=2;i=2', value=42, timestamp='2026-07-13 00:00:00+0000', names=names
)
collection, doc = dm.publish.await_args_list[0].args
assert collection == 'raw_x'
assert doc == {
'tag': 'ns=2;i=2',
'name': 'Counter',
'timestamp': '2026-07-13 00:00:00+0000',
'value': 42,
}
_, kwargs = sink.emit_metric.await_args
assert kwargs['tags']['model_id'] == ''

99
tests/unit/io/test_slots.py
View File

@@ -62,3 +62,102 @@ def test_parse_slot_rejects_tag_without_names():
with pytest.raises((KeyError, ValueError)):
parse_slot(raw)
RAW_V1 = {
'kepserver': {
'name': 'kepserver',
'server_id': '1',
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:sientia:opc-ingestor',
'cert_path': '/app/certs/opc-ingestor-cert.pem',
'private_key_path': '/app/certs/opc-ingestor-key.pem',
'server_cert_path': None,
'subscription_period_ms': 15000.0,
'tags': {
'ns=2;s=A.TargetSin': {
'aggr_func': 'lts',
'frequency': 30000,
'server_id': '1',
'tag_address': 'ns=2;s=A.TargetSin',
'tag_name': 'TargetSin',
'topics': ['raw_scouter-sin-approx', 'raw_other-model'],
}
},
}
}
def test_parse_slot_converts_v1_topics_to_names():
"""Slot v1 (current Orchestrator) is read as v2: one name per topic, no model_id,
so the documents stay identical to the legacy ones (SIENTIAPDE-1974 G1)."""
server = parse_slot(RAW_V1)['kepserver']
tag = server['tags']['ns=2;s=A.TargetSin']
assert tag['names'] == [
{'collection': 'raw_scouter-sin-approx', 'tag_name': 'TargetSin'},
{'collection': 'raw_other-model', 'tag_name': 'TargetSin'},
]
assert tag['frequency'] == 30000
assert tag['active_read'] == {'enabled': False}
def test_parse_slot_v1_cert_path_means_security_but_path_is_dropped():
server = parse_slot(RAW_V1)['kepserver']
assert server['use_security'] is True
assert server['name'] == 'kepserver'
assert server['server_id'] == '1'
assert server['subscription_period_ms'] == 15000.0
# the certificate always comes from MinIO now: no file path survives the parse
assert 'cert_path' not in server
assert 'private_key_path' not in server
def test_parse_slot_v1_without_cert_path_is_insecure():
raw = {
's': {
'server_id': '2',
'url': 'u',
'cert_path': None,
'tags': {'ns=2;i=1': {'tag_name': 't', 'frequency': '1000', 'topics': ['raw_c']}},
}
}
server = parse_slot(raw)['s']
assert server['use_security'] is False
assert server['name'] == 's' # falls back to the slot key
def test_parse_slot_v2_explicit_use_security_wins():
raw = {
's': {
'server_id': '1',
'url': 'u',
'use_security': False,
'cert_path': '/legacy/path',
'tags': {
'ns=2;i=1': {
'frequency': '1000',
'names': [{'collection': 'c', 'tag_name': 't', 'model_id': '1'}],
}
},
}
}
assert parse_slot(raw)['s']['use_security'] is False
def test_parse_slot_rejects_v1_tag_without_tag_name():
raw = {
's': {
'server_id': '1',
'url': 'u',
'tags': {'ns=2;i=1': {'frequency': '1', 'topics': ['c']}},
}
}
import pytest
with pytest.raises(ValueError):
parse_slot(raw)

192
tests/unit/managers/test_ingestor_manager.py
View File

@@ -4,6 +4,9 @@ from unittest.mock import AsyncMock, MagicMock, call, patch
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
from ingestor.config import GatewayConfig
from ingestor.connection.pool import OpcConnectionPool
from ingestor.io.slots import parse_slot
from ingestor.managers.ingestor_manager import IngestorManager
metadata = {
@@ -16,6 +19,21 @@ metadata = {
}
def raw_slot_v1(tag_name='Counter', address='ns=2;i=2'):
"""A slot as the current (v1) Orchestrator writes it to Redis."""
return {
'server1': {
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'server_uri': 'urn:sientia:opc-ingestor',
'cert_path': '/app/certs/opc-ingestor-cert.pem',
'subscription_period_ms': 500.0,
'tags': {address: {'tag_name': tag_name, 'frequency': '1000', 'topics': ['raw_a']}},
}
}
@fixture(autouse=True)
def _collect_garbage_after_test():
# ponytail: IngestorManager.__del__ calls asyncio.run(self.shutdown()); if a
@@ -52,12 +70,12 @@ def ingestor_manager(data_manager_mock, resource_manager_mock):
return ingestor
@patch('ingestor.managers.ingestor_manager.OpcManager')
@patch('ingestor.managers.ingestor_manager.ServerReader')
@patch('ingestor.managers.ingestor_manager.DataManager')
@patch('ingestor.managers.ingestor_manager.ResourceManager')
@patch('ingestor.managers.ingestor_manager.NotificationHandler')
def test___init__(
notification_handler_mock, resource_manager_mock, data_manager_mock, opc_manager_mock
notification_handler_mock, resource_manager_mock, data_manager_mock, server_reader_mock
):
ingestor = IngestorManager(
redis_data={'host': 'localhost', 'port': 6379},
@@ -72,7 +90,7 @@ def test___init__(
metrics_controller=MagicMock(),
)
opc_manager_mock.assert_not_called()
server_reader_mock.assert_not_called()
data_manager_mock.assert_called_once_with(
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
@@ -100,59 +118,86 @@ def test___init__(
assert ingestor.data_manager == data_manager_mock.return_value
assert ingestor.resource_manager == resource_manager_mock.return_value
# Read path (SIENTIAPDE-1974): pool + read sink built here, nothing connected yet.
assert isinstance(ingestor.pool, OpcConnectionPool)
assert ingestor.pool.cert_provider is ingestor.cert_provider
assert ingestor.cert_provider.mongo is data_manager_mock.return_value.mongo_repository
assert ingestor.sink.data_manager is data_manager_mock.return_value
assert ingestor.sink.postgres is None
@mark.asyncio
@patch('ingestor.managers.ingestor_manager.OpcManager')
async def test_initialize_opc_from_config(opc_manager, ingestor_manager):
server_config = {
'name': 'server1',
'url': 'opc.tcp://localhost:4840',
'subscription_period_ms': 1000,
'server_uri': 'http://opcua-server.simulator',
'cert_path': '/path/to/cert',
'private_key_path': '/path/to/private_key',
'server_cert_path': '/path/to/server_cert',
'pod_id': 'test_pod',
}
opc_manager.return_value = MagicMock(connect=AsyncMock())
result = await ingestor_manager.initialize_opc_from_config(server_config)
opc_manager.assert_called_once_with(
name=server_config['name'],
url=server_config['url'],
data_manager=ingestor_manager.data_manager,
logger=ingestor_manager.logger,
subscription_period_ms=server_config['subscription_period_ms'],
server_uri=server_config['server_uri'],
notification_handler=ingestor_manager.notification_handler,
cert_path=server_config['cert_path'],
private_key_path=server_config['private_key_path'],
server_cert_path=server_config['server_cert_path'],
@patch('ingestor.managers.ingestor_manager.DataManager')
@patch('ingestor.managers.ingestor_manager.ResourceManager')
def test___init___cert_provider_uses_gateway_config(resource_manager_mock, data_manager_mock):
ingestor = IngestorManager(
redis_data={'host': 'localhost', 'port': 6379},
lease_ttl=60,
heartbeat_ttl=60,
poll_interval=5,
mongo_connection_string='mongodb://localhost:27017',
mongo_database='sientia',
logger=MagicMock(),
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=ingestor_manager.metrics_controller,
metrics_controller=MagicMock(),
gateway_config=GatewayConfig(cert_bucket='my-bucket', cert_cache_dir='certs-cache'),
)
assert result == opc_manager.return_value
result.connect.assert_called_once()
assert ingestor.cert_provider.bucket == 'my-bucket'
assert ingestor.cert_provider.cache_dir == 'certs-cache'
@mark.asyncio
@patch('ingestor.managers.ingestor_manager.OpcManager')
@patch('ingestor.managers.ingestor_manager.traceback')
async def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, ingestor_manager):
@patch('ingestor.managers.ingestor_manager.ServerReader')
async def test_initialize_opc_from_config(server_reader, ingestor_manager):
server_config = {
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 1000,
}
server_reader.return_value = MagicMock(connect=AsyncMock())
result = await ingestor_manager.initialize_opc_from_config(server_config)
server_reader.assert_called_once_with(
name=server_config['name'],
server_config=server_config,
pool=ingestor_manager.pool,
sink=ingestor_manager.sink,
logger=ingestor_manager.logger,
notification_handler=ingestor_manager.notification_handler,
metrics_controller=ingestor_manager.metrics_controller,
metadata=metadata['metadata'],
)
assert result == server_reader.return_value
assert result.config == server_config
result.connect.assert_awaited_once()
@mark.asyncio
@patch('ingestor.managers.ingestor_manager.ServerReader')
@patch('ingestor.managers.ingestor_manager.traceback')
async def test_initialize_opc_from_config_exception(
traceback_mock, server_reader, ingestor_manager
):
server_config = {
'name': 'server1',
'server_id': '1',
'url': 'opc.tcp://localhost:4840',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 1000,
'server_uri': 'http://opcua-server.simulator',
'cert_path': '/path/to/cert',
'private_key_path': '/path/to/private_key',
'server_cert_path': '/path/to/server_cert',
}
ingestor_manager.logger.error = MagicMock()
opc_manager.side_effect = Exception('Initialization error')
# e.g. the server is unreachable or the certificate can't be fetched from MinIO
server_reader.return_value = MagicMock(
connect=AsyncMock(side_effect=Exception('Initialization error'))
)
result = await ingestor_manager.initialize_opc_from_config(server_config)
@@ -174,11 +219,14 @@ async def test_shutdown(ingestor_manager):
ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()}
ingestor_manager.data_manager.shutdown = MagicMock()
ingestor_manager.pool = MagicMock(close_all=AsyncMock())
await ingestor_manager.shutdown()
ingestor_manager.opc_managers['server1'].shutdown.assert_called_once()
ingestor_manager.opc_managers['server2'].shutdown.assert_called_once()
# The pool is owned by the read path: closed after the readers unsubscribe.
ingestor_manager.pool.close_all.assert_awaited_once()
ingestor_manager.data_manager.shutdown.assert_called_once()
@@ -218,9 +266,8 @@ async def test_remove_server_not_found(ingestor_manager):
@mark.asyncio
@patch('ingestor.managers.ingestor_manager.OpcManager')
@patch('ingestor.managers.ingestor_manager.metrics')
async def test_update_opc_servers(metrics, opc_manager, ingestor_manager):
async def test_update_opc_servers(metrics, ingestor_manager):
manager1 = MagicMock(config={'config': 'config1'})
manager2 = MagicMock(config={'config': 'config2'})
manager3 = MagicMock(config={'config': 'config3'})
@@ -431,25 +478,47 @@ async def test_get_number_of_slots_empty(metrics, ingestor_manager):
@mark.asyncio
async def test_get_slot_leases_1_success(ingestor_manager):
ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True)
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value={'tags': ['tag1']})
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value=raw_slot_v1())
ingestor_manager.number_of_slots = 1
result = await ingestor_manager.get_slot_leases()
assert result == {'1': {'tags': ['tag1']}}
# Stored already normalized to slot v2 (v1 converted by parse_slot).
assert result == {'1': parse_slot(raw_slot_v1())}
assert ingestor_manager.managed_tags == result
@mark.asyncio
async def test_get_slot_leases_2_success(ingestor_manager):
ingestor_manager.resource_manager.lease_tag = AsyncMock(side_effect=[True, True])
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
side_effect=[{'tags': ['tag1']}, {'tags': ['tag2']}]
side_effect=[raw_slot_v1('Counter'), raw_slot_v1('Ramp', 'ns=2;i=3')]
)
ingestor_manager.number_of_slots = 2
result = await ingestor_manager.get_slot_leases(max_slots=2)
assert result == {'1': {'tags': ['tag1']}, '2': {'tags': ['tag2']}}
assert result == {
'1': parse_slot(raw_slot_v1('Counter')),
'2': parse_slot(raw_slot_v1('Ramp', 'ns=2;i=3')),
}
@mark.asyncio
async def test_get_slot_leases_skips_malformed_slot(ingestor_manager):
ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True)
# tag without names/topics/frequency
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
return_value={'server1': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=2': {}}}}
)
ingestor_manager.number_of_slots = 1
result = await ingestor_manager.get_slot_leases()
assert result == {}
_, kwargs = ingestor_manager.send_notification.call_args
assert kwargs['notification_id'] == 'OPC_SLOT_CONFIG_ERROR_1'
assert kwargs['level'] == NotificationLevel.ERROR
@mark.asyncio
@@ -497,20 +566,20 @@ async def test_unsubscribe_slot(ingestor_manager):
@mark.asyncio
async def test_update_slot_config(ingestor_manager):
ingestor_manager.managed_tags = {
'slot1': {'config': 'old_config'},
'slot2': {'config': 'new_config'},
'slot3': {'config': 'old_config'},
'slot1': parse_slot(raw_slot_v1('Old')),
'slot2': parse_slot(raw_slot_v1('Same')),
'slot3': parse_slot(raw_slot_v1('Old')),
}
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
side_effect=[{'config': 'updated_config'}, {'config': 'new_config'}, None]
side_effect=[raw_slot_v1('Updated'), raw_slot_v1('Same'), None]
)
ingestor_manager.resource_manager.renew_tag_lease = AsyncMock()
await ingestor_manager.update_slot_config()
assert ingestor_manager.managed_tags['slot1'] == {'config': 'updated_config'}
assert ingestor_manager.managed_tags['slot2'] == {'config': 'new_config'}
assert ingestor_manager.managed_tags['slot1'] == parse_slot(raw_slot_v1('Updated'))
assert ingestor_manager.managed_tags['slot2'] == parse_slot(raw_slot_v1('Same'))
assert 'slot3' not in ingestor_manager.managed_tags
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot1')
@@ -652,3 +721,20 @@ async def test_subscribe_to_tags(ingestor_manager):
assert ingestor_manager.manage_server.call_count == 3
ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with('server3', None)
@mark.asyncio
async def test_update_slot_config_keeps_last_good_config_when_malformed(ingestor_manager):
good = parse_slot(raw_slot_v1('Counter'))
ingestor_manager.managed_tags = {'slot1': good}
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
return_value={'server1': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=2': {}}}}
)
ingestor_manager.resource_manager.renew_tag_lease = AsyncMock()
await ingestor_manager.update_slot_config()
# Still reading with the last good config; the bad one is notified, not applied.
assert ingestor_manager.managed_tags == {'slot1': good}
_, kwargs = ingestor_manager.send_notification.call_args
assert kwargs['notification_id'] == 'OPC_SLOT_CONFIG_ERROR_slot1'

627
tests/unit/managers/test_opc_manager.py
View File

@@ -1,627 +0,0 @@
import json
from datetime import datetime
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
import pytest
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from pytest import fixture, mark
from sientia_do.notifications.models import NotificationLevel
from ingestor.managers.opc_manager import OpcManager
tags = {
'ns=3;i=1001': {
'aggregation_function': 'LTS',
'frequency': 1000,
'max_value': 100,
'min_value': 0,
'tag_name': 'Counter',
},
'ns=3;i=1003': {
'aggregation_function': 'AVG',
'frequency': 1000,
'max_value': 100,
'min_value': 0,
'tag_name': 'Random',
},
'ns=3;i=1004': {
'aggregation_function': 'MDN',
'frequency': 1000,
'max_value': 100,
'min_value': 0,
'tag_name': 'Sawtooth',
},
}
metadata = {
'metadata': {
'model_id': 'test_model',
'model_name': 'test_model',
'workflow_name': 'test_workflow',
'schema_name': 'test_schedule',
},
}
@fixture
@patch('ingestor.managers.opc_manager.metrics')
def raw_opc_manager(mock_metrics):
opc_manager = OpcManager(
name='TestConnector',
url='opc.tcp://localhost:4840',
data_manager=AsyncMock(),
subscription_period_ms=1000,
logger=MagicMock(),
server_uri='opc.tcp://localhost:4840',
notification_handler=MagicMock(),
metadata=metadata['metadata'],
metrics_controller=AsyncMock(),
)
opc_manager.emit_metric = AsyncMock()
opc_manager.send_notification_async = AsyncMock()
opc_manager.send_notification = MagicMock()
return opc_manager
@fixture
def opc_manager(raw_opc_manager):
raw_opc_manager.client = AsyncMock()
raw_opc_manager.cert_path = 'cert.pem'
raw_opc_manager.private_key_path = 'private_key.pem'
raw_opc_manager.server_cert_path = 'server_cert.pem'
return raw_opc_manager
@fixture
def opc_manager_subscribed(opc_manager):
opc_manager.subscriptions['sub1'] = AsyncMock()
return opc_manager
def test___str__(opc_manager):
assert (
str(opc_manager)
== 'OpcManager(name=TestConnector, url=opc.tcp://localhost:4840, server_uri=opc.tcp://localhost:4840)\nnodes={}, subscriptions={}'
)
@mark.asyncio
async def test_shutdown_success(opc_manager):
opc_manager.disconnect = AsyncMock()
await opc_manager.shutdown()
opc_manager.disconnect.assert_called_once()
@mark.asyncio
async def test_shutdown_error(opc_manager):
opc_manager.disconnect = AsyncMock(side_effect=Exception('Test error'))
await opc_manager.shutdown()
opc_manager.logger.error.assert_called_once_with('Error during cleanup: Test error')
@mark.asyncio
async def test_set_security_success(opc_manager):
await opc_manager.set_security()
assert opc_manager.client.application_uri == opc_manager.server_uri
opc_manager.client.set_security.assert_called_once_with(
SecurityPolicyBasic256,
certificate=opc_manager.cert_path,
private_key=opc_manager.private_key_path,
server_certificate=opc_manager.server_cert_path,
)
assert opc_manager.client.secure_channel_timeout == 10000000
assert opc_manager.client.session_timeout == 10000000
@mark.asyncio
async def test_set_security_no_cert(opc_manager):
opc_manager.cert_path = None
opc_manager.private_key_path = None
try:
await opc_manager.set_security()
except ValueError as e:
assert str(e) == 'Certificate and private key paths must be provided for secure connection.'
else:
raise AssertionError('ValueError not raised')
assert opc_manager.client.set_security.call_count == 0
@mark.asyncio
@patch('ingestor.managers.opc_manager.metrics')
@patch('ingestor.managers.opc_manager.Client')
async def test_connect_no_security(client, mock_metrics, raw_opc_manager):
raw_opc_manager.set_security = AsyncMock()
client.return_value = AsyncMock()
await raw_opc_manager.connect()
client.assert_called_once_with(raw_opc_manager.url, timeout=10, watchdog_intervall=3600000)
raw_opc_manager.client.connect.assert_called_once()
raw_opc_manager.set_security.assert_not_called()
raw_opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=mock_metrics.OPC_CONNECTIONS_TOTAL,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
},
),
call(
metric_object=mock_metrics.OPC_CONNECTION_STATUS,
method='set',
value=1,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
'server_url': raw_opc_manager.url,
},
),
],
any_order=True,
)
@mark.asyncio
@patch('ingestor.managers.opc_manager.Client')
async def test_connect_with_security(client, raw_opc_manager):
raw_opc_manager.cert_path = 'cert.pem'
raw_opc_manager.private_key_path = 'private_key.pem'
raw_opc_manager.server_cert_path = 'server_cert.pem'
raw_opc_manager.set_security = AsyncMock()
client.return_value = AsyncMock()
await raw_opc_manager.connect()
client.assert_called_once_with(raw_opc_manager.url, timeout=10, watchdog_intervall=3600000)
raw_opc_manager.client.connect.assert_called_once()
raw_opc_manager.set_security.assert_called_once()
@mark.asyncio
@patch('ingestor.managers.opc_manager.Client')
@patch('ingestor.managers.opc_manager.metrics')
async def test_connect_exception_handling_and_metrics(
mock_metrics_module, mock_opc_client_class, raw_opc_manager
):
mock_client_instance = mock_opc_client_class.return_value
simulated_error_message = 'Erro de conexão simulado'
mock_client_instance.connect.side_effect = Exception(simulated_error_message)
raw_opc_manager.disconnect = AsyncMock()
opc_manager_instance = raw_opc_manager
opc_manager_instance.cert_path = None
with pytest.raises(Exception, match=simulated_error_message):
await opc_manager_instance.connect()
opc_manager_instance.disconnect.assert_called_once()
@mark.asyncio
async def test_create_subscription_no_client(raw_opc_manager):
try:
await raw_opc_manager.create_subscription('sub1')
except ValueError as e:
assert str(e) == 'Client not connected. Call connect first.'
else:
raise AssertionError('ValueError not raised')
@mark.asyncio
async def test_create_subscription_success_has_period(opc_manager):
await opc_manager.create_subscription('sub1')
opc_manager.client.create_subscription.assert_called_once_with(
opc_manager.subscription_period_ms, opc_manager
)
assert opc_manager.subscriptions['sub1'] is not None
@mark.asyncio
async def test_create_subscription_success_no_period(opc_manager):
await opc_manager.create_subscription('sub1')
opc_manager.client.create_subscription.assert_called_once_with(
opc_manager.subscription_period_ms, opc_manager
)
assert opc_manager.subscriptions['sub1'] is not None
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_create_subscription_with_metrics(metrics, opc_manager):
await opc_manager.create_subscription('sub1')
opc_manager.emit_metric.assert_called_once_with(
metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED,
method='inc',
value=1,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
'slot_name': 'sub1',
},
)
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_create_subscription_exception_during_client_call(
mock_metrics_module, raw_opc_manager
):
opc_manager_instance = raw_opc_manager
opc_manager_instance.client = AsyncMock()
subscription_name = 'test_sub_client_error'
simulated_error_message = 'Falha ao criar subscrição no cliente OPC'
opc_manager_instance.client.create_subscription.side_effect = Exception(simulated_error_message)
with pytest.raises(Exception, match=simulated_error_message):
await opc_manager_instance.create_subscription(subscription_name)
opc_manager_instance.client.create_subscription.assert_called_once_with(
opc_manager_instance.subscription_period_ms, opc_manager_instance
)
opc_manager_instance.logger.error.assert_called_once_with(
f'Failed to create subscription {subscription_name} on {opc_manager_instance.name}: {simulated_error_message}'
)
mock_metrics_module.OPC_SUBSCRIPTIONS_CREATED.labels.assert_not_called()
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_subscribe_no_subscription(metrics, opc_manager):
try:
await opc_manager.subscribe('sub1', tags, 1000)
except ValueError as e:
assert str(e) == 'Subscription not created. Call create_subscription first.'
else:
raise AssertionError('ValueError not raised')
metrics.OPC_TAGS_SUBSCRIBED.labels.assert_not_called()
@mark.asyncio
async def test_subscribe_success(opc_manager_subscribed):
opc_manager_subscribed.client.get_node = MagicMock()
opc_manager_subscribed.nodes = {'ns=3;i=1001': 'data'}
await opc_manager_subscribed.subscribe('sub1', tags, 1000)
assert opc_manager_subscribed.nodes == tags
opc_manager_subscribed.subscriptions['sub1'].subscribe_data_change.assert_called_once_with(
[opc_manager_subscribed.client.get_node(n) for n in tags]
)
@mark.asyncio
async def test_unsubscribe_no_subscription(opc_manager):
await opc_manager.unsubscribe('sub1')
opc_manager.logger.warning.assert_called_once_with(
"Subscription 'sub1' not found. Cannot unsubscribe."
)
assert opc_manager.subscriptions.get('sub1') is None
@mark.asyncio
async def test_unsubscribe_success(opc_manager_subscribed):
await opc_manager_subscribed.unsubscribe('sub1')
assert opc_manager_subscribed.subscriptions.get('sub1') is None
@mark.asyncio
async def test_disconnection_fallback_success(opc_manager):
opc_manager.client = AsyncMock()
opc_manager.client.disconnect.return_value = True
result = await opc_manager.disconnection_fallback()
assert result == []
@mark.asyncio
async def test_disconnection_fallback_fail(opc_manager):
opc_manager.client = AsyncMock()
opc_manager.client.disconnect.side_effect = Exception('Test error')
result = await opc_manager.disconnection_fallback()
assert result == [
{'attempt': 1, 'error': 'Test error', 'traceback': ANY},
{'attempt': 2, 'error': 'Test error', 'traceback': ANY},
{'attempt': 3, 'error': 'Test error', 'traceback': ANY},
{'attempt': 4, 'error': 'Test error', 'traceback': ANY},
{'attempt': 5, 'error': 'Test error', 'traceback': ANY},
]
assert opc_manager.client.disconnect.call_count == 5
@mark.asyncio
async def test_disconnect_success(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock()
await opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
assert opc_manager_subscribed.client is None
@mark.asyncio
async def test_disconnect_no_client(opc_manager_subscribed):
opc_manager_subscribed.client = None
assert await opc_manager_subscribed.disconnect() is None
opc_manager_subscribed.logger.warning.assert_has_calls(
[
call('Client already disconnected.'),
]
)
@mark.asyncio
async def test_disconnect_error_unsubscribe(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock(disconnect=AsyncMock())
opc_manager_subscribed.subscriptions['sub1'] = MagicMock(
delete=AsyncMock(side_effect=Exception('Test error'))
)
await opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
opc_manager_subscribed.client = None
opc_manager_subscribed.logger.error.assert_called_once_with(
'Failed to clean up subscription: Test error'
)
@mark.asyncio
async def test_disconnect_error(opc_manager_subscribed):
opc_manager_subscribed.client = MagicMock()
opc_manager_subscribed.client.disconnect = MagicMock(side_effect=Exception('Test error'))
await opc_manager_subscribed.disconnect()
opc_manager_subscribed.subscriptions['sub1'].delete.assert_called_once()
opc_manager_subscribed.client = None
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_disconnect_metrics_on_successful_path(mock_metrics_module, raw_opc_manager):
mock_metrics_module.OPC_CONNECTION_STATUS.reset_mock()
mock_metrics_module.OPC_TAGS_SUBSCRIBED.reset_mock()
raw_opc_manager.client = MagicMock()
mock_sub1 = MagicMock()
mock_sub2 = MagicMock()
raw_opc_manager.subscriptions = {'sub1': mock_sub1, 'sub2': mock_sub2}
await raw_opc_manager.disconnect()
raw_opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=mock_metrics_module.OPC_CONNECTION_STATUS,
method='set',
value=0,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
'server_url': raw_opc_manager.url,
},
),
call(
metric_object=mock_metrics_module.OPC_TAGS_SUBSCRIBED,
method='set',
value=0,
tags={
'pod_id': raw_opc_manager.pod_id,
'server_name': raw_opc_manager.name,
},
),
],
any_order=True,
)
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_datachange_notification(metrics, opc_manager_subscribed):
data = MagicMock(
monitored_item=MagicMock(
Value=MagicMock(
Value=MagicMock(Value=42),
SourceTimestamp=datetime.strptime('2021-01-01T00:00:00', '%Y-%m-%dT%H:%M:%S'),
)
)
)
opc_manager_subscribed.nodes = {
'ns=3;i=1001': {
'tag_name': 'Counter',
'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 2},
'topics': ['topic1', 'topic2'],
}
}
await opc_manager_subscribed.datachange_notification('ns=3;i=1001', None, data)
opc_manager_subscribed.data_manager.publish.assert_any_call(
'topic1',
{
'tag': 'ns=3;i=1001',
'name': 'Counter',
'timestamp': '2021-01-01 00:00:00-0300',
'value': 42,
},
)
opc_manager_subscribed.data_manager.publish.assert_any_call(
'topic2',
{
'tag': 'ns=3;i=1001',
'name': 'Counter',
'timestamp': '2021-01-01 00:00:00-0300',
'value': 42,
},
)
assert opc_manager_subscribed.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == 0
opc_manager_subscribed.emit_metric.assert_called_once_with(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=0,
tags={
'pod_id': opc_manager_subscribed.pod_id,
'server_name': opc_manager_subscribed.name,
},
)
@mark.asyncio
async def test_check_cycles_no_notification(opc_manager):
# Setup: node with cycle_count just below threshold
opc_manager.nodes = {
'ns=3;i=1001': {
'tag_name': 'Counter',
'cycle_rule': {'cycle_increment': 1.0, 'cycle_count': 3.0},
}
}
await opc_manager.check_cycles()
# After one increment, cycle_count = 4.0, still below threshold
assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(4.0)
opc_manager.send_notification_async.assert_not_called()
@mark.asyncio
async def test_check_cycles_triggers_notification(opc_manager):
# Setup: node with cycle_count just below threshold, increment will cross threshold
opc_manager.nodes = {
'ns=3;i=1001': {
'tag_name': 'Counter',
'cycle_rule': {'cycle_increment': 2.5, 'cycle_count': 3.0},
}
}
opc_manager.notification_handler.build_and_send_notification = MagicMock()
await opc_manager.check_cycles()
# After increment, cycle_count = 5.5, should trigger notification
assert opc_manager.nodes['ns=3;i=1001']['cycle_rule']['cycle_count'] == pytest.approx(5.5)
opc_manager.send_notification_async.assert_called_once_with(
notification_id='TAG_ns=3;i=1001:Counter_LISTENNING_STOPPED',
message='5.5 cycles without receive from ns=3;i=1001:Counter',
block='opc_manager',
level=NotificationLevel.WARNING,
metadata=metadata['metadata'],
)
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_check_opc_listenning_no_notification(metrics, opc_manager):
opc_manager.non_receive_count = 3
result = await opc_manager.check_opc_listenning()
assert opc_manager.non_receive_count == 4
opc_manager.send_notification_async.assert_not_called()
assert result is False
opc_manager.emit_metric.assert_called_once_with(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=opc_manager.non_receive_count,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
},
)
@mark.asyncio
async def test_check_opc_listenning_warning_notification(opc_manager):
opc_manager.non_receive_count = 4
result = await opc_manager.check_opc_listenning()
assert opc_manager.non_receive_count == 5
opc_manager.send_notification_async.assert_called_once_with(
notification_id=f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
message=f'5 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}',
block='opc_manager',
level=NotificationLevel.ERROR,
metadata=metadata['metadata'],
)
assert result is False
@patch('ingestor.managers.opc_manager.metrics')
@mark.asyncio
async def test_check_opc_listenning_error_notification_and_retry(metrics, opc_manager):
opc_manager.non_receive_count = 14
result = await opc_manager.check_opc_listenning()
assert opc_manager.non_receive_count == 15
# Should be called twice: once for 5, once for 15
assert opc_manager.send_notification_async.call_count == 2
calls = opc_manager.send_notification_async.call_args_list
# First call: 5 cycles warning
assert calls[0].kwargs == {
'notification_id': f'OPC_LISTENNING_STOPPED__{opc_manager.name}',
'message': f'15 cycles without receive from OPC {opc_manager.name}. Tags: {json.dumps(opc_manager.nodes)}',
'block': 'opc_manager',
'level': NotificationLevel.ERROR,
'metadata': metadata['metadata'],
}
# Second call: 15 cycles retry
assert calls[1].kwargs == {
'notification_id': f'OPC_CONNECTION_RETRY__{opc_manager.name}',
'message': f'Retrying to connect to server {opc_manager.name}',
'block': 'opc_manager',
'level': NotificationLevel.ERROR,
'metadata': metadata['metadata'],
}
assert result is True
opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=opc_manager.non_receive_count,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
},
),
]
)
opc_manager.emit_metric.assert_has_calls(
[
call(
metric_object=metrics.OPC_RECONNECTIONS_TOTAL,
tags={
'pod_id': opc_manager.pod_id,
'server_name': opc_manager.name,
},
),
]
)

129
tests/unit/managers/test_server_reader.py Normal file
View File

@@ -0,0 +1,129 @@
from unittest.mock import AsyncMock, MagicMock, call
import pytest
import ingestor.metrics as metrics
from ingestor.managers.server_reader import ServerReader
SERVER_CONFIG = {
'name': 'kepserver',
'server_id': 1, # the slot may carry it as int; the pool keys by str
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 15000.0,
}
def make_reader():
pool = MagicMock(get=AsyncMock(), evict=AsyncMock())
reader = ServerReader(
name='kepserver',
server_config=SERVER_CONFIG,
pool=pool,
sink=AsyncMock(),
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
metadata={},
)
reader.engine = MagicMock(
subscriptions={},
pod_id='pod',
create_subscription=AsyncMock(),
subscribe=AsyncMock(),
unsubscribe=AsyncMock(),
check_cycles=AsyncMock(),
check_opc_listenning=AsyncMock(return_value=False),
emit_metric=AsyncMock(),
)
return reader, pool
def test_server_cfg_keeps_only_connection_fields():
reader, _ = make_reader()
assert reader.server_id == '1'
assert reader.server_cfg == {
'url': 'opc.tcp://kepserver:49320',
'server_uri': 'urn:sientia:opc-ingestor',
'use_security': True,
'subscription_period_ms': 15000.0,
}
@pytest.mark.asyncio
async def test_connect_opens_pooled_connection():
reader, pool = make_reader()
await reader.connect()
# use_security=True: the pool fetches the certificate from MinIO (CertProvider)
pool.get.assert_awaited_once_with('1', reader.server_cfg)
@pytest.mark.asyncio
async def test_connect_propagates_failure():
reader, pool = make_reader()
pool.get.side_effect = ConnectionError('unreachable')
with pytest.raises(ConnectionError):
await reader.connect()
@pytest.mark.asyncio
async def test_subscription_calls_delegate_to_engine_per_slot():
reader, _ = make_reader()
nodes = {'ns=2;i=2': {'frequency': '1000', 'names': [{'collection': 'raw_a', 'tag_name': 'a'}]}}
await reader.create_subscription('1')
await reader.subscribe('1', nodes, 10)
await reader.unsubscribe('1')
reader.engine.create_subscription.assert_awaited_once_with('1', reader.server_cfg, '1')
reader.engine.subscribe.assert_awaited_once_with('1', nodes, 10)
reader.engine.unsubscribe.assert_awaited_once_with('1')
@pytest.mark.asyncio
async def test_supervision_delegates_to_engine():
reader, _ = make_reader()
reader.engine.check_opc_listenning.return_value = True
await reader.check_cycles()
lost = await reader.check_opc_listenning()
reader.engine.check_cycles.assert_awaited_once()
reader.engine.check_opc_listenning.assert_awaited_once_with('1')
assert lost is True
def test_subscriptions_are_the_engine_ones():
reader, _ = make_reader()
reader.engine.subscriptions['1'] = MagicMock()
assert reader.subscriptions is reader.engine.subscriptions
@pytest.mark.asyncio
async def test_shutdown_unsubscribes_all_slots_evicts_and_zeroes_metrics():
reader, pool = make_reader()
reader.engine.subscriptions.update({'1': MagicMock(), '2': MagicMock()})
await reader.shutdown()
reader.engine.unsubscribe.assert_has_awaits([call('1'), call('2')])
# Dropping the pooled connection makes the next initialization reconnect.
pool.evict.assert_awaited_once_with('1')
reader.engine.emit_metric.assert_any_await(
metric_object=metrics.OPC_CONNECTION_STATUS,
method='set',
value=0,
tags={'pod_id': 'pod', 'server_name': '1', 'server_url': 'opc.tcp://kepserver:49320'},
)
reader.engine.emit_metric.assert_any_await(
metric_object=metrics.OPC_TAGS_SUBSCRIBED,
method='set',
value=0,
tags={'pod_id': 'pod', 'server_name': '1'},
)

6
tests/unit/test_app.py
View File

@@ -139,6 +139,7 @@ async def test_main_successful_run_one_loop(mock_app_env, capsys):
mock_app_env['GatewayRuntime'].assert_called_once_with(
config=mock_app_env['GatewayConfig'].from_env.return_value,
data_manager=mock_ingestor_instance.ingestor_manager.data_manager,
pool=mock_ingestor_instance.ingestor_manager.pool,
logger=mock_ingestor_instance.logger,
notification_handler=mock_ingestor_instance.notification_handler,
metrics_controller=mock_ingestor_instance.metrics_controller,
@@ -350,8 +351,9 @@ async def test_main_prepare_ingestor_failure(mock_app_env, capsys):
@pytest.mark.asyncio
async def test_main_gateway_runtime_init_failure_keeps_read_path_alive(mock_app_env):
"""GatewayRuntime construction hitting the network (MinIO/Postgres) and failing
must not kill the app: the main loop (read path) still runs and shuts down cleanly."""
"""GatewayRuntime construction failing (e.g. Postgres config) must not kill the
app: the read path lives in the ingestor_manager, so the main loop still runs and
shuts down cleanly."""
mock_ingestor_instance = mock_app_env['Ingestor'].return_value
mock_exit_signal = mock_app_env['mock_exit_signal']

2
tests/unit/test_config.py
View File

@@ -31,7 +31,7 @@ def test_from_env_defaults():
assert cfg.nats_queue == 'gateway-writers'
assert cfg.grpc_enabled is False
assert cfg.grpc_port == 50051
assert cfg.cert_bucket == 'sientia-dataops'
assert cfg.cert_bucket == 'streamlit-connectors'
assert cfg.cert_cache_dir == '/tmp/opc-certs' # noqa: S108
assert cfg.write_timestamp is True
assert cfg.write_quality is True

43
tests/unit/test_gateway_runtime.py
View File

@@ -6,9 +6,6 @@ from ingestor import gateway_runtime as gr
from ingestor.config import GatewayConfig
_PATCHED_CLASSES = (
'MinioRepository',
'CertProvider',
'OpcConnectionPool',
'Postgres',
'build_postgres_config',
'DataSink',
@@ -36,8 +33,10 @@ def mocks(monkeypatch):
'min_connections': 5,
'max_connections': 20,
}
patched['OpcConnectionPool'].return_value.close_all = AsyncMock()
patched['OpcConnectionPool'].return_value.release_idle = AsyncMock()
# The pool is owned by IngestorManager (read path) and only borrowed here.
patched['pool'] = MagicMock()
patched['pool'].close_all = AsyncMock()
patched['pool'].release_idle = AsyncMock()
patched['NatsWriteConsumer'].return_value.start = AsyncMock()
patched['NatsWriteConsumer'].return_value.stop = AsyncMock()
patched['GrpcServer'].return_value.start = AsyncMock()
@@ -45,10 +44,13 @@ def mocks(monkeypatch):
return patched
def make_runtime(**config_overrides):
def make_runtime(pool=None, **config_overrides):
if pool is None:
pool = MagicMock(close_all=AsyncMock(), release_idle=AsyncMock())
return gr.GatewayRuntime(
config=GatewayConfig(**config_overrides),
data_manager=MagicMock(),
pool=pool,
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=MagicMock(),
@@ -57,26 +59,26 @@ def make_runtime(**config_overrides):
def test_builds_object_graph_with_config_flags(mocks):
runtime = make_runtime(
pool=mocks['pool'],
write_timestamp=False,
write_quality=False,
write_min_pause_s=0.25,
cert_bucket='my-bucket',
cert_cache_dir='certs-cache',
grpc_enabled=False,
)
mocks['OpcConnectionPool'].assert_called_once()
# Writes and active reads share the pool built by the read path.
assert runtime.pool is mocks['pool']
mocks['DataSink'].assert_called_once()
mocks['WriteEngine'].assert_called_once()
_, kwargs = mocks['WriteEngine'].call_args
args, kwargs = mocks['WriteEngine'].call_args
assert args[0] is mocks['pool']
assert kwargs['write_timestamp'] is False
assert kwargs['write_quality'] is False
assert kwargs['min_pause_s'] == 0.25
_, cert_kwargs = mocks['CertProvider'].call_args
assert cert_kwargs['bucket'] == 'my-bucket'
assert cert_kwargs['cache_dir'] == 'certs-cache'
args, _ = mocks['ActiveReadEngine'].call_args
assert args[0] is mocks['pool']
mocks['GrpcServer'].assert_not_called()
assert runtime.grpc_server is None
@@ -137,20 +139,21 @@ async def test_nats_start_failure_does_not_raise_and_idle_sweep_retries(mocks):
@pytest.mark.asyncio
async def test_stop_stops_started_components_and_closes_pool(mocks):
runtime = make_runtime(grpc_enabled=True)
async def test_stop_stops_started_components_but_not_the_pool(mocks):
runtime = make_runtime(pool=mocks['pool'], grpc_enabled=True)
await runtime.start()
await runtime.stop()
mocks['GrpcServer'].return_value.stop.assert_awaited_once()
mocks['NatsWriteConsumer'].return_value.stop.assert_awaited_once()
mocks['OpcConnectionPool'].return_value.close_all.assert_awaited_once()
# IngestorManager.shutdown() closes it, after its readers unsubscribe.
mocks['pool'].close_all.assert_not_called()
@pytest.mark.asyncio
async def test_stop_skips_nats_stop_if_never_started(mocks):
runtime = make_runtime()
runtime = make_runtime(pool=mocks['pool'])
runtime.send_notification_async = AsyncMock()
nats_mock = mocks['NatsWriteConsumer'].return_value
nats_mock.start = AsyncMock(side_effect=Exception('boom'))
@@ -159,14 +162,14 @@ async def test_stop_skips_nats_stop_if_never_started(mocks):
await runtime.stop()
nats_mock.stop.assert_not_called()
mocks['OpcConnectionPool'].return_value.close_all.assert_awaited_once()
mocks['pool'].close_all.assert_not_called()
@pytest.mark.asyncio
async def test_idle_sweep_releases_idle_connections(mocks):
runtime = make_runtime(idle_conn_max_s=42.0)
runtime = make_runtime(pool=mocks['pool'], idle_conn_max_s=42.0)
runtime._nats_started = True # isolate from the retry path
await runtime.idle_sweep()
mocks['OpcConnectionPool'].return_value.release_idle.assert_awaited_once_with(42.0)
mocks['pool'].release_idle.assert_awaited_once_with(42.0)