- config: GatewayConfig.certs_from_env() reads only CERT_BUCKET/CERT_CACHE_DIR; IngestorManager uses it, so a malformed write/gRPC env var (GRPC_PORT, WRITE_MIN_PAUSE_S, IDLE_CONN_MAX_S) can no longer stop the reads. - certs: CertProvider raises CertificateNotConfiguredError with an actionable message when no platform_certificates document exists; downloads once under an asyncio.Lock, caches the paths, writes atomically (tmp + os.replace); invalidate() forces a fresh download. - pool: server_id normalized to str (1 and '1' share a session); warns when a get() asks for other url/server_uri/use_security than the pooled client was opened with (client reused, the slot config wins); invalidates the cert cache when a secured connect fails. - SubscriptionEngine: tracks nodes per subscription and forgets them on unsubscribe (no more cycle alarms / tag count for removed tags), keeping addresses still monitored by another slot. - ServerReader.shutdown(evict) / IngestorManager.remove_server(evict): a server that only left this pod's slots keeps the shared connection for NATS writes/gRPC; lost servers and config changes still evict. - IngestorManager warns once per server when a slot still sets server_cert_path (server certificate pinning is no longer applied). - slots: CONNECTION_FIELDS exported and reused by ServerReader. Accepted, not changed: read metrics label server_name with the server_id (same as the write path, which only knows the id); a malformed slot is notified each lease attempt (alerts are deduplicated by notification_id). Tests: 236 unit tests pass (+18); ruff clean; local e2e against an asyncua server incl. slot moved (connection kept and reused by the write path) and lost-server eviction/re-initialization. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
301 lines
9.1 KiB
Python
301 lines
9.1 KiB
Python
from datetime import datetime
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
from sientia_do.temporal.constants import OPC_TIMEZONE
|
|
|
|
import ingestor.metrics as metrics
|
|
from ingestor.engines.subscription import SubscriptionEngine
|
|
|
|
|
|
def make_engine():
|
|
eng = SubscriptionEngine(
|
|
pool=AsyncMock(),
|
|
sink=AsyncMock(),
|
|
logger=MagicMock(),
|
|
notification_handler=MagicMock(),
|
|
metrics_controller=AsyncMock(),
|
|
metadata={},
|
|
)
|
|
eng.emit_metric = AsyncMock()
|
|
eng.send_notification_async = AsyncMock()
|
|
return eng
|
|
|
|
|
|
def make_node(name='ns=2;i=2'):
|
|
node = MagicMock()
|
|
node.__str__ = lambda self: name
|
|
return node
|
|
|
|
|
|
def make_data(value=42, timestamp=None):
|
|
data = MagicMock()
|
|
data.monitored_item.Value.Value.Value = value
|
|
if timestamp is not None:
|
|
data.monitored_item.Value.SourceTimestamp = timestamp
|
|
return data
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_datachange_fans_out():
|
|
eng = make_engine()
|
|
eng.nodes['ns=2;i=2'] = {
|
|
'names': [{'tag_name': 'a'}, {'tag_name': 'b'}],
|
|
'cycle_rule': {'cycle_increment': 1, 'cycle_count': 3},
|
|
}
|
|
node = make_node()
|
|
data = make_data(42)
|
|
|
|
await eng.datachange_notification(node, 42, data)
|
|
|
|
assert eng.sink.publish_reading.await_count == 1
|
|
_, kwargs = eng.sink.publish_reading.call_args
|
|
assert len(kwargs['names']) == 2
|
|
assert kwargs['tag'] == 'ns=2;i=2'
|
|
assert kwargs['value'] == 42
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_datachange_resets_cycle_count_and_non_receive_count():
|
|
eng = make_engine()
|
|
eng.non_receive_count = 7
|
|
eng.nodes['ns=2;i=2'] = {
|
|
'names': [{'tag_name': 'a'}],
|
|
'cycle_rule': {'cycle_increment': 1, 'cycle_count': 3},
|
|
}
|
|
|
|
await eng.datachange_notification(make_node(), 42, make_data(42))
|
|
|
|
assert eng.nodes['ns=2;i=2']['cycle_rule']['cycle_count'] == 0
|
|
assert eng.non_receive_count == 0
|
|
eng.emit_metric.assert_any_await(
|
|
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
|
|
method='set',
|
|
value=0,
|
|
tags={'pod_id': eng.pod_id, 'server_name': 'unknown'},
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_datachange_emits_read_lag():
|
|
eng = make_engine()
|
|
eng.nodes['ns=2;i=2'] = {
|
|
'names': [{'tag_name': 'a'}],
|
|
'cycle_rule': {'cycle_increment': 1, 'cycle_count': 0},
|
|
'server_name': 'srv1',
|
|
}
|
|
source_ts = datetime.now(tz=OPC_TIMEZONE)
|
|
|
|
await eng.datachange_notification(make_node(), 42, make_data(42, timestamp=source_ts))
|
|
|
|
lag_call = next(
|
|
c
|
|
for c in eng.emit_metric.await_args_list
|
|
if c.kwargs['metric_object'] is metrics.OPC_READ_LAG
|
|
)
|
|
assert lag_call.kwargs['tags'] == {'pod_id': eng.pod_id, 'server_name': 'srv1'}
|
|
assert abs(lag_call.kwargs['value']) < 5
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_subscription_uses_pool_client():
|
|
eng = make_engine()
|
|
client = AsyncMock()
|
|
eng.pool.get.return_value = client
|
|
|
|
await eng.create_subscription('srv1', {'subscription_period_ms': 500}, 'sub1')
|
|
|
|
eng.pool.get.assert_awaited_once_with('srv1', {'subscription_period_ms': 500})
|
|
client.create_subscription.assert_awaited_once_with(500, eng)
|
|
assert eng.subscriptions['sub1'] is not None
|
|
eng.emit_metric.assert_any_await(
|
|
metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED,
|
|
method='inc',
|
|
value=1,
|
|
tags={'pod_id': eng.pod_id, 'server_name': 'srv1', 'slot_name': 'sub1'},
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_subscribe_registers_nodes_with_cycle_rule():
|
|
eng = make_engine()
|
|
client = AsyncMock()
|
|
eng.pool.get.return_value = client
|
|
await eng.create_subscription('srv1', {'subscription_period_ms': 500}, 'sub1')
|
|
|
|
nodes = {
|
|
'ns=2;i=2': {
|
|
'frequency': 1000,
|
|
'names': [{'tag_name': 'a'}, {'tag_name': 'b'}],
|
|
'active_read': {'enabled': False},
|
|
}
|
|
}
|
|
|
|
await eng.subscribe('sub1', nodes, collect_period=1)
|
|
|
|
assert eng.nodes['ns=2;i=2']['server_name'] == 'srv1'
|
|
assert eng.nodes['ns=2;i=2']['cycle_rule']['cycle_count'] == 0
|
|
assert eng.nodes['ns=2;i=2']['cycle_rule']['cycle_increment'] == pytest.approx(1.0)
|
|
eng.subscriptions['sub1'].subscribe_data_change.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_subscribe_without_subscription_raises():
|
|
eng = make_engine()
|
|
with pytest.raises(ValueError, match='Subscription not created'):
|
|
await eng.subscribe('sub1', {}, collect_period=1)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unsubscribe_missing_logs_warning():
|
|
eng = make_engine()
|
|
await eng.unsubscribe('sub1')
|
|
eng.logger.warning.assert_called_once_with("Subscription 'sub1' not found. Cannot unsubscribe.")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unsubscribe_deletes_subscription():
|
|
eng = make_engine()
|
|
client = AsyncMock()
|
|
eng.pool.get.return_value = client
|
|
await eng.create_subscription('srv1', {'subscription_period_ms': 500}, 'sub1')
|
|
sub = eng.subscriptions['sub1']
|
|
|
|
await eng.unsubscribe('sub1')
|
|
|
|
sub.delete.assert_awaited_once()
|
|
assert 'sub1' not in eng.subscriptions
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_cycles_triggers_notification_over_threshold():
|
|
eng = make_engine()
|
|
eng.nodes['ns=2;i=2'] = {
|
|
'names': [{'tag_name': 'a'}],
|
|
'cycle_rule': {'cycle_increment': 5.5, 'cycle_count': 0},
|
|
}
|
|
|
|
await eng.check_cycles()
|
|
|
|
eng.send_notification_async.assert_called_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_opc_listenning_below_threshold():
|
|
eng = make_engine()
|
|
eng.non_receive_count = 2
|
|
|
|
result = await eng.check_opc_listenning('srv1')
|
|
|
|
assert eng.non_receive_count == 3
|
|
assert result is False
|
|
eng.send_notification_async.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_opc_listenning_triggers_reconnect_at_15():
|
|
eng = make_engine()
|
|
eng.non_receive_count = 14
|
|
|
|
result = await eng.check_opc_listenning('srv1')
|
|
|
|
assert result is True
|
|
assert eng.send_notification_async.call_count == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_subscription_pins_pool_connection():
|
|
eng = make_engine()
|
|
eng.pool = MagicMock(get=AsyncMock(return_value=AsyncMock()))
|
|
|
|
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
|
|
|
|
eng.pool.pin.assert_called_once_with('1')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_create_subscription_failure_does_not_pin():
|
|
eng = make_engine()
|
|
client = AsyncMock()
|
|
client.create_subscription.side_effect = RuntimeError('boom')
|
|
eng.pool = MagicMock(get=AsyncMock(return_value=client))
|
|
|
|
with pytest.raises(RuntimeError):
|
|
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
|
|
|
|
eng.pool.pin.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unsubscribe_unpins_pool_connection():
|
|
eng = make_engine()
|
|
eng.pool = MagicMock(get=AsyncMock(return_value=AsyncMock()))
|
|
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
|
|
|
|
await eng.unsubscribe('slot1')
|
|
|
|
eng.pool.unpin.assert_called_once_with('1')
|
|
assert 'slot1' not in eng.subscriptions
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unsubscribe_tolerates_dead_connection():
|
|
"""A lost server's subscription can't be deleted on the server side; it must
|
|
still be dropped locally (and unpinned) so the slot can be re-subscribed."""
|
|
eng = make_engine()
|
|
sub = AsyncMock()
|
|
sub.delete.side_effect = ConnectionError('closed')
|
|
client = AsyncMock()
|
|
client.create_subscription.return_value = sub
|
|
eng.pool = MagicMock(get=AsyncMock(return_value=client))
|
|
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
|
|
|
|
await eng.unsubscribe('slot1') # must not raise
|
|
|
|
assert 'slot1' not in eng.subscriptions
|
|
eng.pool.unpin.assert_called_once_with('1')
|
|
eng.logger.warning.assert_called_once()
|
|
|
|
|
|
async def _subscribed_engine(slots):
|
|
"""Engine with one subscription per slot: {slot: [addresses]}."""
|
|
eng = make_engine()
|
|
eng.pool = MagicMock(
|
|
get=AsyncMock(
|
|
return_value=MagicMock(create_subscription=AsyncMock(return_value=AsyncMock()))
|
|
)
|
|
)
|
|
for slot, addresses in slots.items():
|
|
await eng.create_subscription('1', {'subscription_period_ms': 500}, slot)
|
|
nodes = {
|
|
a: {'frequency': '1000', 'names': [{'collection': 'c', 'tag_name': a}]}
|
|
for a in addresses
|
|
}
|
|
await eng.subscribe(slot, nodes, 10)
|
|
return eng
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unsubscribe_forgets_the_slot_nodes():
|
|
eng = await _subscribed_engine({'s1': ['ns=2;i=1', 'ns=2;i=2'], 's2': ['ns=2;i=3']})
|
|
|
|
await eng.unsubscribe('s1')
|
|
|
|
# no more cycle checks / tag counting for addresses nobody reads anymore
|
|
assert set(eng.nodes) == {'ns=2;i=3'}
|
|
eng.emit_metric.assert_any_await(
|
|
metric_object=metrics.OPC_TAGS_SUBSCRIBED,
|
|
method='set',
|
|
value=1,
|
|
tags={'pod_id': eng.pod_id, 'server_name': '1'},
|
|
)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_unsubscribe_keeps_node_shared_with_another_slot():
|
|
eng = await _subscribed_engine({'s1': ['ns=2;i=1', 'ns=2;i=2'], 's2': ['ns=2;i=2']})
|
|
|
|
await eng.unsubscribe('s1')
|
|
|
|
assert set(eng.nodes) == {'ns=2;i=2'}
|