Files
sientia-dataops-opc-ingestor/tests/unit/engines/test_subscription.py
Eduardo Rios 50578fdba8 SIENTIAPDE-1974: feat(read): slot v2 reads via SubscriptionEngine, OPC cert from MinIO
Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over
the shared OpcConnectionPool. The OPC client certificate now always comes from
MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the
Gitea repository, so a new version branch no longer breaks the reads.

- io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy
  v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway
  and Orchestrator can be deployed in any order.
- managers/server_reader.py: new per-server reader with the OpcManager
  interface the IngestorManager already drives (connect, create_subscription,
  subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown).
- IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a
  read DataSink, built without touching the network. Slots from Redis are
  normalized on lease and refresh; a malformed slot is notified and the last
  good config is kept.
- GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no
  longer closes it on stop: a Postgres/NATS failure can't stop the reads.
- pool: pin/unpin per active subscription so release_idle never evicts a
  subscribed connection; public evict() for lost servers / changed config.
- SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a
  subscription locally even when the server-side delete fails.
- DataSink: model_id only written when the name has one, keeping v1 documents
  identical to the legacy ones.
- config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7).
- Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh;
  README/CLAUDE.md document the MinIO certificate and the slot v2 format.

Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against
an asyncua server: v1 slot with two topics -> one doc per collection, legacy
doc shape, idle sweep keeps the subscription, lost-server shutdown and
re-initialization work.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 13:27:18 -03:00

258 lines
7.7 KiB
Python

from datetime import datetime
from unittest.mock import AsyncMock, MagicMock
import pytest
from sientia_do.temporal.constants import OPC_TIMEZONE
import ingestor.metrics as metrics
from ingestor.engines.subscription import SubscriptionEngine
def make_engine():
eng = SubscriptionEngine(
pool=AsyncMock(),
sink=AsyncMock(),
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=AsyncMock(),
metadata={},
)
eng.emit_metric = AsyncMock()
eng.send_notification_async = AsyncMock()
return eng
def make_node(name='ns=2;i=2'):
node = MagicMock()
node.__str__ = lambda self: name
return node
def make_data(value=42, timestamp=None):
data = MagicMock()
data.monitored_item.Value.Value.Value = value
if timestamp is not None:
data.monitored_item.Value.SourceTimestamp = timestamp
return data
@pytest.mark.asyncio
async def test_datachange_fans_out():
eng = make_engine()
eng.nodes['ns=2;i=2'] = {
'names': [{'tag_name': 'a'}, {'tag_name': 'b'}],
'cycle_rule': {'cycle_increment': 1, 'cycle_count': 3},
}
node = make_node()
data = make_data(42)
await eng.datachange_notification(node, 42, data)
assert eng.sink.publish_reading.await_count == 1
_, kwargs = eng.sink.publish_reading.call_args
assert len(kwargs['names']) == 2
assert kwargs['tag'] == 'ns=2;i=2'
assert kwargs['value'] == 42
@pytest.mark.asyncio
async def test_datachange_resets_cycle_count_and_non_receive_count():
eng = make_engine()
eng.non_receive_count = 7
eng.nodes['ns=2;i=2'] = {
'names': [{'tag_name': 'a'}],
'cycle_rule': {'cycle_increment': 1, 'cycle_count': 3},
}
await eng.datachange_notification(make_node(), 42, make_data(42))
assert eng.nodes['ns=2;i=2']['cycle_rule']['cycle_count'] == 0
assert eng.non_receive_count == 0
eng.emit_metric.assert_any_await(
metric_object=metrics.OPC_CYCLES_WITHOUT_DATA,
method='set',
value=0,
tags={'pod_id': eng.pod_id, 'server_name': 'unknown'},
)
@pytest.mark.asyncio
async def test_datachange_emits_read_lag():
eng = make_engine()
eng.nodes['ns=2;i=2'] = {
'names': [{'tag_name': 'a'}],
'cycle_rule': {'cycle_increment': 1, 'cycle_count': 0},
'server_name': 'srv1',
}
source_ts = datetime.now(tz=OPC_TIMEZONE)
await eng.datachange_notification(make_node(), 42, make_data(42, timestamp=source_ts))
lag_call = next(
c
for c in eng.emit_metric.await_args_list
if c.kwargs['metric_object'] is metrics.OPC_READ_LAG
)
assert lag_call.kwargs['tags'] == {'pod_id': eng.pod_id, 'server_name': 'srv1'}
assert abs(lag_call.kwargs['value']) < 5
@pytest.mark.asyncio
async def test_create_subscription_uses_pool_client():
eng = make_engine()
client = AsyncMock()
eng.pool.get.return_value = client
await eng.create_subscription('srv1', {'subscription_period_ms': 500}, 'sub1')
eng.pool.get.assert_awaited_once_with('srv1', {'subscription_period_ms': 500})
client.create_subscription.assert_awaited_once_with(500, eng)
assert eng.subscriptions['sub1'] is not None
eng.emit_metric.assert_any_await(
metric_object=metrics.OPC_SUBSCRIPTIONS_CREATED,
method='inc',
value=1,
tags={'pod_id': eng.pod_id, 'server_name': 'srv1', 'slot_name': 'sub1'},
)
@pytest.mark.asyncio
async def test_subscribe_registers_nodes_with_cycle_rule():
eng = make_engine()
client = AsyncMock()
eng.pool.get.return_value = client
await eng.create_subscription('srv1', {'subscription_period_ms': 500}, 'sub1')
nodes = {
'ns=2;i=2': {
'frequency': 1000,
'names': [{'tag_name': 'a'}, {'tag_name': 'b'}],
'active_read': {'enabled': False},
}
}
await eng.subscribe('sub1', nodes, collect_period=1)
assert eng.nodes['ns=2;i=2']['server_name'] == 'srv1'
assert eng.nodes['ns=2;i=2']['cycle_rule']['cycle_count'] == 0
assert eng.nodes['ns=2;i=2']['cycle_rule']['cycle_increment'] == pytest.approx(1.0)
eng.subscriptions['sub1'].subscribe_data_change.assert_awaited_once()
@pytest.mark.asyncio
async def test_subscribe_without_subscription_raises():
eng = make_engine()
with pytest.raises(ValueError, match='Subscription not created'):
await eng.subscribe('sub1', {}, collect_period=1)
@pytest.mark.asyncio
async def test_unsubscribe_missing_logs_warning():
eng = make_engine()
await eng.unsubscribe('sub1')
eng.logger.warning.assert_called_once_with("Subscription 'sub1' not found. Cannot unsubscribe.")
@pytest.mark.asyncio
async def test_unsubscribe_deletes_subscription():
eng = make_engine()
client = AsyncMock()
eng.pool.get.return_value = client
await eng.create_subscription('srv1', {'subscription_period_ms': 500}, 'sub1')
sub = eng.subscriptions['sub1']
await eng.unsubscribe('sub1')
sub.delete.assert_awaited_once()
assert 'sub1' not in eng.subscriptions
@pytest.mark.asyncio
async def test_check_cycles_triggers_notification_over_threshold():
eng = make_engine()
eng.nodes['ns=2;i=2'] = {
'names': [{'tag_name': 'a'}],
'cycle_rule': {'cycle_increment': 5.5, 'cycle_count': 0},
}
await eng.check_cycles()
eng.send_notification_async.assert_called_once()
@pytest.mark.asyncio
async def test_check_opc_listenning_below_threshold():
eng = make_engine()
eng.non_receive_count = 2
result = await eng.check_opc_listenning('srv1')
assert eng.non_receive_count == 3
assert result is False
eng.send_notification_async.assert_not_called()
@pytest.mark.asyncio
async def test_check_opc_listenning_triggers_reconnect_at_15():
eng = make_engine()
eng.non_receive_count = 14
result = await eng.check_opc_listenning('srv1')
assert result is True
assert eng.send_notification_async.call_count == 2
@pytest.mark.asyncio
async def test_create_subscription_pins_pool_connection():
eng = make_engine()
eng.pool = MagicMock(get=AsyncMock(return_value=AsyncMock()))
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
eng.pool.pin.assert_called_once_with('1')
@pytest.mark.asyncio
async def test_create_subscription_failure_does_not_pin():
eng = make_engine()
client = AsyncMock()
client.create_subscription.side_effect = RuntimeError('boom')
eng.pool = MagicMock(get=AsyncMock(return_value=client))
with pytest.raises(RuntimeError):
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
eng.pool.pin.assert_not_called()
@pytest.mark.asyncio
async def test_unsubscribe_unpins_pool_connection():
eng = make_engine()
eng.pool = MagicMock(get=AsyncMock(return_value=AsyncMock()))
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
await eng.unsubscribe('slot1')
eng.pool.unpin.assert_called_once_with('1')
assert 'slot1' not in eng.subscriptions
@pytest.mark.asyncio
async def test_unsubscribe_tolerates_dead_connection():
"""A lost server's subscription can't be deleted on the server side; it must
still be dropped locally (and unpinned) so the slot can be re-subscribed."""
eng = make_engine()
sub = AsyncMock()
sub.delete.side_effect = ConnectionError('closed')
client = AsyncMock()
client.create_subscription.return_value = sub
eng.pool = MagicMock(get=AsyncMock(return_value=client))
await eng.create_subscription('1', {'subscription_period_ms': 500}, 'slot1')
await eng.unsubscribe('slot1') # must not raise
assert 'slot1' not in eng.subscriptions
eng.pool.unpin.assert_called_once_with('1')
eng.logger.warning.assert_called_once()