SIENTIAPDE-1974: feat(read): slot v2 reads via SubscriptionEngine, OPC cert from MinIO
Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over the shared OpcConnectionPool. The OPC client certificate now always comes from MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the Gitea repository, so a new version branch no longer breaks the reads. - io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway and Orchestrator can be deployed in any order. - managers/server_reader.py: new per-server reader with the OpcManager interface the IngestorManager already drives (connect, create_subscription, subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown). - IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a read DataSink, built without touching the network. Slots from Redis are normalized on lease and refresh; a malformed slot is notified and the last good config is kept. - GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no longer closes it on stop: a Postgres/NATS failure can't stop the reads. - pool: pin/unpin per active subscription so release_idle never evicts a subscribed connection; public evict() for lost servers / changed config. - SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a subscription locally even when the server-side delete fails. - DataSink: model_id only written when the name has one, keeping v1 documents identical to the legacy ones. - config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7). - Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh; README/CLAUDE.md document the MinIO certificate and the slot v2 format. Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against an asyncua server: v1 slot with two topics -> one doc per collection, legacy doc shape, idle sweep keeps the subscription, lost-server shutdown and re-initialization work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Eduardo Rios
@@ -4,6 +4,9 @@ from unittest.mock import AsyncMock, MagicMock, call, patch
|
||||
from pytest import fixture, mark
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
|
||||
from ingestor.config import GatewayConfig
|
||||
from ingestor.connection.pool import OpcConnectionPool
|
||||
from ingestor.io.slots import parse_slot
|
||||
from ingestor.managers.ingestor_manager import IngestorManager
|
||||
|
||||
metadata = {
|
||||
@@ -16,6 +19,21 @@ metadata = {
|
||||
}
|
||||
|
||||
|
||||
def raw_slot_v1(tag_name='Counter', address='ns=2;i=2'):
|
||||
"""A slot as the current (v1) Orchestrator writes it to Redis."""
|
||||
return {
|
||||
'server1': {
|
||||
'name': 'server1',
|
||||
'server_id': '1',
|
||||
'url': 'opc.tcp://localhost:4840',
|
||||
'server_uri': 'urn:sientia:opc-ingestor',
|
||||
'cert_path': '/app/certs/opc-ingestor-cert.pem',
|
||||
'subscription_period_ms': 500.0,
|
||||
'tags': {address: {'tag_name': tag_name, 'frequency': '1000', 'topics': ['raw_a']}},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def _collect_garbage_after_test():
|
||||
# ponytail: IngestorManager.__del__ calls asyncio.run(self.shutdown()); if a
|
||||
@@ -52,12 +70,12 @@ def ingestor_manager(data_manager_mock, resource_manager_mock):
|
||||
return ingestor
|
||||
|
||||
|
||||
@patch('ingestor.managers.ingestor_manager.OpcManager')
|
||||
@patch('ingestor.managers.ingestor_manager.ServerReader')
|
||||
@patch('ingestor.managers.ingestor_manager.DataManager')
|
||||
@patch('ingestor.managers.ingestor_manager.ResourceManager')
|
||||
@patch('ingestor.managers.ingestor_manager.NotificationHandler')
|
||||
def test___init__(
|
||||
notification_handler_mock, resource_manager_mock, data_manager_mock, opc_manager_mock
|
||||
notification_handler_mock, resource_manager_mock, data_manager_mock, server_reader_mock
|
||||
):
|
||||
ingestor = IngestorManager(
|
||||
redis_data={'host': 'localhost', 'port': 6379},
|
||||
@@ -72,7 +90,7 @@ def test___init__(
|
||||
metrics_controller=MagicMock(),
|
||||
)
|
||||
|
||||
opc_manager_mock.assert_not_called()
|
||||
server_reader_mock.assert_not_called()
|
||||
data_manager_mock.assert_called_once_with(
|
||||
mongo_connection_string='mongodb://localhost:27017',
|
||||
mongo_database='sientia',
|
||||
@@ -100,59 +118,86 @@ def test___init__(
|
||||
assert ingestor.data_manager == data_manager_mock.return_value
|
||||
assert ingestor.resource_manager == resource_manager_mock.return_value
|
||||
|
||||
# Read path (SIENTIAPDE-1974): pool + read sink built here, nothing connected yet.
|
||||
assert isinstance(ingestor.pool, OpcConnectionPool)
|
||||
assert ingestor.pool.cert_provider is ingestor.cert_provider
|
||||
assert ingestor.cert_provider.mongo is data_manager_mock.return_value.mongo_repository
|
||||
assert ingestor.sink.data_manager is data_manager_mock.return_value
|
||||
assert ingestor.sink.postgres is None
|
||||
|
||||
@mark.asyncio
|
||||
@patch('ingestor.managers.ingestor_manager.OpcManager')
|
||||
async def test_initialize_opc_from_config(opc_manager, ingestor_manager):
|
||||
server_config = {
|
||||
'name': 'server1',
|
||||
'url': 'opc.tcp://localhost:4840',
|
||||
'subscription_period_ms': 1000,
|
||||
'server_uri': 'http://opcua-server.simulator',
|
||||
'cert_path': '/path/to/cert',
|
||||
'private_key_path': '/path/to/private_key',
|
||||
'server_cert_path': '/path/to/server_cert',
|
||||
'pod_id': 'test_pod',
|
||||
}
|
||||
|
||||
opc_manager.return_value = MagicMock(connect=AsyncMock())
|
||||
result = await ingestor_manager.initialize_opc_from_config(server_config)
|
||||
|
||||
opc_manager.assert_called_once_with(
|
||||
name=server_config['name'],
|
||||
url=server_config['url'],
|
||||
data_manager=ingestor_manager.data_manager,
|
||||
logger=ingestor_manager.logger,
|
||||
subscription_period_ms=server_config['subscription_period_ms'],
|
||||
server_uri=server_config['server_uri'],
|
||||
notification_handler=ingestor_manager.notification_handler,
|
||||
cert_path=server_config['cert_path'],
|
||||
private_key_path=server_config['private_key_path'],
|
||||
server_cert_path=server_config['server_cert_path'],
|
||||
@patch('ingestor.managers.ingestor_manager.DataManager')
|
||||
@patch('ingestor.managers.ingestor_manager.ResourceManager')
|
||||
def test___init___cert_provider_uses_gateway_config(resource_manager_mock, data_manager_mock):
|
||||
ingestor = IngestorManager(
|
||||
redis_data={'host': 'localhost', 'port': 6379},
|
||||
lease_ttl=60,
|
||||
heartbeat_ttl=60,
|
||||
poll_interval=5,
|
||||
mongo_connection_string='mongodb://localhost:27017',
|
||||
mongo_database='sientia',
|
||||
logger=MagicMock(),
|
||||
notification_handler=MagicMock(),
|
||||
metadata=metadata['metadata'],
|
||||
metrics_controller=ingestor_manager.metrics_controller,
|
||||
metrics_controller=MagicMock(),
|
||||
gateway_config=GatewayConfig(cert_bucket='my-bucket', cert_cache_dir='certs-cache'),
|
||||
)
|
||||
|
||||
assert result == opc_manager.return_value
|
||||
result.connect.assert_called_once()
|
||||
assert ingestor.cert_provider.bucket == 'my-bucket'
|
||||
assert ingestor.cert_provider.cache_dir == 'certs-cache'
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('ingestor.managers.ingestor_manager.OpcManager')
|
||||
@patch('ingestor.managers.ingestor_manager.traceback')
|
||||
async def test_initialize_opc_from_config_exception(traceback_mock, opc_manager, ingestor_manager):
|
||||
@patch('ingestor.managers.ingestor_manager.ServerReader')
|
||||
async def test_initialize_opc_from_config(server_reader, ingestor_manager):
|
||||
server_config = {
|
||||
'name': 'server1',
|
||||
'server_id': '1',
|
||||
'url': 'opc.tcp://localhost:4840',
|
||||
'server_uri': 'urn:sientia:opc-ingestor',
|
||||
'use_security': True,
|
||||
'subscription_period_ms': 1000,
|
||||
}
|
||||
|
||||
server_reader.return_value = MagicMock(connect=AsyncMock())
|
||||
result = await ingestor_manager.initialize_opc_from_config(server_config)
|
||||
|
||||
server_reader.assert_called_once_with(
|
||||
name=server_config['name'],
|
||||
server_config=server_config,
|
||||
pool=ingestor_manager.pool,
|
||||
sink=ingestor_manager.sink,
|
||||
logger=ingestor_manager.logger,
|
||||
notification_handler=ingestor_manager.notification_handler,
|
||||
metrics_controller=ingestor_manager.metrics_controller,
|
||||
metadata=metadata['metadata'],
|
||||
)
|
||||
|
||||
assert result == server_reader.return_value
|
||||
assert result.config == server_config
|
||||
result.connect.assert_awaited_once()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('ingestor.managers.ingestor_manager.ServerReader')
|
||||
@patch('ingestor.managers.ingestor_manager.traceback')
|
||||
async def test_initialize_opc_from_config_exception(
|
||||
traceback_mock, server_reader, ingestor_manager
|
||||
):
|
||||
server_config = {
|
||||
'name': 'server1',
|
||||
'server_id': '1',
|
||||
'url': 'opc.tcp://localhost:4840',
|
||||
'server_uri': 'urn:sientia:opc-ingestor',
|
||||
'use_security': True,
|
||||
'subscription_period_ms': 1000,
|
||||
'server_uri': 'http://opcua-server.simulator',
|
||||
'cert_path': '/path/to/cert',
|
||||
'private_key_path': '/path/to/private_key',
|
||||
'server_cert_path': '/path/to/server_cert',
|
||||
}
|
||||
|
||||
ingestor_manager.logger.error = MagicMock()
|
||||
opc_manager.side_effect = Exception('Initialization error')
|
||||
# e.g. the server is unreachable or the certificate can't be fetched from MinIO
|
||||
server_reader.return_value = MagicMock(
|
||||
connect=AsyncMock(side_effect=Exception('Initialization error'))
|
||||
)
|
||||
|
||||
result = await ingestor_manager.initialize_opc_from_config(server_config)
|
||||
|
||||
@@ -174,11 +219,14 @@ async def test_shutdown(ingestor_manager):
|
||||
ingestor_manager.opc_managers = {'server1': AsyncMock(), 'server2': AsyncMock()}
|
||||
|
||||
ingestor_manager.data_manager.shutdown = MagicMock()
|
||||
ingestor_manager.pool = MagicMock(close_all=AsyncMock())
|
||||
|
||||
await ingestor_manager.shutdown()
|
||||
|
||||
ingestor_manager.opc_managers['server1'].shutdown.assert_called_once()
|
||||
ingestor_manager.opc_managers['server2'].shutdown.assert_called_once()
|
||||
# The pool is owned by the read path: closed after the readers unsubscribe.
|
||||
ingestor_manager.pool.close_all.assert_awaited_once()
|
||||
|
||||
ingestor_manager.data_manager.shutdown.assert_called_once()
|
||||
|
||||
@@ -218,9 +266,8 @@ async def test_remove_server_not_found(ingestor_manager):
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('ingestor.managers.ingestor_manager.OpcManager')
|
||||
@patch('ingestor.managers.ingestor_manager.metrics')
|
||||
async def test_update_opc_servers(metrics, opc_manager, ingestor_manager):
|
||||
async def test_update_opc_servers(metrics, ingestor_manager):
|
||||
manager1 = MagicMock(config={'config': 'config1'})
|
||||
manager2 = MagicMock(config={'config': 'config2'})
|
||||
manager3 = MagicMock(config={'config': 'config3'})
|
||||
@@ -431,25 +478,47 @@ async def test_get_number_of_slots_empty(metrics, ingestor_manager):
|
||||
@mark.asyncio
|
||||
async def test_get_slot_leases_1_success(ingestor_manager):
|
||||
ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True)
|
||||
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value={'tags': ['tag1']})
|
||||
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(return_value=raw_slot_v1())
|
||||
|
||||
ingestor_manager.number_of_slots = 1
|
||||
result = await ingestor_manager.get_slot_leases()
|
||||
|
||||
assert result == {'1': {'tags': ['tag1']}}
|
||||
# Stored already normalized to slot v2 (v1 converted by parse_slot).
|
||||
assert result == {'1': parse_slot(raw_slot_v1())}
|
||||
assert ingestor_manager.managed_tags == result
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_get_slot_leases_2_success(ingestor_manager):
|
||||
ingestor_manager.resource_manager.lease_tag = AsyncMock(side_effect=[True, True])
|
||||
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
|
||||
side_effect=[{'tags': ['tag1']}, {'tags': ['tag2']}]
|
||||
side_effect=[raw_slot_v1('Counter'), raw_slot_v1('Ramp', 'ns=2;i=3')]
|
||||
)
|
||||
|
||||
ingestor_manager.number_of_slots = 2
|
||||
result = await ingestor_manager.get_slot_leases(max_slots=2)
|
||||
|
||||
assert result == {'1': {'tags': ['tag1']}, '2': {'tags': ['tag2']}}
|
||||
assert result == {
|
||||
'1': parse_slot(raw_slot_v1('Counter')),
|
||||
'2': parse_slot(raw_slot_v1('Ramp', 'ns=2;i=3')),
|
||||
}
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_get_slot_leases_skips_malformed_slot(ingestor_manager):
|
||||
ingestor_manager.resource_manager.lease_tag = AsyncMock(return_value=True)
|
||||
# tag without names/topics/frequency
|
||||
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
|
||||
return_value={'server1': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=2': {}}}}
|
||||
)
|
||||
|
||||
ingestor_manager.number_of_slots = 1
|
||||
result = await ingestor_manager.get_slot_leases()
|
||||
|
||||
assert result == {}
|
||||
_, kwargs = ingestor_manager.send_notification.call_args
|
||||
assert kwargs['notification_id'] == 'OPC_SLOT_CONFIG_ERROR_1'
|
||||
assert kwargs['level'] == NotificationLevel.ERROR
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@@ -497,20 +566,20 @@ async def test_unsubscribe_slot(ingestor_manager):
|
||||
@mark.asyncio
|
||||
async def test_update_slot_config(ingestor_manager):
|
||||
ingestor_manager.managed_tags = {
|
||||
'slot1': {'config': 'old_config'},
|
||||
'slot2': {'config': 'new_config'},
|
||||
'slot3': {'config': 'old_config'},
|
||||
'slot1': parse_slot(raw_slot_v1('Old')),
|
||||
'slot2': parse_slot(raw_slot_v1('Same')),
|
||||
'slot3': parse_slot(raw_slot_v1('Old')),
|
||||
}
|
||||
|
||||
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
|
||||
side_effect=[{'config': 'updated_config'}, {'config': 'new_config'}, None]
|
||||
side_effect=[raw_slot_v1('Updated'), raw_slot_v1('Same'), None]
|
||||
)
|
||||
ingestor_manager.resource_manager.renew_tag_lease = AsyncMock()
|
||||
|
||||
await ingestor_manager.update_slot_config()
|
||||
|
||||
assert ingestor_manager.managed_tags['slot1'] == {'config': 'updated_config'}
|
||||
assert ingestor_manager.managed_tags['slot2'] == {'config': 'new_config'}
|
||||
assert ingestor_manager.managed_tags['slot1'] == parse_slot(raw_slot_v1('Updated'))
|
||||
assert ingestor_manager.managed_tags['slot2'] == parse_slot(raw_slot_v1('Same'))
|
||||
assert 'slot3' not in ingestor_manager.managed_tags
|
||||
|
||||
ingestor_manager.resource_manager.renew_tag_lease.assert_any_call('slot1')
|
||||
@@ -652,3 +721,20 @@ async def test_subscribe_to_tags(ingestor_manager):
|
||||
assert ingestor_manager.manage_server.call_count == 3
|
||||
|
||||
ingestor_manager.managed_tags['slot1'].pop.assert_called_once_with('server3', None)
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
async def test_update_slot_config_keeps_last_good_config_when_malformed(ingestor_manager):
|
||||
good = parse_slot(raw_slot_v1('Counter'))
|
||||
ingestor_manager.managed_tags = {'slot1': good}
|
||||
ingestor_manager.resource_manager.get_tag_slot = AsyncMock(
|
||||
return_value={'server1': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=2': {}}}}
|
||||
)
|
||||
ingestor_manager.resource_manager.renew_tag_lease = AsyncMock()
|
||||
|
||||
await ingestor_manager.update_slot_config()
|
||||
|
||||
# Still reading with the last good config; the bad one is notified, not applied.
|
||||
assert ingestor_manager.managed_tags == {'slot1': good}
|
||||
_, kwargs = ingestor_manager.send_notification.call_args
|
||||
assert kwargs['notification_id'] == 'OPC_SLOT_CONFIG_ERROR_slot1'
|
||||
|
||||
Reference in New Issue
Block a user