Files
sientia-dataops-opc-ingestor/tests/unit/test_gateway_runtime.py
Eduardo Rios 50578fdba8 SIENTIAPDE-1974: feat(read): slot v2 reads via SubscriptionEngine, OPC cert from MinIO
Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over
the shared OpcConnectionPool. The OPC client certificate now always comes from
MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the
Gitea repository, so a new version branch no longer breaks the reads.

- io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy
  v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway
  and Orchestrator can be deployed in any order.
- managers/server_reader.py: new per-server reader with the OpcManager
  interface the IngestorManager already drives (connect, create_subscription,
  subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown).
- IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a
  read DataSink, built without touching the network. Slots from Redis are
  normalized on lease and refresh; a malformed slot is notified and the last
  good config is kept.
- GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no
  longer closes it on stop: a Postgres/NATS failure can't stop the reads.
- pool: pin/unpin per active subscription so release_idle never evicts a
  subscribed connection; public evict() for lost servers / changed config.
- SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a
  subscription locally even when the server-side delete fails.
- DataSink: model_id only written when the name has one, keeping v1 documents
  identical to the legacy ones.
- config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7).
- Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh;
  README/CLAUDE.md document the MinIO certificate and the slot v2 format.

Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against
an asyncua server: v1 slot with two topics -> one doc per collection, legacy
doc shape, idle sweep keeps the subscription, lost-server shutdown and
re-initialization work.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 13:27:18 -03:00

176 lines
5.4 KiB
Python

from unittest.mock import AsyncMock, MagicMock
import pytest
from ingestor import gateway_runtime as gr
from ingestor.config import GatewayConfig
_PATCHED_CLASSES = (
'Postgres',
'build_postgres_config',
'DataSink',
'WriteEngine',
'NatsWriteConsumer',
'ActiveReadEngine',
'GrpcServer',
)
@pytest.fixture
def mocks(monkeypatch):
patched = {}
for name in _PATCHED_CLASSES:
m = MagicMock()
monkeypatch.setattr(gr, name, m)
patched[name] = m
patched['build_postgres_config'].return_value = {
'host': 'localhost',
'port': 5432,
'user': 'u',
'password': 'p',
'dbname': 'd',
'min_connections': 5,
'max_connections': 20,
}
# The pool is owned by IngestorManager (read path) and only borrowed here.
patched['pool'] = MagicMock()
patched['pool'].close_all = AsyncMock()
patched['pool'].release_idle = AsyncMock()
patched['NatsWriteConsumer'].return_value.start = AsyncMock()
patched['NatsWriteConsumer'].return_value.stop = AsyncMock()
patched['GrpcServer'].return_value.start = AsyncMock()
patched['GrpcServer'].return_value.stop = AsyncMock()
return patched
def make_runtime(pool=None, **config_overrides):
if pool is None:
pool = MagicMock(close_all=AsyncMock(), release_idle=AsyncMock())
return gr.GatewayRuntime(
config=GatewayConfig(**config_overrides),
data_manager=MagicMock(),
pool=pool,
logger=MagicMock(),
notification_handler=MagicMock(),
metrics_controller=MagicMock(),
)
def test_builds_object_graph_with_config_flags(mocks):
runtime = make_runtime(
pool=mocks['pool'],
write_timestamp=False,
write_quality=False,
write_min_pause_s=0.25,
grpc_enabled=False,
)
# Writes and active reads share the pool built by the read path.
assert runtime.pool is mocks['pool']
mocks['DataSink'].assert_called_once()
mocks['WriteEngine'].assert_called_once()
args, kwargs = mocks['WriteEngine'].call_args
assert args[0] is mocks['pool']
assert kwargs['write_timestamp'] is False
assert kwargs['write_quality'] is False
assert kwargs['min_pause_s'] == 0.25
args, _ = mocks['ActiveReadEngine'].call_args
assert args[0] is mocks['pool']
mocks['GrpcServer'].assert_not_called()
assert runtime.grpc_server is None
def test_builds_grpc_server_when_enabled(mocks):
runtime = make_runtime(grpc_enabled=True, grpc_port=12345)
mocks['GrpcServer'].assert_called_once()
assert runtime.grpc_server is mocks['GrpcServer'].return_value
@pytest.mark.asyncio
async def test_start_starts_nats_consumer(mocks):
runtime = make_runtime(grpc_enabled=False)
await runtime.start()
mocks['NatsWriteConsumer'].return_value.start.assert_awaited_once()
mocks['GrpcServer'].return_value.start.assert_not_called()
assert runtime._nats_started is True
@pytest.mark.asyncio
async def test_start_also_starts_grpc_when_enabled(mocks):
runtime = make_runtime(grpc_enabled=True)
await runtime.start()
mocks['NatsWriteConsumer'].return_value.start.assert_awaited_once()
mocks['GrpcServer'].return_value.start.assert_awaited_once()
@pytest.mark.asyncio
async def test_nats_start_failure_does_not_raise_and_idle_sweep_retries(mocks):
runtime = make_runtime()
runtime.send_notification_async = AsyncMock()
nats_mock = mocks['NatsWriteConsumer'].return_value
nats_mock.start = AsyncMock(side_effect=[Exception('boom'), Exception('boom'), None])
await runtime.start() # must not raise
assert runtime._nats_started is False
runtime.logger.error.assert_called_once()
runtime.send_notification_async.assert_awaited_once()
await runtime.idle_sweep() # second failure: retry connects, but no re-notify
assert runtime._nats_started is False
runtime.send_notification_async.assert_awaited_once() # still just one notification
await runtime.idle_sweep()
assert runtime._nats_started is True
assert nats_mock.start.await_count == 3
# flag reset on success: a future outage notifies again
assert runtime._nats_notified is False
@pytest.mark.asyncio
async def test_stop_stops_started_components_but_not_the_pool(mocks):
runtime = make_runtime(pool=mocks['pool'], grpc_enabled=True)
await runtime.start()
await runtime.stop()
mocks['GrpcServer'].return_value.stop.assert_awaited_once()
mocks['NatsWriteConsumer'].return_value.stop.assert_awaited_once()
# IngestorManager.shutdown() closes it, after its readers unsubscribe.
mocks['pool'].close_all.assert_not_called()
@pytest.mark.asyncio
async def test_stop_skips_nats_stop_if_never_started(mocks):
runtime = make_runtime(pool=mocks['pool'])
runtime.send_notification_async = AsyncMock()
nats_mock = mocks['NatsWriteConsumer'].return_value
nats_mock.start = AsyncMock(side_effect=Exception('boom'))
await runtime.start() # nats fails to start
await runtime.stop()
nats_mock.stop.assert_not_called()
mocks['pool'].close_all.assert_not_called()
@pytest.mark.asyncio
async def test_idle_sweep_releases_idle_connections(mocks):
runtime = make_runtime(pool=mocks['pool'], idle_conn_max_s=42.0)
runtime._nats_started = True # isolate from the retry path
await runtime.idle_sweep()
mocks['pool'].release_idle.assert_awaited_once_with(42.0)