SIENTIAPDE-2202

SIENTIAPDE-2202 Add gateway end-to-end test suite (e2e/)

End-to-end suite at the repo root, following the Laborious/Orchestrator
pattern: numbered scenarios per topic in e2e/scenarios.md (1:1 with
test_scenario_* functions), scenario inputs as JSON, Postgres DDL mirrored
from production, `integration` marker, excluded from the default pytest run.

Every dependency runs for real: MongoDB, Redis, PostgreSQL, MinIO and NATS
via testcontainers, OPC UA servers in process (asyncua). Pods are built from
the environment like ingestor/app.py and driven one loop iteration at a time.

69 scenarios across 7 topics: slot leasing and failover, subscription reads,
NATS writes and write_status (incl. the SIENTIAPDE-2202 timestamp/quality
matrix, checked on the server and on the wire), gRPC ReadTag, active reads,
client certificates/security, pod lifecycle.

Replaces tests/functional (legacy slot v1 tests, fully commented out) and
tests/functional/test_write_e2e.py (now scenarios 3.1.x).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

e2e/test_slot_leasing.py

@@ -0,0 +1,193 @@
+"""E2E scenarios — 1. Slot leasing and load balancing (Redis). See ``e2e/scenarios.md``."""
+
+import asyncio
+
+import pytest
+
+from e2e.helpers import (
+    load_scenario_input,
+    notifications,
+    readings,
+    redis_json,
+    stays_false,
+    wait_until,
+)
+
+pytestmark = [pytest.mark.integration, pytest.mark.asyncio]
+
+
+async def _two_slots(opc_servers, put_slot):
+    """Slot 1 reads plant-a, slot 2 reads plant-b."""
+    plant_a = await opc_servers('plant-a', {'Temperature': 20.0})
+    plant_b = await opc_servers('plant-b', {'Flow': 3.0})
+    both = load_scenario_input('slot_two_servers.json', {'plant-a': plant_a, 'plant-b': plant_b})
+    put_slot(1, {'plant-a': both['plant-a']})
+    put_slot(2, {'plant-b': both['plant-b']})
+    return plant_a, plant_b
+
+
+# 1.1 Acquisition ----------------------------------------------------------------------
+
+
+async def test_scenario_1_1_1_single_pod_leases_the_slot_and_heartbeats(
+    opc_servers, pods, put_slot, redis_client, mongo_db
+):
+    server = await opc_servers('plant-a', {'Temperature': 20.0})
+    put_slot(1, load_scenario_input('slot_single_server.json', {'plant-a': server}))
+
+    pod = await pods('pod-a')
+    await pod.tick()
+
+    assert pod.slots == {'1'}
+    assert redis_json(redis_client, 'lease:opc_tags:1') == 'pod-a'
+    assert 0 < redis_client.ttl('lease:opc_tags:1') <= 10
+    assert redis_json(redis_client, 'heartbeat:ingestor:pod-a') == 1
+    assert 0 < redis_client.ttl('heartbeat:ingestor:pod-a') <= 20
+    await wait_until(lambda: readings(mongo_db, 'raw_plant_a'))
+
+
+async def test_scenario_1_1_2_idle_pod_takes_a_slot_created_after_startup(
+    opc_servers, pods, put_slot, redis_client, mongo_db
+):
+    server = await opc_servers('plant-a', {'Temperature': 20.0})
+    pod = await pods('pod-a')
+    await pod.tick()
+    assert pod.slots == set()
+
+    put_slot(1, load_scenario_input('slot_single_server.json', {'plant-a': server}))
+    await pod.tick()
+
+    assert pod.slots == {'1'}
+    assert redis_json(redis_client, 'lease:opc_tags:1') == 'pod-a'
+    await wait_until(lambda: readings(mongo_db, 'raw_plant_a'))
+
+
+# 1.2 Balancing and failover ------------------------------------------------------------
+
+
+async def test_scenario_1_2_1_two_pods_split_two_slots(
+    opc_servers, pods, put_slot, redis_client, mongo_db
+):
+    plant_a, plant_b = await _two_slots(opc_servers, put_slot)
+
+    pod_a = await pods('pod-a')
+    await pod_a.tick()
+    assert pod_a.slots == {'1', '2'}  # alone: takes every slot
+
+    pod_b = await pods('pod-b')
+    await pod_b.tick()
+    assert pod_b.slots == set()  # every slot is leased
+    await pod_a.tick()  # pod-a now sees a pod without slot and gives one back
+    assert pod_a.slots == {'1'}
+    await pod_b.tick()
+    assert pod_b.slots == {'2'}
+
+    assert redis_json(redis_client, 'lease:opc_tags:1') == 'pod-a'
+    assert redis_json(redis_client, 'lease:opc_tags:2') == 'pod-b'
+    assert set(pod_a.manager.opc_managers) == {'plant-a'}
+    assert set(pod_b.manager.opc_managers) == {'plant-b'}
+    await plant_a.set('Temperature', 25.0)
+    await plant_b.set('Flow', 4.5)
+    await wait_until(lambda: readings(mongo_db, 'raw_plant_a', value=25.0))
+    await wait_until(lambda: readings(mongo_db, 'raw_plant_b', value=4.5))
+
+
+async def test_scenario_1_2_2_extra_pod_stays_idle(opc_servers, pods, put_slot, redis_client):
+    await _two_slots(opc_servers, put_slot)
+    pod_a, pod_b = await pods('pod-a'), await pods('pod-b')
+    await pod_a.tick()
+    await pod_b.tick()
+    await pod_a.tick()
+    await pod_b.tick()
+
+    pod_c = await pods('pod-c')
+    for _ in range(2):
+        for pod in (pod_a, pod_b, pod_c):
+            await pod.tick()
+
+    assert pod_c.slots == set()
+    assert pod_a.slots | pod_b.slots == {'1', '2'}
+    assert pod_a.slots.isdisjoint(pod_b.slots)
+    assert {
+        redis_json(redis_client, 'lease:opc_tags:1'),
+        redis_json(redis_client, 'lease:opc_tags:2'),
+    } == {
+        'pod-a',
+        'pod-b',
+    }
+
+
+async def test_scenario_1_2_3_failover_after_lease_expiry(
+    opc_servers, pods, put_slot, redis_client, mongo_db
+):
+    server = await opc_servers('plant-a', {'Temperature': 20.0})
+    put_slot(1, load_scenario_input('slot_single_server.json', {'plant-a': server}))
+    pod_a = await pods('pod-a', LEASE_TTL='2', HEARTBEAT_TTL='2')
+    await pod_a.tick()
+    pod_b = await pods('pod-b')
+    await pod_b.tick()
+    assert (pod_a.slots, pod_b.slots) == ({'1'}, set())
+
+    await pod_a.stop()  # crash-like: the lease is not released, it expires
+    assert redis_json(redis_client, 'lease:opc_tags:1') == 'pod-a'
+    await wait_until(lambda: redis_json(redis_client, 'lease:opc_tags:1') is None, timeout=5)
+    await pod_b.tick()
+
+    assert pod_b.slots == {'1'}
+    assert redis_json(redis_client, 'lease:opc_tags:1') == 'pod-b'
+    await server.set('Temperature', 30.0)
+    await wait_until(lambda: readings(mongo_db, 'raw_plant_a', value=30.0))
+
+
+# 1.3 Slot changes and invalid slots ----------------------------------------------------
+
+
+async def test_scenario_1_3_1_deleted_slot_stops_the_reads(
+    opc_servers, pods, put_slot, redis_client, mongo_db
+):
+    server = await opc_servers('plant-a', {'Temperature': 20.0})
+    put_slot(1, load_scenario_input('slot_single_server.json', {'plant-a': server}))
+    pod = await pods('pod-a')
+    await pod.tick()
+    await wait_until(lambda: readings(mongo_db, 'raw_plant_a'))
+
+    redis_client.delete('slot:opc_tags:1')
+    await pod.tick()
+
+    assert pod.slots == set()
+    assert pod.manager.opc_managers == {}
+    await server.set('Temperature', 99.0)
+    assert await stays_false(lambda: readings(mongo_db, 'raw_plant_a', value=99.0))
+
+
+async def test_scenario_1_3_2_invalid_slot_is_notified_and_not_read(
+    opc_servers, pods, put_slot, mongo_db
+):
+    server = await opc_servers('plant-a', {'Temperature': 20.0})
+    put_slot(1, load_scenario_input('slot_v1_invalid.json', {'plant-a': server}))
+
+    pod = await pods('pod-a')
+    await pod.tick()
+
+    assert pod.slots == set()
+    assert notifications(mongo_db, 'OPC_SLOT_CONFIG_ERROR_1')
+    assert await stays_false(lambda: readings(mongo_db, 'raw_plant_a'))
+
+
+async def test_scenario_1_3_3_fixed_slot_is_read_once_its_lease_expires(
+    opc_servers, pods, put_slot, mongo_db
+):
+    server = await opc_servers('plant-a', {'Temperature': 20.0})
+    put_slot(1, load_scenario_input('slot_v1_invalid.json', {'plant-a': server}))
+    pod = await pods('pod-a', LEASE_TTL='2')
+    await pod.tick()
+    assert pod.slots == set()
+
+    put_slot(1, load_scenario_input('slot_single_server.json', {'plant-a': server}))
+    await pod.tick()
+    assert pod.slots == set()  # still leased by the failed acquisition
+
+    await asyncio.sleep(2.5)
+    await pod.tick()
+    assert pod.slots == {'1'}
+    await wait_until(lambda: readings(mongo_db, 'raw_plant_a'))