"""Pytest configuration and fixtures for the gateway E2E tests. Every external dependency runs for real: MongoDB, Redis, PostgreSQL, MinIO and NATS via testcontainers (session-scoped), OPC UA servers in process (``e2e/opc_server.py``, function-scoped). Gateway code under ``ingestor/**`` is not mocked: pods are built from the environment exactly like ``ingestor/app.py`` builds them (``e2e/helpers.Pod``). State is reset before every test. """ import io import json import os from pathlib import Path import nats import pytest import pytest_asyncio from asyncua import ua from asyncua.client import ua_client from minio import Minio from pymongo import MongoClient from redis import Redis from sqlalchemy import create_engine, text from testcontainers.community.minio import MinioContainer from testcontainers.community.mongodb import MongoDbContainer from testcontainers.community.postgres import PostgresContainer from testcontainers.community.redis import RedisContainer from testcontainers.core.container import DockerContainer from testcontainers.core.wait_strategies import LogMessageWaitStrategy from e2e.helpers import CERT_BUCKET, Pod from e2e.opc_server import OpcTestServer, free_port, make_certificate DB_SCHEMA_SQL_PATH = Path(__file__).parent / 'db_schema.sql' MONGO_USER = MONGO_PASSWORD = 'sientia' MONGO_DATABASE = 'sientia_e2e' WRITE_SUBJECT = 'opc.write' # -------------------------------------------------------------------------------------- # Session infrastructure # -------------------------------------------------------------------------------------- @pytest.fixture(scope='session') def mongo_container(): """MongoDB with root credentials: the gateway always connects as user:password@url.""" mongo = MongoDbContainer('mongo:7', username=MONGO_USER, password=MONGO_PASSWORD) mongo.start() yield mongo mongo.stop() @pytest.fixture(scope='session') def redis_container(): redis = RedisContainer('redis:7-alpine') redis.start() yield redis redis.stop() @pytest.fixture(scope='session') def postgres_container(): postgres = PostgresContainer( 'postgres:15', username='sientia', password='sientia', dbname='sientia' ) postgres.start() yield postgres postgres.stop() @pytest.fixture(scope='session') def minio_container(): """ MinIO holding the OPC client certificates. Same Bitnami "legacy" image as the Laborious E2E suite and production: Docker Hub no longer serves ``minio/minio``. """ minio = MinioContainer(image='docker.io/bitnamilegacy/minio:2025.7.23-debian-12-r3') minio.with_env('MINIO_ROOT_USER', minio.access_key) minio.with_env('MINIO_ROOT_PASSWORD', minio.secret_key) minio.with_kwargs(user='root') minio.start() yield minio minio.stop() NATS_IMAGE = 'nats:2.10-alpine' def nats_server_container(host_port: int | None = None) -> DockerContainer: """NATS server; ``host_port`` pins the published port (to start it after the gateway).""" container = DockerContainer(NATS_IMAGE).waiting_for(LogMessageWaitStrategy('Server is ready')) if host_port is None: return container.with_exposed_ports(4222) return container.with_bind_ports(4222, host_port) @pytest.fixture(scope='session') def nats_container(): nats_server = nats_server_container() nats_server.start() yield nats_server nats_server.stop() @pytest.fixture(scope='session') def mongo_client(mongo_container): port = mongo_container.get_exposed_port(27017) client = MongoClient(f'mongodb://{MONGO_USER}:{MONGO_PASSWORD}@localhost:{port}') yield client client.close() @pytest.fixture(scope='session') def postgres_engine(postgres_container): engine = create_engine(postgres_container.get_connection_url()) yield engine engine.dispose() @pytest.fixture(scope='session') def redis_client(redis_container): client = Redis( host=redis_container.get_container_host_ip(), port=int(redis_container.get_exposed_port(6379)), decode_responses=True, ) yield client client.close() @pytest.fixture(scope='session') def minio_client(minio_container): return minio_container.get_client() @pytest.fixture(scope='session') def nats_url(nats_container) -> str: host = nats_container.get_container_host_ip() return f'nats://{host}:{nats_container.get_exposed_port(4222)}' # -------------------------------------------------------------------------------------- # Per-test environment and state reset # -------------------------------------------------------------------------------------- @pytest.fixture(autouse=True) def gateway_env( monkeypatch, tmp_path, mongo_container, redis_container, postgres_container, minio_container, nats_url, ): """Point every gateway setting at the containers, with the app's defaults otherwise.""" minio_cfg = minio_container.get_config() env = { 'HOSTNAME': 'pod-a', 'REDIS_HOST': redis_container.get_container_host_ip(), 'REDIS_PORT': str(redis_container.get_exposed_port(6379)), 'LEASE_TTL': '10', 'HEARTBEAT_TTL': '20', 'POLL_INTERVAL': '1', 'MONGODB_URL': f'localhost:{mongo_container.get_exposed_port(27017)}', 'MONGODB_USERNAME': MONGO_USER, 'MONGODB_PASSWORD': MONGO_PASSWORD, 'MONGODB_DATABASE': MONGO_DATABASE, 'POSTGRES_HOST': postgres_container.get_container_host_ip(), 'POSTGRES_PORT': str(postgres_container.get_exposed_port(5432)), 'POSTGRES_USER': 'sientia', 'POSTGRES_PASSWORD': 'sientia', 'POSTGRES_DBNAME': 'sientia', 'POSTGRES_MIN_CONNECTIONS': '1', 'POSTGRES_MAX_CONNECTIONS': '2', 'MINIO_ENDPOINT': minio_cfg['endpoint'], 'MINIO_ACCESS_KEY': minio_cfg['access_key'], 'MINIO_SECRET_KEY': minio_cfg['secret_key'], 'MINIO_SECURE': 'false', 'NATS_URL': nats_url, 'NATS_WRITE_SUBJECT': WRITE_SUBJECT, 'NATS_QUEUE': 'gateway-writers', 'GRPC_ENABLED': 'false', 'GRPC_PORT': str(free_port()), 'CERT_CACHE_DIR': str(tmp_path / 'cert-cache'), 'WRITE_MIN_PAUSE_S': '0', 'IDLE_CONN_MAX_S': '300', } for name, value in env.items(): monkeypatch.setenv(name, value) for name in ('REDIS_USERNAME', 'REDIS_PASSWORD', 'WRITE_TIMESTAMP', 'WRITE_QUALITY'): monkeypatch.delenv(name, raising=False) return env @pytest.fixture(autouse=True) def reset_state(redis_client, mongo_client, postgres_engine, minio_client): """Known-empty baseline: Redis flushed, Mongo database dropped, Postgres schema recreated from ``e2e/db_schema.sql``, certificate bucket emptied.""" redis_client.flushdb() mongo_client.drop_database(MONGO_DATABASE) with postgres_engine.begin() as conn: conn.execute(text('DROP SCHEMA IF EXISTS sientia_data CASCADE')) conn.exec_driver_sql(DB_SCHEMA_SQL_PATH.read_text(encoding='utf-8')) if minio_client.bucket_exists(CERT_BUCKET): for obj in minio_client.list_objects(CERT_BUCKET, recursive=True): minio_client.remove_object(CERT_BUCKET, obj.object_name) else: minio_client.make_bucket(CERT_BUCKET) yield @pytest.fixture def mongo_db(mongo_client): return mongo_client[MONGO_DATABASE] # -------------------------------------------------------------------------------------- # Factories: OPC servers, pods, slots, certificates, NATS commands # -------------------------------------------------------------------------------------- @pytest_asyncio.fixture async def opc_servers(tmp_path): """ Factory of started in-process OPC servers. ``await opc_servers('plant-a', {'Temperature': 20.0}, secured=False)`` declares the variables (``{name: value}`` or ``{name: (value, data_type, writable)}``) and starts it. """ started: dict[str, OpcTestServer] = {} async def start(name: str, variables: dict, secured: bool = False) -> OpcTestServer: server = OpcTestServer(name, cert_dir=tmp_path / 'server-certs', secured=secured) for var, spec in variables.items(): value, data_type, writable = spec if isinstance(spec, tuple) else (spec, 'float', True) server.add_variable(var, value, data_type, writable) await server.start() started[name] = server return server yield start for server in started.values(): await server.stop() @pytest_asyncio.fixture async def pods(monkeypatch): """ Factory of started gateway pods: ``await pods('pod-a', runtime=True, **env)``. Extra keyword arguments are environment overrides applied before the pod is built (e.g. ``WRITE_TIMESTAMP='false'``). Every pod is stopped at teardown. """ started: list[Pod] = [] async def start(name: str = 'pod-a', runtime: bool = True, **env: str) -> Pod: for key, value in env.items(): monkeypatch.setenv(key, value) pod = Pod(name, runtime=runtime) started.append(pod) return await pod.start() yield start for pod in reversed(started): await pod.stop() @pytest.fixture def put_slot(redis_client): """Write a slot v2 to Redis (``slot:opc_tags:``), as the Orchestrator does.""" def put(number: int, slot: dict) -> None: redis_client.set(f'slot:opc_tags:{number}', json.dumps(slot)) return put @pytest_asyncio.fixture async def publish_command(nats_url): """Publish C2 write commands (dicts) or raw payloads (bytes) on the write subject.""" client = await nats.connect(nats_url) async def publish(payload) -> None: data = payload if isinstance(payload, bytes) else json.dumps(payload).encode() await client.publish(WRITE_SUBJECT, data) await client.flush() yield publish await client.drain() @pytest_asyncio.fixture async def client_certificate(tmp_path, minio_client: Minio, mongo_db): """ Create an OPC client certificate the way the platform stores it: cert (DER) and private key (PEM) uploaded to MinIO, and an ``opc_certificates`` document (active, kind client). Returns the certificate in the slot format (contract C1). """ async def create( certificate_id: int, upload: bool = True, status: str = 'active', app_uri: str | None = None, ) -> dict: app_uri = app_uri or f'urn:sientia:e2e:client:{certificate_id}' cert, key = await make_certificate( tmp_path / 'client-certs', f'client-{certificate_id}', app_uri, server=False ) objects = { 'cert': {'bucket': CERT_BUCKET, 'object_name': f'{certificate_id}/client.der'}, 'private_key': {'bucket': CERT_BUCKET, 'object_name': f'{certificate_id}/client.pem'}, } if upload: for path, field in ((cert, 'cert'), (key, 'private_key')): data = path.read_bytes() minio_client.put_object( CERT_BUCKET, objects[field]['object_name'], io.BytesIO(data), len(data) ) mongo_db['opc_certificates'].insert_one( { 'certificate_id': certificate_id, 'kind': 'client', 'status': status, 'application_uri': app_uri, **objects, } ) return {'certificate_id': certificate_id, 'application_uri': app_uri, **objects} return create @pytest.fixture def cert_cache_dir() -> Path: return Path(os.environ['CERT_CACHE_DIR']) @pytest.fixture def opc_wire(monkeypatch): """Serialized WriteRequests, exactly as the gateway's OPC client sends them.""" sent: list[bytes] = [] original = ua_client.struct_to_binary def capture(obj): data = original(obj) if isinstance(obj, ua.WriteRequest): sent.append(data) return data monkeypatch.setattr(ua_client, 'struct_to_binary', capture) return sent