gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

SIENTIAPDE-2202

Browse Source 
SIENTIAPDE-2202 Add gateway end-to-end test suite (e2e/)

End-to-end suite at the repo root, following the Laborious/Orchestrator
pattern: numbered scenarios per topic in e2e/scenarios.md (1:1 with
test_scenario_* functions), scenario inputs as JSON, Postgres DDL mirrored
from production, `integration` marker, excluded from the default pytest run.

Every dependency runs for real: MongoDB, Redis, PostgreSQL, MinIO and NATS
via testcontainers, OPC UA servers in process (asyncua). Pods are built from
the environment like ingestor/app.py and driven one loop iteration at a time.

69 scenarios across 7 topics: slot leasing and failover, subscription reads,
NATS writes and write_status (incl. the SIENTIAPDE-2202 timestamp/quality
matrix, checked on the server and on the wire), gRPC ReadTag, active reads,
client certificates/security, pod lifecycle.

Replaces tests/functional (legacy slot v1 tests, fully commented out) and
tests/functional/test_write_e2e.py (now scenarios 3.1.x).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-29 17:31:15 -03:00
parent ca9f4ce188
commit 4ee672a715
30 changed files with 2893 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
tests/functional/__init__.py
View File

31
tests/functional/conftest.py
View File

@@ -1,31 +0,0 @@
# import subprocess
# from time import sleep
# import pytest
# from redis import Redis
# @pytest.fixture(scope="session", autouse=True)
# def docker_compose():
# """Sobe os containers antes dos testes e derruba depois."""
# print("\n🚀 Subindo Docker Compose...")
# subprocess.run(["docker", "compose", "up", "-d"], check=True)
# print("⏳ Aguardando containers ficarem prontos...")
# sleep(15) # ajuste conforme necessário
# yield # os testes rodam aqui
# print("\n🧹 Derrubando Docker Compose...")
# subprocess.run(["docker", "compose", "down"], check=True)
# @pytest.fixture()
# def redis_client():
# redis = Redis(host="localhost", port=6379, decode_responses=True)
# redis.flushdb()
# yield redis
# # Limpa o banco de dados após os testes
# redis.flushdb()
# redis.close()

93
tests/functional/test_single_node.py
View File

@@ -1,93 +0,0 @@
# import json
# import subprocess
# from time import sleep
# new_data = {
# "slot:opc_tags:1": {
# "server1": {
# "name": "server1",
# "url": "opc.tcp://simulator:4840",
# "server_uri": "http://opcua-server.simulator",
# "tags": {
# 'ns=2;i=2': {
# 'tag_name': 'Counter',
# 'frequency': 1000,
# 'topics': [],
# },
# 'ns=2;i=3': {
# 'tag_name': 'Rollout',
# 'frequency': 1000,
# "topics": [],
# },
# 'ns=2;i=4': {
# 'tag_name': 'Square',
# 'frequency': 1000,
# "topics": [],
# },
# }
# }
# },
# "slot:opc_tags:2": {
# "server2": {
# "name": "server2",
# "url": "opc.tcp://simulator:4840",
# "server_uri": "http://opcua-server.simulator",
# "tags": {
# 'ns=2;i=2': {
# 'tag_name': 'Counter',
# 'frequency': 1000,
# 'topics': [],
# },
# 'ns=2;i=3': {
# 'tag_name': 'Rollout',
# 'frequency': 1000,
# "topics": [],
# },
# 'ns=2;i=4': {
# 'tag_name': 'Square',
# 'frequency': 1000,
# "topics": [],
# },
# }
# }
# }
# }
# def test_simple(redis_client):
# new_data['slot:opc_tags:1']['server1']['tags']['ns=2;i=2']['topics'] = [
# 'test_topic_1']
# redis_client.set("slot:opc_tags:1",
# json.dumps(new_data['slot:opc_tags:1']))
# sleep(20) # Espera o Ingestor processar os dados
# # Check if lease is in Redis
# assert redis_client.get("lease:opc_tags:1") == 'ingestor'
# assert redis_client.get("heartbeat:ingestor:ingestor") == '1'
# def test_simple_double_slot(redis_client):
# new_data['slot:opc_tags:1']['server1']['tags']['ns=2;i=2']['topics'] = [
# 'test_topic_double_slot1']
# redis_client.set("slot:opc_tags:1",
# json.dumps(new_data['slot:opc_tags:1']))
# sleep(20) # Espera o Ingestor processar os dados
# assert redis_client.get("lease:opc_tags:1") == 'ingestor'
# assert redis_client.get("heartbeat:ingestor:ingestor") == '1'
# new_data['slot:opc_tags:2']['server2']['tags']['ns=2;i=2']['topics'] = [
# 'test_topic_double_slot2']
# redis_client.set("slot:opc_tags:2",
# json.dumps(new_data['slot:opc_tags:2']))
# sleep(20) # Espera o Ingestor processar os dados
# # Check if lease is in Redis
# assert redis_client.get("lease:opc_tags:2") == 'ingestor'
# assert redis_client.get("lease:opc_tags:1") == 'ingestor'
# assert redis_client.get("heartbeat:ingestor:ingestor") == '1'

173
tests/functional/test_write_e2e.py
View File

@@ -1,173 +0,0 @@
"""End-to-end OPC write (SIENTIAPDE-2202): env → GatewayConfig → WriteEngine →
OpcConnectionPool → a real asyncua server, in process.
The server keeps the SourceTimestamp it receives, so the timestamp is checked on the
server. An omitted StatusCode decodes as Good on the server (OPC UA default), so the
quality is checked on the wire: the DataValue encoding mask of the WriteRequest the
client sent.
"""
import socket
from datetime import datetime
from unittest.mock import AsyncMock, MagicMock
import pytest
import pytest_asyncio
from asyncua import Server, ua
from asyncua.client import ua_client
from asyncua.ua.ua_binary import Primitives, nodeid_to_binary
from ingestor.config import GatewayConfig
from ingestor.connection.pool import OpcConnectionPool
from ingestor.engines.write import WriteEngine
TIMESTAMP = '2026-07-13T12:00:00Z'
# DataValue encoding mask bits (OPC UA Part 6, 5.2.2.17).
_STATUS_CODE_BIT = 0x02
_SOURCE_TIMESTAMP_BIT = 0x04
def _free_port() -> int:
with socket.socket() as s:
s.bind(('127.0.0.1', 0))
return s.getsockname()[1]
@pytest_asyncio.fixture
async def opc_server():
url = f'opc.tcp://127.0.0.1:{_free_port()}'
server = Server()
await server.init()
server.set_endpoint(url)
idx = await server.register_namespace('urn:sientia:e2e')
var = await server.nodes.objects.add_variable(idx, 'Pred', 0.0, ua.VariantType.Float)
await var.set_writable()
async with server:
yield url, var
@pytest.fixture
def wire(monkeypatch):
"""Serialized WriteRequests, as sent by the client."""
sent: list[bytes] = []
original = ua_client.struct_to_binary
def capture(obj):
data = original(obj)
if isinstance(obj, ua.WriteRequest):
sent.append(data)
return data
monkeypatch.setattr(ua_client, 'struct_to_binary', capture)
return sent
def _datavalue_mask(request: bytes, addr: str) -> int:
# WriteValue = NodeId, AttributeId, IndexRange, DataValue: the byte right after
# the first three is the DataValue encoding mask. The NodeId is parsed from the
# address as the client does (its binary encoding depends on how it was built).
prefix = (
nodeid_to_binary(ua.NodeId.from_string(addr))
+ Primitives.UInt32.pack(ua.AttributeIds.Value)
+ Primitives.String.pack(None)
)
return request[request.index(prefix) + len(prefix)]
async def _engine(url: str, pools: list) -> WriteEngine:
config = GatewayConfig.from_env()
logger, notification_handler = MagicMock(), MagicMock()
# emit() is awaited, start()/shutdown_sync() are not.
metrics_controller = MagicMock(emit=AsyncMock(), shutdown=AsyncMock())
pool = OpcConnectionPool(MagicMock(), logger, notification_handler, metrics_controller)
pools.append(pool)
servers = MagicMock()
servers.resolve = AsyncMock(
return_value={
'server_id': 'e2e',
'url': url,
'security': {'mode': 'None', 'policy': 'None'},
'certificate': None,
}
)
return WriteEngine(
pool,
AsyncMock(),
logger,
notification_handler,
metrics_controller,
write_timestamp=config.write_timestamp,
write_quality=config.write_quality,
servers=servers,
)
@pytest_asyncio.fixture
async def pools():
created: list[OpcConnectionPool] = []
yield created
for pool in created:
for client, _ in pool._connections.values():
await client.disconnect()
@pytest.mark.integration
@pytest.mark.asyncio
@pytest.mark.parametrize(
'env,cmd_flags,expect_timestamp,expect_quality',
[
pytest.param({}, {}, True, True, id='defaults-on'),
pytest.param({'WRITE_TIMESTAMP': 'false'}, {}, False, True, id='timestamp-off'),
pytest.param({'WRITE_QUALITY': 'false'}, {}, True, False, id='quality-off'),
pytest.param(
{'WRITE_TIMESTAMP': 'false', 'WRITE_QUALITY': 'false'}, {}, False, False, id='both-off'
),
pytest.param(
{'WRITE_TIMESTAMP': 'true'},
{'write_timestamp': False},
False,
True,
id='command-overrides-env',
),
pytest.param(
{'WRITE_TIMESTAMP': 'false', 'WRITE_QUALITY': 'false'},
{'write_timestamp': True, 'write_quality': True},
True,
True,
id='command-turns-on-over-env',
),
],
)
async def test_write_datavalue_parts(
monkeypatch, opc_server, wire, pools, env, cmd_flags, expect_timestamp, expect_quality
):
monkeypatch.delenv('WRITE_TIMESTAMP', raising=False)
monkeypatch.delenv('WRITE_QUALITY', raising=False)
for name, value in env.items():
monkeypatch.setenv(name, value)
url, var = opc_server
engine = await _engine(url, pools)
cmd = {
'server_id': 'e2e',
'addr': var.nodeid.to_string(),
'value': 1.5,
'data_type': 'float',
'model_id': 'm1',
'timestamp': TIMESTAMP,
**cmd_flags,
}
result = await engine.write(cmd)
assert result.status == 'ok', result.message
stored = await var.read_data_value()
assert stored.Value.Value == 1.5
if expect_timestamp:
assert stored.SourceTimestamp == datetime.fromisoformat(TIMESTAMP)
else:
assert stored.SourceTimestamp is None
assert len(wire) == 1
mask = _datavalue_mask(wire[0], cmd['addr'])
assert bool(mask & _SOURCE_TIMESTAMP_BIT) is expect_timestamp
assert bool(mask & _STATUS_CODE_BIT) is expect_quality