gitea_admin/sientia-dataops-opc-ingestor
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'feature/SIENTIAPDE-1972-gateway-write-read' into feature/SIENTIAPDE-1972-gateway-wiring

Browse Source
This commit is contained in:
PedroHMCosme
2026-09-22 11:28:06 -03:00
parent b8d3a5118b 6f4b04e8c0
commit 0d20f6d123
8 changed files with 132 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
ingestor/engines/active_read.py
View File

@@ -40,7 +40,11 @@ class ActiveReadEngine(SientiaMonitoring):
)
async def read_now(
self, server_id: str, addr: str, names: list[dict] | None = None, server_cfg: dict | None = None
self,
server_id: str,
addr: str,
names: list[dict] | None = None,
server_cfg: dict | None = None,
) -> dict:
"""Read `addr` on `server_id` right now; persist via sink only if `names` given."""
tags = {'pod_id': self.pod_id, 'server_name': server_id}
@@ -72,7 +76,9 @@ class ActiveReadEngine(SientiaMonitoring):
source_str = self._format_ts(source_ts)
persisted = bool(names)
if persisted:
await self.sink.publish_reading(tag=addr, value=value, timestamp=source_str, names=names)
await self.sink.publish_reading(
tag=addr, value=value, timestamp=source_str, names=names
)
return {
'value': value,
'opc_status_code': status_name,
@@ -96,7 +102,9 @@ class ActiveReadEngine(SientiaMonitoring):
if source_ts is None:
return
lag = MonotonicClock.now_wall() - source_ts.replace(tzinfo=OPC_TIMEZONE).timestamp()
await self.emit_metric(metric_object=metrics.OPC_READ_LAG, method='set', value=lag, tags=tags)
await self.emit_metric(
metric_object=metrics.OPC_READ_LAG, method='set', value=lag, tags=tags
)
async def scheduled_tick(self, slot: dict) -> None:
"""Sweep every server/tag in a parsed slot, reading tags due for an active read."""

31
ingestor/engines/write.py
View File

@@ -37,13 +37,24 @@ _VARIANT_TYPES = {
# ponytail: same reconnect-worthy Bad*-status subset as laborious's
# opc_repository.py (different repo, not importable here); keep in sync by
# hand or promote to a shared lib if it drifts.
_SESSION_BAD_NAMES = frozenset({
'BadSessionIdInvalid', 'BadSessionClosed', 'BadSessionNotActivated',
'BadSecureChannelIdInvalid', 'BadSecureChannelClosed', 'BadSecureChannelTokenUnknown',
'BadTcpSecureChannelUnknown', 'BadServerNotConnected', 'BadConnectionClosed',
'BadConnectionRejected', 'BadNotConnected', 'BadTimeout', 'BadRequestTimeout',
'BadTooManySessions',
})
_SESSION_BAD_NAMES = frozenset(
{
'BadSessionIdInvalid',
'BadSessionClosed',
'BadSessionNotActivated',
'BadSecureChannelIdInvalid',
'BadSecureChannelClosed',
'BadSecureChannelTokenUnknown',
'BadTcpSecureChannelUnknown',
'BadServerNotConnected',
'BadConnectionClosed',
'BadConnectionRejected',
'BadNotConnected',
'BadTimeout',
'BadRequestTimeout',
'BadTooManySessions',
}
)
@dataclass
@@ -174,7 +185,11 @@ class WriteEngine(SientiaMonitoring):
metric_object=metrics.OPC_WRITE_STATUS_BY_MODEL,
method='set',
value=1,
tags={'pod_id': self.pod_id, 'model_id': cmd.get('model_id', 'unknown'), 'status': 'ok'},
tags={
'pod_id': self.pod_id,
'model_id': cmd.get('model_id', 'unknown'),
'status': 'ok',
},
)
result = WriteResult(ok=True, status='ok', opc_status_code='Good', message='')
await self.sink.update_write_status(cmd, result)

6
ingestor/io/data_sink.py
View File

@@ -57,7 +57,11 @@ class DataSink:
f'UPDATE {schema}.{table} SET write_status = :status ' # noqa: S608
'WHERE timestamp = :timestamp AND model_id = :model_id'
)
params = {'status': result.status, 'timestamp': cmd['timestamp'], 'model_id': cmd['model_id']}
params = {
'status': result.status,
'timestamp': cmd['timestamp'],
'model_id': cmd['model_id'],
}
# ponytail: sync engine call inside async def — same pattern sientia_do's own
# Postgres activities use (e.g. export_data_to_postgres); no async pg driver here.
with self.postgres.engine.begin() as connection:

32
ingestor/metrics.py
View File

@@ -142,35 +142,49 @@ NOTIFICATIONS_SENT = Counter(
# --- Gateway: Write metrics ---
OPC_WRITE_TOTAL = Counter(
'opc_write_total', 'OPC active write attempts',
'opc_write_total',
'OPC active write attempts',
['pod_id', 'server_name', 'model_id', 'result'], # result: success|error
)
OPC_WRITE_LATENCY = Histogram(
'opc_write_latency_seconds', 'OPC active write latency', ['pod_id', 'server_name'],
'opc_write_latency_seconds',
'OPC active write latency',
['pod_id', 'server_name'],
)
OPC_WRITE_STATUS_BY_MODEL = Gauge(
'opc_write_status_by_model', 'Last write status per model (1=ok,0=error)',
'opc_write_status_by_model',
'Last write status per model (1=ok,0=error)',
['pod_id', 'model_id', 'status'],
)
# --- Gateway: Active read metrics ---
OPC_ACTIVE_READ_TOTAL = Counter(
'opc_active_read_total', 'OPC active read attempts',
'opc_active_read_total',
'OPC active read attempts',
['pod_id', 'server_name', 'result'],
)
OPC_ACTIVE_READ_LATENCY = Histogram(
'opc_active_read_latency_seconds', 'OPC active read latency', ['pod_id', 'server_name'],
'opc_active_read_latency_seconds',
'OPC active read latency',
['pod_id', 'server_name'],
)
OPC_READ_LAG = Gauge(
'opc_read_lag_seconds', 'Lag between OPC source timestamp and machine time',
'opc_read_lag_seconds',
'Lag between OPC source timestamp and machine time',
['pod_id', 'server_name'],
)
OPC_TAG_VALUE = Gauge(
'opc_tag_value', 'Last value read per tag/model', ['pod_id', 'tag_name', 'model_id'],
'opc_tag_value',
'Last value read per tag/model',
['pod_id', 'tag_name', 'model_id'],
)
# --- Gateway: connection/error metrics ---
OPC_DISCONNECTIONS_TOTAL = Counter(
'opc_disconnections_total', 'OPC disconnections', ['pod_id', 'server_name'],
'opc_disconnections_total',
'OPC disconnections',
['pod_id', 'server_name'],
)
OPC_ERROR_BY_TYPE = Counter(
'opc_error_by_type_total', 'OPC errors by type', ['pod_id', 'server_name', 'error_type'],
'opc_error_by_type_total',
'OPC errors by type',
['pod_id', 'server_name', 'error_type'],
)

4
tests/unit/engines/test_active_read.py
View File

@@ -74,7 +74,9 @@ async def test_read_now_persists_and_returns_fields_when_names_given():
assert latency_call.kwargs['method'] == 'observe'
assert latency_call.kwargs['value'] >= 0
lag_call = next(
c for c in eng.emit_metric.await_args_list if c.kwargs['metric_object'] is metrics.OPC_READ_LAG
c
for c in eng.emit_metric.await_args_list
if c.kwargs['metric_object'] is metrics.OPC_READ_LAG
)
assert lag_call.kwargs['method'] == 'set'

4
tests/unit/engines/test_subscription.py
View File

@@ -150,9 +150,7 @@ async def test_subscribe_without_subscription_raises():
async def test_unsubscribe_missing_logs_warning():
eng = make_engine()
await eng.unsubscribe('sub1')
eng.logger.warning.assert_called_once_with(
"Subscription 'sub1' not found. Cannot unsubscribe."
)
eng.logger.warning.assert_called_once_with("Subscription 'sub1' not found. Cannot unsubscribe.")
@pytest.mark.asyncio

50
tests/unit/io/test_slots.py
View File

@@ -1,10 +1,33 @@
from ingestor.io.slots import parse_slot
RAW = {'server1': {'server_id': '1', 'url': 'opc.tcp://h:4840', 'use_security': True,
'subscription_period_ms': 1000, 'tags': {'ns=2;i=2': {'frequency': '15000',
'active_read': {'enabled': True, 'cycle_multiple': 3},
'names': [{'collection': 'raw_x', 'tag_name': 'Counter', 'model_name': 'm1', 'model_id': '1'},
{'collection': 'raw_y', 'tag_name': 'Contador', 'model_name': 'm2', 'model_id': '2'}]}}}}
RAW = {
'server1': {
'server_id': '1',
'url': 'opc.tcp://h:4840',
'use_security': True,
'subscription_period_ms': 1000,
'tags': {
'ns=2;i=2': {
'frequency': '15000',
'active_read': {'enabled': True, 'cycle_multiple': 3},
'names': [
{
'collection': 'raw_x',
'tag_name': 'Counter',
'model_name': 'm1',
'model_id': '1',
},
{
'collection': 'raw_y',
'tag_name': 'Contador',
'model_name': 'm2',
'model_id': '2',
},
],
}
},
}
}
def test_parse_slot_multi_name():
@@ -15,8 +38,20 @@ def test_parse_slot_multi_name():
def test_parse_slot_defaults_active_read_disabled():
raw = {'s': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=1': {
'frequency': '1000', 'names': [{'collection': 'c', 'tag_name': 't', 'model_name': 'm', 'model_id': '1'}]}}}}
raw = {
's': {
'server_id': '1',
'url': 'u',
'tags': {
'ns=2;i=1': {
'frequency': '1000',
'names': [
{'collection': 'c', 'tag_name': 't', 'model_name': 'm', 'model_id': '1'}
],
}
},
}
}
tag = parse_slot(raw)['s']['tags']['ns=2;i=1']
assert tag['active_read'] == {'enabled': False}
@@ -24,5 +59,6 @@ def test_parse_slot_defaults_active_read_disabled():
def test_parse_slot_rejects_tag_without_names():
raw = {'s': {'server_id': '1', 'url': 'u', 'tags': {'ns=2;i=1': {'frequency': '1000'}}}}
import pytest
with pytest.raises((KeyError, ValueError)):
parse_slot(raw)

30
tests/unit/test_converters.py
View File

@@ -3,10 +3,20 @@ import pytest
from ingestor.converters import ConversionError, to_typed_value
@pytest.mark.parametrize('raw,expected', [
('false', False), ('False', False), ('0', False), ('no', False),
('off', False), ('', False), ('true', True), ('1', True), (True, True),
])
@pytest.mark.parametrize(
'raw,expected',
[
('false', False),
('False', False),
('0', False),
('no', False),
('off', False),
('', False),
('true', True),
('1', True),
(True, True),
],
)
def test_bool_from_json_string(raw, expected):
assert to_typed_value(raw, 'boolean') is expected
@@ -16,9 +26,15 @@ def test_float_and_int():
assert to_typed_value('7', 'int') == 7
@pytest.mark.parametrize('raw,dtype', [
(None, 'int'), ('abc', 'float'), ('123.123', 'int'), (None, 'float'),
])
@pytest.mark.parametrize(
'raw,dtype',
[
(None, 'int'),
('abc', 'float'),
('123.123', 'int'),
(None, 'float'),
],
)
def test_invalid_raises_conversion_error(raw, dtype):
with pytest.raises(ConversionError):
to_typed_value(raw, dtype)