50578fdba8
Swaps the legacy OpcManager read path for the gateway's SubscriptionEngine over the shared OpcConnectionPool. The OPC client certificate now always comes from MinIO + Mongo (CertProvider, Basic256Sha256) — no more certs/ shipped in the Gitea repository, so a new version branch no longer breaks the reads. - io/slots.py: parse_slot accepts slot v2 (names[]) and converts the legacy v1 (tag_name + topics; cert_path => use_security, path dropped), so gateway and Orchestrator can be deployed in any order. - managers/server_reader.py: new per-server reader with the OpcManager interface the IngestorManager already drives (connect, create_subscription, subscribe, unsubscribe, check_cycles, check_opc_listenning, shutdown). - IngestorManager owns the read path: lazy-MinIO CertProvider, the pool and a read DataSink, built without touching the network. Slots from Redis are normalized on lease and refresh; a malformed slot is notified and the last good config is kept. - GatewayRuntime borrows the pool (writes/gRPC) instead of building it, and no longer closes it on stop: a Postgres/NATS failure can't stop the reads. - pool: pin/unpin per active subscription so release_idle never evicts a subscribed connection; public evict() for lost servers / changed config. - SubscriptionEngine pins on subscribe, unpins on unsubscribe, and drops a subscription locally even when the server-side delete fails. - DataSink: model_id only written when the name has one, keeping v1 documents identical to the legacy ones. - config: CERT_BUCKET default is the legacy streamlit-connectors bucket (D7). - Removes OpcManager, its tests and scripts/generate-and-push-opc-cert.sh; README/CLAUDE.md document the MinIO certificate and the slot v2 format. Tests: 218 unit tests pass; ruff check/format clean. Local end-to-end against an asyncua server: v1 slot with two topics -> one doc per collection, legacy doc shape, idle sweep keeps the subscription, lost-server shutdown and re-initialization work. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
76 lines
3.2 KiB
Python
76 lines
3.2 KiB
Python
import re
|
|
|
|
from sqlalchemy import text
|
|
|
|
import ingestor.metrics as metrics
|
|
|
|
# fullmatch (no ^/$ anchors) so a trailing newline can't slip through — `$` would
|
|
# match before a trailing '\n', which is not what we want for a SQL identifier.
|
|
_IDENTIFIER_RE = re.compile(r'\w+', re.ASCII)
|
|
|
|
|
|
class DataSink:
|
|
"""Gateway egress wrapper: fans a reading out to every configured name/collection."""
|
|
|
|
def __init__(self, data_manager, postgres) -> None:
|
|
self.data_manager = data_manager
|
|
self.postgres = postgres
|
|
|
|
async def emit_metric(self, **kwargs) -> None:
|
|
# ponytail: delegate to data_manager, it's already a SientiaMonitoring
|
|
await self.data_manager.emit_metric(**kwargs)
|
|
|
|
async def publish_reading(self, tag: str, value, timestamp, names: list[dict]) -> None:
|
|
"""Publish one MongoDB document per name entry and record its tag value metric.
|
|
|
|
`model_id` is only written when the name carries one: names converted from a
|
|
slot v1 have none, and their documents must stay identical to the legacy ones."""
|
|
for name in names:
|
|
doc = {'tag': tag, 'name': name['tag_name']}
|
|
if 'model_id' in name:
|
|
doc['model_id'] = name['model_id']
|
|
doc['timestamp'] = timestamp
|
|
doc['value'] = value
|
|
await self.data_manager.publish(name['collection'], doc)
|
|
await self.emit_metric(
|
|
metric_object=metrics.OPC_TAG_VALUE,
|
|
method='set',
|
|
value=value,
|
|
tags={
|
|
'pod_id': self.data_manager.pod_id,
|
|
'tag_name': name['tag_name'],
|
|
'model_id': name.get('model_id', ''),
|
|
},
|
|
)
|
|
|
|
async def update_write_status(self, cmd: dict, result) -> None: # NOSONAR
|
|
"""UPDATE the per-model prediction row's write_status (C3), by timestamp+model_id.
|
|
|
|
schema/table_name are per-model SQL identifiers, not values — they can't be
|
|
bound params, so they're checked against an allowlist before interpolation.
|
|
status/timestamp/model_id ARE bound params, never f-stringed.
|
|
"""
|
|
schema = self._validate_identifier(cmd['schema'])
|
|
table = self._validate_identifier(cmd['table_name'])
|
|
# schema/table are allowlist-validated identifiers above; status/timestamp/model_id
|
|
# are bound params below, never interpolated — not a real injection vector.
|
|
stmt = text(
|
|
f'UPDATE {schema}.{table} SET write_status = :status ' # noqa: S608 # nosec B608
|
|
'WHERE timestamp = :timestamp AND model_id = :model_id'
|
|
)
|
|
params = {
|
|
'status': result.status,
|
|
'timestamp': cmd['timestamp'],
|
|
'model_id': cmd['model_id'],
|
|
}
|
|
# ponytail: sync engine call inside async def — same pattern sientia_do's own
|
|
# Postgres activities use (e.g. export_data_to_postgres); no async pg driver here.
|
|
with self.postgres.engine.begin() as connection:
|
|
connection.execute(stmt, params)
|
|
|
|
@staticmethod
|
|
def _validate_identifier(name: str) -> str:
|
|
if not isinstance(name, str) or not _IDENTIFIER_RE.fullmatch(name):
|
|
raise ValueError(f'invalid SQL identifier: {name!r}')
|
|
return name
|