SIENTIAPDE-1646
SIENTIAPDE-1646 Enhance OPC integration and E2E testing framework - Updated .gitignore to exclude '.cursor/*' for better file management. - Added new marker in pyproject.toml for E2E tests using OPC. - Introduced async OPC server fixtures and improved connection handling in conftest.py. - Enhanced error handling in OPC activities and added metrics for session management in metrics.py. - Updated E2E tests to cover new OPC scenarios, including session errors and reconnect handling. - Refactored helper functions to improve prediction assertion logic in helpers.py. - Documented new OPC scenarios in scenarios.md for clarity on expected outcomes.
This commit is contained in:
vitor-aignosi
@@ -1,12 +1,14 @@
|
||||
"""
|
||||
Synchronous OPC UA client repository using asyncua (opcua-asyncio) ``sync`` API.
|
||||
Synchronous OPC UA client repository using asyncua ``sync`` API.
|
||||
|
||||
``asyncua.sync.Client`` runs the asyncio client on a background thread so callers
|
||||
stay synchronous. Connect/disconnect, optional Basic256 security, session checks,
|
||||
and typed writes mirror the previous python-opcua integration.
|
||||
``asyncua.sync.Client`` runs the asyncio stack on a background thread so Temporal
|
||||
activities and other callers stay blocking while preserving the same session
|
||||
lifecycle, security policy, reconnect semantics, and write error classification
|
||||
as the async ``origin/main`` implementation.
|
||||
"""
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
@@ -16,6 +18,7 @@ from typing import Any
|
||||
from asyncua import ua
|
||||
from asyncua.crypto import security_policies
|
||||
from asyncua.sync import Client
|
||||
from asyncua.ua.uaerrors import UaStatusCodeError
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
@@ -24,6 +27,119 @@ from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
|
||||
from laborious import metrics
|
||||
|
||||
# Requested session and secure channel lifetime (ms) before server revision; 10 minutes.
|
||||
OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS = 10 * 60 * 1000
|
||||
|
||||
|
||||
class OpcClientAlreadyExistsError(RuntimeError):
|
||||
"""Raised when _create_client is called while self.client is already set."""
|
||||
|
||||
|
||||
class OpcSessionAlreadyConnectedError(RuntimeError):
|
||||
"""Raised when _open_session is called while a UA session is already open."""
|
||||
|
||||
|
||||
class OpcClientNotInitializedError(RuntimeError):
|
||||
"""Raised when _open_session is called before _create_client."""
|
||||
|
||||
|
||||
RECONNECTABLE_OPC_BAD_NAMES: frozenset[str] = frozenset(
|
||||
{
|
||||
'BadSessionIdInvalid',
|
||||
'BadSessionClosed',
|
||||
'BadSessionNotActivated',
|
||||
'BadSecureChannelIdInvalid',
|
||||
'BadSecureChannelClosed',
|
||||
'BadSecureChannelTokenUnknown',
|
||||
'BadTcpSecureChannelUnknown',
|
||||
'BadServerNotConnected',
|
||||
'BadConnectionClosed',
|
||||
'BadDisconnect',
|
||||
'BadConnectionRejected',
|
||||
'BadCommunicationError',
|
||||
'BadRequestInterrupted',
|
||||
'BadUnknownResponse',
|
||||
'BadTimeout',
|
||||
'BadRequestTimeout',
|
||||
'BadSequenceNumberInvalid',
|
||||
'BadSequenceNumberUnknown',
|
||||
'BadSecurityModeInsufficient',
|
||||
'BadRequestHeaderInvalid',
|
||||
'BadInvalidState',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _opc_authentication_token_str(client: Client | None) -> str:
|
||||
"""
|
||||
Serialize the current OPC UA authentication token (session handle) for logging and metrics.
|
||||
|
||||
Return:
|
||||
str: Token string, or "unknown" if unavailable.
|
||||
"""
|
||||
if client is None:
|
||||
return 'unknown'
|
||||
try:
|
||||
proto = client.aio_obj.uaclient.protocol
|
||||
if proto is None:
|
||||
return 'unknown'
|
||||
tok = getattr(proto, 'authentication_token', None)
|
||||
if tok is None:
|
||||
return 'unknown'
|
||||
return str(tok)
|
||||
except Exception:
|
||||
return 'unknown'
|
||||
|
||||
|
||||
def _opc_status_from_exception(exc: BaseException) -> str:
|
||||
"""
|
||||
Resolve OPC UA status name from an exception, including chained UaStatusCodeError causes.
|
||||
|
||||
Args:
|
||||
exc (BaseException): Raised error from asyncua.
|
||||
|
||||
Return:
|
||||
str: Status class name or generic Python exception name.
|
||||
"""
|
||||
current: BaseException | None = exc
|
||||
while current is not None:
|
||||
if isinstance(current, UaStatusCodeError):
|
||||
return type(current).__name__
|
||||
current = current.__cause__
|
||||
return type(exc).__name__
|
||||
|
||||
|
||||
def is_reconnectable_opcua_bad(exc: BaseException) -> bool:
|
||||
"""
|
||||
Return whether the exception is a Tier-1 OPC UA Bad* that should trigger reconnect.
|
||||
|
||||
Args:
|
||||
exc (BaseException): Raised error from get_node or write_value.
|
||||
|
||||
Return:
|
||||
bool: True if reconnect should be scheduled.
|
||||
"""
|
||||
return _opc_status_from_exception(exc) in RECONNECTABLE_OPC_BAD_NAMES
|
||||
|
||||
|
||||
def _model_labels_from_write_metadata(metadata: dict[str, Any] | None) -> dict[str, str]:
|
||||
"""
|
||||
Extract model_id and model_name from write metadata for Prometheus labels.
|
||||
|
||||
Args:
|
||||
metadata (dict[str, Any] | None): Context passed into write_data; may omit keys.
|
||||
|
||||
Return:
|
||||
dict[str, str]: Labels model_id and model_name, defaulting to "unknown".
|
||||
"""
|
||||
if not metadata:
|
||||
return {'model_id': 'unknown', 'model_name': 'unknown'}
|
||||
return {
|
||||
'model_id': str(metadata.get('model_id', 'unknown')),
|
||||
'model_name': str(metadata.get('model_name', 'unknown')),
|
||||
}
|
||||
|
||||
|
||||
data_type_map = {
|
||||
'float': {
|
||||
'converter': float,
|
||||
@@ -52,11 +168,8 @@ class OpcRepository(SientiaMonitoring):
|
||||
"""
|
||||
Synchronous OPC UA repository for connect/disconnect and typed writes.
|
||||
|
||||
Attributes:
|
||||
url: OPC UA endpoint URL.
|
||||
id: Server identifier used in metrics and notifications.
|
||||
server_name: Human-readable server name for labels.
|
||||
client: Active ``asyncua.sync.Client`` while connected.
|
||||
Uses ``asyncua.sync.Client`` with the same session metrics, Tier-1 Bad* reconnect,
|
||||
and structured write error payloads as the async repository on ``origin/main``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -80,10 +193,10 @@ class OpcRepository(SientiaMonitoring):
|
||||
self.cert_path = cert_path
|
||||
self.private_key_path = private_key_path
|
||||
self.server_cert_path = server_cert_path
|
||||
self.error_count = 0
|
||||
self.reconnection_interval = reconnection_interval
|
||||
self.last_reconnection_time: datetime | None = None
|
||||
self.disconnection_interval = 10.0
|
||||
self.notification_handler = notification_handler
|
||||
self.client: Client | None = None
|
||||
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
@@ -94,15 +207,63 @@ class OpcRepository(SientiaMonitoring):
|
||||
'workflow_name': 'opc_repository',
|
||||
'schedule_name': '-',
|
||||
}
|
||||
self._last_write_mono: float | None = None
|
||||
self._connection_lock = threading.Lock()
|
||||
self._session_ready = threading.Event()
|
||||
self._reconnect_thread: threading.Thread | None = None
|
||||
|
||||
def _opc_debug_tags(self, session_id: str) -> dict[str, str]:
|
||||
return {
|
||||
'pod_id': str(getattr(self, 'pod_id', 'unknown')),
|
||||
'server_name': self.server_name,
|
||||
'runtime': str(getattr(self, 'runtime', 'unknown')),
|
||||
'opc_server_id': self.id,
|
||||
'session_id': session_id,
|
||||
}
|
||||
|
||||
def _is_session_open(self) -> bool:
|
||||
"""
|
||||
Return whether the asyncua client has an open transport session.
|
||||
|
||||
Return:
|
||||
bool: True when protocol exists and is not closed.
|
||||
"""
|
||||
if self.client is None:
|
||||
return False
|
||||
try:
|
||||
proto = self.client.aio_obj.uaclient.protocol
|
||||
return proto is not None and proto.state != 'closed'
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _reconnection_window_elapsed(self) -> bool:
|
||||
"""
|
||||
Return whether enough time has passed since the last reconnect attempt.
|
||||
|
||||
Return:
|
||||
bool: True if a new reconnect is allowed.
|
||||
"""
|
||||
if self.last_reconnection_time is None:
|
||||
return True
|
||||
return (
|
||||
datetime.now() - self.last_reconnection_time
|
||||
).total_seconds() > self.reconnection_interval
|
||||
|
||||
def _not_connected_error(self) -> dict[str, Any]:
|
||||
return {
|
||||
'notification_id': f'OPC_CONNECTION_NOT_READY_{self.id}',
|
||||
'message': f'OPC server {self.id} is not connected',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.WARNING,
|
||||
}
|
||||
|
||||
def set_security(self) -> None:
|
||||
"""
|
||||
Configure Basic256 security policy, certificates, and long channel/session timeouts.
|
||||
Configure certificates and timeouts on the sync asyncua client.
|
||||
|
||||
Raises:
|
||||
ValueError: If certificate paths are missing or client is not initialized.
|
||||
ValueError: If cert paths or client are missing.
|
||||
"""
|
||||
|
||||
if self.cert_path is None or self.private_key_path is None:
|
||||
raise ValueError(
|
||||
'Certificate and private key paths must be provided for secure connection.'
|
||||
@@ -115,7 +276,7 @@ class OpcRepository(SientiaMonitoring):
|
||||
if self.client is None:
|
||||
raise ValueError('Client must be initialized before setting security')
|
||||
|
||||
self.client.application_uri = self.server_uri or self.client.application_uri
|
||||
self.client.application_uri = self.server_uri
|
||||
self.info('Setting security...', self.metadata)
|
||||
self.client.set_security(
|
||||
security_policies.SecurityPolicyBasic256,
|
||||
@@ -124,75 +285,105 @@ class OpcRepository(SientiaMonitoring):
|
||||
None,
|
||||
str(server_cert) if server_cert else None,
|
||||
)
|
||||
self.client.aio_obj.secure_channel_timeout = 10000000
|
||||
self.client.aio_obj.session_timeout = 10000000
|
||||
aio = self.client.aio_obj
|
||||
aio.secure_channel_timeout = OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS
|
||||
aio.session_timeout = OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS
|
||||
|
||||
def connect(self) -> tuple[bool, dict[str, Any]]:
|
||||
def _create_client(self) -> None:
|
||||
"""
|
||||
Create the synchronous client, optionally apply security, and connect to the server.
|
||||
Instantiate the sync Client and apply security when configured.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: Success flag and error payload when False.
|
||||
Caller must hold _connection_lock. Does not open a UA session.
|
||||
|
||||
Raises:
|
||||
OpcClientAlreadyExistsError: If self.client is already set.
|
||||
"""
|
||||
if self.client is not None:
|
||||
raise OpcClientAlreadyExistsError(
|
||||
f'OPC client already exists for server {self.id}; '
|
||||
'call disconnect() before creating a new client'
|
||||
)
|
||||
|
||||
self.client = Client(self.url, timeout=10)
|
||||
|
||||
self.client.aio_obj.name = self.pod_id
|
||||
self.client.aio_obj.description = self.pod_id
|
||||
aio = self.client.aio_obj
|
||||
if hasattr(aio, 'watchdog_intervall'):
|
||||
aio.watchdog_intervall = 50
|
||||
aio.name = self.pod_id
|
||||
aio.description = self.pod_id
|
||||
pod_uri = self.pod_id.replace('-', ':')
|
||||
self.client.application_uri = pod_uri
|
||||
self.client.aio_obj.product_uri = pod_uri
|
||||
|
||||
aio.product_uri = pod_uri
|
||||
if self.cert_path:
|
||||
self.set_security()
|
||||
self.info(
|
||||
f'Starting connection to OPC server {self.id}:{self.server_name}...', self.metadata
|
||||
)
|
||||
return self.try_connect()
|
||||
|
||||
def try_connect(self) -> tuple[bool, dict[str, Any]]:
|
||||
def _open_session(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Perform the TCP/session handshake and emit connection metrics.
|
||||
Open the OPC UA session on the existing client.
|
||||
|
||||
Caller must hold _connection_lock.
|
||||
|
||||
Raises:
|
||||
OpcClientNotInitializedError: If self.client is None.
|
||||
OpcSessionAlreadyConnectedError: If a session is already open.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: Success flag and structured error when False.
|
||||
tuple[bool, dict[str, Any]]: Success flag and error payload on connect failure.
|
||||
"""
|
||||
if self.client is None:
|
||||
raise OpcClientNotInitializedError(
|
||||
f'OPC client is not initialized for server {self.id}; '
|
||||
'call _create_client() before opening a session'
|
||||
)
|
||||
if self._is_session_open():
|
||||
raise OpcSessionAlreadyConnectedError(
|
||||
f'OPC session already connected for server {self.id}; '
|
||||
'call disconnect() before connecting again'
|
||||
)
|
||||
|
||||
tags = {
|
||||
'pod_id': self.pod_id,
|
||||
'server_name': self.server_name,
|
||||
}
|
||||
self.emit_metric_sync(metrics.OPC_CONNECTIONS_TOTAL, tags)
|
||||
|
||||
try:
|
||||
self.last_reconnection_time = datetime.now()
|
||||
if self.client is None:
|
||||
return False, {
|
||||
'notification_id': f'OPC_CONNECTION_ERROR_{self.id}',
|
||||
'message': 'Client is not initialized',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.ERROR,
|
||||
}
|
||||
self.client.connect()
|
||||
|
||||
aio = self.client.aio_obj
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
revised_session_timeout_ms = int(aio.session_timeout)
|
||||
revised_secure_channel_timeout_ms = int(aio.secure_channel_timeout)
|
||||
self.info(
|
||||
f'OPC new session connected opc_server_id={self.id} session_id={session_id} '
|
||||
f'revised_session_timeout_ms={revised_session_timeout_ms} '
|
||||
f'revised_secure_channel_timeout_ms={revised_secure_channel_timeout_ms}',
|
||||
self.metadata,
|
||||
)
|
||||
self.emit_metric_sync(
|
||||
metrics.OPC_SESSION_CREATED_TOTAL, self._opc_debug_tags(session_id)
|
||||
)
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.OPC_SESSION_REVISED_TIMEOUT_MS,
|
||||
method='set',
|
||||
tags=self._opc_debug_tags(session_id),
|
||||
value=revised_session_timeout_ms,
|
||||
)
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.OPC_CONNECTION_STATUS,
|
||||
method='set',
|
||||
tags={
|
||||
**tags,
|
||||
'server_url': self.url,
|
||||
},
|
||||
tags={**tags, 'server_url': self.url},
|
||||
value=1,
|
||||
)
|
||||
|
||||
self._last_write_mono = None
|
||||
self._session_ready.set()
|
||||
return True, {}
|
||||
except Exception as e:
|
||||
self.disconnect()
|
||||
|
||||
except Exception as e:
|
||||
self._disconnect_locked()
|
||||
trace = traceback.format_exc()
|
||||
self.error(trace, self.metadata)
|
||||
|
||||
self.emit_metric_sync(metrics.OPC_CONNECTIONS_FAILED, tags)
|
||||
|
||||
return False, {
|
||||
'notification_id': f'OPC_CONNECTION_ERROR_{self.id}',
|
||||
'message': f'Failed to connect to OPC server: {e}',
|
||||
@@ -201,20 +392,38 @@ class OpcRepository(SientiaMonitoring):
|
||||
'attachment_content': trace,
|
||||
}
|
||||
|
||||
def disconnection_fallback(self) -> list[dict[str, Any]]:
|
||||
def _connect_locked(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Retry disconnect up to five times with linear backoff.
|
||||
Create the client when absent, then open a UA session.
|
||||
|
||||
Caller must hold _connection_lock.
|
||||
|
||||
Raises:
|
||||
OpcSessionAlreadyConnectedError: If a session is already open.
|
||||
|
||||
Return:
|
||||
list[dict[str, Any]]: Empty on success, otherwise error records per attempt.
|
||||
tuple[bool, dict[str, Any]]: Result from _open_session on connect failure.
|
||||
"""
|
||||
if self._is_session_open():
|
||||
raise OpcSessionAlreadyConnectedError(
|
||||
f'OPC session already connected for server {self.id}; '
|
||||
'call disconnect() before connecting again'
|
||||
)
|
||||
if self.client is None:
|
||||
self._create_client()
|
||||
return self._open_session()
|
||||
|
||||
def _disconnection_fallback(self) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Try up to five times to disconnect from the OPC UA server.
|
||||
"""
|
||||
assert self.client is not None
|
||||
error_stack = []
|
||||
error_stack: list[dict[str, Any]] = []
|
||||
for i in range(5):
|
||||
try:
|
||||
self.info(
|
||||
f'Disconnecting from OPC UA server, attempt {i + 1} of 5', self.metadata
|
||||
f'Disconnecting from OPC UA server, attempt {i + 1} of 5',
|
||||
self.metadata,
|
||||
)
|
||||
self.client.disconnect()
|
||||
return []
|
||||
@@ -233,15 +442,26 @@ class OpcRepository(SientiaMonitoring):
|
||||
time.sleep(self.disconnection_interval * i)
|
||||
return error_stack
|
||||
|
||||
def disconnect(self) -> None:
|
||||
def _disconnect_locked(self) -> None:
|
||||
"""
|
||||
Tear down the UA session and reset connection metrics.
|
||||
Tear down the current session and client.
|
||||
|
||||
Caller must hold _connection_lock.
|
||||
"""
|
||||
self._last_write_mono = None
|
||||
self._session_ready.clear()
|
||||
|
||||
if self.client is None:
|
||||
return
|
||||
|
||||
errors = self.disconnection_fallback()
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
self.info(
|
||||
f'OPC disconnecting opc_server_id={self.id} session_id={session_id}',
|
||||
self.metadata,
|
||||
)
|
||||
self.emit_metric_sync(metrics.OPC_SESSION_CLOSED_TOTAL, self._opc_debug_tags(session_id))
|
||||
|
||||
errors = self._disconnection_fallback()
|
||||
if errors:
|
||||
self.send_notification(
|
||||
metadata=self.metadata,
|
||||
@@ -252,9 +472,8 @@ class OpcRepository(SientiaMonitoring):
|
||||
attachment_content=json.dumps(errors, indent=4),
|
||||
)
|
||||
else:
|
||||
self.warning(
|
||||
f'Disconnected from OPC server {self.id} successfully', self.metadata
|
||||
)
|
||||
self.warning(f'Disconnected from OPC server {self.id} successfully', self.metadata)
|
||||
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.OPC_CONNECTION_STATUS,
|
||||
method='set',
|
||||
@@ -265,139 +484,313 @@ class OpcRepository(SientiaMonitoring):
|
||||
},
|
||||
value=0,
|
||||
)
|
||||
|
||||
self.client = None
|
||||
|
||||
def _session_alive(self) -> bool:
|
||||
def _reconnect_locked(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Best-effort check that the synchronous client still has a working session.
|
||||
Close the current session and open a new one.
|
||||
|
||||
Caller must hold _connection_lock. Records last_reconnection_time for interval gating.
|
||||
|
||||
Return:
|
||||
bool: True if a root browse succeeds, False otherwise.
|
||||
tuple[bool, dict[str, Any]]: Result from _connect_locked after teardown.
|
||||
"""
|
||||
self.last_reconnection_time = datetime.now()
|
||||
self._disconnect_locked()
|
||||
return self._connect_locked()
|
||||
|
||||
if self.client is None:
|
||||
return False
|
||||
try:
|
||||
self.client.get_root_node()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
def connect(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Open an OPC UA session under the connection lock (worker initialization).
|
||||
"""
|
||||
with self._connection_lock:
|
||||
self.info(
|
||||
f'Starting connection to OPC server {self.id}:{self.server_name}...',
|
||||
self.metadata,
|
||||
)
|
||||
return self._connect_locked()
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""
|
||||
Gracefully disconnect from the OPC server under the connection lock.
|
||||
"""
|
||||
with self._connection_lock:
|
||||
self._disconnect_locked()
|
||||
|
||||
def validate_connection(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Ensure the UA session is usable; reconnect when outside the backoff window.
|
||||
Read-only check that the asyncua protocol is open.
|
||||
|
||||
Caller must ensure _session_ready before writing. Does not connect or reconnect.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: Whether the session is ready and optional error payload.
|
||||
tuple[bool, dict[str, Any]]: (True, {}) when open, otherwise (False, error).
|
||||
"""
|
||||
|
||||
if self.client is None:
|
||||
return self.connect()
|
||||
|
||||
try:
|
||||
if not self._session_alive():
|
||||
self.error(f'OPC server {self.id} is not connected', self.metadata)
|
||||
if (
|
||||
self.last_reconnection_time is None
|
||||
or (datetime.now() - self.last_reconnection_time).total_seconds()
|
||||
> self.reconnection_interval
|
||||
):
|
||||
self.disconnect()
|
||||
self.info(
|
||||
f'Trying to reconnect to OPC server {self.id}...', self.metadata
|
||||
)
|
||||
return self.connect()
|
||||
|
||||
return False, {
|
||||
'notification_id': f'OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}',
|
||||
'message': (
|
||||
f'OPC server {self.id} is not connected, waiting for next reconnection window...'
|
||||
),
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.WARNING,
|
||||
}
|
||||
if self._is_session_open():
|
||||
return True, {}
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
message = f'Failed to validate connection to OPC server: {e}'
|
||||
self.error(message, self.metadata)
|
||||
return False, {
|
||||
'notification_id': f'OPC_CONNECTION_CHECK_ERROR_{self.id}',
|
||||
'message': message,
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.ERROR,
|
||||
'attachment_content': trace,
|
||||
}
|
||||
self.error(f'OPC server {self.id} is not connected', self.metadata)
|
||||
return False, self._not_connected_error()
|
||||
|
||||
def _start_reconnect_on_bad(self, opc_status: str, session_id: str) -> None:
|
||||
"""
|
||||
Schedule a background reconnect if interval and thread state allow it.
|
||||
|
||||
Args:
|
||||
opc_status (str): OPC UA status name that triggered reconnect.
|
||||
session_id (str): Session token before failure.
|
||||
"""
|
||||
if not self._reconnection_window_elapsed():
|
||||
self.warning(
|
||||
f'OPC reconnect skipped reason=reconnection_window opc_server_id={self.id} '
|
||||
f'opc_status={opc_status}',
|
||||
self.metadata,
|
||||
)
|
||||
return
|
||||
if self._reconnect_thread is not None and self._reconnect_thread.is_alive():
|
||||
self.warning(
|
||||
f'OPC reconnect skipped reason=in_progress opc_server_id={self.id} '
|
||||
f'opc_status={opc_status}',
|
||||
self.metadata,
|
||||
)
|
||||
return
|
||||
|
||||
self._session_ready.clear()
|
||||
self.info(
|
||||
f'OPC reconnect scheduled after opc_status={opc_status} opc_server_id={self.id} '
|
||||
f'old_session_id={session_id}',
|
||||
self.metadata,
|
||||
)
|
||||
self._reconnect_thread = threading.Thread(
|
||||
target=self._run_reconnect_on_bad,
|
||||
args=(opc_status, session_id),
|
||||
daemon=True,
|
||||
)
|
||||
self._reconnect_thread.start()
|
||||
|
||||
def _run_reconnect_on_bad(self, opc_status: str, session_id: str) -> None:
|
||||
"""
|
||||
Tear down and re-establish the OPC UA session under the connection lock.
|
||||
|
||||
Args:
|
||||
opc_status (str): OPC UA status that triggered reconnect.
|
||||
session_id (str): Previous session token string.
|
||||
"""
|
||||
try:
|
||||
with self._connection_lock:
|
||||
self.info(
|
||||
f'OPC reconnect started opc_status={opc_status} opc_server_id={self.id} '
|
||||
f'old_session_id={session_id}',
|
||||
self.metadata,
|
||||
)
|
||||
self._reconnect_locked()
|
||||
except Exception:
|
||||
self.error(
|
||||
f'OPC reconnect task failed opc_server_id={self.id} opc_status={opc_status}',
|
||||
self.metadata,
|
||||
)
|
||||
self.error(traceback.format_exc(), self.metadata)
|
||||
|
||||
def _log_write_inter_arrival(self, session_id: str, node: str) -> None:
|
||||
"""
|
||||
Log elapsed wall time since the previous successful OPC write on this repository.
|
||||
|
||||
Args:
|
||||
session_id (str): Current OPC UA session token string.
|
||||
node (str): Node id written in this operation.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
if self._last_write_mono is not None:
|
||||
delta_s = now - self._last_write_mono
|
||||
self.info(
|
||||
f'OPC write inter-arrival_s={delta_s:.6f} opc_server_id={self.id} '
|
||||
f'session_id={session_id} node={node}',
|
||||
self.metadata,
|
||||
)
|
||||
if self.client is not None:
|
||||
session_timeout_ms = float(self.client.aio_obj.session_timeout)
|
||||
if session_timeout_ms > 0 and delta_s > (session_timeout_ms / 1000.0):
|
||||
self.emit_metric_sync(
|
||||
metrics.OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL,
|
||||
self._opc_debug_tags(session_id),
|
||||
)
|
||||
self._last_write_mono = now
|
||||
|
||||
def _emit_opc_write_metric(
|
||||
self, session_id: str, result: str, metadata: dict[str, Any] | None
|
||||
) -> None:
|
||||
self.emit_metric_sync(
|
||||
metrics.OPC_WRITE_ATTEMPTS_TOTAL,
|
||||
{
|
||||
**self._opc_debug_tags(session_id),
|
||||
**_model_labels_from_write_metadata(metadata),
|
||||
'result': result,
|
||||
},
|
||||
)
|
||||
|
||||
def _write_failure_payload(
|
||||
self,
|
||||
notification_id: str,
|
||||
message: str,
|
||||
level: NotificationLevel = NotificationLevel.ERROR,
|
||||
attachment_content: str | None = None,
|
||||
opc_error_kind: str | None = None,
|
||||
opc_status: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
'notification_id': notification_id,
|
||||
'message': message,
|
||||
'block': 'opc_repository',
|
||||
'level': level,
|
||||
}
|
||||
if attachment_content is not None:
|
||||
payload['attachment_content'] = attachment_content
|
||||
if opc_error_kind is not None:
|
||||
payload['opc_error_kind'] = opc_error_kind
|
||||
if opc_status is not None:
|
||||
payload['opc_status'] = opc_status
|
||||
return payload
|
||||
|
||||
def _handle_tier1_bad(
|
||||
self,
|
||||
exc: BaseException,
|
||||
session_id: str,
|
||||
node: str,
|
||||
metadata: dict[str, Any],
|
||||
phase: str,
|
||||
) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Record metrics/logs and schedule reconnect after a Tier-1 Bad* error.
|
||||
|
||||
Args:
|
||||
exc (BaseException): Tier-1 OPC UA error.
|
||||
session_id (str): Session token at failure time.
|
||||
node (str): Node id being written.
|
||||
metadata (dict[str, Any]): Write context.
|
||||
phase (str): get_node or write_value.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: Always (False, error payload).
|
||||
"""
|
||||
opc_status = _opc_status_from_exception(exc)
|
||||
trace = traceback.format_exc()
|
||||
self.error(trace, metadata)
|
||||
self._emit_opc_write_metric(session_id, opc_status, metadata)
|
||||
self.error(
|
||||
f'OPC write failed opc_status={opc_status} opc_server_id={self.id} '
|
||||
f'session_id={session_id} model_id={metadata.get("model_id", "unknown")} '
|
||||
f'model_name={metadata.get("model_name", "unknown")} node={node} phase={phase}',
|
||||
metadata,
|
||||
)
|
||||
self._start_reconnect_on_bad(opc_status, session_id)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}',
|
||||
message=f'Failed to {phase} on OPC server: {exc} | metadata: {metadata}',
|
||||
attachment_content=trace,
|
||||
opc_error_kind='session_bad',
|
||||
opc_status=opc_status,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _write_node_value(
|
||||
node_obj: Any,
|
||||
ua_data: ua.DataValue,
|
||||
data: Any,
|
||||
variant_type: ua.VariantType,
|
||||
) -> None:
|
||||
"""
|
||||
Write a DataValue to a node, falling back to set_value when write_value is unavailable.
|
||||
|
||||
Args:
|
||||
node_obj: Sync or async node wrapper from asyncua.
|
||||
ua_data (ua.DataValue): Encoded value for write_value.
|
||||
data: Scalar converted value for set_value fallback.
|
||||
variant_type (ua.VariantType): OPC UA type for set_value fallback.
|
||||
"""
|
||||
if hasattr(node_obj, 'write_value'):
|
||||
try:
|
||||
node_obj.write_value(ua_data)
|
||||
return
|
||||
except (AttributeError, TypeError):
|
||||
pass
|
||||
node_obj.set_value(data, variant_type)
|
||||
|
||||
def write_data(
|
||||
self, node: str, value: Any, data_type: str, metadata: dict[str, Any]
|
||||
) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Write a typed value to an OPC UA node after validating connectivity.
|
||||
|
||||
Args:
|
||||
node: Node id string accepted by ``Client.get_node``.
|
||||
value: Scalar value to encode.
|
||||
data_type: Key into ``data_type_map`` (e.g. float, str).
|
||||
metadata: Workflow metadata for error context.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: Success flag and either ``response_time`` or error fields.
|
||||
Write data to OPC server with a single attempt and Tier-1 Bad* reconnect scheduling.
|
||||
"""
|
||||
if not self._session_ready.is_set():
|
||||
self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata)
|
||||
self.warning(
|
||||
f'OPC write rejected reconnect_in_progress opc_server_id={self.id} '
|
||||
f'model_id={metadata.get("model_id", "unknown")} '
|
||||
f'model_name={metadata.get("model_name", "unknown")}',
|
||||
metadata,
|
||||
)
|
||||
return False, {
|
||||
'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}',
|
||||
'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.WARNING,
|
||||
'opc_error_kind': 'reconnect_in_progress',
|
||||
}
|
||||
|
||||
is_connected, error = self.validate_connection()
|
||||
|
||||
if not is_connected:
|
||||
self._emit_opc_write_metric('unknown', 'NotConnected', metadata)
|
||||
return False, error
|
||||
|
||||
start_time = time.time()
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
|
||||
try:
|
||||
assert self.client is not None
|
||||
node_obj = self.client.get_node(node)
|
||||
except Exception as e:
|
||||
if is_reconnectable_opcua_bad(e):
|
||||
return self._handle_tier1_bad(e, session_id, node, metadata, 'get_node')
|
||||
trace = traceback.format_exc()
|
||||
self.error(trace, metadata)
|
||||
self.error_count += 1
|
||||
return False, {
|
||||
'notification_id': f'OPC_WRITE_GET_NODE_ERROR_{self.id}',
|
||||
'message': f'Failed to get node from OPC server: {e} | metadata: {metadata}',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.ERROR,
|
||||
'attachment_content': trace,
|
||||
}
|
||||
self._emit_opc_write_metric(session_id, f'GetNodeError:{type(e).__name__}', metadata)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_GET_NODE_ERROR_{self.id}',
|
||||
message=f'Failed to get node from OPC server: {e} | metadata: {metadata}',
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
if data_type not in data_type_map:
|
||||
return False, {
|
||||
'notification_id': f'OPC_WRITE_DATA_TYPE_ERROR_{self.id}',
|
||||
'message': f'Unsupported data type: {data_type} | metadata: {metadata}',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.ERROR,
|
||||
}
|
||||
self._emit_opc_write_metric(session_id, 'UnsupportedDataType', metadata)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_DATA_TYPE_ERROR_{self.id}',
|
||||
message=f'Unsupported data type: {data_type} | metadata: {metadata}',
|
||||
)
|
||||
|
||||
data = data_type_map[data_type]['converter'](value)
|
||||
self.info(f'Writing {data} - {type(data)} to {node}', metadata)
|
||||
variant_type = data_type_map[data_type]['opc_type']
|
||||
ua_data = ua.DataValue(
|
||||
ua.Variant(data, variant_type),
|
||||
)
|
||||
|
||||
try:
|
||||
node_obj.set_value(data, variant_type)
|
||||
|
||||
self._write_node_value(node_obj, ua_data, data, variant_type)
|
||||
end_time = time.time()
|
||||
response_time = end_time - start_time
|
||||
|
||||
except Exception as e:
|
||||
if is_reconnectable_opcua_bad(e):
|
||||
return self._handle_tier1_bad(e, session_id, node, metadata, 'write_value')
|
||||
trace = traceback.format_exc()
|
||||
self.error(trace, metadata)
|
||||
self.error_count += 1
|
||||
return False, {
|
||||
'notification_id': f'OPC_WRITE_DATA_ERROR_{self.id}',
|
||||
'message': f'Failed to write data to OPC server: {e} | metadata: {metadata}',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.ERROR,
|
||||
'attachment_content': trace,
|
||||
}
|
||||
self.error_count = 0
|
||||
self._emit_opc_write_metric(session_id, type(e).__name__, metadata)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}',
|
||||
message=f'Failed to write data to OPC server: {e} | metadata: {metadata}',
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
self._emit_opc_write_metric(session_id, 'OK', metadata)
|
||||
self._log_write_inter_arrival(session_id, node)
|
||||
|
||||
return True, {
|
||||
'response_time': response_time,
|
||||
|
||||
Reference in New Issue
Block a user