SIENTIAPDE-1811
Update .gitignore, requirements, and enhance OPC UA error handling - Added new entries to .gitignore for openspec and cursor directories. - Updated sientia-dataops-library dependency version in requirements-light.txt from 1.10.4 to 1.12.0. - Enhanced OPC UA communication by refining reconnect logic and error handling in opc_repository.py, including the introduction of a reconnect flag and improved session management. - Updated tests to cover new reconnect scenarios and ensure robust error handling for protocol states.
This commit is contained in:
vitor-aignosi
@@ -195,8 +195,18 @@ class OpcRepository(SientiaMonitoring):
|
||||
self._connection_lock = asyncio.Lock()
|
||||
self._session_ready = asyncio.Event()
|
||||
self._reconnect_task: asyncio.Task[None] | None = None
|
||||
self._allow_reconnect = True
|
||||
|
||||
def _opc_debug_tags(self, session_id: str) -> dict[str, str]:
|
||||
"""
|
||||
Build Prometheus/log label tags for OPC session-scoped metrics.
|
||||
|
||||
Args:
|
||||
session_id (str): OPC UA session token string.
|
||||
|
||||
Return:
|
||||
dict[str, str]: Labels pod_id, server_name, runtime, opc_server_id, session_id.
|
||||
"""
|
||||
return {
|
||||
'pod_id': str(getattr(self, 'pod_id', 'unknown')),
|
||||
'server_name': self.server_name,
|
||||
@@ -234,6 +244,12 @@ class OpcRepository(SientiaMonitoring):
|
||||
).total_seconds() > self.reconnection_interval
|
||||
|
||||
def _not_connected_error(self) -> dict[str, Any]:
|
||||
"""
|
||||
Build the standard error payload when validate_connection finds no open protocol.
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Notification fields for OPC_CONNECTION_NOT_READY.
|
||||
"""
|
||||
return {
|
||||
'notification_id': f'OPC_CONNECTION_NOT_READY_{self.id}',
|
||||
'message': f'OPC server {self.id} is not connected',
|
||||
@@ -491,8 +507,12 @@ class OpcRepository(SientiaMonitoring):
|
||||
async def disconnect(self) -> None:
|
||||
"""
|
||||
Gracefully disconnect from the OPC server under the connection lock.
|
||||
|
||||
Disables background reconnect so late writes during worker shutdown do not
|
||||
respawn sessions.
|
||||
"""
|
||||
async with self._connection_lock:
|
||||
self._allow_reconnect = False
|
||||
await self._disconnect_locked()
|
||||
|
||||
async def validate_connection(self) -> tuple[bool, dict[str, Any]]:
|
||||
@@ -509,58 +529,77 @@ class OpcRepository(SientiaMonitoring):
|
||||
self.error(f'OPC server {self.id} is not connected', self.metadata)
|
||||
return False, self._not_connected_error()
|
||||
|
||||
async def _start_reconnect_on_bad(self, opc_status: str, session_id: str) -> None:
|
||||
def _reconnect_task_in_progress(self) -> bool:
|
||||
"""
|
||||
Schedule a background reconnect if interval and task state allow it.
|
||||
Return whether a background reconnect task is currently running.
|
||||
|
||||
Return:
|
||||
bool: True when a reconnect task exists and has not finished.
|
||||
"""
|
||||
return self._reconnect_task is not None and not self._reconnect_task.done()
|
||||
|
||||
async def _start_reconnect(self, reason: str, session_id: str) -> None:
|
||||
"""
|
||||
Schedule a background reconnect when allowed by interval and task state.
|
||||
|
||||
Clears _session_ready before starting the task. No-op when _allow_reconnect is
|
||||
False, the reconnection window has not elapsed, or a reconnect is already running.
|
||||
|
||||
Args:
|
||||
opc_status (str): OPC UA status name that triggered reconnect.
|
||||
reason (str): Trigger for reconnect (OPC status name or synthetic reason).
|
||||
session_id (str): Session token before failure.
|
||||
"""
|
||||
if not self._allow_reconnect:
|
||||
return
|
||||
if not self._reconnection_window_elapsed():
|
||||
self.warning(
|
||||
f'OPC reconnect skipped reason=reconnection_window opc_server_id={self.id} '
|
||||
f'opc_status={opc_status}',
|
||||
f'reconnect_reason={reason}',
|
||||
self.metadata,
|
||||
)
|
||||
return
|
||||
if self._reconnect_task is not None and not self._reconnect_task.done():
|
||||
if self._reconnect_task_in_progress():
|
||||
self.warning(
|
||||
f'OPC reconnect skipped reason=in_progress opc_server_id={self.id} '
|
||||
f'opc_status={opc_status}',
|
||||
f'reconnect_reason={reason}',
|
||||
self.metadata,
|
||||
)
|
||||
return
|
||||
|
||||
self._session_ready.clear()
|
||||
self.info(
|
||||
f'OPC reconnect scheduled after opc_status={opc_status} opc_server_id={self.id} '
|
||||
f'OPC reconnect scheduled reconnect_reason={reason} opc_server_id={self.id} '
|
||||
f'old_session_id={session_id}',
|
||||
self.metadata,
|
||||
)
|
||||
self._reconnect_task = asyncio.create_task(
|
||||
self._run_reconnect_on_bad(opc_status, session_id)
|
||||
)
|
||||
self._reconnect_task = asyncio.create_task(self._run_reconnect(reason, session_id))
|
||||
|
||||
async def _run_reconnect_on_bad(self, opc_status: str, session_id: str) -> None:
|
||||
async def _run_reconnect(self, reason: str, session_id: str) -> None:
|
||||
"""
|
||||
Tear down and re-establish the OPC UA session under the connection lock.
|
||||
Background task that tears down and re-establishes the OPC UA session.
|
||||
|
||||
Args:
|
||||
opc_status (str): OPC UA status that triggered reconnect.
|
||||
session_id (str): Previous session token string.
|
||||
reason (str): Trigger for reconnect (OPC status or ProtocolClosed).
|
||||
session_id (str): Previous session token string for logging.
|
||||
"""
|
||||
try:
|
||||
async with self._connection_lock:
|
||||
self.info(
|
||||
f'OPC reconnect started opc_status={opc_status} opc_server_id={self.id} '
|
||||
f'OPC reconnect started reconnect_reason={reason} opc_server_id={self.id} '
|
||||
f'old_session_id={session_id}',
|
||||
self.metadata,
|
||||
)
|
||||
await self._reconnect_locked()
|
||||
success, error = await self._reconnect_locked()
|
||||
if not success:
|
||||
self.error(
|
||||
f'OPC reconnect failed reconnect_reason={reason} opc_server_id={self.id}',
|
||||
self.metadata,
|
||||
)
|
||||
if error:
|
||||
self.error(error.get('message', ''), self.metadata)
|
||||
except Exception:
|
||||
self.error(
|
||||
f'OPC reconnect task failed opc_server_id={self.id} opc_status={opc_status}',
|
||||
f'OPC reconnect task failed opc_server_id={self.id} reconnect_reason={reason}',
|
||||
self.metadata,
|
||||
)
|
||||
self.error(traceback.format_exc(), self.metadata)
|
||||
@@ -593,6 +632,14 @@ class OpcRepository(SientiaMonitoring):
|
||||
async def _emit_opc_write_metric(
|
||||
self, session_id: str, result: str, metadata: dict[str, Any] | None
|
||||
) -> None:
|
||||
"""
|
||||
Emit opc_write_attempts_total for a single write attempt outcome.
|
||||
|
||||
Args:
|
||||
session_id (str): OPC UA session token string, or "unknown".
|
||||
result (str): Outcome label (OK, OPC status name, ProtocolClosed, etc.).
|
||||
metadata (dict[str, Any] | None): Write context for model_id/model_name labels.
|
||||
"""
|
||||
await self.emit_metric(
|
||||
metrics.OPC_WRITE_ATTEMPTS_TOTAL,
|
||||
{
|
||||
@@ -611,6 +658,20 @@ class OpcRepository(SientiaMonitoring):
|
||||
opc_error_kind: str | None = None,
|
||||
opc_status: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Build a structured error dict returned from failed write_data paths.
|
||||
|
||||
Args:
|
||||
notification_id (str): Stable notification identifier.
|
||||
message (str): Human-readable failure message.
|
||||
level (NotificationLevel): Severity for downstream notifications.
|
||||
attachment_content (str | None): Optional traceback or diagnostic text.
|
||||
opc_error_kind (str | None): Classifier (session_bad, connection_lost, etc.).
|
||||
opc_status (str | None): OPC UA status name or synthetic reason.
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Error payload consumed by the OPC activity layer.
|
||||
"""
|
||||
payload: dict[str, Any] = {
|
||||
'notification_id': notification_id,
|
||||
'message': message,
|
||||
@@ -656,7 +717,7 @@ class OpcRepository(SientiaMonitoring):
|
||||
f'model_name={metadata.get("model_name", "unknown")} node={node} phase={phase}',
|
||||
metadata,
|
||||
)
|
||||
await self._start_reconnect_on_bad(opc_status, session_id)
|
||||
await self._start_reconnect(opc_status, session_id)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}',
|
||||
message=f'Failed to {phase} on OPC server: {exc} | metadata: {metadata}',
|
||||
@@ -665,32 +726,89 @@ class OpcRepository(SientiaMonitoring):
|
||||
opc_status=opc_status,
|
||||
)
|
||||
|
||||
async def _write_reconnect_in_progress(
|
||||
self, metadata: dict[str, Any]
|
||||
) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Fail a write because a background reconnect task is already running.
|
||||
|
||||
Args:
|
||||
metadata (dict[str, Any]): Write context passed through to the activity.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind reconnect_in_progress).
|
||||
"""
|
||||
await self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata)
|
||||
self.warning(
|
||||
f'OPC write rejected reconnect_in_progress opc_server_id={self.id} '
|
||||
f'model_id={metadata.get("model_id", "unknown")} '
|
||||
f'model_name={metadata.get("model_name", "unknown")}',
|
||||
metadata,
|
||||
)
|
||||
return False, {
|
||||
'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}',
|
||||
'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.WARNING,
|
||||
'opc_error_kind': 'reconnect_in_progress',
|
||||
}
|
||||
|
||||
async def _write_connection_lost(
|
||||
self, metadata: dict[str, Any], opc_status: str
|
||||
) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Fail a write after scheduling reconnect for a closed or stale session.
|
||||
|
||||
Args:
|
||||
metadata (dict[str, Any]): Write context passed through to the activity.
|
||||
opc_status (str): Synthetic reason (ProtocolClosed, SessionNotReady).
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind connection_lost).
|
||||
"""
|
||||
await self._emit_opc_write_metric('unknown', opc_status, metadata)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_CONNECTION_LOST_{self.id}',
|
||||
message=f'OPC write skipped: connection lost ({opc_status}) | metadata: {metadata}',
|
||||
level=NotificationLevel.WARNING,
|
||||
opc_error_kind='connection_lost',
|
||||
opc_status=opc_status,
|
||||
)
|
||||
|
||||
async def write_data(
|
||||
self, node: str, value: Any, data_type: str, metadata: dict[str, Any]
|
||||
) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Write data to OPC server with a single attempt and Tier-1 Bad* reconnect scheduling.
|
||||
"""
|
||||
if not self._session_ready.is_set():
|
||||
await self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata)
|
||||
self.warning(
|
||||
f'OPC write rejected reconnect_in_progress opc_server_id={self.id} '
|
||||
f'model_id={metadata.get("model_id", "unknown")} '
|
||||
f'model_name={metadata.get("model_name", "unknown")}',
|
||||
metadata,
|
||||
)
|
||||
return False, {
|
||||
'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}',
|
||||
'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.WARNING,
|
||||
'opc_error_kind': 'reconnect_in_progress',
|
||||
}
|
||||
Write data to OPC server with a single attempt and background reconnect scheduling.
|
||||
|
||||
is_connected, error = await self.validate_connection()
|
||||
Reconnect is scheduled on Tier-1 Bad*, closed protocol, or stale session readiness.
|
||||
There is no retry within the same call.
|
||||
|
||||
Args:
|
||||
node (str): OPC UA node id to write.
|
||||
value (Any): Value to convert and send.
|
||||
data_type (str): Logical type key (float, int, bool, str, double).
|
||||
metadata (dict[str, Any]): Activity context (model_id, model_name, etc.).
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: (True, {response_time}) on success, or
|
||||
(False, structured error info) on failure.
|
||||
"""
|
||||
if self._reconnect_task_in_progress():
|
||||
return await self._write_reconnect_in_progress(metadata)
|
||||
|
||||
if not self._session_ready.is_set():
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
await self._start_reconnect('SessionNotReady', session_id)
|
||||
if self._reconnect_task_in_progress():
|
||||
return await self._write_reconnect_in_progress(metadata)
|
||||
return await self._write_connection_lost(metadata, 'SessionNotReady')
|
||||
|
||||
is_connected, _error = await self.validate_connection()
|
||||
if not is_connected:
|
||||
await self._emit_opc_write_metric('unknown', 'NotConnected', metadata)
|
||||
return False, error
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
await self._start_reconnect('ProtocolClosed', session_id)
|
||||
return await self._write_connection_lost(metadata, 'ProtocolClosed')
|
||||
|
||||
start_time = time.time()
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
|
||||
Reference in New Issue
Block a user