gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1811

Browse Source 
Enhance OPC UA communication and metrics tracking

- Updated README.md to include new OPC UA Communication section and detailed metrics for session and write diagnostics.
- Added new metrics in laborious/metrics.py for tracking OPC UA session states and write attempts.
- Refactored OPC activity in laborious/activities/opc.py to handle session errors and improve error reporting.
- Updated e2e tests to cover new scenarios for OPC session/channel errors and reconnect handling.
- Modified .gitignore to include relatorio files and mlruns directory.
- Added ipykernel to requirements-dev.txt for Jupyter notebook support.
This commit is contained in:
vitor-aignosi
2026-05-15 15:28:28 -03:00
parent 4edf70c4a0
commit 7bf7848a93
8 changed files with 266 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
laborious/utils/repository/opc_repository.py
View File

@@ -211,6 +211,7 @@ class OpcRepository(SientiaMonitoring):
self._connection_lock = threading.Lock()
self._session_ready = threading.Event()
self._reconnect_thread: threading.Thread | None = None
self._allow_reconnect = True
def _opc_debug_tags(self, session_id: str) -> dict[str, str]:
return {
@@ -513,8 +514,12 @@ class OpcRepository(SientiaMonitoring):
def disconnect(self) -> None:
"""
Gracefully disconnect from the OPC server under the connection lock.
Disables background reconnect so late writes during worker shutdown do not
respawn sessions.
"""
with self._connection_lock:
self._allow_reconnect = False
self._disconnect_locked()
def validate_connection(self) -> tuple[bool, dict[str, Any]]:
@@ -531,61 +536,82 @@ class OpcRepository(SientiaMonitoring):
self.error(f'OPC server {self.id} is not connected', self.metadata)
return False, self._not_connected_error()
def _start_reconnect_on_bad(self, opc_status: str, session_id: str) -> None:
def _reconnect_thread_in_progress(self) -> bool:
"""
Schedule a background reconnect if interval and thread state allow it.
Return whether a background reconnect thread is currently running.
Return:
bool: True when a reconnect thread exists and is alive.
"""
return self._reconnect_thread is not None and self._reconnect_thread.is_alive()
def _start_reconnect(self, reason: str, session_id: str) -> None:
"""
Schedule a background reconnect when allowed by interval and thread state.
Clears _session_ready before starting the thread. No-op when _allow_reconnect is
False, the reconnection window has not elapsed, or a reconnect is already running.
Args:
opc_status (str): OPC UA status name that triggered reconnect.
reason (str): Trigger for reconnect (OPC status name or synthetic reason).
session_id (str): Session token before failure.
"""
if not self._allow_reconnect:
return
if not self._reconnection_window_elapsed():
self.warning(
f'OPC reconnect skipped reason=reconnection_window opc_server_id={self.id} '
f'opc_status={opc_status}',
f'reconnect_reason={reason}',
self.metadata,
)
return
if self._reconnect_thread is not None and self._reconnect_thread.is_alive():
if self._reconnect_thread_in_progress():
self.warning(
f'OPC reconnect skipped reason=in_progress opc_server_id={self.id} '
f'opc_status={opc_status}',
f'reconnect_reason={reason}',
self.metadata,
)
return
self._session_ready.clear()
self.info(
f'OPC reconnect scheduled after opc_status={opc_status} opc_server_id={self.id} '
f'OPC reconnect scheduled reconnect_reason={reason} opc_server_id={self.id} '
f'old_session_id={session_id}',
self.metadata,
)
self._reconnect_thread = threading.Thread(
target=self._run_reconnect_on_bad,
args=(opc_status, session_id),
target=self._run_reconnect,
args=(reason, session_id),
daemon=True,
)
self._reconnect_thread.start()
def _run_reconnect_on_bad(self, opc_status: str, session_id: str) -> None:
def _run_reconnect(self, reason: str, session_id: str) -> None:
"""
Tear down and re-establish the OPC UA session under the connection lock.
Args:
opc_status (str): OPC UA status that triggered reconnect.
reason (str): Trigger for reconnect (OPC status or ProtocolClosed).
session_id (str): Previous session token string.
"""
try:
with self._connection_lock:
self.info(
f'OPC reconnect started opc_status={opc_status} opc_server_id={self.id} '
f'OPC reconnect started reconnect_reason={reason} opc_server_id={self.id} '
f'old_session_id={session_id}',
self.metadata,
)
self._reconnect_locked()
success, error = self._reconnect_locked()
if not success:
self.error(
f'OPC reconnect failed reconnect_reason={reason} opc_server_id={self.id}',
self.metadata,
)
if error:
self.error(error.get('message', ''), self.metadata)
except Exception:
self.error(
f'OPC reconnect task failed opc_server_id={self.id} opc_status={opc_status}',
f'OPC reconnect task failed opc_server_id={self.id} reconnect_reason={reason}',
self.metadata,
)
self.error(traceback.format_exc(), self.metadata)
@@ -681,7 +707,7 @@ class OpcRepository(SientiaMonitoring):
f'model_name={metadata.get("model_name", "unknown")} node={node} phase={phase}',
metadata,
)
self._start_reconnect_on_bad(opc_status, session_id)
self._start_reconnect(opc_status, session_id)
return False, self._write_failure_payload(
notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}',
message=f'Failed to {phase} on OPC server: {exc} | metadata: {metadata}',
@@ -714,32 +740,89 @@ class OpcRepository(SientiaMonitoring):
pass
node_obj.set_value(data, variant_type)
def _write_reconnect_in_progress(
self, metadata: dict[str, Any]
) -> tuple[bool, dict[str, Any]]:
"""
Fail a write because a background reconnect thread is already running.
Args:
metadata (dict[str, Any]): Write context passed through to the activity.
Return:
tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind reconnect_in_progress).
"""
self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata)
self.warning(
f'OPC write rejected reconnect_in_progress opc_server_id={self.id} '
f'model_id={metadata.get("model_id", "unknown")} '
f'model_name={metadata.get("model_name", "unknown")}',
metadata,
)
return False, {
'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}',
'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}',
'block': 'opc_repository',
'level': NotificationLevel.WARNING,
'opc_error_kind': 'reconnect_in_progress',
}
def _write_connection_lost(
self, metadata: dict[str, Any], opc_status: str
) -> tuple[bool, dict[str, Any]]:
"""
Fail a write after scheduling reconnect for a closed or stale session.
Args:
metadata (dict[str, Any]): Write context passed through to the activity.
opc_status (str): Synthetic reason (ProtocolClosed, SessionNotReady).
Return:
tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind connection_lost).
"""
self._emit_opc_write_metric('unknown', opc_status, metadata)
return False, self._write_failure_payload(
notification_id=f'OPC_WRITE_CONNECTION_LOST_{self.id}',
message=f'OPC write skipped: connection lost ({opc_status}) | metadata: {metadata}',
level=NotificationLevel.WARNING,
opc_error_kind='connection_lost',
opc_status=opc_status,
)
def write_data(
self, node: str, value: Any, data_type: str, metadata: dict[str, Any]
) -> tuple[bool, dict[str, Any]]:
"""
Write data to OPC server with a single attempt and Tier-1 Bad* reconnect scheduling.
"""
if not self._session_ready.is_set():
self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata)
self.warning(
f'OPC write rejected reconnect_in_progress opc_server_id={self.id} '
f'model_id={metadata.get("model_id", "unknown")} '
f'model_name={metadata.get("model_name", "unknown")}',
metadata,
)
return False, {
'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}',
'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}',
'block': 'opc_repository',
'level': NotificationLevel.WARNING,
'opc_error_kind': 'reconnect_in_progress',
}
Write data to OPC server with a single attempt and background reconnect scheduling.
is_connected, error = self.validate_connection()
Reconnect is scheduled on Tier-1 Bad*, closed protocol, or stale session readiness.
There is no retry within the same call.
Args:
node (str): OPC UA node id to write.
value (Any): Value to convert and send.
data_type (str): Logical type key (float, int, bool, str, double).
metadata (dict[str, Any]): Activity context (model_id, model_name, etc.).
Return:
tuple[bool, dict[str, Any]]: (True, {response_time}) on success, or
(False, structured error info) on failure.
"""
if self._reconnect_thread_in_progress():
return self._write_reconnect_in_progress(metadata)
if not self._session_ready.is_set():
session_id = _opc_authentication_token_str(self.client)
self._start_reconnect('SessionNotReady', session_id)
if self._reconnect_thread_in_progress():
return self._write_reconnect_in_progress(metadata)
return self._write_connection_lost(metadata, 'SessionNotReady')
is_connected, _error = self.validate_connection()
if not is_connected:
self._emit_opc_write_metric('unknown', 'NotConnected', metadata)
return False, error
session_id = _opc_authentication_token_str(self.client)
self._start_reconnect('ProtocolClosed', session_id)
return self._write_connection_lost(metadata, 'ProtocolClosed')
start_time = time.time()
session_id = _opc_authentication_token_str(self.client)