gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1811

Browse Source 
Enhance OPC UA communication and metrics tracking

- Updated README.md to include new OPC UA Communication section and detailed metrics for session and write diagnostics.
- Added new metrics in laborious/metrics.py for tracking OPC UA session states and write attempts.
- Refactored OPC activity in laborious/activities/opc.py to handle session errors and improve error reporting.
- Updated e2e tests to cover new scenarios for OPC session/channel errors and reconnect handling.
- Modified .gitignore to include relatorio files and mlruns directory.
- Added ipykernel to requirements-dev.txt for Jupyter notebook support.
This commit is contained in:
vitor-aignosi
2026-05-15 15:28:28 -03:00
parent 473bd0b03f
commit 638d5b70b4
15 changed files with 1370 additions and 424 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
laborious/activities/opc.py
View File

@@ -15,6 +15,16 @@ with workflow.unsafe.imports_passed_through():
from laborious.utils.repository.opc_repository import OpcRepository
OPC_WRITTING_ERROR_CONFIDENCE = 12
OPC_SESSION_BAD_CONFIDENCE = 14
OPC_SESSION_BAD_COMMENT_PREFIX = 'OPC UA session/channel error:'
OPC_WRITTING_ERROR_MESSAGE = 'Some data could not be written to OPC servers'
OPC_RECONNECT_IN_PROGRESS_COMMENT = 'OPC UA reconnect in progress'
OPC_COMMENT_SEPARATOR = ' | '
def _opc_session_bad_comment(opc_status: str | None) -> str:
status = opc_status or 'Unknown'
return f'{OPC_SESSION_BAD_COMMENT_PREFIX} {status}'
class OPC(SientiaMonitoring):
@@ -118,29 +128,18 @@ class OPC(SientiaMonitoring):
data_type: str,
tag_type: str,
metadata: dict[str, Any],
) -> float | None:
) -> tuple[float | None, dict[str, Any] | None]:
"""
Write data to a specific OPC server tag with comprehensive error handling.
This method provides a secure and reliable way to write data to OPC servers
with automatic error handling, notification integration, and detailed logging.
It validates server availability before attempting write operations and
provides comprehensive error reporting for operational monitoring.
Args:
- server_id (str): The id of the OPC server.
- tag (str): The tag to write to.
- data (Any): The data to write.
- data_type (str): The data type.
- tag_type (str): The tag type.
Returns:
- bool: True if the data was written successfully, False otherwise.
Return:
tuple[float | None, dict[str, Any] | None]: Response time on success, or
(None, error info_data) on repository failure.
"""
try:
is_success, info_data = await self.opc_repository[server_id].write_data(
tag, data, data_type, self.logger, metadata
tag, data, data_type, metadata
)
if not is_success:
await self.send_notification_async(
@@ -151,8 +150,8 @@ class OPC(SientiaMonitoring):
level=info_data.get('level', NotificationLevel.ERROR),
attachment_content=info_data.get('attachment_content', None),
)
return None
return info_data['response_time']
return None, info_data
return info_data['response_time'], None
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
@@ -205,7 +204,7 @@ class OPC(SientiaMonitoring):
config: dict[str, Any],
data: DataFrame,
metadata: dict[str, Any],
) -> tuple[bool, dict[str, float | None]]:
) -> tuple[bool, dict[str, float | None], bool, str | None, bool]:
"""
Manage the writing of prediction and confidence data to OPC server tags.
@@ -234,10 +233,13 @@ class OPC(SientiaMonitoring):
"""
response_times: dict[str, float | None] = {}
session_bad_seen = False
session_bad_status: str | None = None
reconnect_in_progress_seen = False
if 'prediction_tags' in config:
for tag, tag_config in config['prediction_tags'].items():
response_time = await self.write_data(
response_time, error_info = await self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction'].values[0],
@@ -245,6 +247,13 @@ class OPC(SientiaMonitoring):
tag_type='prediction',
metadata=metadata,
)
if error_info:
kind = error_info.get('opc_error_kind')
if kind == 'session_bad':
session_bad_seen = True
session_bad_status = error_info.get('opc_status', session_bad_status)
elif kind == 'reconnect_in_progress':
reconnect_in_progress_seen = True
if response_time is not None:
self.info(
f'Prediction data written to OPC server {server_id} for tag {tag}.',
@@ -254,7 +263,7 @@ class OPC(SientiaMonitoring):
if 'confidence_tags' in config:
for tag, tag_config in config['confidence_tags'].items():
response_time = await self.write_data(
response_time, error_info = await self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction_confidence'].values[0],
@@ -262,6 +271,13 @@ class OPC(SientiaMonitoring):
tag_type='confidence',
metadata=metadata,
)
if error_info:
kind = error_info.get('opc_error_kind')
if kind == 'session_bad':
session_bad_seen = True
session_bad_status = error_info.get('opc_status', session_bad_status)
elif kind == 'reconnect_in_progress':
reconnect_in_progress_seen = True
if response_time is not None:
self.info(
f'Confidence data written to OPC server {server_id} for tag {tag}.',
@@ -271,7 +287,13 @@ class OPC(SientiaMonitoring):
success = None not in response_times.values()
return success, response_times
return (
success,
response_times,
session_bad_seen,
session_bad_status,
reconnect_in_progress_seen,
)
@activity.defn(name='write_opc_data')
async def write_opc_data(
@@ -301,6 +323,9 @@ class OPC(SientiaMonitoring):
self.info(f'Data to write: {data.size} rows', metadata)
success = True
session_bad_seen = False
session_bad_status: str | None = None
reconnect_in_progress_seen = False
metrics: dict[str, dict[str, float | None]] = {}
@@ -309,22 +334,48 @@ class OPC(SientiaMonitoring):
success = False
continue
local_success, local_response_times = await self.manage_output_tags(
server_id, config, data, metadata
)
(
local_success,
local_response_times,
local_session_bad,
local_status,
local_reconnect_in_progress,
) = await self.manage_output_tags(server_id, config, data, metadata)
metrics[server_id] = local_response_times
local_count = len(local_response_times)
success = success and local_success
if local_session_bad:
session_bad_seen = True
session_bad_status = local_status or session_bad_status
if local_reconnect_in_progress:
reconnect_in_progress_seen = True
self.info(
f'Process completed for OPC server {server_id}: {local_count} of {len(config.get("prediction_tags", []))} prediction tags and {len(config.get("confidence_tags", []))} confidence tags',
metadata,
)
return self.process_confidence(data, success, metadata), metrics
return (
self.process_confidence(
data,
success,
metadata,
session_bad=session_bad_seen,
opc_status=session_bad_status,
reconnect_in_progress=reconnect_in_progress_seen,
),
metrics,
)
def process_confidence(
self, data: DataFrame, success: bool, metadata: dict[str, Any]
self,
data: DataFrame,
success: bool,
metadata: dict[str, Any],
*,
session_bad: bool = False,
opc_status: str | None = None,
reconnect_in_progress: bool = False,
) -> dict[Hashable, Any]:
"""
Process prediction confidence based on OPC write operation success.
@@ -352,16 +403,26 @@ class OPC(SientiaMonitoring):
This allows downstream systems to handle data quality appropriately.
"""
message = 'Some data could not be written to OPC servers'
if not success:
data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE
data['comments'] = message
comment_parts: list[str] = []
confidence = OPC_WRITTING_ERROR_CONFIDENCE
if session_bad:
comment_parts.append(_opc_session_bad_comment(opc_status))
confidence = OPC_SESSION_BAD_CONFIDENCE
if reconnect_in_progress:
comment_parts.append(OPC_RECONNECT_IN_PROGRESS_COMMENT)
confidence = OPC_SESSION_BAD_CONFIDENCE
if not comment_parts:
comment_parts.append(OPC_WRITTING_ERROR_MESSAGE)
comments = OPC_COMMENT_SEPARATOR.join(comment_parts)
data['prediction_confidence'] = confidence
data['comments'] = comments
self.debug(
f'{message}, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.',
f'OPC write issues, confidence={confidence}, comments={comments}',
metadata,
)
else:
self.debug('Data written to OPC servers successfully.', metadata)