gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1231

Browse Source 
Update .gitignore and refactor metrics.py for improved logging and consistency

- Added coverage.xml to .gitignore to prevent tracking of coverage reports.
- Refactored metric labels in metrics.py for consistency in string formatting and improved readability.
- Enhanced logging messages in various activities to ensure uniformity in message formatting.
This commit is contained in:
vitor-aignosi
2025-10-15 16:00:18 -03:00
parent a5d2b0d3fd
commit ac795c7c53
39 changed files with 4122 additions and 2602 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
laborious/activities/opc.py
View File

@@ -1,15 +1,16 @@
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import traceback
from typing import Any
from pandas import DataFrame
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
from laborious.utils.repository.opc_repository import OpcRepository
from typing import Any
import traceback
from pandas import DataFrame
OPC_WRITTING_ERROR_CONFIDENCE = 12
@@ -33,15 +34,17 @@ class OPC(BaseActivity):
notification_handler (NotificationHandler): Notification management instance
"""
def __init__(self, opc_servers: dict[str, dict[str, Any]],
logger: Logger, notification_handler: NotificationHandler):
def __init__(
self,
opc_servers: dict[str, dict[str, Any]],
logger: Logger,
notification_handler: NotificationHandler,
):
self.logger = logger
self.notification_handler = notification_handler
self.opc_servers = opc_servers
BaseActivity.__init__(
self, logger, notification_handler, set_error_counter=True)
BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True)
self.opc_repository: dict[str, OpcRepository] = {}
self.opc_servers = opc_servers
@@ -70,10 +73,10 @@ class OPC(BaseActivity):
the initialization of other OPC servers. Each server is handled
independently to ensure maximum availability.
"""
self.logger.info("Initializing OPC servers...")
for id, server in self.opc_servers.items():
self.opc_repository[id] = OpcRepository(
id=server['id'],
self.logger.info('Initializing OPC servers...')
for opc_id, server in self.opc_servers.items():
self.opc_repository[opc_id] = OpcRepository(
opc_id=server['id'],
url=server['url'],
logger=self.logger,
server_uri=server['server_uri'],
@@ -82,30 +85,35 @@ class OPC(BaseActivity):
server_cert_path=server['server_cert_path'],
notification_handler=self.notification_handler,
reconnection_interval=server['reconnection_interval'],
pod_id=self.pod_id
pod_id=self.pod_id,
)
is_connected, error_data = await self.opc_repository[id].connect()
is_connected, error_data = await self.opc_repository[opc_id].connect()
if not is_connected:
self.send_notification(
metadata={
'model_id': '-',
'model_name': '-',
'workflow_name': '-',
'schedule_name': 'INITIALIZATION'
'schedule_name': 'INITIALIZATION',
},
notification_id=error_data['notification_id'],
message=error_data['message'],
block=error_data['block'],
level=error_data.get('level', NotificationLevel.ERROR),
attachment_content=error_data.get(
'attachment_content', None)
attachment_content=error_data.get('attachment_content', None),
)
else:
self.logger.info(
f"OPC server {id} connected successfully.")
self.logger.info(f'OPC server {opc_id} connected successfully.')
async def write_data(self, server_id: str, tag: str, data: Any,
data_type: str, tag_type: str, metadata: dict[str, Any]) -> bool:
async def write_data(
self,
server_id: str,
tag: str,
data: Any,
data_type: str,
tag_type: str,
metadata: dict[str, Any],
) -> bool:
"""
Write data to a specific OPC server tag with comprehensive error handling.
@@ -127,7 +135,8 @@ class OPC(BaseActivity):
try:
is_success, error_data = await self.opc_repository[server_id].write_data(
tag, data, data_type, self.logger, metadata)
tag, data, data_type, self.logger, metadata
)
if not is_success:
self.send_notification(
metadata=metadata,
@@ -135,8 +144,7 @@ class OPC(BaseActivity):
message=error_data['message'],
block=error_data['block'],
level=error_data.get('level', NotificationLevel.ERROR),
attachment_content=error_data.get(
'attachment_content', None)
attachment_content=error_data.get('attachment_content', None),
)
return False
return True
@@ -144,11 +152,11 @@ class OPC(BaseActivity):
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id=f"WRITE_OPC_{tag_type.upper()}_ERROR",
message=f"Error writing data to OPC server: {e}",
block="write_opc_data",
notification_id=f'WRITE_OPC_{tag_type.upper()}_ERROR',
message=f'Error writing data to OPC server: {e}',
block='write_opc_data',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
raise e
@@ -174,21 +182,26 @@ class OPC(BaseActivity):
This helps operators quickly identify configuration issues.
"""
if self.opc_repository.get(server_id) is None:
message = f"OPC server {server_id} not found to perform write operation."
message = f'OPC server {server_id} not found to perform write operation.'
self.send_notification(
metadata=metadata,
notification_id="OPC_SERVER_NOT_FOUND",
notification_id='OPC_SERVER_NOT_FOUND',
message=message,
block="write_opc_data",
block='write_opc_data',
level=NotificationLevel.ERROR,
attachment_content=f"OPC servers: {list(self.opc_repository.keys())}"
attachment_content=f'OPC servers: {list(self.opc_repository.keys())}',
)
return False
return True
async def manage_output_tags(
self, server_id: str, config: dict[str, Any], data: DataFrame,
metadata: dict[str, Any], success: bool) -> tuple[bool, int]:
self,
server_id: str,
config: dict[str, Any],
data: DataFrame,
metadata: dict[str, Any],
success: bool,
) -> tuple[bool, int]:
"""
Manage the writing of prediction and confidence data to OPC server tags.
@@ -225,11 +238,13 @@ class OPC(BaseActivity):
data=data.head(1)['prediction'].values[0],
data_type=tag_config['data_type'],
tag_type='prediction',
metadata=metadata
metadata=metadata,
)
if local_success:
self.info(
f"Prediction data written to OPC server {server_id} for tag {tag}.", metadata)
f'Prediction data written to OPC server {server_id} for tag {tag}.',
metadata,
)
count += 1
success = success and local_success
@@ -241,11 +256,13 @@ class OPC(BaseActivity):
data=data.head(1)['prediction_confidence'].values[0],
data_type=tag_config['data_type'],
tag_type='confidence',
metadata=metadata
metadata=metadata,
)
if local_success:
self.info(
f"Confidence data written to OPC server {server_id} for tag {tag}.", metadata)
f'Confidence data written to OPC server {server_id} for tag {tag}.',
metadata,
)
count += 1
success = success and local_success
@@ -271,29 +288,33 @@ class OPC(BaseActivity):
"""
metadata = input_data['metadata']
self.info("Writing data to OPC servers...", metadata)
self.info('Writing data to OPC servers...', metadata)
data = DataFrame(input_data['data'])
opc_output_config = input_data['opc_output_config']
self.info(f"Data to write: {data.size} rows", metadata)
self.info(f'Data to write: {data.size} rows', metadata)
success = True
for server_id, config in opc_output_config.items():
if not self.validate_server(server_id, metadata):
success = False
continue
local_success, local_count = await self.manage_output_tags(
server_id, config, data, metadata, success)
server_id, config, data, metadata, success
)
success = success and local_success
self.info(
f"Process completed for OPC server {server_id}: {local_count} of {len(config.get('prediction_tags', []))} prediction tags and {len(config.get('confidence_tags', []))} confidence tags", metadata)
f'Process completed for OPC server {server_id}: {local_count} of {len(config.get("prediction_tags", []))} prediction tags and {len(config.get("confidence_tags", []))} confidence tags',
metadata,
)
return self.process_confidence(data, success, metadata)
def process_confidence(self, data: DataFrame, success: bool, metadata: dict[str, Any]) -> dict[Any, Any]:
def process_confidence(
self, data: DataFrame, success: bool, metadata: dict[str, Any]
) -> dict[Any, Any]:
"""
Process prediction confidence based on OPC write operation success.
@@ -323,12 +344,12 @@ class OPC(BaseActivity):
if not success:
data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE
self.debug(
f"Some data could not be written to OPC servers, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.",
metadata
f'Some data could not be written to OPC servers, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.',
metadata,
)
else:
self.debug("Data written to OPC servers successfully.", metadata)
self.debug('Data written to OPC servers successfully.', metadata)
return data.to_dict()