SIENTIAPDE-994
Refactor and enhance the laborious workflow and utilities - Removed outdated test file `test_predictions_batch.py` from workflows. - Added `input_sample.json` for standardized input configuration. - Introduced `connectors_config.py` to manage database and service configurations. - Implemented a logging utility in `logger.py` for consistent logging across the application. - Created `policies.py` to define retry policies for workflows. - Developed comprehensive tests for `MLFlowRepository` in `test_model_repository.py`. - Added extensive tests for `OpcRepository` in `test_opc_repository.py`. - Updated `test_predictions_batch.py` to reflect new workflow structure and testing methodology.
This commit is contained in:
vitor-aignosi
@@ -4,40 +4,55 @@ from temporalio import activity, workflow
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from logging import Logger
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from laborious.activities.base import BaseActivity
|
||||
from laborious.utils.repository.opc_repository import OpcRepository
|
||||
from typing import Any
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
import traceback
|
||||
from pandas import DataFrame
|
||||
|
||||
|
||||
class OPC(BaseActivity):
|
||||
def __init__(self,
|
||||
name: str, url: str, server_uri: str,
|
||||
cert_path: str, private_key_path: str, server_cert_path: str,
|
||||
def __init__(self, opc_servers: dict[str, dict[str, Any]],
|
||||
logger: Logger, notification_handler: NotificationHandler):
|
||||
|
||||
self.logger = logger
|
||||
self.notification_handler = notification_handler
|
||||
self.name = name
|
||||
self.url = url
|
||||
self.server_uri = server_uri
|
||||
self.cert_path = cert_path
|
||||
self.private_key_path = private_key_path
|
||||
self.server_cert_path = server_cert_path
|
||||
self.opc_servers = opc_servers
|
||||
|
||||
self.opc_repository = OpcRepository(
|
||||
name=self.name,
|
||||
url=self.url,
|
||||
logger=self.logger,
|
||||
server_uri=self.server_uri,
|
||||
cert_path=self.cert_path,
|
||||
private_key_path=self.private_key_path,
|
||||
server_cert_path=self.server_cert_path
|
||||
)
|
||||
self.opc_repository = {}
|
||||
for name, server in opc_servers.items():
|
||||
self.opc_repository[name] = OpcRepository(
|
||||
name=name,
|
||||
url=server['url'],
|
||||
logger=self.logger,
|
||||
server_uri=server['server_uri'],
|
||||
cert_path=server['cert_path'],
|
||||
private_key_path=server['private_key_path'],
|
||||
server_cert_path=server['server_cert_path'],
|
||||
notification_handler=self.notification_handler,
|
||||
reconnection_interval=server['reconnection_interval'],
|
||||
)
|
||||
self.opc_repository[name].connect()
|
||||
|
||||
self.opc_repository.connect()
|
||||
BaseActivity.__init__(self, logger, notification_handler)
|
||||
|
||||
def write_data(self, server: str, tag: str, data: Any,
|
||||
data_type: str, tag_type: str):
|
||||
try:
|
||||
self.opc_repository[server].write_data(
|
||||
tag, data, data_type)
|
||||
self.logger.debug(f"Wrote {tag_type} to {tag}")
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id=f"WRITE_OPC_{tag_type.upper()}_ERROR",
|
||||
message=f"Error writing data to OPC server: {e}",
|
||||
block="write_opc_data",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
self.logger.error(trace)
|
||||
|
||||
@activity.defn(name='write_opc_data')
|
||||
async def write_opc_data(self, input_data: dict[str, Any]):
|
||||
@@ -47,46 +62,40 @@ class OPC(BaseActivity):
|
||||
|
||||
Args:
|
||||
input_data (dict[str, Any]): The input data. Contains the following keys:
|
||||
data (dict[str, Any]): The dataframe that contains the data to write to the OPC servers.
|
||||
opc_servers (list[str]): The OPC servers to write to.
|
||||
opc_output_config (dict[str, Any]): The OPC writing configuration. Contains:
|
||||
- data (dict[str, Any]): The dataframe that contains the data to write
|
||||
to the OPC servers.
|
||||
- opc_output_config (dict[str, Any]): The OPC writing configuration.
|
||||
The keys are the OPC server names and the values contain:
|
||||
prediction_tags (dict[str, Any]): The tags to write to the OPC servers.
|
||||
confidence_tags (dict[str, Any]): The tags to write to the OPC servers.
|
||||
|
||||
Returns:
|
||||
"""
|
||||
self.logger.debug("Writing data to OPC servers...")
|
||||
data = DataFrame(input_data['data'])
|
||||
_opc_servers = input_data['opc_servers']
|
||||
opc_output_config = input_data['opc_output_config']
|
||||
self.logger.debug(data)
|
||||
|
||||
if 'prediction_tags' in opc_output_config:
|
||||
for tag, config in opc_output_config['prediction_tags'].items():
|
||||
try:
|
||||
self.opc_repository.write_data(
|
||||
tag, data.head(1)['prediction'].values[0], config['data_type'])
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id="WRITE_OPC_PREDICTION_ERROR",
|
||||
message=f"Error writing data to OPC server: {e}",
|
||||
block="write_opc_data",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
self.logger.error(trace)
|
||||
for server, config in opc_output_config.items():
|
||||
if self.opc_repository.get(server) is None:
|
||||
self.logger.error(f"OPC server {server} not found")
|
||||
continue
|
||||
|
||||
if 'confidence_tags' in opc_output_config:
|
||||
for tag, config in opc_output_config['confidence_tags'].items():
|
||||
try:
|
||||
self.opc_repository.write_data(
|
||||
tag, data.head(1)['prediction_confidence'].values[0], config['data_type'])
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id="WRITE_OPC_CONFIDENCE_ERROR",
|
||||
message=f"Error writing data to OPC server: {e}",
|
||||
block="write_opc_data",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
if 'prediction_tags' in config:
|
||||
for tag, tag_config in config['prediction_tags'].items():
|
||||
self.write_data(
|
||||
server=server,
|
||||
tag=tag,
|
||||
data=data.head(1)['prediction'].values[0],
|
||||
data_type=tag_config['data_type'],
|
||||
tag_type='prediction'
|
||||
)
|
||||
if 'confidence_tags' in config:
|
||||
for tag, tag_config in config['confidence_tags'].items():
|
||||
self.write_data(
|
||||
server=server,
|
||||
tag=tag,
|
||||
data=data.head(1)['prediction_confidence'].values[0],
|
||||
data_type=tag_config['data_type'],
|
||||
tag_type='confidence'
|
||||
)
|
||||
self.logger.error(trace)
|
||||
|
||||
Reference in New Issue
Block a user