SIENTIAPDE-994
Refactor and enhance the laborious workflow and utilities - Removed outdated test file `test_predictions_batch.py` from workflows. - Added `input_sample.json` for standardized input configuration. - Introduced `connectors_config.py` to manage database and service configurations. - Implemented a logging utility in `logger.py` for consistent logging across the application. - Created `policies.py` to define retry policies for workflows. - Developed comprehensive tests for `MLFlowRepository` in `test_model_repository.py`. - Added extensive tests for `OpcRepository` in `test_opc_repository.py`. - Updated `test_predictions_batch.py` to reflect new workflow structure and testing methodology.
This commit is contained in:
vitor-aignosi
42
laborious/utils/connectors_config.py
Normal file
42
laborious/utils/connectors_config.py
Normal file
@@ -0,0 +1,42 @@
|
||||
from os import getenv
|
||||
import json
|
||||
|
||||
|
||||
def build_postgres_config():
|
||||
return {
|
||||
'host': getenv('POSTGRES_HOST', 'localhost'),
|
||||
'port': int(getenv('POSTGRES_PORT', '5432')),
|
||||
'user': getenv('POSTGRES_USER', 'sientia'),
|
||||
'password': getenv('POSTGRES_PASSWORD', 'sientia'),
|
||||
'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
|
||||
'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
|
||||
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20'))
|
||||
}
|
||||
|
||||
|
||||
def build_mlflow_config():
|
||||
return {
|
||||
'host': getenv('MLFLOW_HOST', 'http://localhost'),
|
||||
'port': int(getenv('MLFLOW_PORT', '5080')),
|
||||
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
||||
'password': getenv('MLFLOW_PASSWORD', 'aignosi')
|
||||
}
|
||||
|
||||
|
||||
def build_opc_config():
|
||||
opc_raw = getenv('OPC_CONFIG', None)
|
||||
|
||||
if opc_raw:
|
||||
return json.loads(opc_raw)
|
||||
|
||||
return {
|
||||
'opc': {
|
||||
'name': getenv('OPC_NAME', 'opc'),
|
||||
'url': getenv('OPC_URL', 'opc.tcp://localhost:4840'),
|
||||
'server_uri': getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'),
|
||||
'cert_path': getenv('OPC_CERT_PATH', None),
|
||||
'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None),
|
||||
'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None),
|
||||
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120'))
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,11 @@
|
||||
from typing import List
|
||||
|
||||
from pandas import DataFrame
|
||||
|
||||
|
||||
def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool:
|
||||
"""
|
||||
Returns True if the data is empty, False otherwise.
|
||||
Returns True if the specific columns have null values, False otherwise.
|
||||
"""
|
||||
return data[
|
||||
return not data[
|
||||
data['variable'].isin(config['VARIABLES']) & data['value'].isna()].empty
|
||||
|
||||
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
import os
|
||||
from git import Repo
|
||||
from urllib.parse import quote
|
||||
|
||||
# Lê variáveis de ambiente
|
||||
GIT_TOKEN = os.getenv("GIT_TOKEN")
|
||||
GIT_EMAIL = os.getenv("GIT_EMAIL")
|
||||
REPO_URL = os.getenv("REPO_URL") # ex: "github.com/usuario/repositorio.git"
|
||||
CLONE_DIR = os.getenv("CLONE_DIR", "./repo_clonado")
|
||||
|
||||
if not GIT_TOKEN or not GIT_EMAIL or not REPO_URL:
|
||||
raise EnvironmentError("As variáveis GIT_TOKEN, GIT_EMAIL e REPO_URL devem estar definidas.")
|
||||
|
||||
# Escapa o token (caso contenha caracteres especiais)
|
||||
safe_token = quote(GIT_TOKEN)
|
||||
|
||||
# Monta URL com autenticação via token
|
||||
repo_url_with_auth = f"https://{safe_token}@{REPO_URL}"
|
||||
|
||||
# Clona o repositório
|
||||
print(f"Clonando repositório em {CLONE_DIR}...")
|
||||
Repo.clone_from(repo_url_with_auth, CLONE_DIR)
|
||||
print("Repositório clonado com sucesso.")
|
||||
|
||||
# Opcional: configura o e-mail globalmente no Git (ou dentro do repo)
|
||||
repo = Repo(CLONE_DIR)
|
||||
with repo.config_writer() as git_config:
|
||||
git_config.set_value("user", "email", GIT_EMAIL)
|
||||
|
||||
print(f"E-mail configurado como {GIT_EMAIL}.")
|
||||
22
laborious/utils/logger.py
Normal file
22
laborious/utils/logger.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from os import getenv
|
||||
import logging
|
||||
import sys
|
||||
|
||||
|
||||
def get_logger(name: str):
|
||||
log_level = getenv('LOG_LEVEL', 'INFO').upper()
|
||||
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(log_level)
|
||||
stream_handler = logging.StreamHandler(sys.stdout)
|
||||
stream_handler.setLevel(log_level)
|
||||
|
||||
stream_handler.setFormatter(
|
||||
logging.Formatter(
|
||||
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
)
|
||||
|
||||
logger.addHandler(stream_handler)
|
||||
|
||||
return logger
|
||||
9
laborious/utils/policies.py
Normal file
9
laborious/utils/policies.py
Normal file
@@ -0,0 +1,9 @@
|
||||
from datetime import timedelta
|
||||
from temporalio.common import RetryPolicy
|
||||
|
||||
retry_policy = RetryPolicy(
|
||||
initial_interval=timedelta(seconds=1),
|
||||
backoff_coefficient=2.0,
|
||||
maximum_interval=timedelta(minutes=1),
|
||||
maximum_attempts=1
|
||||
)
|
||||
@@ -3,20 +3,39 @@ from asyncua.sync import Client
|
||||
from asyncua.crypto.security_policies import SecurityPolicyBasic256
|
||||
from asyncua.ua import DataValue, Variant, VariantType
|
||||
from logging import Logger
|
||||
from datetime import datetime
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
import traceback
|
||||
|
||||
data_type_map = {
|
||||
'float': VariantType.Float,
|
||||
'double': VariantType.Double,
|
||||
'int': VariantType.Int32,
|
||||
'bool': VariantType.Boolean,
|
||||
'str': VariantType.String,
|
||||
'datetime': VariantType.DateTime,
|
||||
'float': {
|
||||
'converter': float,
|
||||
'opc_type': VariantType.Float,
|
||||
},
|
||||
'double': {
|
||||
'converter': float,
|
||||
'opc_type': VariantType.Double,
|
||||
},
|
||||
'int': {
|
||||
'converter': int,
|
||||
'opc_type': VariantType.Int32,
|
||||
},
|
||||
'bool': {
|
||||
'converter': bool,
|
||||
'opc_type': VariantType.Boolean,
|
||||
},
|
||||
'str': {
|
||||
'converter': str,
|
||||
'opc_type': VariantType.String,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class OpcRepository():
|
||||
def __init__(self, name: str, url: str, logger: Logger, server_uri: str,
|
||||
cert_path: str = None, private_key_path: str = None, server_cert_path: str = None):
|
||||
def __init__(self, name: str, url: str, logger: Logger, notification_handler: NotificationHandler,
|
||||
reconnection_interval: int = 60, server_uri: str = None, cert_path: str = None,
|
||||
private_key_path: str = None, server_cert_path: str = None):
|
||||
self.url = url
|
||||
self.name = name
|
||||
self.server_uri = server_uri
|
||||
@@ -24,8 +43,10 @@ class OpcRepository():
|
||||
self.private_key_path = private_key_path
|
||||
self.server_cert_path = server_cert_path
|
||||
self.logger = logger
|
||||
self.non_receive_count = 0
|
||||
|
||||
self.error_count = 0
|
||||
self.reconnection_interval = reconnection_interval
|
||||
self.last_reconnection_time = None
|
||||
self.notification_handler = notification_handler
|
||||
self.client = None
|
||||
|
||||
def set_security(self):
|
||||
@@ -67,13 +88,6 @@ class OpcRepository():
|
||||
self.client.secure_channel_timeout = 10000000
|
||||
self.client.session_timeout = 10000000
|
||||
|
||||
def connect(self):
|
||||
self.client = Client(self.url)
|
||||
if self.security:
|
||||
self.set_security()
|
||||
self.logger.info('Starting connection...')
|
||||
self.client.connect()
|
||||
|
||||
def connect(self):
|
||||
"""
|
||||
Establishes a connection to the OPC server.
|
||||
@@ -88,7 +102,24 @@ class OpcRepository():
|
||||
if self.cert_path:
|
||||
self.set_security()
|
||||
self.logger.info('Starting connection...')
|
||||
self.client.connect()
|
||||
return self.try_connect()
|
||||
|
||||
def try_connect(self):
|
||||
try:
|
||||
self.last_reconnection_time = datetime.now()
|
||||
self.client.connect()
|
||||
return True
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id=f"OPC_CONNECTION_ERROR_{self.name}",
|
||||
message=f"Failed to connect to OPC server: {e}",
|
||||
block="opc_repository",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
self.logger.error(trace)
|
||||
return False
|
||||
|
||||
def disconnect(self):
|
||||
self.client.disconnect()
|
||||
@@ -98,9 +129,74 @@ class OpcRepository():
|
||||
def __del__(self):
|
||||
self.disconnect()
|
||||
|
||||
def write_data(self, node, value, data_type, logger):
|
||||
node = self.client.get_node(node)
|
||||
data = float(value)
|
||||
logger.info(f'Writing {data} - {type(data)} to {node}')
|
||||
ua_data = DataValue(Variant(data, data_type_map[data_type]))
|
||||
node.write_value(ua_data)
|
||||
def validate_connection(self):
|
||||
if self.client is None:
|
||||
return self.connect()
|
||||
|
||||
if self.error_count > 5:
|
||||
self.logger.warning(
|
||||
f"OPC server {self.name} will be disconnected due to multiple errors")
|
||||
try:
|
||||
self.disconnect()
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.logger.error(f"Failed to disconnect from OPC server: {e}")
|
||||
self.logger.error(trace)
|
||||
self.logger.info(
|
||||
f"Attempting to reconnect to OPC server {self.name}...")
|
||||
return self.connect()
|
||||
|
||||
if hasattr(self.client, 'aio_obj') and self.client.aio_obj.uaclient.protocol is None or \
|
||||
(hasattr(self.client.aio_obj.uaclient, 'protocol') and
|
||||
self.client.aio_obj.uaclient.protocol.state == "closed"):
|
||||
|
||||
self.logger.error(
|
||||
f"OPC server {self.name} is not connected")
|
||||
if (datetime.now() - self.last_reconnection_time).total_seconds(
|
||||
) > self.reconnection_interval:
|
||||
self.logger.error(
|
||||
f"Trying to reconnect to OPC server {self.name}...")
|
||||
return self.try_connect()
|
||||
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def write_data(self, node, value, data_type):
|
||||
if not self.validate_connection():
|
||||
return
|
||||
try:
|
||||
node = self.client.get_node(node)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id=f"OPC_WRITE_GET_NODE_ERROR_{self.name}",
|
||||
message=f"Failed to get node from OPC server: {e}",
|
||||
block="opc_repository",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
self.logger.error(trace)
|
||||
self.error_count += 1
|
||||
return
|
||||
|
||||
data = data_type_map[data_type]['converter'](value)
|
||||
self.logger.info(f'Writing {data} - {type(data)} to {node}')
|
||||
ua_data = DataValue(
|
||||
Variant(data, data_type_map[data_type]['opc_type']))
|
||||
|
||||
try:
|
||||
node.write_value(ua_data)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.notification_handler.build_and_send_notification(
|
||||
notification_id=f"OPC_WRITE_DATA_ERROR_{self.name}",
|
||||
message=f"Failed to write data to OPC server: {e}",
|
||||
block="opc_repository",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
self.logger.error(trace)
|
||||
self.error_count += 1
|
||||
return
|
||||
self.error_count = 0
|
||||
|
||||
Reference in New Issue
Block a user