gitea_admin/sientia-dataops-model-manager
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1243: Remove OPC server integration and add code quality tools.

Browse Source 
This commit removes the OPC server integration from the Model Manager, including related activities, repositories, metrics, and configuration. It also adds code quality tools such as Ruff (linting/formatting), mypy (type checking), and Bandit (security analysis) along with a validation script and CI/CD integration for automated code validation. The README has been updated to reflect these changes.
This commit is contained in:
Bruno Domingues
2025-10-01 16:25:08 -03:00
parent bc4d98f78d
commit b102f79087
24 changed files with 453 additions and 1810 deletions
Download Patch File Download Diff File Expand all files Collapse all files

359
model_manager/utils/repository/opc_repository.py
View File

@@ -1,359 +0,0 @@
import asyncio
import traceback
import time
from datetime import datetime
from pathlib import Path
from typing import Any
from asyncua import Client
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.ua import DataValue, Variant, VariantType, DateTime
from regex import F
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from model_manager import metrics
data_type_map = {
'float': {
'converter': float,
'opc_type': VariantType.Float,
},
'double': {
'converter': float,
'opc_type': VariantType.Double,
},
'int': {
'converter': int,
'opc_type': VariantType.Int32,
},
'bool': {
'converter': bool,
'opc_type': VariantType.Boolean,
},
'str': {
'converter': str,
'opc_type': VariantType.String,
}
}
class OpcRepository():
def __init__(self, id: str, url: str, logger: Logger,
notification_handler: NotificationHandler,
reconnection_interval: int = 60, server_uri: str = None, cert_path: str = None,
private_key_path: str = None, server_cert_path: str = None, pod_id: str = None):
self.url = url
self.id = id
self.server_uri = server_uri
self.cert_path = cert_path
self.private_key_path = private_key_path
self.server_cert_path = server_cert_path
self.logger = logger
self.error_count = 0
self.reconnection_interval = reconnection_interval
self.last_reconnection_time = None
self.notification_handler = notification_handler
self.client = None
self.pod_id = pod_id
self.metadata = {
'model_name': '-',
'model_id': '-',
'workflow_name': 'opc_repository',
'schedule_name': '-'
}
async def set_security(self):
"""
Configures the security settings for the OPC UA client.
This method sets up the security policy, certificates, and timeouts
required for establishing a secure connection with the OPC UA server.
Raises:
ValueError: If either the certificate path or private key path is not provided.
Attributes:
- cert_path (str): Path to the client's certificate file.
- private_key_path (str): Path to the client's private key file.
- server_cert_path (str, optional): Path to the server's certificate file.
- server_uri (str): The URI of the server to be used as the application URI.
- client (opcua.Client): The OPC UA client instance.
- logger (logging.Logger): Logger instance for logging information.
Security Settings:
- Security Policy: Basic256
- Secure Channel Timeout: 10,000,000 ms
- Session Timeout: 10,000,000 ms
"""
if not all([self.cert_path, self.private_key_path]):
raise ValueError(
"Certificate and private key paths must be provided for secure connection.")
cert = Path(self.cert_path)
private_key = Path(self.private_key_path)
server_cert = Path(
self.server_cert_path) if self.server_cert_path else None
self.client.application_uri = self.server_uri
self.logger.custom_info('Setting security...', self.metadata)
await self.client.set_security(
SecurityPolicyBasic256,
certificate=str(cert),
private_key=str(private_key),
server_certificate=str(server_cert)
)
self.client.secure_channel_timeout = 10000000
self.client.session_timeout = 10000000
async def connect(self) -> tuple[bool, dict[str, Any]]:
"""
Establishes a connection to the OPC server.
This method initializes the OPC client using the provided URL and
sets up security if a certificate path is specified. It then
attempts to connect to the server and logs the connection status.
Raises:
Exception: If the connection to the OPC server fails.
"""
self.client = Client(self.url)
if self.cert_path:
await self.set_security()
self.logger.custom_info(
f'Starting connection to OPC server {self.id}...', self.metadata)
return await self.try_connect()
async def try_connect(self) -> tuple[bool, dict[str, Any]]:
"""
Attempt to establish connection to the OPC server.
This method performs the actual connection attempt to the OPC server
and handles connection failures with comprehensive error reporting.
It updates reconnection timing and provides detailed error information
for operational monitoring and debugging.
Returns:
tuple[bool, dict[str, Any]]: Connection result
- bool: True if connection successful, False otherwise
- dict: Error information if connection failed
"""
try:
self.last_reconnection_time = datetime.now()
await self.client.connect()
return True, {}
except Exception as e:
trace = traceback.format_exc()
self.logger.custom_error(trace, self.metadata)
return False, {
"notification_id": f"OPC_CONNECTION_ERROR_{self.id}",
"message": f"Failed to connect to OPC server: {e}",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
}
async def disconnect(self):
"""
Gracefully disconnect from the OPC server.
This method safely terminates the connection to the OPC server
and cleans up client resources. It handles disconnection errors
gracefully and ensures proper resource cleanup.
"""
if self.client is None:
return
try:
await self.client.disconnect()
self.logger.custom_info(
'Disconnected from OPC server', self.metadata)
except Exception as e:
self.logger.custom_error(
f"Failed to disconnect from OPC server: {e}", self.metadata)
self.client = None
async def validate_connection(self) -> tuple[bool, dict[str, Any]]:
"""
Validate and maintain OPC server connection health.
This method performs comprehensive connection validation and
implements automatic reconnection logic for production reliability.
It handles various connection states and implements intelligent
reconnection strategies with error counting and timing controls.
Connection Validation:
1. Checks client existence and connection state
2. Implements error counting with automatic disconnection
3. Enforces reconnection timing windows
4. Provides detailed error reporting and notifications
Reconnection Strategy:
- Error Count Threshold: Disconnects after 5 consecutive errors
- Reconnection Window: Enforces minimum intervals between attempts
- Automatic Recovery: Attempts reconnection when conditions allow
- State Monitoring: Continuously monitors connection health
Args:
None
Returns:
tuple[bool, dict[str, Any]]: Connection validation result
- bool: True if connection is healthy, False otherwise
- dict: Error information if validation fails
"""
if self.client is None:
return await self.connect()
if self.error_count > 5:
self.logger.custom_warning(
f"OPC server {self.id} will be disconnected due to multiple errors", self.metadata)
try:
await self.disconnect()
except Exception as e:
trace = traceback.format_exc()
self.logger.custom_error(
f"Failed to disconnect from OPC server: {e}", self.metadata)
self.logger.custom_error(trace, self.metadata)
self.logger.custom_info(
f"Attempting to reconnect to OPC server {self.id}...", self.metadata)
return await self.connect()
# Check if client is connected using asyncua's connection state
try:
if self.client.uaclient.protocol is None or self.client.uaclient.protocol.state == "closed":
# OPC server is not connected
self.logger.custom_error(
f"OPC server {self.id} is not connected", self.metadata)
if self.last_reconnection_time is None or (datetime.now() - self.last_reconnection_time).total_seconds(
) > self.reconnection_interval:
await self.disconnect()
self.logger.custom_info(
f"Trying to reconnect to OPC server {self.id}...", self.metadata)
return await self.connect()
return False, {
"notification_id": f"OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}",
"message": f"OPC server {self.id} is not connected, waiting for next reconnection window...",
"block": "opc_repository",
"level": NotificationLevel.WARNING
}
return True, {}
except Exception as e:
trace = traceback.format_exc()
message = f"Failed to validate connection to OPC server: {e}"
self.logger.custom_error(message, self.metadata)
return False, {
"notification_id": f"OPC_CONNECTION_CHECK_ERROR_{self.id}",
"message": message,
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
}
async def write_data(self, node: str, value: Any, data_type: str,
logger: Logger, metadata: dict[str, Any]) -> tuple[bool, dict[str, Any]]:
"""
Write data to OPC server with comprehensive validation and monitoring.
This method provides secure and reliable data writing to OPC servers
with automatic connection validation, data type conversion, and
comprehensive error handling. It implements performance monitoring
and metrics collection for operational visibility.
Data Writing Process:
1. Connection validation and automatic reconnection
2. Node validation and error handling
3. Data type conversion and validation
4. OPC data writing with timestamp
5. Performance metrics collection
6. Error handling and notification
Args:
node (str): OPC node identifier to write data to
value (Any): Data value to write to the OPC node
data_type (str): Data type for OPC conversion
logger (Logger): Logger instance for operation logging
metadata (dict[str, Any]): Context metadata for logging and metrics
Returns:
tuple[bool, dict[str, Any]]: Write operation result
- bool: True if write successful, False otherwise
- dict: Error information if write failed
"""
is_connected, error = await self.validate_connection()
if not is_connected:
return False, error
start_time = time.time()
try:
node_obj = self.client.get_node(node)
except Exception as e:
trace = traceback.format_exc()
logger.custom_error(trace, metadata.get('schedule_name', 'N/A'))
self.error_count += 1
return False, {
"notification_id": f"OPC_WRITE_GET_NODE_ERROR_{self.id}",
"message": f"Failed to get node from OPC server: {e} | metadata: {metadata}",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
}
if data_type not in data_type_map:
return False, {
"notification_id": f"OPC_WRITE_DATA_TYPE_ERROR_{self.id}",
"message": f"Unsupported data type: {data_type} | metadata: {metadata}",
"block": "opc_repository",
"level": NotificationLevel.ERROR
}
data = data_type_map[data_type]['converter'](value)
logger.custom_info(
f'Writing {data} - {type(data)} to {node}', metadata)
now = datetime.now()
ua_data = DataValue(
Variant(data, data_type_map[data_type]['opc_type']),
SourceTimestamp=DateTime(
now.year,
now.month,
now.day,
now.hour,
now.minute,
now.second,
now.microsecond
)
)
try:
await node_obj.write_value(ua_data)
metrics.PREDICTION_OPC_WRITING_COUNT.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=self.id
).inc()
end_time = time.time()
response_time = end_time - start_time
metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=self.id
).observe(response_time)
except Exception as e:
trace = traceback.format_exc()
logger.custom_error(trace, metadata)
self.error_count += 1
return False, {
"notification_id": f"OPC_WRITE_DATA_ERROR_{self.id}",
"message": f"Failed to write data to OPC server: {e} | metadata: {metadata}",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
}
self.error_count = 0
return True, {}