gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1231

Browse Source 
Update .gitignore and refactor metrics.py for improved logging and consistency

- Added coverage.xml to .gitignore to prevent tracking of coverage reports.
- Refactored metric labels in metrics.py for consistency in string formatting and improved readability.
- Enhanced logging messages in various activities to ensure uniformity in message formatting.
This commit is contained in:
vitor-aignosi
2025-10-15 16:00:18 -03:00
parent a5d2b0d3fd
commit ac795c7c53
39 changed files with 4122 additions and 2602 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
laborious/utils/repository/minio_repository.py
View File

@@ -1,40 +1,41 @@
from io import BytesIO
import traceback
from typing import Any
import boto3
from botocore.config import Config
from pandas import DataFrame, read_parquet
from sientia_do.observability.logger import Logger
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from typing import Any
from botocore.exceptions import ClientError
from pandas import DataFrame, read_parquet
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
class MinioRepository():
def __init__(self,
minio_endpoint_url: str,
minio_access_key: str,
minio_secret_key: str,
minio_region_name: str,
minio_default_bucket: str,
logger: Logger,
notification_handler: NotificationHandler):
class MinioRepository:
def __init__(
self,
minio_endpoint_url: str,
minio_access_key: str,
minio_secret_key: str,
minio_region_name: str,
minio_default_bucket: str,
logger: Logger,
notification_handler: NotificationHandler,
):
# MinIO settings shared with pandas s3fs
self.storage_options = {
'key': minio_access_key,
'secret': minio_secret_key,
'client_kwargs': {'endpoint_url': minio_endpoint_url}
'client_kwargs': {'endpoint_url': minio_endpoint_url},
}
self.minio_bucket = minio_default_bucket
self.minio_endpoint_url = minio_endpoint_url
self.minio_region_name = minio_region_name
logger.info(
f"Connecting to MinIO at {self.minio_endpoint_url}, default bucket: {self.minio_bucket}")
f'Connecting to MinIO at {self.minio_endpoint_url}, default bucket: {self.minio_bucket}'
)
# Reusable MinIO client
self.s3_client = boto3.client(
self.s3_client: Any = boto3.client(
's3',
endpoint_url=self.minio_endpoint_url,
aws_access_key_id=self.storage_options['key'],
@@ -48,67 +49,46 @@ class MinioRepository():
read_timeout=120,
),
)
self._bucket_checked = False
self.logger = logger
self.notification_handler = notification_handler
def close(self):
self.s3_client.close()
def ensure_bucket_exists(self, metadata: dict[str, Any]) -> bool:
"""
Ensure the MinIO bucket exists; create it if necessary.
"""
if self._bucket_checked:
return True
try:
self.logger.custom_info(
f"Checking if bucket '{self.minio_bucket}' exists", metadata)
self.logger.custom_info(f"Checking if bucket '{self.minio_bucket}' exists", metadata)
self.s3_client.head_bucket(Bucket=self.minio_bucket)
self._bucket_checked = True
return True
except ClientError:
try:
self.logger.custom_info(
f"Creating bucket '{self.minio_bucket}'", metadata)
self.s3_client.create_bucket(Bucket=self.minio_bucket)
self._bucket_checked = True
return True
except ClientError as ce:
trace = traceback.format_exc()
self.notification_handler.send_notification(
metadata=metadata,
notification_id="ERROR_CREATING_MINIO_BUCKET",
message=f"Failed to ensure bucket '{self.minio_bucket}': {ce}",
block="ensure_bucket_exists",
level=NotificationLevel.ERROR,
attachment_content=str(ce)
)
self.logger.custom_error(trace, metadata)
return False
self.logger.custom_info(f"Creating bucket '{self.minio_bucket}'", metadata)
self.s3_client.create_bucket(Bucket=self.minio_bucket)
def store_dataframe_as_parquet(self, dataframe: DataFrame, uri: str,
object_name: str, metadata: dict[str, Any]):
return True
def store_dataframe_as_parquet(
self, dataframe: DataFrame, uri: str, object_name: str, metadata: dict[str, Any]
):
self.ensure_bucket_exists(metadata)
self.logger.custom_info(
f"Storing dataframe as parquet in {uri}", metadata)
self.logger.custom_info(f'Storing dataframe as parquet in {uri}', metadata)
buffer = BytesIO()
dataframe.to_parquet(buffer, engine='pyarrow', index=True)
buffer.seek(0)
self.s3_client.put_object(
Bucket=self.minio_bucket, Key=object_name, Body=buffer.getvalue())
self.s3_client.put_object(Bucket=self.minio_bucket, Key=object_name, Body=buffer.getvalue())
self.logger.custom_info(
f"Dataframe stored as parquet in {uri}", metadata)
self.logger.custom_info(f'Dataframe stored as parquet in {uri}', metadata)
def get_parquet_as_dataframe(self, object_key: str, metadata: dict[str, Any]) -> DataFrame:
self.logger.custom_info(
f"Getting parquet as dataframe from {object_key}", metadata)
self.logger.custom_info(f'Getting parquet as dataframe from {object_key}', metadata)
response = self.s3_client.get_object(
Bucket=self.minio_bucket, Key=object_key)
response = self.s3_client.get_object(Bucket=self.minio_bucket, Key=object_key)
# Read the content into a BytesIO buffer to support seek operations
buffer = BytesIO(response['Body'].read())

765
laborious/utils/repository/model_repository.py
View File

File diff suppressed because it is too large Load Diff

182
laborious/utils/repository/opc_repository.py
View File

@@ -1,16 +1,16 @@
import asyncio
import traceback
import time
import traceback
from datetime import datetime
from pathlib import Path
from typing import Any
from asyncua import Client
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.ua import DataValue, Variant, VariantType, DateTime
from regex import F
from asyncua.ua import DataValue, DateTime, Variant, VariantType
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from laborious import metrics
data_type_map = {
@@ -33,17 +33,26 @@ data_type_map = {
'str': {
'converter': str,
'opc_type': VariantType.String,
}
},
}
class OpcRepository():
def __init__(self, id: str, url: str, logger: Logger,
notification_handler: NotificationHandler,
reconnection_interval: int = 60, server_uri: str = None, cert_path: str = None,
private_key_path: str = None, server_cert_path: str = None, pod_id: str = None):
class OpcRepository:
def __init__(
self,
opc_id: str,
url: str,
logger: Logger,
notification_handler: NotificationHandler,
reconnection_interval: int = 60,
server_uri: str | None = None,
cert_path: str | None = None,
private_key_path: str | None = None,
server_cert_path: str | None = None,
pod_id: str | None = None,
):
self.url = url
self.id = id
self.id = opc_id
self.server_uri = server_uri
self.cert_path = cert_path
self.private_key_path = private_key_path
@@ -51,16 +60,16 @@ class OpcRepository():
self.logger = logger
self.error_count = 0
self.reconnection_interval = reconnection_interval
self.last_reconnection_time = None
self.last_reconnection_time: None | datetime = None
self.notification_handler = notification_handler
self.client = None
self.client: None | Client = None
self.pod_id = pod_id
self.metadata = {
'model_name': '-',
'model_id': '-',
'workflow_name': 'opc_repository',
'schedule_name': '-'
'schedule_name': '-',
}
async def set_security(self):
@@ -85,11 +94,18 @@ class OpcRepository():
if not all([self.cert_path, self.private_key_path]):
raise ValueError(
"Certificate and private key paths must be provided for secure connection.")
'Certificate and private key paths must be provided for secure connection.'
)
if self.cert_path is None or self.private_key_path is None:
raise ValueError('Certificate and private key paths cannot be None')
cert = Path(self.cert_path)
private_key = Path(self.private_key_path)
server_cert = Path(
self.server_cert_path) if self.server_cert_path else None
server_cert = Path(self.server_cert_path) if self.server_cert_path else None
if self.client is None:
raise ValueError('Client must be initialized before setting security')
self.client.application_uri = self.server_uri
self.logger.custom_info('Setting security...', self.metadata)
@@ -97,7 +113,7 @@ class OpcRepository():
SecurityPolicyBasic256,
certificate=str(cert),
private_key=str(private_key),
server_certificate=str(server_cert)
server_certificate=str(server_cert) if server_cert else None,
)
self.client.secure_channel_timeout = 10000000
self.client.session_timeout = 10000000
@@ -115,8 +131,7 @@ class OpcRepository():
self.client = Client(self.url)
if self.cert_path:
await self.set_security()
self.logger.custom_info(
f'Starting connection to OPC server {self.id}...', self.metadata)
self.logger.custom_info(f'Starting connection to OPC server {self.id}...', self.metadata)
return await self.try_connect()
async def try_connect(self) -> tuple[bool, dict[str, Any]]:
@@ -136,6 +151,13 @@ class OpcRepository():
try:
self.last_reconnection_time = datetime.now()
if self.client is None:
return False, {
'notification_id': f'OPC_CONNECTION_ERROR_{self.id}',
'message': 'Client is not initialized',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
}
await self.client.connect()
return True, {}
except Exception as e:
@@ -143,11 +165,11 @@ class OpcRepository():
self.logger.custom_error(trace, self.metadata)
return False, {
"notification_id": f"OPC_CONNECTION_ERROR_{self.id}",
"message": f"Failed to connect to OPC server: {e}",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
'notification_id': f'OPC_CONNECTION_ERROR_{self.id}',
'message': f'Failed to connect to OPC server: {e}',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': trace,
}
async def disconnect(self):
@@ -162,11 +184,9 @@ class OpcRepository():
return
try:
await self.client.disconnect()
self.logger.custom_info(
'Disconnected from OPC server', self.metadata)
self.logger.custom_info('Disconnected from OPC server', self.metadata)
except Exception as e:
self.logger.custom_error(
f"Failed to disconnect from OPC server: {e}", self.metadata)
self.logger.custom_error(f'Failed to disconnect from OPC server: {e}', self.metadata)
self.client = None
async def validate_connection(self) -> tuple[bool, dict[str, Any]]:
@@ -203,52 +223,62 @@ class OpcRepository():
if self.error_count > 5:
self.logger.custom_warning(
f"OPC server {self.id} will be disconnected due to multiple errors", self.metadata)
f'OPC server {self.id} will be disconnected due to multiple errors', self.metadata
)
try:
await self.disconnect()
except Exception as e:
trace = traceback.format_exc()
self.logger.custom_error(
f"Failed to disconnect from OPC server: {e}", self.metadata)
f'Failed to disconnect from OPC server: {e}', self.metadata
)
self.logger.custom_error(trace, self.metadata)
self.logger.custom_info(
f"Attempting to reconnect to OPC server {self.id}...", self.metadata)
f'Attempting to reconnect to OPC server {self.id}...', self.metadata
)
return await self.connect()
# Check if client is connected using asyncua's connection state
try:
if self.client.uaclient.protocol is None or self.client.uaclient.protocol.state == "closed":
if (
self.client.uaclient.protocol is None
or self.client.uaclient.protocol.state == 'closed'
):
# OPC server is not connected
self.logger.custom_error(
f"OPC server {self.id} is not connected", self.metadata)
if self.last_reconnection_time is None or (datetime.now() - self.last_reconnection_time).total_seconds(
) > self.reconnection_interval:
self.logger.custom_error(f'OPC server {self.id} is not connected', self.metadata)
if (
self.last_reconnection_time is None
or (datetime.now() - self.last_reconnection_time).total_seconds()
> self.reconnection_interval
):
await self.disconnect()
self.logger.custom_info(
f"Trying to reconnect to OPC server {self.id}...", self.metadata)
f'Trying to reconnect to OPC server {self.id}...', self.metadata
)
return await self.connect()
return False, {
"notification_id": f"OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}",
"message": f"OPC server {self.id} is not connected, waiting for next reconnection window...",
"block": "opc_repository",
"level": NotificationLevel.WARNING
'notification_id': f'OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}',
'message': f'OPC server {self.id} is not connected, waiting for next reconnection window...',
'block': 'opc_repository',
'level': NotificationLevel.WARNING,
}
return True, {}
except Exception as e:
trace = traceback.format_exc()
message = f"Failed to validate connection to OPC server: {e}"
message = f'Failed to validate connection to OPC server: {e}'
self.logger.custom_error(message, self.metadata)
return False, {
"notification_id": f"OPC_CONNECTION_CHECK_ERROR_{self.id}",
"message": message,
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
'notification_id': f'OPC_CONNECTION_CHECK_ERROR_{self.id}',
'message': message,
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': trace,
}
async def write_data(self, node: str, value: Any, data_type: str,
logger: Logger, metadata: dict[str, Any]) -> tuple[bool, dict[str, Any]]:
async def write_data(
self, node: str, value: Any, data_type: str, logger: Logger, metadata: dict[str, Any]
) -> tuple[bool, dict[str, Any]]:
"""
Write data to OPC server with comprehensive validation and monitoring.
@@ -286,42 +316,42 @@ class OpcRepository():
start_time = time.time()
try:
if self.client is None:
return False, {
'notification_id': f'OPC_WRITE_GET_NODE_ERROR_{self.id}',
'message': 'Client is not initialized',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
}
node_obj = self.client.get_node(node)
except Exception as e:
trace = traceback.format_exc()
logger.custom_error(trace, metadata.get('schedule_name', 'N/A'))
self.error_count += 1
return False, {
"notification_id": f"OPC_WRITE_GET_NODE_ERROR_{self.id}",
"message": f"Failed to get node from OPC server: {e} | metadata: {metadata}",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
'notification_id': f'OPC_WRITE_GET_NODE_ERROR_{self.id}',
'message': f'Failed to get node from OPC server: {e} | metadata: {metadata}',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': trace,
}
if data_type not in data_type_map:
return False, {
"notification_id": f"OPC_WRITE_DATA_TYPE_ERROR_{self.id}",
"message": f"Unsupported data type: {data_type} | metadata: {metadata}",
"block": "opc_repository",
"level": NotificationLevel.ERROR
'notification_id': f'OPC_WRITE_DATA_TYPE_ERROR_{self.id}',
'message': f'Unsupported data type: {data_type} | metadata: {metadata}',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
}
data = data_type_map[data_type]['converter'](value)
logger.custom_info(
f'Writing {data} - {type(data)} to {node}', metadata)
logger.custom_info(f'Writing {data} - {type(data)} to {node}', metadata)
now = datetime.now()
ua_data = DataValue(
Variant(data, data_type_map[data_type]['opc_type']),
SourceTimestamp=DateTime(
now.year,
now.month,
now.day,
now.hour,
now.minute,
now.second,
now.microsecond
)
now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond
),
)
try:
@@ -331,7 +361,7 @@ class OpcRepository():
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=self.id
opc_server_id=self.id,
).inc()
end_time = time.time()
@@ -340,7 +370,7 @@ class OpcRepository():
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=self.id
opc_server_id=self.id,
).observe(response_time)
except Exception as e:
@@ -348,11 +378,11 @@ class OpcRepository():
logger.custom_error(trace, metadata)
self.error_count += 1
return False, {
"notification_id": f"OPC_WRITE_DATA_ERROR_{self.id}",
"message": f"Failed to write data to OPC server: {e} | metadata: {metadata}",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
'notification_id': f'OPC_WRITE_DATA_ERROR_{self.id}',
'message': f'Failed to write data to OPC server: {e} | metadata: {metadata}',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': trace,
}
self.error_count = 0