gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Update README, requirements, and E2E tests for improved configuration and functionality

- Enhanced the README with updated model configuration examples, including the addition of an alias for production.
- Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`.
- Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity.
- Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs.
- Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.
This commit is contained in:
vitor-aignosi
2026-05-07 17:02:25 -03:00
parent aaf647efdf
commit e6018af23f
51 changed files with 4408 additions and 2660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
laborious/utils/models/minio_dataframe_payload.py
View File

@@ -21,7 +21,7 @@ from typing import Any, Literal
from pandas import DataFrame, read_parquet
from sientia_do.observability.logger import Logger
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
# Keys that are part of the serialized wire format (not arbitrary metadata).
@@ -89,7 +89,7 @@ class MinioDataFramePayload:
metadata: dict[str, Any] | None = None,
) -> None:
"""
Emit debug logs only when logger is provided
Emit a debug message only when a logger instance is available.
Args:
- logger (Logger | None): Logger instance used for debug messages
@@ -182,7 +182,14 @@ class MinioDataFramePayload:
def cleanup_prefix(self) -> str | None:
"""
Return True if cleanup is enabled for this payload.
Return the MinIO prefix eligible for retention cleanup.
Cleanup is only applicable when payload data was offloaded to MinIO
(``object_key`` present and inline ``data`` absent). Inline-only payloads
return ``None`` because there is no object tree to prune.
Return:
str | None: Prefix used by cleanup listing, or ``None`` when cleanup does not apply.
"""
if self.object_key is not None and self.data is None:
return self.object_prefix
@@ -190,12 +197,19 @@ class MinioDataFramePayload:
def has_data(self) -> bool:
"""
Return True if the payload has some data internally or in MinIO.
Indicate whether the payload contains retrievable tabular content.
A payload is considered non-empty when either inline ``data`` exists
(and is not an empty dict) or an ``object_key`` is available for MinIO
download.
Return:
bool: ``True`` when data can be retrieved, ``False`` otherwise.
"""
return (self.data is not None and self.data != {}) or self.object_key is not None
@classmethod
async def from_dataframe(
def from_dataframe(
cls,
dataframe: DataFrame | None,
minio_repo: MinioRepository,
@@ -274,7 +288,7 @@ class MinioDataFramePayload:
dataframe.to_parquet(parquet_buffer, engine='pyarrow', index=True)
file_bytes = parquet_buffer.getvalue()
upload_result = await minio_repo.upload_file(
upload_result = minio_repo.upload_file(
file_bytes=file_bytes,
relative_key=object_key,
metadata=workflow_metadata,
@@ -299,7 +313,7 @@ class MinioDataFramePayload:
status=status,
)
async def retrieve(
def retrieve(
self,
minio_repo: MinioRepository,
workflow_metadata: dict[str, Any] | None = None,
@@ -336,7 +350,7 @@ class MinioDataFramePayload:
f'MinioDataFramePayload.retrieve downloading object from MinIO: {self.object_key}',
workflow_metadata,
)
file_bytes = await minio_repo.download_file(
file_bytes = minio_repo.download_file(
object_name=self.object_key, metadata=workflow_metadata
)
df = read_parquet(BytesIO(file_bytes))

32
laborious/utils/repository/minio_manager.py
View File

@@ -1,32 +0,0 @@
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository import MinioRepository
class MinioManager(SientiaMonitoring):
minio_repository: MinioRepository | None = None
def __init__(
self,
minio_repository: MinioRepository | None = None,
logger: Logger | None = None,
notification_handler: NotificationHandler | None = None,
metrics_controller: MetricsController | None = None,
):
if self.minio_repository is None:
self.minio_repository = minio_repository
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
def close(self) -> None:
"""
Close the MinioManager and clean up resources.
"""
if self.minio_repository is not None:
try:
self.minio_repository.close()
finally:
self.minio_repository = None
SientiaMonitoring.shutdown(self)

281
laborious/utils/repository/opc_repository.py
View File

@@ -1,4 +1,10 @@
import asyncio
"""
Synchronous OPC UA client repository using python-opcua (opcua package).
Connects to OPC UA servers, optionally configures Basic256 security, validates sessions,
and writes node values with typed variants and Prometheus-compatible metrics.
"""
import json
import time
import traceback
@@ -6,9 +12,8 @@ from datetime import datetime
from pathlib import Path
from typing import Any
from asyncua import Client
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.ua import DataValue, Variant, VariantType
from opcua import Client, ua
from opcua.crypto import security_policies
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
@@ -20,28 +25,38 @@ from laborious import metrics
data_type_map = {
'float': {
'converter': float,
'opc_type': VariantType.Float,
'opc_type': ua.VariantType.Float,
},
'double': {
'converter': float,
'opc_type': VariantType.Double,
'opc_type': ua.VariantType.Double,
},
'int': {
'converter': int,
'opc_type': VariantType.Int32,
'opc_type': ua.VariantType.Int32,
},
'bool': {
'converter': bool,
'opc_type': VariantType.Boolean,
'opc_type': ua.VariantType.Boolean,
},
'str': {
'converter': str,
'opc_type': VariantType.String,
'opc_type': ua.VariantType.String,
},
}
class OpcRepository(SientiaMonitoring):
"""
Synchronous OPC UA repository for connect/disconnect and typed writes.
Attributes:
url: OPC UA endpoint URL.
id: Server identifier used in metrics and notifications.
server_name: Human-readable server name for labels.
client: Active opcua.Client instance while connected.
"""
def __init__(
self,
opc_id: str,
@@ -66,10 +81,10 @@ class OpcRepository(SientiaMonitoring):
self.logger = logger
self.error_count = 0
self.reconnection_interval = reconnection_interval
self.last_reconnection_time: None | datetime = None
self.last_reconnection_time: datetime | None = None
self.disconnection_interval = 10.0
self.notification_handler = notification_handler
self.client: None | Client = None
self.client: Client | None = None
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
@@ -80,24 +95,12 @@ class OpcRepository(SientiaMonitoring):
'schedule_name': '-',
}
async def set_security(self):
def set_security(self) -> None:
"""
Configures the security settings for the OPC UA client.
This method sets up the security policy, certificates, and timeouts
required for establishing a secure connection with the OPC UA server.
Configure Basic256 security policy, certificates, and long channel/session timeouts.
Raises:
ValueError: If either the certificate path or private key path is not provided.
Attributes:
- cert_path (str): Path to the client's certificate file.
- private_key_path (str): Path to the client's private key file.
- server_cert_path (str, optional): Path to the server's certificate file.
- server_uri (str): The URI of the server to be used as the application URI.
- client (opcua.Client): The OPC UA client instance.
- logger (logging.Logger): Logger instance for logging information.
Security Settings:
- Security Policy: Basic256
- Secure Channel Timeout: 10,000,000 ms
- Session Timeout: 10,000,000 ms
ValueError: If certificate paths are missing or client is not initialized.
"""
if self.cert_path is None or self.private_key_path is None:
@@ -114,26 +117,24 @@ class OpcRepository(SientiaMonitoring):
self.client.application_uri = self.server_uri
self.logger.custom_info('Setting security...', self.metadata)
await self.client.set_security(
SecurityPolicyBasic256,
certificate=str(cert),
private_key=str(private_key),
server_certificate=str(server_cert) if server_cert else None,
self.client.set_security(
security_policies.SecurityPolicyBasic256,
str(cert),
str(private_key),
str(server_cert) if server_cert else None,
)
self.client.secure_channel_timeout = 10000000
self.client.session_timeout = 10000000
async def connect(self) -> tuple[bool, dict[str, Any]]:
def connect(self) -> tuple[bool, dict[str, Any]]:
"""
Establishes a connection to the OPC server.
This method initializes the OPC client using the provided URL and
sets up security if a certificate path is specified. It then
attempts to connect to the server and logs the connection status.
Raises:
Exception: If the connection to the OPC server fails.
Create the synchronous client, optionally apply security, and connect to the server.
Return:
tuple[bool, dict[str, Any]]: Success flag and error payload when False.
"""
self.client = Client(self.url, timeout=10, watchdog_intervall=3600000) # type: ignore[attr-defined]
self.client = Client(self.url, timeout=10)
self.client.name = self.pod_id
self.client.application_name = self.pod_id
@@ -142,32 +143,25 @@ class OpcRepository(SientiaMonitoring):
self.client.product_uri = pod_uri
if self.cert_path:
await self.set_security()
self.set_security()
self.logger.custom_info(
f'Starting connection to OPC server {self.id}:{self.server_name}...', self.metadata
)
return await self.try_connect()
return self.try_connect()
async def try_connect(self) -> tuple[bool, dict[str, Any]]:
def try_connect(self) -> tuple[bool, dict[str, Any]]:
"""
Attempt to establish connection to the OPC server.
Perform the TCP/session handshake and emit connection metrics.
This method performs the actual connection attempt to the OPC server
and handles connection failures with comprehensive error reporting.
It updates reconnection timing and provides detailed error information
for operational monitoring and debugging.
Returns:
tuple[bool, dict[str, Any]]: Connection result
- bool: True if connection successful, False otherwise
- dict: Error information if connection failed
Return:
tuple[bool, dict[str, Any]]: Success flag and structured error when False.
"""
tags = {
'pod_id': self.pod_id,
'server_name': self.server_name,
}
await self.emit_metric(metrics.OPC_CONNECTIONS_TOTAL, tags)
self.emit_metric_sync(metrics.OPC_CONNECTIONS_TOTAL, tags)
try:
self.last_reconnection_time = datetime.now()
if self.client is None:
@@ -177,9 +171,9 @@ class OpcRepository(SientiaMonitoring):
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
}
await self.client.connect()
self.client.connect()
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.OPC_CONNECTION_STATUS,
method='set',
tags={
@@ -191,12 +185,12 @@ class OpcRepository(SientiaMonitoring):
return True, {}
except Exception as e:
await self.disconnect()
self.disconnect()
trace = traceback.format_exc()
self.logger.custom_error(trace, self.metadata)
await self.emit_metric(metrics.OPC_CONNECTIONS_FAILED, tags)
self.emit_metric_sync(metrics.OPC_CONNECTIONS_FAILED, tags)
return False, {
'notification_id': f'OPC_CONNECTION_ERROR_{self.id}',
@@ -206,21 +200,27 @@ class OpcRepository(SientiaMonitoring):
'attachment_content': trace,
}
async def disconnection_fallback(self) -> list:
def disconnection_fallback(self) -> list[dict[str, Any]]:
"""
Tries 5 times to disconnect from the OPC UA server, with a delay of 100ms x try.
Retry disconnect up to five times with linear backoff.
Return:
list[dict[str, Any]]: Empty on success, otherwise error records per attempt.
"""
assert self.client is not None
error_stack = []
for i in range(5):
try:
self.logger.info(f'Disconnecting from OPC UA server, attempt {i + 1} of 5')
await self.client.disconnect()
self.logger.custom_info(
f'Disconnecting from OPC UA server, attempt {i + 1} of 5', self.metadata
)
self.client.disconnect()
return []
except Exception as e:
self.logger.error(
f'Failed to disconnect from OPC UA server in attempt {i + 1} of 5: {e}'
self.logger.custom_error(
f'Failed to disconnect from OPC UA server in attempt {i + 1} of 5: {e}',
self.metadata,
)
error_stack.append(
{
@@ -229,23 +229,20 @@ class OpcRepository(SientiaMonitoring):
'traceback': traceback.format_exc(),
}
)
await asyncio.sleep(self.disconnection_interval * i)
time.sleep(self.disconnection_interval * i)
return error_stack
async def disconnect(self):
def disconnect(self) -> None:
"""
Tear down the UA session and reset connection metrics.
"""
Gracefully disconnect from the OPC server.
This method safely terminates the connection to the OPC server
and cleans up client resources. It handles disconnection errors
gracefully and ensures proper resource cleanup.
"""
if self.client is None:
return
errors = await self.disconnection_fallback()
errors = self.disconnection_fallback()
if errors:
await self.send_notification_async(
self.send_notification(
metadata=self.metadata,
notification_id=f'OPC_DISCONNECTION_ERROR_{self.id}',
message='Failed to disconnect from OPC server in 5 attempts.',
@@ -254,8 +251,10 @@ class OpcRepository(SientiaMonitoring):
attachment_content=json.dumps(errors, indent=4),
)
else:
self.logger.warning(f'Disconnected from OPC server {self.id} successfully')
await self.emit_metric(
self.logger.custom_warning(
f'Disconnected from OPC server {self.id} successfully', self.metadata
)
self.emit_metric_sync(
metric_object=metrics.OPC_CONNECTION_STATUS,
method='set',
tags={
@@ -268,77 +267,52 @@ class OpcRepository(SientiaMonitoring):
self.client = None
async def validate_connection(self) -> tuple[bool, dict[str, Any]]:
def _session_alive(self) -> bool:
"""
Validate and maintain OPC server connection health.
Best-effort check that the synchronous client still has a working session.
This method performs comprehensive connection validation and
implements automatic reconnection logic for production reliability.
It handles various connection states and implements intelligent
reconnection strategies with error counting and timing controls.
Connection Validation:
1. Checks client existence and connection state
2. Implements error counting with automatic disconnection
3. Enforces reconnection timing windows
4. Provides detailed error reporting and notifications
Reconnection Strategy:
- Error Count Threshold: Disconnects after 5 consecutive errors
- Reconnection Window: Enforces minimum intervals between attempts
- Automatic Recovery: Attempts reconnection when conditions allow
- State Monitoring: Continuously monitors connection health
Args:
None
Returns:
tuple[bool, dict[str, Any]]: Connection validation result
- bool: True if connection is healthy, False otherwise
- dict: Error information if validation fails
Return:
bool: True if a root browse succeeds, False otherwise.
"""
if self.client is None:
return await self.connect()
# if self.error_count > 5: # NOSONAR
# self.logger.custom_warning(
# f'OPC server {self.id} will be disconnected due to multiple errors', self.metadata
# )
# try:
# await self.disconnect()
# except Exception as e:
# trace = traceback.format_exc()
# self.logger.custom_error(
# f'Failed to disconnect from OPC server: {e}', self.metadata
# )
# self.logger.custom_error(trace, self.metadata)
# self.logger.custom_info(
# f'Attempting to reconnect to OPC server {self.id}...', self.metadata
# )
# return await self.connect()
# Check if client is connected using asyncua's connection state
return False
try:
if (
self.client.uaclient.protocol is None
or self.client.uaclient.protocol.state == 'closed'
):
# OPC server is not connected
self.client.get_root_node()
return True
except Exception:
return False
def validate_connection(self) -> tuple[bool, dict[str, Any]]:
"""
Ensure the UA session is usable; reconnect when outside the backoff window.
Return:
tuple[bool, dict[str, Any]]: Whether the session is ready and optional error payload.
"""
if self.client is None:
return self.connect()
try:
if not self._session_alive():
self.logger.custom_error(f'OPC server {self.id} is not connected', self.metadata)
if (
self.last_reconnection_time is None
or (datetime.now() - self.last_reconnection_time).total_seconds()
> self.reconnection_interval
):
await self.disconnect()
self.disconnect()
self.logger.custom_info(
f'Trying to reconnect to OPC server {self.id}...', self.metadata
)
return await self.connect()
return self.connect()
return False, {
'notification_id': f'OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}',
'message': f'OPC server {self.id} is not connected, waiting for next reconnection window...',
'message': (
f'OPC server {self.id} is not connected, waiting for next reconnection window...'
),
'block': 'opc_repository',
'level': NotificationLevel.WARNING,
}
@@ -355,39 +329,24 @@ class OpcRepository(SientiaMonitoring):
'attachment_content': trace,
}
async def write_data(
def write_data(
self, node: str, value: Any, data_type: str, logger: Logger, metadata: dict[str, Any]
) -> tuple[bool, dict[str, Any]]:
"""
Write data to OPC server with comprehensive validation and monitoring.
This method provides secure and reliable data writing to OPC servers
with automatic connection validation, data type conversion, and
comprehensive error handling. It implements performance monitoring
and metrics collection for operational visibility.
Data Writing Process:
1. Connection validation and automatic reconnection
2. Node validation and error handling
3. Data type conversion and validation
4. OPC data writing with timestamp
5. Performance metrics collection
6. Error handling and notification
Write a typed value to an OPC UA node after validating connectivity.
Args:
node (str): OPC node identifier to write data to
value (Any): Data value to write to the OPC node
data_type (str): Data type for OPC conversion
logger (Logger): Logger instance for operation logging
metadata (dict[str, Any]): Context metadata for logging and metrics
node: Node id string accepted by opcua Client.get_node.
value: Scalar value to encode.
data_type: Key into ``data_type_map`` (e.g. float, str).
logger: Caller logger for per-write traces.
metadata: Workflow metadata for error context.
Returns:
tuple[bool, dict[str, Any]]: Write operation result
- bool: True if write successful, False otherwise
- dict: Error information if write failed
Return:
tuple[bool, dict[str, Any]]: Success flag and either ``response_time`` or error fields.
"""
is_connected, error = await self.validate_connection()
is_connected, error = self.validate_connection()
if not is_connected:
return False, error
@@ -395,8 +354,8 @@ class OpcRepository(SientiaMonitoring):
start_time = time.time()
try:
# ignored because self.validate_connection is called before, so we know self.client is not None
node_obj = self.client.get_node(node) # type: ignore[union-attr]
assert self.client is not None
node_obj = self.client.get_node(node)
except Exception as e:
trace = traceback.format_exc()
logger.custom_error(trace, metadata.get('schedule_name', 'N/A'))
@@ -419,16 +378,10 @@ class OpcRepository(SientiaMonitoring):
data = data_type_map[data_type]['converter'](value)
logger.custom_info(f'Writing {data} - {type(data)} to {node}', metadata)
# now = datetime.now() # NOSONAR
ua_data = DataValue(
Variant(data, data_type_map[data_type]['opc_type']),
# SourceTimestamp=DateTime( # NOSONAR
# now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond # NOSONAR
# ), # NOSONAR
)
variant_type = data_type_map[data_type]['opc_type']
try:
await node_obj.write_value(ua_data)
node_obj.set_value(data, variant_type)
end_time = time.time()
response_time = end_time - start_time