gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Update README, requirements, and E2E tests for improved configuration and functionality

- Enhanced the README with updated model configuration examples, including the addition of an alias for production.
- Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`.
- Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity.
- Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs.
- Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.
This commit is contained in:
vitor-aignosi
2026-05-07 17:02:25 -03:00
parent aaf647efdf
commit e6018af23f
51 changed files with 4408 additions and 2660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

210
laborious/activities/opc.py
View File

@@ -51,7 +51,7 @@ class OPC(SientiaMonitoring):
self.opc_repository: dict[str, OpcRepository] = {}
async def init_opc(self):
def init_opc(self) -> None:
"""
Initialize OPC server connections and establish communication channels.
@@ -59,58 +59,43 @@ class OPC(SientiaMonitoring):
establish secure connections using certificate-based authentication.
Each server connection is managed independently, and connection failures
are reported through the notification system.
The method performs the following operations:
1. Creates OpcRepository instances for each configured server
2. Establishes secure connections with certificate validation
3. Reports connection success/failure through notifications
4. Logs connection status for operational visibility
Raises:
Exception: If OPC repository initialization fails or connection
establishment encounters critical errors
Note:
Connection failures are logged and reported but do not prevent
the initialization of other OPC servers. Each server is handled
independently to ensure maximum availability.
"""
self.logger.info('Initializing OPC servers...')
for opc_id, server in self.opc_servers.items():
self.opc_repository[opc_id] = OpcRepository(
opc_id=server['id'],
server_name=server['server_name'],
opc_id=opc_id,
url=server['url'],
server_name=server['server_name'],
logger=self.logger,
notification_handler=self.notification_handler,
metrics_controller=self.metrics_controller,
server_uri=server['server_uri'],
cert_path=server['cert_path'],
private_key_path=server['private_key_path'],
server_cert_path=server['server_cert_path'],
notification_handler=self.notification_handler,
reconnection_interval=server['reconnection_interval'],
metrics_controller=self.metrics_controller,
)
is_connected, error_data = await self.opc_repository[opc_id].connect()
if not is_connected:
await self.send_notification_async(
ok, err = self.opc_repository[opc_id].connect()
if not ok:
self.send_notification(
metadata={
'model_id': '-',
'model_name': '-',
'workflow_name': '-',
'schedule_name': 'INITIALIZATION',
},
notification_id=error_data['notification_id'],
message=error_data['message'],
block=error_data['block'],
level=error_data.get('level', NotificationLevel.ERROR),
attachment_content=error_data.get('attachment_content', None),
notification_id=f'OPC_CONNECTION_ERROR_{server.get("id", opc_id)}',
message=err.get('message', 'Failed to connect to OPC server'),
block='opc_repository',
level=NotificationLevel.ERROR,
attachment_content=err.get('attachment_content', traceback.format_exc()),
)
else:
self.logger.info(
f'OPC server {opc_id}:{server["server_name"]} connected successfully.'
)
async def write_data(
def write_data(
self,
server_id: str,
tag: str,
@@ -122,11 +107,6 @@ class OPC(SientiaMonitoring):
"""
Write data to a specific OPC server tag with comprehensive error handling.
This method provides a secure and reliable way to write data to OPC servers
with automatic error handling, notification integration, and detailed logging.
It validates server availability before attempting write operations and
provides comprehensive error reporting for operational monitoring.
Args:
- server_id (str): The id of the OPC server.
- tag (str): The tag to write to.
@@ -135,15 +115,15 @@ class OPC(SientiaMonitoring):
- tag_type (str): The tag type.
Returns:
- bool: True if the data was written successfully, False otherwise.
- float | None: Response time in seconds if successful, None otherwise.
"""
try:
is_success, info_data = await self.opc_repository[server_id].write_data(
is_success, info_data = self.opc_repository[server_id].write_data(
tag, data, data_type, self.logger, metadata
)
if not is_success:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=info_data['notification_id'],
message=info_data['message'],
@@ -155,7 +135,7 @@ class OPC(SientiaMonitoring):
return info_data['response_time']
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'WRITE_OPC_{tag_type.upper()}_ERROR',
message=f'Error writing data to OPC server: {e}',
@@ -165,30 +145,25 @@ class OPC(SientiaMonitoring):
)
raise e
async def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool:
def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool:
"""
Validate that an OPC server is available and configured for write operations.
Validate that an OPC repository exists for the requested server identifier.
This method checks if the specified OPC server exists in the active
repository and is available for data writing operations. It provides
immediate feedback for server availability and logs validation failures
for operational monitoring.
This guard prevents write attempts against unknown/uninitialized servers.
When the server is missing, it emits an error notification with the list
of available repositories to help operators diagnose configuration drift.
Args:
server_id (str): Unique identifier for the OPC server to validate
metadata (dict[str, Any]): Context metadata for logging and notifications
- server_id (str): OPC server identifier from workflow output config.
- metadata (dict[str, Any]): Workflow metadata used for logs/alerts.
Returns:
bool: True if server is available, False otherwise
Note:
Server validation failures are automatically reported through the
notification system with detailed information about available servers.
This helps operators quickly identify configuration issues.
Return:
bool: ``True`` when the server repository is available; ``False`` otherwise.
"""
if self.opc_repository.get(server_id) is None:
message = f'OPC server {server_id} not found to perform write operation.'
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='OPC_SERVER_NOT_FOUND',
message=message,
@@ -199,7 +174,7 @@ class OPC(SientiaMonitoring):
return False
return True
async def manage_output_tags(
def manage_output_tags(
self,
server_id: str,
config: dict[str, Any],
@@ -207,37 +182,30 @@ class OPC(SientiaMonitoring):
metadata: dict[str, Any],
) -> tuple[bool, dict[str, float | None]]:
"""
Manage the writing of prediction and confidence data to OPC server tags.
Write prediction and confidence values for one OPC server configuration.
This method orchestrates the writing of multiple data types to OPC servers
based on configuration. It handles both prediction data and confidence
values independently, allowing for flexible tag configuration and
comprehensive error handling.
The method supports two main tag types:
1. Prediction tags: Write actual prediction values to configured OPC tags
2. Confidence tags: Write confidence scores to separate OPC tags
The method iterates through optional ``prediction_tags`` and
``confidence_tags``, performs synchronous writes for each tag, collects
per-tag response times, and returns an aggregate success flag
(all tags successful) with a metrics-friendly response map.
Args:
server_id (str): Unique identifier for the target OPC server
config (dict[str, Any]): OPC tag configuration containing:
- prediction_tags (dict, optional): Prediction tag configurations
- confidence_tags (dict, optional): Confidence tag configurations
data (DataFrame): DataFrame containing prediction and confidence data
metadata (dict[str, Any]): Context metadata for logging and notifications
success (bool): Current success status to maintain across operations
- server_id (str): Target OPC server id.
- config (dict[str, Any]): Server output configuration containing optional
``prediction_tags`` and ``confidence_tags`` sections.
- data (DataFrame): Prediction dataframe used as source values.
- metadata (dict[str, Any]): Workflow metadata for logging/notifications.
Returns:
tuple[bool, int]: (overall_success, total_tags_written)
- overall_success: True if all configured tags were written successfully
- total_tags_written: Count of successfully written tags
Return:
tuple[bool, dict[str, float | None]]: Global success flag and response-time
map per tag (``None`` for failed writes).
"""
response_times: dict[str, float | None] = {}
if 'prediction_tags' in config:
for tag, tag_config in config['prediction_tags'].items():
response_time = await self.write_data(
response_time = self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction'].values[0],
@@ -254,7 +222,7 @@ class OPC(SientiaMonitoring):
if 'confidence_tags' in config:
for tag, tag_config in config['confidence_tags'].items():
response_time = await self.write_data(
response_time = self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction_confidence'].values[0],
@@ -274,25 +242,25 @@ class OPC(SientiaMonitoring):
return success, response_times
@activity.defn(name='write_opc_data')
async def write_opc_data(
def write_opc_data(
self, input_data: dict[str, Any]
) -> tuple[dict[Hashable, Any], dict[str, dict[str, float | None]]]:
"""
Write prediction and confidence data to OPC servers. The two writing
operations are optional and independent of each other.
Execute OPC writes across all configured servers and collect per-tag metrics.
For each server in ``opc_output_config``, this activity validates server
availability, writes enabled prediction/confidence tags, accumulates
response-time metrics, and then normalizes confidence/comments in the
returned prediction payload when at least one write fails.
Args:
- input_data(dict[str, Any]): The input data. Contains the following keys:
- data(dict[str, Any]): The dataframe that contains the data to write
to the OPC servers.
- opc_output_config(dict[str, Any]): The OPC writing configuration.
The keys are the OPC server names and the values contain:
- prediction_tags(dict[str, Any]): The tags to write to the OPC servers.
- confidence_tags(dict[str, Any]): The tags to write to the OPC servers.
Returns:
- dict[Any, Any]: The data that was written to the OPC servers.
- input_data (dict[str, Any]): Payload containing workflow metadata, data
to write, and ``opc_output_config`` server/tag definitions.
Return:
tuple[dict[Hashable, Any], dict[str, dict[str, float | None]]]: Updated
prediction payload dict and nested metrics
``{server_id: {tag_name: response_time_or_none}}``.
"""
metadata = input_data['metadata']
self.info('Writing data to OPC servers...', metadata)
@@ -305,19 +273,21 @@ class OPC(SientiaMonitoring):
metrics: dict[str, dict[str, float | None]] = {}
for server_id, config in opc_output_config.items():
if not await self.validate_server(server_id, metadata):
if not self.validate_server(server_id, metadata):
success = False
continue
local_success, local_response_times = await self.manage_output_tags(
local_success, local_response_times = self.manage_output_tags(
server_id, config, data, metadata
)
metrics[server_id] = local_response_times
local_count = len(local_response_times)
success = success and local_success
n_pred = len(config.get('prediction_tags') or {})
n_conf = len(config.get('confidence_tags') or {})
self.info(
f'Process completed for OPC server {server_id}: {local_count} of {len(config.get("prediction_tags", []))} prediction tags and {len(config.get("confidence_tags", []))} confidence tags',
f'Process completed for OPC server {server_id}: {local_count} of {n_pred} prediction tags and {n_conf} confidence tags',
metadata,
)
@@ -327,29 +297,16 @@ class OPC(SientiaMonitoring):
self, data: DataFrame, success: bool, metadata: dict[str, Any]
) -> dict[Hashable, Any]:
"""
Process prediction confidence based on OPC write operation success.
This method updates the prediction confidence values in the DataFrame
based on the success status of OPC server write operations. If any
write operations failed, it sets the confidence to a predefined error
value to indicate data quality issues.
The method implements a confidence degradation strategy:
- Success: Maintains original confidence values
- Failure: Sets confidence to error value for operational awareness
Apply fallback confidence/comment values when OPC writes are not fully successful.
Args:
data (DataFrame): DataFrame containing prediction and confidence data
success (bool): Overall success status of OPC write operations
metadata (dict[str, Any]): Context metadata for logging and notifications
- data (DataFrame): Prediction dataframe to be returned to downstream steps.
- success (bool): Aggregate write status across all attempted OPC tags.
- metadata (dict[str, Any]): Workflow metadata used for debug logs.
Returns:
dict[Any, Any]: Processed data as a dictionary with updated confidence values
Note:
The error confidence value (OPC_WRITTING_ERROR_CONFIDENCE = 12) is
used to indicate that data was not successfully exported to OPC servers.
This allows downstream systems to handle data quality appropriately.
Return:
dict[Hashable, Any]: Serialized dataframe dict with original values on success,
or downgraded confidence/comment fields on failure.
"""
message = 'Some data could not be written to OPC servers'
@@ -367,25 +324,14 @@ class OPC(SientiaMonitoring):
return data.to_dict()
async def close(self):
def close(self) -> None:
"""
Gracefully shutdown all OPC server connections and cleanup resources.
Disconnect all tracked OPC repositories and clear in-memory references.
This method ensures proper cleanup of all active OPC server connections
by calling the disconnect method on each repository instance. It's
designed to be called during application shutdown to prevent resource
leaks and ensure clean termination.
The method performs the following cleanup operations:
1. Iterates through all active OPC repository connections
2. Calls disconnect() on each repository instance
3. Allows for graceful connection termination
4. Prevents resource leaks and connection hanging
Note:
This method should be called during application shutdown to ensure
proper cleanup. It handles all active connections regardless of
their current state and provides a clean shutdown experience.
This method should be called during worker shutdown to ensure every
synchronous OPC session is explicitly closed before process exit.
"""
for opc in self.opc_repository.values():
await opc.disconnect()
opc.disconnect()
self.opc_repository.clear()