SIENTIAPDE-1243: Refactor: Rename 'laborious' package to 'model_manager'
This commit renames the 'laborious' package to 'model_manager' across the entire project. This includes renaming directories, modules, references in code, configuration files, and documentation to reflect the new package name. This change improves clarity and consistency within the project.
This commit is contained in:
Bruno Domingues
0
model_manager/activities/__init__.py
Normal file
0
model_manager/activities/__init__.py
Normal file
99
model_manager/activities/activities.py
Normal file
99
model_manager/activities/activities.py
Normal file
@@ -0,0 +1,99 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.temporal.activities.postgres import Postgres
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.observability.logger import Logger
|
||||
from model_manager.activities.mlflow import MLFlow
|
||||
from model_manager.activities.gates import Gates
|
||||
from model_manager.activities.opc import OPC
|
||||
from typing import Any
|
||||
|
||||
|
||||
class Activities(Postgres, MLFlow, Gates, OPC):
|
||||
"""
|
||||
Main activities orchestrator for the Laborious system.
|
||||
|
||||
This class combines functionality from multiple activity classes to provide
|
||||
a unified interface for all workflow operations. It manages database connections,
|
||||
MLFlow model interactions, data quality validation, and OPC server communications.
|
||||
|
||||
The class implements multiple inheritance to combine specialized functionality:
|
||||
- Postgres: Database operations and data persistence
|
||||
- MLFlow: Model inference and transformation operations
|
||||
- Gates: Data quality validation and filtering mechanisms
|
||||
- OPC: Real-time data export to OPC servers
|
||||
|
||||
Attributes:
|
||||
postgres_config (dict): PostgreSQL connection configuration
|
||||
mlflow_config (dict): MLFlow server configuration
|
||||
opc_config (dict): OPC server configuration
|
||||
logger (Logger): Logging and observability instance
|
||||
notification_handler (NotificationHandler): Notification management instance
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
postgres_config: dict[str, Any],
|
||||
mlflow_config: dict[str, Any],
|
||||
opc_config: dict[str, Any],
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler):
|
||||
"""
|
||||
Initialize the Activities orchestrator with all required configurations.
|
||||
|
||||
This constructor initializes all parent classes with their respective
|
||||
configurations and sets up the foundation for all activity operations.
|
||||
|
||||
Args:
|
||||
postgres_config: PostgreSQL connection configuration dictionary
|
||||
Required keys: host, port, user, password, dbname, min_connections, max_connections
|
||||
mlflow_config: MLFlow server configuration dictionary
|
||||
Required keys: host, port, username, password
|
||||
opc_config: OPC server configuration dictionary
|
||||
Can contain multiple server configurations
|
||||
logger: Logger instance for observability and debugging
|
||||
notification_handler: Notification handler for alerts and monitoring
|
||||
|
||||
Raises:
|
||||
Exception: If any parent class initialization fails
|
||||
"""
|
||||
# Initialize parent classes
|
||||
Postgres.__init__(self, host=postgres_config['host'],
|
||||
port=postgres_config['port'],
|
||||
user=postgres_config['user'],
|
||||
password=postgres_config['password'],
|
||||
dbname=postgres_config['dbname'],
|
||||
min_connections=postgres_config['min_connections'],
|
||||
max_connections=postgres_config['max_connections'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler)
|
||||
|
||||
MLFlow.__init__(self, mlflow_host=mlflow_config['host'],
|
||||
mlflow_port=mlflow_config['port'],
|
||||
mlflow_username=mlflow_config['username'],
|
||||
mlflow_password=mlflow_config['password'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler)
|
||||
|
||||
Gates.__init__(self, logger=logger,
|
||||
notification_handler=notification_handler)
|
||||
|
||||
OPC.__init__(self,
|
||||
opc_servers=opc_config,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler)
|
||||
|
||||
async def shutdown(self):
|
||||
"""
|
||||
Gracefully shutdown all activities and clean up resources.
|
||||
|
||||
This method ensures proper cleanup of all resources including:
|
||||
- PostgreSQL connection pools
|
||||
- OPC server connections
|
||||
- Any other resources that need explicit cleanup
|
||||
|
||||
The method should be called before the application terminates to ensure
|
||||
proper resource cleanup and prevent resource leaks.
|
||||
"""
|
||||
Postgres.close(self)
|
||||
await OPC.shutdown(self)
|
||||
587
model_manager/activities/gates.py
Normal file
587
model_manager/activities/gates.py
Normal file
@@ -0,0 +1,587 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
import traceback
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
|
||||
from sientia_do.formatters import create_sample_dict
|
||||
from model_manager.utils.filters.mlflow_filters import nan_values_filter, api_error_filter
|
||||
|
||||
from model_manager.utils.filters.conditional_filters import (
|
||||
filter_empty_data,
|
||||
filter_specific_variables_null_values
|
||||
)
|
||||
from pandas import DataFrame
|
||||
from model_manager import metrics
|
||||
|
||||
# Input filter function mappings
|
||||
input_filter_functions = {
|
||||
'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values,
|
||||
'EMPTY_DATA': filter_empty_data,
|
||||
'path_confidence': {
|
||||
'STOP': -1,
|
||||
'CONTINUE': 2,
|
||||
'REPEAT': -1
|
||||
}
|
||||
}
|
||||
|
||||
# MLFlow response filter function mappings
|
||||
mlflow_response_filter_functions = {
|
||||
'API_ERROR': api_error_filter,
|
||||
'path_confidence': {
|
||||
'STOP': -1,
|
||||
'CONTINUE': 10,
|
||||
'REPEAT': -1
|
||||
},
|
||||
}
|
||||
|
||||
# MLFlow content filter function mappings
|
||||
mlflow_content_filter_functions = {
|
||||
'NAN_VALUES': nan_values_filter,
|
||||
'EMPTY_DATA': filter_empty_data,
|
||||
'path_confidence': {
|
||||
'STOP': -1,
|
||||
'CONTINUE': 18,
|
||||
'REPEAT': -1
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class Gates(BaseActivity):
|
||||
"""
|
||||
Data quality gates and filtering activities for the Laborious system.
|
||||
|
||||
This class implements comprehensive data quality validation and filtering
|
||||
mechanisms that can be applied at different stages of the prediction pipeline.
|
||||
It provides configurable filters with policy-based decision making to ensure
|
||||
data integrity and quality throughout the ML workflow.
|
||||
|
||||
The class supports multiple filter types and implements a flexible policy
|
||||
system that can be configured for different validation requirements. Each
|
||||
filter returns a path decision (STOP, CONTINUE, REPEAT) along with confidence
|
||||
scores and detailed comments for monitoring and debugging.
|
||||
|
||||
Attributes:
|
||||
input_filter_functions (dict): Mapping of input filter names to functions
|
||||
mlflow_response_filter_functions (dict): Mapping of MLFlow response filter names to functions
|
||||
mlflow_content_filter_functions (dict): Mapping of MLFlow content filter names to functions
|
||||
"""
|
||||
|
||||
def __init__(self, logger: Logger, notification_handler: NotificationHandler):
|
||||
"""
|
||||
Initialize data quality gates with logging and notification capabilities.
|
||||
|
||||
Args:
|
||||
logger: Logger instance for observability and debugging
|
||||
notification_handler: Notification handler for alerts and monitoring
|
||||
|
||||
Raises:
|
||||
Exception: If BaseActivity initialization fails
|
||||
"""
|
||||
BaseActivity.__init__(
|
||||
self, logger, notification_handler, set_error_counter=True)
|
||||
|
||||
@activity.defn(name="input_gate")
|
||||
async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
"""
|
||||
Apply input data quality filters and validation.
|
||||
|
||||
This activity validates input data quality using configurable filters
|
||||
before proceeding with ML operations. It applies multiple filter types
|
||||
and returns a path decision based on the filter results and configured
|
||||
policies.
|
||||
|
||||
The method implements a comprehensive filtering system that:
|
||||
1. Applies configured filters to input data
|
||||
2. Evaluates filter results against policy configurations
|
||||
3. Determines appropriate path decisions (STOP, CONTINUE, REPEAT)
|
||||
4. Provides confidence scores and detailed comments
|
||||
5. Handles errors gracefully with notification integration
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for input validation
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- filters (dict): Filter configuration and policies
|
||||
- data (dict): Input data to validate
|
||||
- path_priority (list[str]): Priority order for path decisions
|
||||
|
||||
Returns:
|
||||
tuple: (path_flag, confidence, comment)
|
||||
- path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
|
||||
- confidence (int): Confidence score for the decision
|
||||
- comment (str): Detailed explanation of the decision
|
||||
|
||||
Raises:
|
||||
Exception: If filter execution fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
self.info("Performing input gate...", metadata)
|
||||
|
||||
filters = input_data['filters']
|
||||
data = DataFrame(input_data['data'])
|
||||
path_priority = input_data['path_priority']
|
||||
|
||||
filter_output = []
|
||||
|
||||
self.debug(f"Input data: {data.head(5).to_string()}", metadata)
|
||||
self.debug(f"Filters: {filters}", metadata)
|
||||
|
||||
# Apply each configured filter
|
||||
for fil, config in filters.items():
|
||||
if fil not in input_filter_functions:
|
||||
self.error(f"Filter {fil} not found", metadata)
|
||||
continue
|
||||
try:
|
||||
if input_filter_functions[fil](data, config['config']):
|
||||
self.debug(
|
||||
f"Data not passed the input filter {fil}:{config}", metadata)
|
||||
filter_output.append(config['policy'])
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"INTPUT_GATE_ERROR__{fil}",
|
||||
message=f"Error in filter {fil}:{config}: \n {e}",
|
||||
block="input_gate",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
|
||||
for path_flag in path_priority:
|
||||
if path_flag in filter_output:
|
||||
self.info(f"Input gate result: {path_flag}", metadata)
|
||||
return path_flag, input_filter_functions['path_confidence'][path_flag], \
|
||||
"Input data with bad quality"
|
||||
|
||||
self.info("Nothing was filtered by the input gate", metadata)
|
||||
return None, 0, ""
|
||||
|
||||
@activity.defn(name="mlflow_response_gate")
|
||||
async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
"""
|
||||
Validate MLFlow API response quality and integrity.
|
||||
|
||||
This activity validates MLFlow API responses to ensure they meet quality
|
||||
standards before proceeding with further processing. It applies response-specific
|
||||
filters and determines appropriate path decisions based on response quality.
|
||||
|
||||
The method implements response validation that:
|
||||
1. Applies MLFlow response-specific filters
|
||||
2. Evaluates API response quality and integrity
|
||||
3. Determines path decisions based on response validation results
|
||||
4. Provides confidence scores and detailed validation comments
|
||||
5. Handles API errors and response validation failures
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for response validation
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- filters (dict): Response filter configuration and policies
|
||||
- data (dict): MLFlow API response data to validate
|
||||
- type (str): Type of MLFlow operation (transform, predict)
|
||||
- path_priority (list[str]): Priority order for path decisions
|
||||
|
||||
Returns:
|
||||
tuple: (path_flag, confidence, comment)
|
||||
- path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
|
||||
- confidence (int): Confidence score for the decision
|
||||
- comment (str): Detailed explanation of the decision
|
||||
|
||||
Raises:
|
||||
Exception: If response validation fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info("Performing mlflow response gate...", metadata)
|
||||
|
||||
filters = input_data['filters']
|
||||
data = input_data['data']
|
||||
gate_type = input_data['type']
|
||||
path_priority = input_data['path_priority']
|
||||
|
||||
filter_output = []
|
||||
|
||||
self.debug(
|
||||
f"Input data: \n {create_sample_dict(data, max_items=5, max_depth=2)}", metadata)
|
||||
self.debug(f"Filters: {filters}", metadata)
|
||||
|
||||
comments = []
|
||||
for fil, config in filters.items():
|
||||
if fil not in mlflow_response_filter_functions:
|
||||
self.error(f"Filter {fil} not found", metadata)
|
||||
continue
|
||||
try:
|
||||
if mlflow_response_filter_functions[fil](data, config):
|
||||
filter_output.append(config['policy'])
|
||||
comments.append(data['content']['message'])
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}",
|
||||
message=data['content']['message'],
|
||||
block="mlflow_gate",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=data['content']['traceback']
|
||||
)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"MLFLOW_GATE_RESPONSE_FILTER__{fil}",
|
||||
message=f"Error in filter {fil}:{config}: \n {e}",
|
||||
block="mlflow_gate",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
|
||||
for path_flag in path_priority:
|
||||
if path_flag in filter_output:
|
||||
self.info(
|
||||
f"Mlflow response gate result: {path_flag}", metadata)
|
||||
return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag], \
|
||||
", ".join(comments)
|
||||
|
||||
self.info("Nothing was filtered by the mlflow response gate", metadata)
|
||||
return None, 0, ""
|
||||
|
||||
@activity.defn(name="mlflow_content_gate")
|
||||
async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
"""
|
||||
Validate MLFlow prediction content quality and integrity.
|
||||
|
||||
This activity validates the content of MLFlow predictions to ensure they
|
||||
meet quality standards before export and persistence. It applies content-specific
|
||||
filters and determines appropriate path decisions based on content quality.
|
||||
|
||||
The method implements content validation that:
|
||||
1. Applies MLFlow content-specific filters
|
||||
2. Evaluates prediction content quality and integrity
|
||||
3. Determines path decisions based on content validation results
|
||||
4. Provides confidence scores and detailed validation comments
|
||||
5. Handles content validation failures and quality issues
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for content validation
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- filters (dict): Content filter configuration and policies
|
||||
- data (dict): MLFlow prediction content to validate
|
||||
- type (str): Type of MLFlow operation (transform, predict)
|
||||
- path_priority (list[str]): Priority order for path decisions
|
||||
|
||||
Returns:
|
||||
tuple: (path_flag, confidence, comment)
|
||||
- path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
|
||||
- confidence (int): Confidence score for the decision
|
||||
- comment (str): Detailed explanation of the decision
|
||||
|
||||
Raises:
|
||||
Exception: If content validation fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info("Performing mlflow content gate...", metadata)
|
||||
|
||||
filters = input_data['filters']
|
||||
data = DataFrame(input_data['data'])
|
||||
gate_type = input_data['type']
|
||||
path_priority = input_data['path_priority']
|
||||
|
||||
filter_output = []
|
||||
|
||||
self.debug(f"Input data:\n {data.head(5).to_string()}", metadata)
|
||||
self.debug(f"Filters: \n {create_sample_dict(filters)}", metadata)
|
||||
|
||||
for fil, config in filters.items():
|
||||
if fil not in mlflow_content_filter_functions:
|
||||
continue
|
||||
try:
|
||||
if mlflow_content_filter_functions[fil](data, config):
|
||||
filter_output.append(config['policy'])
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}",
|
||||
message=f"Data not passed the content filter {fil}:{config}",
|
||||
block="mlflow_gate",
|
||||
level=NotificationLevel.WARNING,
|
||||
attachment_content=data.to_string()
|
||||
)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"MLFLOW_GATE_CONTENT_FILTER__{fil}",
|
||||
message=f"Error in filter {fil}:{config}: \n {e}",
|
||||
block="mlflow_gate",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
|
||||
for path_flag in path_priority:
|
||||
if path_flag in filter_output:
|
||||
self.info(
|
||||
f"Mlflow content gate result: {path_flag}", metadata)
|
||||
return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag], \
|
||||
"Transformed data not passed the content filter"
|
||||
|
||||
self.info("Nothing was filtered by the mlflow content gate", metadata)
|
||||
return None, 0, ""
|
||||
|
||||
def get_prediction_store_policy(self,
|
||||
prediction_store_policy: str,
|
||||
metadata: dict[str, Any]) -> tuple[str, int]:
|
||||
"""
|
||||
Parse and validate prediction store policy configuration.
|
||||
|
||||
This method parses prediction store policy strings in the format 'type:value'
|
||||
and validates them against allowed policy types and values. It provides
|
||||
sensible defaults for invalid configurations and logs policy validation
|
||||
failures for operational monitoring.
|
||||
|
||||
Supported Policy Types:
|
||||
- 'lts': Latest timestamp - sorts data by timestamp descending
|
||||
- 'erl': Earliest timestamp - sorts data by timestamp ascending
|
||||
|
||||
Args:
|
||||
prediction_store_policy (str): Policy string in format 'type:value'
|
||||
metadata (dict[str, Any]): Context metadata for logging and notifications
|
||||
|
||||
Returns:
|
||||
tuple[str, int]: (policy_type, policy_value)
|
||||
- policy_type (str): Validated policy type ('lts' or 'erl')
|
||||
- policy_value (int): Number of rows to retain
|
||||
"""
|
||||
policy_elements = prediction_store_policy.split(':')
|
||||
|
||||
if len(policy_elements) < 2:
|
||||
self.error(
|
||||
f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata)
|
||||
return 'lts', 1
|
||||
|
||||
policy_type = policy_elements[0]
|
||||
policy_value = policy_elements[1]
|
||||
|
||||
# If the policy_type is not lts or erl, we use the default policy
|
||||
# If the policty_value is not a number or 0, we use the default policy
|
||||
if policy_type not in ['lts', 'erl'] or not policy_value.isdigit() or int(policy_value) == 0:
|
||||
self.error(
|
||||
f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata)
|
||||
return 'lts', 1
|
||||
|
||||
return policy_type, int(policy_value)
|
||||
|
||||
@activity.defn(name="format_prediction")
|
||||
async def format_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
"""
|
||||
Format prediction data according to configured storage policies.
|
||||
|
||||
This method formats prediction data for storage and export operations.
|
||||
It applies timestamp-based sorting policies, adds metadata fields,
|
||||
and ensures data consistency before persistence. The method supports
|
||||
multiple storage policies for flexible data retention strategies.
|
||||
|
||||
Storage Policies:
|
||||
- 'lts:N': Latest timestamp - retains N most recent predictions
|
||||
- 'erl:N': Earliest timestamp - retains N oldest predictions
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- data (dict[str, Any]): Raw prediction data to format
|
||||
- timestamp (str): Default timestamp if data lacks timestamp column
|
||||
- model_id (str): Unique identifier for the ML model
|
||||
- prediction_confidence (float): Confidence score for the prediction
|
||||
- prediction_store_policy (str): Storage policy in format 'type:value'
|
||||
|
||||
Returns:
|
||||
dict: Formatted prediction data ready for storage and export
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
prediction_store_policy = input_data['prediction_store_policy']
|
||||
self.info("Formatting prediction...", metadata)
|
||||
|
||||
data = DataFrame(input_data['data'])
|
||||
|
||||
# Create timestamp column from index and reset index
|
||||
data['timestamp'] = data.index
|
||||
data = data.reset_index(drop=True)
|
||||
|
||||
self.debug(
|
||||
f"Prediction store policy: {prediction_store_policy}", metadata)
|
||||
self.debug(f"Prediction data: {data.head(5).to_string()}", metadata)
|
||||
|
||||
policy_type, policy_value = self.get_prediction_store_policy(
|
||||
prediction_store_policy, metadata)
|
||||
|
||||
# If data has no timestamp, we use the default timestamp and not sort the data
|
||||
self.info(
|
||||
f"Sorting data by timestamp and applying policy: {policy_type}:{policy_value}", metadata)
|
||||
|
||||
# If policy_type is lts, we need to sort the data by timestamp descending and take the first policy_value rows
|
||||
if policy_type == 'lts':
|
||||
self.debug(
|
||||
"Sorting data by timestamp descending", metadata)
|
||||
data = data.sort_values(by='timestamp', ascending=False)
|
||||
# If policy_type is erl, we need to sort the data by timestamp ascending and take the first policy_value rows
|
||||
elif policy_type == 'erl':
|
||||
self.debug(
|
||||
"Sorting data by timestamp ascending", metadata)
|
||||
data = data.sort_values(by='timestamp', ascending=True)
|
||||
else:
|
||||
self.error(
|
||||
f"Invalid policy type: {policy_type}, using default policy", metadata)
|
||||
raise ValueError(
|
||||
f"Invalid policy type: {policy_type}")
|
||||
|
||||
data = data.head(int(policy_value))
|
||||
|
||||
data['model_id'] = input_data['model_id']
|
||||
data['prediction_confidence'] = input_data['prediction_confidence']
|
||||
data['prediction_status'] = 'Good'
|
||||
data['comments'] = ""
|
||||
data = data.sort_values(by='timestamp', ascending=False)
|
||||
data = data.reset_index(drop=True)
|
||||
|
||||
self.info(f"Prediction formatted: {len(data)} rows", metadata)
|
||||
self.debug(f"Prediction data: {data.head(5).to_string()}", metadata)
|
||||
|
||||
return data.to_dict()
|
||||
|
||||
@activity.defn(name="format_default_prediction")
|
||||
async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
"""
|
||||
Create and format default prediction data for error conditions.
|
||||
|
||||
This method generates default prediction data when the main prediction
|
||||
pipeline encounters errors or quality issues. It creates a standardized
|
||||
data structure with zero values for predictions and useful metadata
|
||||
for operational monitoring and debugging.
|
||||
|
||||
The default prediction serves as a fallback mechanism to:
|
||||
1. Maintain data pipeline continuity during failures
|
||||
2. Provide operational visibility into prediction quality issues
|
||||
3. Enable downstream systems to handle error conditions gracefully
|
||||
4. Support debugging and troubleshooting efforts
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- timestamp (str): Timestamp for the default prediction
|
||||
- model_id (str): Unique identifier for the ML model
|
||||
- prediction_confidence (float): Confidence score (typically low for errors)
|
||||
- comment (str): Error description or operational comment
|
||||
|
||||
Returns:
|
||||
dict: Formatted default prediction data with error indicators
|
||||
"""
|
||||
|
||||
metadata = input_data['metadata']
|
||||
self.debug("Formatting default prediction...", metadata)
|
||||
|
||||
data = DataFrame({
|
||||
'prediction': [0],
|
||||
'response_time': [0],
|
||||
'timestamp': [input_data['timestamp']],
|
||||
'model_id': [input_data['model_id']],
|
||||
'prediction_confidence': [input_data['prediction_confidence']],
|
||||
'prediction_status': ['Bad'],
|
||||
'comments': [input_data['comment']]
|
||||
})
|
||||
|
||||
self.info(f"Default prediction formatted: {data.size} rows", metadata)
|
||||
return data.to_dict()
|
||||
|
||||
@activity.defn(name="get_last_timestamp")
|
||||
async def get_last_timestamp(self, input_data: dict[str, Any]) -> str:
|
||||
"""
|
||||
Extract the most recent timestamp from prediction data.
|
||||
|
||||
This method analyzes prediction data to find the latest timestamp,
|
||||
enabling incremental processing and data continuity tracking.
|
||||
It handles empty datasets gracefully by returning the current time
|
||||
as a fallback timestamp.
|
||||
|
||||
The method is essential for:
|
||||
1. Incremental data processing workflows
|
||||
2. Data continuity validation
|
||||
3. Timestamp-based data loading optimization
|
||||
4. Workflow execution tracking
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- data (dict[str, Any]): Prediction data to analyze
|
||||
|
||||
Returns:
|
||||
str: Formatted timestamp string in UTC with timezone
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
self.info("Getting last timestamp...", metadata)
|
||||
|
||||
data = DataFrame(input_data['data'])
|
||||
|
||||
self.debug(f"Input data: {data.head(5).to_string()}", metadata)
|
||||
|
||||
if data.empty:
|
||||
return now().strftime(DATETIME_FORMAT_WITH_TZ)
|
||||
|
||||
max_timestamp = max(
|
||||
data['timestamp'].values.tolist())
|
||||
|
||||
self.info(
|
||||
f"Last timestamp: {max_timestamp}", metadata)
|
||||
|
||||
return max_timestamp
|
||||
|
||||
@activity.defn(name="write_metrics")
|
||||
async def write_metrics(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Write prediction performance metrics to Prometheus monitoring system.
|
||||
|
||||
This method records comprehensive metrics for prediction operations,
|
||||
enabling operational monitoring, performance analysis, and alerting.
|
||||
It tracks prediction counts, confidence levels, and response times
|
||||
for each model and pipeline combination.
|
||||
|
||||
Metrics Recorded:
|
||||
1. Prediction Count: Incremental counter for successful predictions
|
||||
2. Confidence Monitor: Current confidence level for predictions
|
||||
3. Response Time Monitor: Histogram of prediction response times
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict[str, Any]): Workflow execution metadata
|
||||
- prediction (dict[str, Any]): Prediction data with metrics
|
||||
|
||||
Raises:
|
||||
Exception: If metrics writing fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
prediction = DataFrame(input_data['prediction'])
|
||||
prediction_confidence = prediction['prediction_confidence'].values[0]
|
||||
response_time = prediction['response_time'].values[0]
|
||||
|
||||
self.info(
|
||||
f"Writing metrics for model {metadata['model_name']}", metadata)
|
||||
|
||||
metrics.PREDICTIONS_WRITTEN_COUNT.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name']
|
||||
).inc()
|
||||
|
||||
metrics.PREDICTION_CONFIDENCE_MONITOR.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name']
|
||||
).set(prediction_confidence)
|
||||
|
||||
metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name']
|
||||
).observe(response_time)
|
||||
|
||||
self.info(
|
||||
f"Metrics written for model {metadata['model_name']}", metadata)
|
||||
343
model_manager/activities/mlflow.py
Normal file
343
model_manager/activities/mlflow.py
Normal file
@@ -0,0 +1,343 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import datetime
|
||||
from pandas import Timestamp, to_datetime
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.formatters import create_sample_dict
|
||||
from model_manager.utils.repository.model_repository import MLFlowRepository
|
||||
from typing import Any
|
||||
import numpy as np
|
||||
from pandas import DataFrame
|
||||
import traceback
|
||||
|
||||
|
||||
class MLFlow(BaseActivity):
|
||||
"""
|
||||
MLFlow integration activities for model inference operations.
|
||||
|
||||
This class provides activities for interacting with MLFlow models, including
|
||||
data transformation and prediction operations. It handles authentication,
|
||||
data preprocessing, and model management with configurable retention policies.
|
||||
|
||||
The class implements comprehensive error handling and logging for all
|
||||
MLFlow operations, ensuring reliable model inference in production environments.
|
||||
|
||||
Attributes:
|
||||
mlflow_host (str): MLFlow server hostname
|
||||
mlflow_port (int): MLFlow server port
|
||||
mlflow_username (str): MLFlow authentication username
|
||||
mlflow_password (str): MLFlow authentication password
|
||||
model_monitoring_repository (MLFlowRepository): Repository for MLFlow operations
|
||||
"""
|
||||
|
||||
def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str,
|
||||
mlflow_password: str, logger: Logger, notification_handler: NotificationHandler):
|
||||
"""
|
||||
Initialize MLFlow activities with server configuration.
|
||||
|
||||
Args:
|
||||
mlflow_host: MLFlow server hostname or IP address
|
||||
mlflow_port: MLFlow server port number
|
||||
mlflow_username: Username for MLFlow authentication
|
||||
mlflow_password: Password for MLFlow authentication
|
||||
logger: Logger instance for observability and debugging
|
||||
notification_handler: Notification handler for alerts and monitoring
|
||||
|
||||
Raises:
|
||||
Exception: If MLFlowRepository initialization fails
|
||||
"""
|
||||
BaseActivity.__init__(
|
||||
self, logger, notification_handler, set_error_counter=True)
|
||||
self.mlflow_host = mlflow_host
|
||||
self.mlflow_port = mlflow_port
|
||||
self.mlflow_username = mlflow_username
|
||||
self.mlflow_password = mlflow_password
|
||||
|
||||
self.model_monitoring_repository = MLFlowRepository(
|
||||
f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger
|
||||
)
|
||||
|
||||
@activity.defn(name="request_transform")
|
||||
async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Transform input data using MLFlow models.
|
||||
|
||||
This activity processes input data through MLFlow model transformation,
|
||||
including data preprocessing, format conversion, and validation. It handles
|
||||
data deduplication, pivoting, and cleanup to ensure optimal model performance.
|
||||
|
||||
The transformation process includes:
|
||||
1. Data deduplication based on variable and timestamp
|
||||
2. Data pivoting for model input format
|
||||
3. Null value handling and cleanup
|
||||
4. MLFlow model transformation request
|
||||
5. Response validation and logging
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for transformation
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict): Input data for transformation
|
||||
- model_name (str): Name of the MLFlow model to use
|
||||
- model_retention (int): Model retention period in minutes
|
||||
|
||||
Returns:
|
||||
dict: Transformed data from MLFlow model
|
||||
|
||||
Raises:
|
||||
Exception: If transformation fails or MLFlow model is unavailable
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info('Transforming data...', metadata)
|
||||
data = DataFrame(input_data['data'])
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
|
||||
self.debug("Raw input data:", metadata)
|
||||
self.debug(data.head(5).to_string(), metadata)
|
||||
|
||||
# Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
|
||||
data = data.sort_values('created_at', ascending=False).drop_duplicates(
|
||||
subset=['variable', 'timestamp'], keep='first'
|
||||
)
|
||||
|
||||
# Pivot data for model input format
|
||||
data = data.pivot(
|
||||
index='timestamp', columns='variable',
|
||||
values='value')
|
||||
data.fillna(np.nan, inplace=True)
|
||||
# data.reset_index(inplace=True)
|
||||
data.columns.name = None
|
||||
|
||||
self.debug("Processed input data:", metadata)
|
||||
self.debug(data.head(5).to_string(), metadata)
|
||||
|
||||
# Request transformation from MLFlow model
|
||||
response_data = self.model_monitoring_repository.transform(
|
||||
model_name, data, model_config, metadata
|
||||
)
|
||||
|
||||
self.debug(
|
||||
f"Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata)
|
||||
|
||||
self.debug(
|
||||
f"Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata)
|
||||
|
||||
self.info("Data transformed successfully", metadata)
|
||||
|
||||
return response_data
|
||||
|
||||
@activity.defn(name="request_predict")
|
||||
async def request_predict(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Execute predictions using MLFlow models.
|
||||
|
||||
This activity performs ML model inference using MLFlow models with the
|
||||
transformed data. It handles data format conversion, null value processing,
|
||||
and model prediction requests with comprehensive error handling.
|
||||
|
||||
The prediction process includes:
|
||||
1. Data format validation and cleanup
|
||||
2. Null value handling for model compatibility
|
||||
3. MLFlow model prediction request
|
||||
4. Response validation and logging
|
||||
5. Performance monitoring and metrics
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for prediction
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict): Transformed data for prediction
|
||||
- model_name (str): Name of the MLFlow model to use
|
||||
- model_retention (int): Model retention period in minutes
|
||||
|
||||
Returns:
|
||||
dict: Prediction results from MLFlow model
|
||||
|
||||
Raises:
|
||||
Exception: If prediction fails or MLFlow model is unavailable
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info('Predicting data...', metadata)
|
||||
data = DataFrame(input_data['data'])
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
|
||||
self.debug(f"Input data for: \n {data.head(5).to_string()}", metadata)
|
||||
|
||||
# Convert numpy.nan to None for model compatibility
|
||||
data.replace(np.nan, None, inplace=True)
|
||||
|
||||
data['timestamp'] = data.index
|
||||
data['timestamp'] = to_datetime(
|
||||
data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT)
|
||||
|
||||
# Request prediction from MLFlow model
|
||||
response_data = self.model_monitoring_repository.predict(
|
||||
model_name, data, model_config, metadata
|
||||
)
|
||||
|
||||
self.debug(
|
||||
f"Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata)
|
||||
|
||||
self.info("Data predicted successfully", metadata)
|
||||
|
||||
return response_data
|
||||
|
||||
@activity.defn(name="retrain_model")
|
||||
async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Retrain MLFlow models with updated training data.
|
||||
|
||||
This activity orchestrates the complete model retraining process,
|
||||
including data preparation, model retraining execution, and result
|
||||
validation. It handles data preprocessing, column cleanup, and
|
||||
comprehensive error handling for production model management.
|
||||
|
||||
The retraining process includes:
|
||||
1. Data timestamp extraction and validation
|
||||
2. Column cleanup and data preparation
|
||||
3. Data pivoting for model input format
|
||||
4. MLFlow model retraining execution
|
||||
5. Result validation and error handling
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict[str, Any]): Training data for model retraining
|
||||
- model_name (str): Name of the MLFlow model to retrain
|
||||
|
||||
Returns:
|
||||
dict: Retraining results containing:
|
||||
- status (str): Retraining operation status
|
||||
- timestamp (str): Timestamp of the retraining operation
|
||||
- experiment (str): MLFlow experiment identifier
|
||||
|
||||
Raises:
|
||||
Exception: If retraining fails or encounters critical errors
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
data = DataFrame(input_data['data'])
|
||||
model_name = input_data['model_name']
|
||||
|
||||
self.info(f'Retraining model {model_name}...', metadata)
|
||||
|
||||
timestamp = data['timestamp'].max()
|
||||
self.debug(f'Timestamp: {timestamp}', metadata)
|
||||
|
||||
data.drop(columns=['model_id'], inplace=True, errors='ignore')
|
||||
data.drop(columns=['created_at'], inplace=True, errors='ignore')
|
||||
|
||||
data = data.pivot(index='timestamp', columns='variable',
|
||||
values='value')
|
||||
data.sort_index(inplace=True)
|
||||
data.reset_index(inplace=True)
|
||||
|
||||
data = data.dropna()
|
||||
data.columns.name = None
|
||||
|
||||
try:
|
||||
retrain_output, experiment = self.model_monitoring_repository.retrain_model(
|
||||
data=data,
|
||||
model_name=model_name
|
||||
)
|
||||
|
||||
return {
|
||||
'status': retrain_output,
|
||||
'timestamp': timestamp,
|
||||
'experiment': experiment
|
||||
}
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='RETRAIN_MODEL_ERROR',
|
||||
message=f'Error retraining model {model_name}: {e}',
|
||||
block='retrain_model',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
self.error(trace, metadata=metadata)
|
||||
raise e
|
||||
|
||||
@activity.defn(name="update_production_model")
|
||||
async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
"""
|
||||
Update production model with newly trained model version.
|
||||
|
||||
This activity manages the critical process of updating production
|
||||
models with newly trained versions. It handles model deployment,
|
||||
status tracking, and comprehensive reporting for operational
|
||||
visibility and audit trails.
|
||||
|
||||
The update process includes:
|
||||
1. Production model update execution
|
||||
2. Status and metadata tracking
|
||||
3. Comprehensive reporting and logging
|
||||
4. Error handling and notification
|
||||
5. Audit trail maintenance
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- model_name (str): Name of the MLFlow model to update
|
||||
- experiment (str): MLFlow experiment identifier
|
||||
- model_id (str): Unique identifier for the model version
|
||||
- timestamp (str): Timestamp of the update operation
|
||||
- status (str): Current status of the model update
|
||||
|
||||
Returns:
|
||||
dict[Any, Any]: Comprehensive update report containing:
|
||||
- model_id (str): Model version identifier
|
||||
- model_name (str): Name of the updated model
|
||||
- timestamp (str): Update operation timestamp
|
||||
- status (str): Update operation status
|
||||
- Additional MLFlow response metadata
|
||||
|
||||
Raises:
|
||||
Exception: If production model update fails
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
model_name = input_data['model_name']
|
||||
model_id = input_data['model_id']
|
||||
experiment = input_data['experiment']
|
||||
timestamp = input_data['timestamp']
|
||||
status = input_data['status']
|
||||
|
||||
self.info(
|
||||
f'Updating production model {model_name} from experiment {experiment}...', metadata)
|
||||
|
||||
try:
|
||||
response = self.model_monitoring_repository.update_production_model(
|
||||
experiment=experiment,
|
||||
model_name=model_name
|
||||
)
|
||||
|
||||
report = DataFrame([response])
|
||||
report['model_id'] = model_id
|
||||
report['model_name'] = model_name
|
||||
report['timestamp'] = timestamp
|
||||
report['status'] = status
|
||||
|
||||
self.info(
|
||||
f'Production model {model_name} updated successfully', metadata)
|
||||
return report.to_dict()
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='UPDATE_PRODUCTION_MODEL_ERROR',
|
||||
message=f'Error updating production model {model_name}: {e}',
|
||||
block='update_production_model',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
self.error(trace, metadata=metadata)
|
||||
raise e
|
||||
356
model_manager/activities/opc.py
Normal file
356
model_manager/activities/opc.py
Normal file
@@ -0,0 +1,356 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from sientia_do.observability.logger import Logger
|
||||
from model_manager.utils.repository.opc_repository import OpcRepository
|
||||
from typing import Any
|
||||
import traceback
|
||||
from pandas import DataFrame
|
||||
|
||||
OPC_WRITTING_ERROR_CONFIDENCE = 12
|
||||
|
||||
|
||||
class OPC(BaseActivity):
|
||||
"""
|
||||
OPC server integration activities for real-time data export.
|
||||
|
||||
This class provides comprehensive OPC UA client functionality for connecting
|
||||
to multiple OPC servers and writing prediction data in real-time. It implements
|
||||
secure communication with certificate-based authentication and automatic
|
||||
reconnection capabilities.
|
||||
|
||||
The class supports multiple OPC servers with individual configurations and
|
||||
provides robust error handling and monitoring for production environments.
|
||||
|
||||
Attributes:
|
||||
opc_servers (dict): Configuration for multiple OPC servers
|
||||
opc_repository (dict): Active OPC repository connections
|
||||
logger (Logger): Logging and observability instance
|
||||
notification_handler (NotificationHandler): Notification management instance
|
||||
"""
|
||||
|
||||
def __init__(self, opc_servers: dict[str, dict[str, Any]],
|
||||
logger: Logger, notification_handler: NotificationHandler):
|
||||
|
||||
self.logger = logger
|
||||
self.notification_handler = notification_handler
|
||||
self.opc_servers = opc_servers
|
||||
|
||||
BaseActivity.__init__(
|
||||
self, logger, notification_handler, set_error_counter=True)
|
||||
|
||||
self.opc_repository: dict[str, OpcRepository] = {}
|
||||
self.opc_servers = opc_servers
|
||||
|
||||
async def init_opc(self):
|
||||
"""
|
||||
Initialize OPC server connections and establish communication channels.
|
||||
|
||||
This method iterates through all configured OPC servers and attempts to
|
||||
establish secure connections using certificate-based authentication.
|
||||
Each server connection is managed independently, and connection failures
|
||||
are reported through the notification system.
|
||||
|
||||
The method performs the following operations:
|
||||
1. Creates OpcRepository instances for each configured server
|
||||
2. Establishes secure connections with certificate validation
|
||||
3. Reports connection success/failure through notifications
|
||||
4. Logs connection status for operational visibility
|
||||
|
||||
Raises:
|
||||
Exception: If OPC repository initialization fails or connection
|
||||
establishment encounters critical errors
|
||||
|
||||
Note:
|
||||
Connection failures are logged and reported but do not prevent
|
||||
the initialization of other OPC servers. Each server is handled
|
||||
independently to ensure maximum availability.
|
||||
"""
|
||||
self.logger.info("Initializing OPC servers...")
|
||||
for id, server in self.opc_servers.items():
|
||||
self.opc_repository[id] = OpcRepository(
|
||||
id=server['id'],
|
||||
url=server['url'],
|
||||
logger=self.logger,
|
||||
server_uri=server['server_uri'],
|
||||
cert_path=server['cert_path'],
|
||||
private_key_path=server['private_key_path'],
|
||||
server_cert_path=server['server_cert_path'],
|
||||
notification_handler=self.notification_handler,
|
||||
reconnection_interval=server['reconnection_interval'],
|
||||
pod_id=self.pod_id
|
||||
)
|
||||
is_connected, error_data = await self.opc_repository[id].connect()
|
||||
if not is_connected:
|
||||
self.send_notification(
|
||||
metadata={
|
||||
'model_id': '-',
|
||||
'model_name': '-',
|
||||
'workflow_name': '-',
|
||||
'schedule_name': 'INITIALIZATION'
|
||||
},
|
||||
notification_id=error_data['notification_id'],
|
||||
message=error_data['message'],
|
||||
block=error_data['block'],
|
||||
level=error_data.get('level', NotificationLevel.ERROR),
|
||||
attachment_content=error_data.get(
|
||||
'attachment_content', None)
|
||||
)
|
||||
else:
|
||||
self.logger.info(
|
||||
f"OPC server {id} connected successfully.")
|
||||
|
||||
async def write_data(self, server_id: str, tag: str, data: Any,
|
||||
data_type: str, tag_type: str, metadata: dict[str, Any]) -> bool:
|
||||
"""
|
||||
Write data to a specific OPC server tag with comprehensive error handling.
|
||||
|
||||
This method provides a secure and reliable way to write data to OPC servers
|
||||
with automatic error handling, notification integration, and detailed logging.
|
||||
It validates server availability before attempting write operations and
|
||||
provides comprehensive error reporting for operational monitoring.
|
||||
|
||||
Args:
|
||||
- server_id (str): The id of the OPC server.
|
||||
- tag (str): The tag to write to.
|
||||
- data (Any): The data to write.
|
||||
- data_type (str): The data type.
|
||||
- tag_type (str): The tag type.
|
||||
|
||||
Returns:
|
||||
- bool: True if the data was written successfully, False otherwise.
|
||||
"""
|
||||
|
||||
try:
|
||||
is_success, error_data = await self.opc_repository[server_id].write_data(
|
||||
tag, data, data_type, self.logger, metadata)
|
||||
if not is_success:
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=error_data['notification_id'],
|
||||
message=error_data['message'],
|
||||
block=error_data['block'],
|
||||
level=error_data.get('level', NotificationLevel.ERROR),
|
||||
attachment_content=error_data.get(
|
||||
'attachment_content', None)
|
||||
)
|
||||
return False
|
||||
return True
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"WRITE_OPC_{tag_type.upper()}_ERROR",
|
||||
message=f"Error writing data to OPC server: {e}",
|
||||
block="write_opc_data",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
raise e
|
||||
|
||||
def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool:
|
||||
"""
|
||||
Validate that an OPC server is available and configured for write operations.
|
||||
|
||||
This method checks if the specified OPC server exists in the active
|
||||
repository and is available for data writing operations. It provides
|
||||
immediate feedback for server availability and logs validation failures
|
||||
for operational monitoring.
|
||||
|
||||
Args:
|
||||
server_id (str): Unique identifier for the OPC server to validate
|
||||
metadata (dict[str, Any]): Context metadata for logging and notifications
|
||||
|
||||
Returns:
|
||||
bool: True if server is available, False otherwise
|
||||
|
||||
Note:
|
||||
Server validation failures are automatically reported through the
|
||||
notification system with detailed information about available servers.
|
||||
This helps operators quickly identify configuration issues.
|
||||
"""
|
||||
if self.opc_repository.get(server_id) is None:
|
||||
message = f"OPC server {server_id} not found to perform write operation."
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id="OPC_SERVER_NOT_FOUND",
|
||||
message=message,
|
||||
block="write_opc_data",
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=f"OPC servers: {list(self.opc_repository.keys())}"
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
async def manage_output_tags(
|
||||
self, server_id: str, config: dict[str, Any], data: DataFrame,
|
||||
metadata: dict[str, Any], success: bool) -> tuple[bool, int]:
|
||||
"""
|
||||
Manage the writing of prediction and confidence data to OPC server tags.
|
||||
|
||||
This method orchestrates the writing of multiple data types to OPC servers
|
||||
based on configuration. It handles both prediction data and confidence
|
||||
values independently, allowing for flexible tag configuration and
|
||||
comprehensive error handling.
|
||||
|
||||
The method supports two main tag types:
|
||||
1. Prediction tags: Write actual prediction values to configured OPC tags
|
||||
2. Confidence tags: Write confidence scores to separate OPC tags
|
||||
|
||||
Args:
|
||||
server_id (str): Unique identifier for the target OPC server
|
||||
config (dict[str, Any]): OPC tag configuration containing:
|
||||
- prediction_tags (dict, optional): Prediction tag configurations
|
||||
- confidence_tags (dict, optional): Confidence tag configurations
|
||||
data (DataFrame): DataFrame containing prediction and confidence data
|
||||
metadata (dict[str, Any]): Context metadata for logging and notifications
|
||||
success (bool): Current success status to maintain across operations
|
||||
|
||||
Returns:
|
||||
tuple[bool, int]: (overall_success, total_tags_written)
|
||||
- overall_success: True if all configured tags were written successfully
|
||||
- total_tags_written: Count of successfully written tags
|
||||
"""
|
||||
|
||||
count = 0
|
||||
if 'prediction_tags' in config:
|
||||
for tag, tag_config in config['prediction_tags'].items():
|
||||
local_success = await self.write_data(
|
||||
server_id=server_id,
|
||||
tag=tag,
|
||||
data=data.head(1)['prediction'].values[0],
|
||||
data_type=tag_config['data_type'],
|
||||
tag_type='prediction',
|
||||
metadata=metadata
|
||||
)
|
||||
if local_success:
|
||||
self.info(
|
||||
f"Prediction data written to OPC server {server_id} for tag {tag}.", metadata)
|
||||
count += 1
|
||||
success = success and local_success
|
||||
|
||||
if 'confidence_tags' in config:
|
||||
for tag, tag_config in config['confidence_tags'].items():
|
||||
local_success = await self.write_data(
|
||||
server_id=server_id,
|
||||
tag=tag,
|
||||
data=data.head(1)['prediction_confidence'].values[0],
|
||||
data_type=tag_config['data_type'],
|
||||
tag_type='confidence',
|
||||
metadata=metadata
|
||||
)
|
||||
if local_success:
|
||||
self.info(
|
||||
f"Confidence data written to OPC server {server_id} for tag {tag}.", metadata)
|
||||
count += 1
|
||||
success = success and local_success
|
||||
|
||||
return success, count
|
||||
|
||||
@activity.defn(name='write_opc_data')
|
||||
async def write_opc_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
"""
|
||||
Write prediction and confidence data to OPC servers. The two writing
|
||||
operations are optional and independent of each other.
|
||||
|
||||
Args:
|
||||
- input_data(dict[str, Any]): The input data. Contains the following keys:
|
||||
- data(dict[str, Any]): The dataframe that contains the data to write
|
||||
to the OPC servers.
|
||||
- opc_output_config(dict[str, Any]): The OPC writing configuration.
|
||||
The keys are the OPC server names and the values contain:
|
||||
- prediction_tags(dict[str, Any]): The tags to write to the OPC servers.
|
||||
- confidence_tags(dict[str, Any]): The tags to write to the OPC servers.
|
||||
|
||||
Returns:
|
||||
- dict[Any, Any]: The data that was written to the OPC servers.
|
||||
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info("Writing data to OPC servers...", metadata)
|
||||
data = DataFrame(input_data['data'])
|
||||
opc_output_config = input_data['opc_output_config']
|
||||
self.info(f"Data to write: {data.size} rows", metadata)
|
||||
|
||||
success = True
|
||||
|
||||
for server_id, config in opc_output_config.items():
|
||||
|
||||
if not self.validate_server(server_id, metadata):
|
||||
success = False
|
||||
continue
|
||||
|
||||
local_success, local_count = await self.manage_output_tags(
|
||||
server_id, config, data, metadata, success)
|
||||
success = success and local_success
|
||||
|
||||
self.info(
|
||||
f"Process completed for OPC server {server_id}: {local_count} of {len(config['prediction_tags'])} prediction tags and {len(config['confidence_tags'])} confidence tags", metadata)
|
||||
|
||||
return self.process_confidence(data, success, metadata)
|
||||
|
||||
def process_confidence(self, data: DataFrame, success: bool, metadata: dict[str, Any]) -> dict[Any, Any]:
|
||||
"""
|
||||
Process prediction confidence based on OPC write operation success.
|
||||
|
||||
This method updates the prediction confidence values in the DataFrame
|
||||
based on the success status of OPC server write operations. If any
|
||||
write operations failed, it sets the confidence to a predefined error
|
||||
value to indicate data quality issues.
|
||||
|
||||
The method implements a confidence degradation strategy:
|
||||
- Success: Maintains original confidence values
|
||||
- Failure: Sets confidence to error value for operational awareness
|
||||
|
||||
Args:
|
||||
data (DataFrame): DataFrame containing prediction and confidence data
|
||||
success (bool): Overall success status of OPC write operations
|
||||
metadata (dict[str, Any]): Context metadata for logging and notifications
|
||||
|
||||
Returns:
|
||||
dict[Any, Any]: Processed data as a dictionary with updated confidence values
|
||||
|
||||
Note:
|
||||
The error confidence value (OPC_WRITTING_ERROR_CONFIDENCE = 12) is
|
||||
used to indicate that data was not successfully exported to OPC servers.
|
||||
This allows downstream systems to handle data quality appropriately.
|
||||
"""
|
||||
|
||||
if not success:
|
||||
data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE
|
||||
self.debug(
|
||||
f"Some data could not be written to OPC servers, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.",
|
||||
metadata
|
||||
)
|
||||
|
||||
else:
|
||||
self.debug("Data written to OPC servers successfully.", metadata)
|
||||
|
||||
return data.to_dict()
|
||||
|
||||
async def shutdown(self):
|
||||
"""
|
||||
Gracefully shutdown all OPC server connections and cleanup resources.
|
||||
|
||||
This method ensures proper cleanup of all active OPC server connections
|
||||
by calling the disconnect method on each repository instance. It's
|
||||
designed to be called during application shutdown to prevent resource
|
||||
leaks and ensure clean termination.
|
||||
|
||||
The method performs the following cleanup operations:
|
||||
1. Iterates through all active OPC repository connections
|
||||
2. Calls disconnect() on each repository instance
|
||||
3. Allows for graceful connection termination
|
||||
4. Prevents resource leaks and connection hanging
|
||||
|
||||
Note:
|
||||
This method should be called during application shutdown to ensure
|
||||
proper cleanup. It handles all active connections regardless of
|
||||
their current state and provides a clean shutdown experience.
|
||||
"""
|
||||
for opc in self.opc_repository.values():
|
||||
await opc.disconnect()
|
||||
Reference in New Issue
Block a user