Code import - branch release/SIENTIAPDE-1646
This commit is contained in:
0
laborious/utils/__init__.py
Normal file
0
laborious/utils/__init__.py
Normal file
127
laborious/utils/connectors_config.py
Normal file
127
laborious/utils/connectors_config.py
Normal file
@@ -0,0 +1,127 @@
|
||||
import json
|
||||
from os import getenv
|
||||
from typing import Any
|
||||
|
||||
|
||||
def build_mlflow_config() -> dict[str, Any]:
|
||||
"""
|
||||
Read MLflow tracking and registry credentials from the environment.
|
||||
|
||||
Used by ``Activities`` when constructing ``SientiaMLflowRepository``. The ``url`` value is the
|
||||
same string workers and notebooks should use for ``MLFLOW_TRACKING_URI``-style clients.
|
||||
|
||||
Environment Variables:
|
||||
MLFLOW_URL: Host with scheme
|
||||
MLFLOW_USERNAME: Basic-auth or service user (default: aignosi)
|
||||
MLFLOW_PASSWORD: Password or token (default: aignosi)
|
||||
|
||||
Return:
|
||||
dict[str, Any]: ``url``, ``username``, ``password``.
|
||||
"""
|
||||
return {
|
||||
'url': getenv('MLFLOW_URL', 'http://localhost:5080'),
|
||||
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
||||
'password': getenv('MLFLOW_PASSWORD', 'aignosi'),
|
||||
}
|
||||
|
||||
|
||||
def build_plugin_store_config() -> dict[str, Any]:
|
||||
"""
|
||||
Collect settings for ``PluginStore`` (Git-backed catalog + runtime install via pip).
|
||||
|
||||
Mirrors the model-manager service: the worker passes these kwargs into ``PluginStore`` after
|
||||
``install_runtime`` resolves wheels from the configured PyPI index. Missing optional env vars
|
||||
become ``None`` so the store can run without auth in local dev.
|
||||
|
||||
Environment Variables:
|
||||
STORE_BASE_URL: Git HTTP(S) server (e.g. Gitea) base URL (default: http://localhost:3000)
|
||||
STORE_OWNER: Namespace or org owning the store repo (default: sientia)
|
||||
STORE_REPO: Repository name (default: model-library-store)
|
||||
STORE_BRANCH: Checkout branch; unset lets the client use default
|
||||
STORE_USERNAME / STORE_PASSWORD: HTTP basic credentials for Git fetch
|
||||
STORE_CACHE_TTL_SECONDS: Optional integer seconds for metadata cache TTL
|
||||
PYPI_SERVER: Index URL for ``pip install`` during runtime install (default: http://localhost:5000)
|
||||
PYPI_USERNAME / PYPI_PASSWORD: Optional index authentication
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Keys aligned with ``PluginStore`` constructor parameter names.
|
||||
"""
|
||||
|
||||
cache_ttl_seconds = getenv('STORE_CACHE_TTL_SECONDS')
|
||||
return {
|
||||
'base_url': getenv('STORE_BASE_URL', 'http://localhost:3000'),
|
||||
'owner': getenv('STORE_OWNER', 'sientia'),
|
||||
'repo': getenv('STORE_REPO', 'model-library-store'),
|
||||
'username': getenv('STORE_USERNAME'),
|
||||
'password': getenv('STORE_PASSWORD'),
|
||||
'branch': getenv('STORE_BRANCH'),
|
||||
'cache_ttl_seconds': int(cache_ttl_seconds) if cache_ttl_seconds else None,
|
||||
'pypi_index_url': getenv('PYPI_SERVER', 'http://localhost:5000'),
|
||||
'pypi_username': getenv('PYPI_USERNAME'),
|
||||
'pypi_password': getenv('PYPI_PASSWORD'),
|
||||
}
|
||||
|
||||
|
||||
def build_opc_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build OPC server configuration from environment variables.
|
||||
|
||||
This function constructs an OPC server configuration dictionary from
|
||||
environment variables. It supports both single server and multi-server
|
||||
configurations with flexible parameter handling.
|
||||
|
||||
Environment Variables:
|
||||
OPC_CONFIG: JSON string containing multiple OPC server configurations
|
||||
OPC_ID: OPC server ID (fallback, default: 1)
|
||||
OPC_URL: Single OPC server URL (fallback, default: opc.tcp://localhost:4840)
|
||||
OPC_SERVER_URI: Single OPC server URI (fallback, default: opc.tcp://localhost:4840)
|
||||
OPC_CERT_PATH: Client certificate path (fallback, default: None)
|
||||
OPC_PRIVATE_KEY_PATH: Client private key path (fallback, default: None)
|
||||
OPC_SERVER_CERT_PATH: Server certificate path (fallback, default: None)
|
||||
OPC_RECONNECTION_INTERVAL: Reconnection interval in seconds (fallback, default: 120)
|
||||
|
||||
Returns:
|
||||
dict: OPC server configuration dictionary
|
||||
"""
|
||||
opc_raw = getenv('OPC_CONFIG', None)
|
||||
|
||||
if opc_raw:
|
||||
return json.loads(opc_raw)
|
||||
|
||||
return {
|
||||
getenv('OPC_ID', '1'): {
|
||||
'id': getenv('OPC_ID', '1'),
|
||||
'server_name': getenv('OPC_SERVER_NAME', 'default_server'),
|
||||
'url': getenv('OPC_URL', 'opc.tcp://localhost:4840'),
|
||||
'server_uri': getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'),
|
||||
'cert_path': getenv('OPC_CERT_PATH', None),
|
||||
'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None),
|
||||
'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None),
|
||||
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120')),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def build_minio_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build MinIO (S3-compatible) configuration from environment variables.
|
||||
|
||||
Environment Variables:
|
||||
MINIO_ENDPOINT_URL: Host:port or URL for the S3 API (default: http://localhost:9000)
|
||||
MINIO_ACCESS_KEY: Access key (default: minioadmin)
|
||||
MINIO_SECRET_KEY: Secret key (default: minioadmin)
|
||||
MINIO_DEFAULT_BUCKET: Default bucket for Laborious payloads (default: laborious)
|
||||
MINIO_RETENTION_HOURS: Offloaded object retention window (default: 24)
|
||||
MINIO_SECURE: If ``true``, use HTTPS (default: false)
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Keys consumed by ``Activities`` / ``MinioRepository``.
|
||||
"""
|
||||
return {
|
||||
'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'),
|
||||
'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'),
|
||||
'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'),
|
||||
'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious'),
|
||||
'retention_hours': int(getenv('MINIO_RETENTION_HOURS', '24')),
|
||||
'secure': getenv('MINIO_SECURE', 'false') == 'true',
|
||||
}
|
||||
34
laborious/utils/dataframe_debug.py
Normal file
34
laborious/utils/dataframe_debug.py
Normal file
@@ -0,0 +1,34 @@
|
||||
from typing import Any
|
||||
|
||||
from pandas import DataFrame
|
||||
|
||||
DEFAULT_MAX_DEBUG_DATAFRAME_ROWS = 100
|
||||
|
||||
|
||||
def build_dataframe_debug_message(
|
||||
message: str,
|
||||
data: Any,
|
||||
max_rows: int = DEFAULT_MAX_DEBUG_DATAFRAME_ROWS,
|
||||
) -> str:
|
||||
"""
|
||||
Build a safe debug message for dataframe payloads
|
||||
|
||||
Args:
|
||||
- message (str): Base message to identify the logged payload
|
||||
- data (Any): Payload to evaluate for dataframe-aware logging
|
||||
- max_rows (int): Maximum dataframe row count allowed for full payload logging
|
||||
|
||||
Return:
|
||||
Formatted debug message with full dataframe content or compact summary
|
||||
"""
|
||||
if not isinstance(data, DataFrame):
|
||||
return f'{message} {data}'
|
||||
|
||||
rows = data.shape[0]
|
||||
if rows <= max_rows:
|
||||
return f'{message}\n{data.to_csv()}'
|
||||
|
||||
return (
|
||||
f'{message} skipped because dataframe has {rows} rows '
|
||||
f'(max: {max_rows}). Shape: {data.shape}'
|
||||
)
|
||||
0
laborious/utils/filters/__init__.py
Normal file
0
laborious/utils/filters/__init__.py
Normal file
48
laborious/utils/filters/conditional_filters.py
Normal file
48
laborious/utils/filters/conditional_filters.py
Normal file
@@ -0,0 +1,48 @@
|
||||
from pandas import DataFrame
|
||||
|
||||
|
||||
def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool:
|
||||
"""
|
||||
Filter to check if specific variables contain null values.
|
||||
|
||||
This function examines a DataFrame to determine if any of the specified variables
|
||||
contain null (NaN) values. It returns True if null values are found for any of
|
||||
the specified variables, False otherwise.
|
||||
|
||||
Args:
|
||||
data (DataFrame): The pandas DataFrame to be examined. Must contain columns
|
||||
named 'variable' and 'value'.
|
||||
config (dict): Configuration dictionary containing the following key:
|
||||
- variables (list): List of variable names to check for null values
|
||||
|
||||
Returns:
|
||||
bool: True if any of the specified variables contain null values,
|
||||
False if none of the specified variables contain null values.
|
||||
|
||||
"""
|
||||
|
||||
if data.empty:
|
||||
return False
|
||||
|
||||
return not data[data['variable'].isin(config['variables']) & data['value'].isna()].empty
|
||||
|
||||
|
||||
def filter_empty_data(data: DataFrame, _config: dict) -> bool:
|
||||
"""
|
||||
Filter to check if the DataFrame is empty.
|
||||
|
||||
This function determines whether the provided DataFrame contains any data.
|
||||
It's a simple utility function that can be used in conditional logic to
|
||||
handle cases where no data is available.
|
||||
|
||||
Args:
|
||||
data (DataFrame): The pandas DataFrame to be checked for emptiness.
|
||||
_config (dict): Configuration dictionary (unused in this function).
|
||||
The underscore prefix indicates this parameter is required for
|
||||
interface consistency but not used in the implementation.
|
||||
|
||||
Returns:
|
||||
bool: True if the DataFrame is empty (has no rows), False if it contains data.
|
||||
|
||||
"""
|
||||
return data.empty
|
||||
64
laborious/utils/filters/mlflow_filters.py
Normal file
64
laborious/utils/filters/mlflow_filters.py
Normal file
@@ -0,0 +1,64 @@
|
||||
import numpy as np
|
||||
from pandas import DataFrame
|
||||
|
||||
|
||||
def api_error_filter(response: dict, _config: dict) -> bool:
|
||||
"""
|
||||
Filter MLFlow API responses for error conditions.
|
||||
|
||||
This function analyzes MLFlow API responses to detect error conditions
|
||||
and determine if the response should be filtered out due to quality
|
||||
or reliability issues.
|
||||
|
||||
|
||||
Args:
|
||||
response: MLFlow API response data (dict)
|
||||
_config: Filter configuration dictionary
|
||||
Required keys:
|
||||
- error_codes (list, optional): List of error codes to detect
|
||||
- error_keywords (list, optional): List of error keywords to detect
|
||||
- check_structure (bool, optional): Whether to validate response structure
|
||||
|
||||
Returns:
|
||||
bool: True if data should be filtered (contains errors), False otherwise
|
||||
|
||||
"""
|
||||
if not response:
|
||||
return True
|
||||
|
||||
if not response['success']:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def nan_values_filter(predictions: DataFrame, _config: dict) -> bool:
|
||||
"""
|
||||
Filter data for NaN (Not a Number) values.
|
||||
|
||||
This function detects NaN values in MLFlow prediction results and
|
||||
determines if the data quality is sufficient for further processing
|
||||
or export operations.
|
||||
|
||||
Args:
|
||||
predictions: DataFrame containing prediction data to check for NaN values
|
||||
_config: Filter configuration dictionary
|
||||
Required keys:
|
||||
- max_nan_ratio (float, optional): Maximum allowed NaN value ratio (0.0 to 1.0)
|
||||
- max_nan_count (int, optional): Maximum allowed NaN value count
|
||||
- check_nested (bool, optional): Whether to check nested data structures
|
||||
|
||||
Returns:
|
||||
bool: True if data should be filtered (too many NaN values), False otherwise
|
||||
|
||||
"""
|
||||
data = (
|
||||
predictions.replace({None: np.nan})
|
||||
.drop(columns=['timestamp'], errors='ignore')
|
||||
.infer_objects()
|
||||
)
|
||||
|
||||
if data.isna().all().all():
|
||||
return True
|
||||
|
||||
return False
|
||||
0
laborious/utils/models/__init__.py
Normal file
0
laborious/utils/models/__init__.py
Normal file
362
laborious/utils/models/minio_dataframe_payload.py
Normal file
362
laborious/utils/models/minio_dataframe_payload.py
Normal file
@@ -0,0 +1,362 @@
|
||||
"""
|
||||
MinIO-backed DataFrame payload for Temporal workflows.
|
||||
|
||||
Data is never stored as a pandas ``DataFrame`` field on the dataclass.
|
||||
Instead, the DataFrame is only provided as an input to:
|
||||
`from_dataframe` / `from_dataframe_to_dict`.
|
||||
|
||||
At build time, the DataFrame is evaluated for its serialized size; if it exceeds
|
||||
the configured threshold, it is serialized to parquet bytes and uploaded to MinIO.
|
||||
Otherwise, it is inlined as a Temporal-friendly ``dict``.
|
||||
"""
|
||||
|
||||
import pickle
|
||||
import re
|
||||
from collections.abc import Hashable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from os import getenv
|
||||
from typing import Any, Literal
|
||||
|
||||
from pandas import DataFrame, read_parquet
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.repository.minio_repository_sync import MinioRepository
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
|
||||
|
||||
# Keys that are part of the serialized wire format (not arbitrary metadata).
|
||||
_SERIALIZED_FIELD_KEYS = frozenset({'data', 'bucket', 'object_key', 'object_prefix', 'uri'})
|
||||
|
||||
_OBJECT_TIMESTAMP_PATTERN = re.compile(
|
||||
r'-(?:initial|transform)-(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.parquet$'
|
||||
)
|
||||
|
||||
OFFLOAD_THRESHOLD_BYTES = int(
|
||||
float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1.5')) * 1024 * 1024
|
||||
)
|
||||
|
||||
# Relative prefix used for storing offloaded prediction datasets in MinIO.
|
||||
# It is also the root directory for retention cleanup listing.
|
||||
PREDICTION_DATASETS_PREFIX = 'prediction_datasets'
|
||||
|
||||
OperationKind = Literal['initial', 'transform', 'predict']
|
||||
|
||||
|
||||
def _build_object_key(
|
||||
model_name: str, operation: OperationKind, timestamp: str
|
||||
) -> tuple[str, str | None]:
|
||||
"""
|
||||
Build the MinIO object key and the directory prefix used for retention listing.
|
||||
|
||||
Args:
|
||||
model_name: Registered model name used in the pipeline.
|
||||
operation: Either initial (pre-transform load) or transform (post-MLFlow transform).
|
||||
timestamp: Filename timestamp segment from DATETIME_FORMAT_FILENAME.
|
||||
|
||||
Return:
|
||||
tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root).
|
||||
"""
|
||||
# Naming convention:
|
||||
# - Directory is always `prediction_datasets/<model_name>`
|
||||
# - Filename follows the retention-parsing pattern
|
||||
basename = f'{model_name}-{operation}-{timestamp}.parquet'
|
||||
model_dir = model_name.strip().strip('/')
|
||||
prefix = f'{PREDICTION_DATASETS_PREFIX}/{model_dir}'
|
||||
return f'{prefix}/{basename}', prefix
|
||||
|
||||
|
||||
@dataclass
|
||||
class MinioDataFramePayload:
|
||||
"""
|
||||
Serializable payload after a DataFrame was evaluated: inline tabular dict and/or MinIO keys.
|
||||
|
||||
Build from a live DataFrame only via `from_dataframe` / `from_dataframe_to_dict`.
|
||||
Rehydrate from Temporal via `from_dict`. The DataFrame is not a field on this class.
|
||||
"""
|
||||
|
||||
last_timestamp: str
|
||||
status: dict[str, Any] | None = None
|
||||
data: dict[Hashable, Any] | None = None
|
||||
bucket: str | None = None
|
||||
object_key: str | None = None
|
||||
object_prefix: str | None = None
|
||||
uri: str | None = None
|
||||
|
||||
@staticmethod
|
||||
def _debug(
|
||||
logger: Logger | None,
|
||||
message: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Emit a debug message only when a logger instance is available.
|
||||
|
||||
Args:
|
||||
- logger (Logger | None): Logger instance used for debug messages
|
||||
- message (str): Message to be logged
|
||||
- metadata (dict[str, Any] | None): Optional workflow metadata context
|
||||
"""
|
||||
if logger is None:
|
||||
return
|
||||
logger.custom_debug(message, metadata)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: 'dict[str, Any] | MinioDataFramePayload') -> 'MinioDataFramePayload':
|
||||
"""
|
||||
Reconstruct a MinioDataFramePayload from a plain dict produced by Temporal serialization.
|
||||
|
||||
Temporal converts dataclass return values into plain dicts when crossing
|
||||
workflow/activity boundaries. This method rebuilds the typed instance so
|
||||
that methods like ``retrieve``, ``cleanup_prefix`` and ``has_data`` are
|
||||
available on the receiving side.
|
||||
|
||||
If the argument is already a MinioDataFramePayload, it is returned as-is.
|
||||
|
||||
Args:
|
||||
raw: Dict with keys matching the dataclass fields
|
||||
(last_timestamp, status, data, bucket, object_key, object_prefix, uri),
|
||||
or an existing MinioDataFramePayload instance.
|
||||
|
||||
Return:
|
||||
MinioDataFramePayload: Reconstructed (or original) instance.
|
||||
"""
|
||||
if isinstance(raw, MinioDataFramePayload):
|
||||
return raw
|
||||
return cls(
|
||||
last_timestamp=raw['last_timestamp'],
|
||||
status=raw.get('status'),
|
||||
data=raw.get('data'),
|
||||
bucket=raw.get('bucket'),
|
||||
object_key=raw.get('object_key'),
|
||||
object_prefix=raw.get('object_prefix'),
|
||||
uri=raw.get('uri'),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def estimate_size_bytes(
|
||||
df: DataFrame,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
logger: Logger | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Approximate serialized size of the DataFrame as the default-orient dict.
|
||||
|
||||
Args:
|
||||
df: DataFrame whose tabular content size is estimated.
|
||||
|
||||
Return:
|
||||
int: Estimated size in bytes (pickle of dict representation).
|
||||
"""
|
||||
try:
|
||||
size = len(pickle.dumps(df.to_dict()))
|
||||
except Exception:
|
||||
size = len(pickle.dumps(df))
|
||||
|
||||
MinioDataFramePayload._debug(
|
||||
logger,
|
||||
f'DataFrame size: {size} bytes',
|
||||
metadata,
|
||||
)
|
||||
return size
|
||||
|
||||
@staticmethod
|
||||
def parse_object_timestamp(object_key: str) -> datetime | None:
|
||||
"""
|
||||
Parse the timestamp embedded in the object key basename (before .parquet).
|
||||
|
||||
Args:
|
||||
object_key: S3/MinIO object key whose basename follows
|
||||
``{model}-{initial|transform}-{DATETIME_FORMAT_FILENAME}.parquet``.
|
||||
|
||||
Return:
|
||||
datetime | None: Parsed UTC-naive datetime from the key, or None if not matched.
|
||||
"""
|
||||
basename = object_key.rsplit('/', 1)[-1]
|
||||
match = _OBJECT_TIMESTAMP_PATTERN.search(basename)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(match.group(1), DATETIME_FORMAT_FILENAME)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def cleanup_prefix(self) -> str | None:
|
||||
"""
|
||||
Return the MinIO prefix eligible for retention cleanup.
|
||||
|
||||
Cleanup is only applicable when payload data was offloaded to MinIO
|
||||
(``object_key`` present and inline ``data`` absent). Inline-only payloads
|
||||
return ``None`` because there is no object tree to prune.
|
||||
|
||||
Return:
|
||||
str | None: Prefix used by cleanup listing, or ``None`` when cleanup does not apply.
|
||||
"""
|
||||
if self.object_key is not None and self.data is None:
|
||||
return self.object_prefix
|
||||
return None
|
||||
|
||||
def has_data(self) -> bool:
|
||||
"""
|
||||
Indicate whether the payload contains retrievable tabular content.
|
||||
|
||||
A payload is considered non-empty when either inline ``data`` exists
|
||||
(and is not an empty dict) or an ``object_key`` is available for MinIO
|
||||
download.
|
||||
|
||||
Return:
|
||||
bool: ``True`` when data can be retrieved, ``False`` otherwise.
|
||||
"""
|
||||
return (self.data is not None and self.data != {}) or self.object_key is not None
|
||||
|
||||
@classmethod
|
||||
def from_dataframe(
|
||||
cls,
|
||||
dataframe: DataFrame | None,
|
||||
minio_repo: MinioRepository,
|
||||
model_name: str,
|
||||
operation: OperationKind,
|
||||
status: dict[str, Any] | None = None,
|
||||
workflow_metadata: dict | None = None,
|
||||
last_timestamp: str | None = None,
|
||||
logger: Logger | None = None,
|
||||
) -> 'MinioDataFramePayload':
|
||||
"""
|
||||
Evaluate the DataFrame size, then either inline dict or upload parquet to MinIO.
|
||||
|
||||
The DataFrame is not stored on the returned instance.
|
||||
|
||||
Args:
|
||||
dataframe: Tabular data to evaluate and persist (inline or MinIO).
|
||||
metadata: Small metadata dict merged into the payload (e.g. success, message).
|
||||
minio_repo: sientia_do MinioRepository (or compatible) with `upload_file()`.
|
||||
workflow_metadata: Metadata passed to MinIO store for logging/metrics.
|
||||
model_name: Registered model name used in the object basename.
|
||||
operation: Either ``initial`` (query load) or ``transform`` (post-transform).
|
||||
key_prefix: Backward-compatible parameter (currently ignored for object naming).
|
||||
size_threshold_bytes: Byte limit before offload. When None, the module-level
|
||||
environment-derived default is used.
|
||||
|
||||
Return:
|
||||
MinioDataFramePayload: Instance with data and/or MinIO fields set.
|
||||
"""
|
||||
|
||||
if dataframe is None or dataframe.empty:
|
||||
cls._debug(
|
||||
logger,
|
||||
'MinioDataFramePayload.from_dataframe received empty dataframe, returning empty payload',
|
||||
workflow_metadata,
|
||||
)
|
||||
return cls(
|
||||
data=None, last_timestamp=now().strftime(DATETIME_FORMAT_WITH_TZ), status=status
|
||||
)
|
||||
|
||||
if last_timestamp is None:
|
||||
last_timestamp = max(dataframe['timestamp'].values.tolist())
|
||||
|
||||
dataframe_size = cls.estimate_size_bytes(dataframe, workflow_metadata, logger)
|
||||
cls._debug(
|
||||
logger,
|
||||
(
|
||||
f'MinioDataFramePayload.from_dataframe estimated size: {dataframe_size} bytes '
|
||||
f'(threshold: {OFFLOAD_THRESHOLD_BYTES} bytes)'
|
||||
),
|
||||
workflow_metadata,
|
||||
)
|
||||
|
||||
if dataframe_size <= OFFLOAD_THRESHOLD_BYTES:
|
||||
cls._debug(
|
||||
logger,
|
||||
'MinioDataFramePayload.from_dataframe using inline payload',
|
||||
workflow_metadata,
|
||||
)
|
||||
return cls(data=dataframe.to_dict(), last_timestamp=last_timestamp, status=status)
|
||||
|
||||
timestamp = now().strftime(DATETIME_FORMAT_FILENAME)
|
||||
object_key, object_prefix = _build_object_key(model_name, operation, timestamp)
|
||||
cls._debug(
|
||||
logger,
|
||||
(
|
||||
'MinioDataFramePayload.from_dataframe offloading payload to MinIO '
|
||||
f'with key {object_key}'
|
||||
),
|
||||
workflow_metadata,
|
||||
)
|
||||
|
||||
# Upload using the relative object key. The upstream repository will
|
||||
# prefix it internally under its MinIO namespace.
|
||||
parquet_buffer = BytesIO()
|
||||
dataframe.to_parquet(parquet_buffer, engine='pyarrow', index=True)
|
||||
file_bytes = parquet_buffer.getvalue()
|
||||
|
||||
upload_result = minio_repo.upload_file(
|
||||
file_bytes=file_bytes,
|
||||
relative_key=object_key,
|
||||
metadata=workflow_metadata,
|
||||
)
|
||||
|
||||
bucket = minio_repo.bucket
|
||||
object_key_full = upload_result.get('minio_object_name', object_key)
|
||||
uri = f's3://{bucket}/{object_key_full}' if bucket else None
|
||||
cls._debug(
|
||||
logger,
|
||||
f'MinioDataFramePayload.from_dataframe upload completed: {uri}',
|
||||
workflow_metadata,
|
||||
)
|
||||
|
||||
return cls(
|
||||
data=None,
|
||||
bucket=bucket,
|
||||
object_key=object_key_full,
|
||||
object_prefix=object_prefix,
|
||||
uri=uri,
|
||||
last_timestamp=last_timestamp,
|
||||
status=status,
|
||||
)
|
||||
|
||||
def retrieve(
|
||||
self,
|
||||
minio_repo: MinioRepository,
|
||||
workflow_metadata: dict[str, Any] | None = None,
|
||||
logger: Logger | None = None,
|
||||
) -> DataFrame:
|
||||
"""
|
||||
Load parquet from MinIO when object_key is set and populate inline data.
|
||||
|
||||
Args:
|
||||
minio_repo: sientia_do MinioRepository (or compatible) with download_file().
|
||||
workflow_metadata: Metadata passed to MinIO read for logging/metrics.
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Flat dict with data filled (same keys as to_dict after load).
|
||||
"""
|
||||
if self.data is not None:
|
||||
self._debug(
|
||||
logger,
|
||||
'MinioDataFramePayload.retrieve using inline payload data',
|
||||
workflow_metadata,
|
||||
)
|
||||
return DataFrame(self.data)
|
||||
|
||||
if not self.has_data():
|
||||
self._debug(
|
||||
logger,
|
||||
'MinioDataFramePayload.retrieve found no payload data, returning empty dataframe',
|
||||
workflow_metadata,
|
||||
)
|
||||
return DataFrame()
|
||||
|
||||
self._debug(
|
||||
logger,
|
||||
f'MinioDataFramePayload.retrieve downloading object from MinIO: {self.object_key}',
|
||||
workflow_metadata,
|
||||
)
|
||||
file_bytes = minio_repo.download_file(
|
||||
object_name=self.object_key, metadata=workflow_metadata
|
||||
)
|
||||
df = read_parquet(BytesIO(file_bytes))
|
||||
self._debug(
|
||||
logger,
|
||||
f'MinioDataFramePayload.retrieve loaded dataframe from MinIO with shape {df.shape}',
|
||||
workflow_metadata,
|
||||
)
|
||||
return df
|
||||
893
laborious/utils/repository/opc_repository.py
Normal file
893
laborious/utils/repository/opc_repository.py
Normal file
@@ -0,0 +1,893 @@
|
||||
"""
|
||||
Synchronous OPC UA client repository using asyncua ``sync`` API.
|
||||
|
||||
``asyncua.sync.Client`` runs the asyncio stack on a background thread so Temporal
|
||||
activities and other callers stay blocking while preserving the same session
|
||||
lifecycle, security policy, reconnect semantics, and write error classification
|
||||
as the async ``main`` implementation at ``fcc8920a8be4`` (async → sync/thread conversion).
|
||||
"""
|
||||
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from asyncua import ua
|
||||
from asyncua.crypto import security_policies
|
||||
from asyncua.sync import Client
|
||||
from asyncua.ua.uaerrors import UaStatusCodeError
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
|
||||
from laborious import metrics
|
||||
|
||||
# Requested session and secure channel lifetime (ms) before server revision; 10 minutes.
|
||||
OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS = 10 * 60 * 1000
|
||||
|
||||
|
||||
class OpcClientAlreadyExistsError(RuntimeError):
|
||||
"""Raised when _create_client is called while self.client is already set."""
|
||||
|
||||
|
||||
class OpcSessionAlreadyConnectedError(RuntimeError):
|
||||
"""Raised when _open_session is called while a UA session is already open."""
|
||||
|
||||
|
||||
class OpcClientNotInitializedError(RuntimeError):
|
||||
"""Raised when _open_session is called before _create_client."""
|
||||
|
||||
|
||||
RECONNECTABLE_OPC_BAD_NAMES: frozenset[str] = frozenset(
|
||||
{
|
||||
'BadSessionIdInvalid',
|
||||
'BadSessionClosed',
|
||||
'BadSessionNotActivated',
|
||||
'BadSecureChannelIdInvalid',
|
||||
'BadSecureChannelClosed',
|
||||
'BadSecureChannelTokenUnknown',
|
||||
'BadTcpSecureChannelUnknown',
|
||||
'BadServerNotConnected',
|
||||
'BadConnectionClosed',
|
||||
'BadDisconnect',
|
||||
'BadConnectionRejected',
|
||||
'BadCommunicationError',
|
||||
'BadRequestInterrupted',
|
||||
'BadUnknownResponse',
|
||||
'BadTimeout',
|
||||
'BadRequestTimeout',
|
||||
'BadSequenceNumberInvalid',
|
||||
'BadSequenceNumberUnknown',
|
||||
'BadSecurityModeInsufficient',
|
||||
'BadRequestHeaderInvalid',
|
||||
'BadInvalidState',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _opc_authentication_token_str(client: Client | None) -> str:
|
||||
"""
|
||||
Serialize the current OPC UA authentication token (session handle) for logging and metrics.
|
||||
|
||||
Return:
|
||||
str: Token string, or "unknown" if unavailable.
|
||||
"""
|
||||
if client is None:
|
||||
return 'unknown'
|
||||
try:
|
||||
proto = client.aio_obj.uaclient.protocol
|
||||
if proto is None:
|
||||
return 'unknown'
|
||||
tok = getattr(proto, 'authentication_token', None)
|
||||
if tok is None:
|
||||
return 'unknown'
|
||||
return str(tok)
|
||||
except Exception:
|
||||
return 'unknown'
|
||||
|
||||
|
||||
def _opc_status_from_exception(exc: BaseException) -> str:
|
||||
"""
|
||||
Resolve OPC UA status name from an exception, including chained UaStatusCodeError causes.
|
||||
|
||||
Args:
|
||||
exc (BaseException): Raised error from asyncua.
|
||||
|
||||
Return:
|
||||
str: Status class name or generic Python exception name.
|
||||
"""
|
||||
current: BaseException | None = exc
|
||||
while current is not None:
|
||||
if isinstance(current, UaStatusCodeError):
|
||||
return type(current).__name__
|
||||
current = current.__cause__
|
||||
return type(exc).__name__
|
||||
|
||||
|
||||
def is_reconnectable_opcua_bad(exc: BaseException) -> bool:
|
||||
"""
|
||||
Return whether the exception is a Tier-1 OPC UA Bad* that should trigger reconnect.
|
||||
|
||||
Args:
|
||||
exc (BaseException): Raised error from get_node or write_value.
|
||||
|
||||
Return:
|
||||
bool: True if reconnect should be scheduled.
|
||||
"""
|
||||
return _opc_status_from_exception(exc) in RECONNECTABLE_OPC_BAD_NAMES
|
||||
|
||||
|
||||
def _model_labels_from_write_metadata(metadata: dict[str, Any] | None) -> dict[str, str]:
|
||||
"""
|
||||
Extract model_id and model_name from write metadata for Prometheus labels.
|
||||
|
||||
Args:
|
||||
metadata (dict[str, Any] | None): Context passed into write_data; may omit keys.
|
||||
|
||||
Return:
|
||||
dict[str, str]: Labels model_id and model_name, defaulting to "unknown".
|
||||
"""
|
||||
if not metadata:
|
||||
return {'model_id': 'unknown', 'model_name': 'unknown'}
|
||||
return {
|
||||
'model_id': str(metadata.get('model_id', 'unknown')),
|
||||
'model_name': str(metadata.get('model_name', 'unknown')),
|
||||
}
|
||||
|
||||
|
||||
data_type_map = {
|
||||
'float': {
|
||||
'converter': float,
|
||||
'opc_type': ua.VariantType.Float,
|
||||
},
|
||||
'double': {
|
||||
'converter': float,
|
||||
'opc_type': ua.VariantType.Double,
|
||||
},
|
||||
'int': {
|
||||
'converter': int,
|
||||
'opc_type': ua.VariantType.Int32,
|
||||
},
|
||||
'bool': {
|
||||
'converter': bool,
|
||||
'opc_type': ua.VariantType.Boolean,
|
||||
},
|
||||
'str': {
|
||||
'converter': str,
|
||||
'opc_type': ua.VariantType.String,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class OpcRepository(SientiaMonitoring):
|
||||
"""
|
||||
Synchronous OPC UA repository for connect/disconnect and typed writes.
|
||||
|
||||
Uses ``asyncua.sync.Client`` with the same session metrics, Tier-1 Bad* reconnect,
|
||||
and structured write error payloads as the async repository on ``main``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
opc_id: str,
|
||||
url: str,
|
||||
server_name: str,
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
reconnection_interval: int = 60,
|
||||
server_uri: str | None = None,
|
||||
cert_path: str | None = None,
|
||||
private_key_path: str | None = None,
|
||||
server_cert_path: str | None = None,
|
||||
):
|
||||
self.url = url
|
||||
self.id = opc_id
|
||||
self.server_name = server_name
|
||||
self.server_uri = server_uri
|
||||
self.cert_path = cert_path
|
||||
self.private_key_path = private_key_path
|
||||
self.server_cert_path = server_cert_path
|
||||
self.reconnection_interval = reconnection_interval
|
||||
self.last_reconnection_time: None | datetime = None
|
||||
self.disconnection_interval = 10.0
|
||||
self.notification_handler = notification_handler
|
||||
self.client: None | Client = None
|
||||
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
|
||||
self.metadata = {
|
||||
'model_name': '-',
|
||||
'model_id': '-',
|
||||
'workflow_name': 'opc_repository',
|
||||
'schedule_name': '-',
|
||||
}
|
||||
self._last_write_mono: float | None = None
|
||||
self._connection_lock = threading.Lock()
|
||||
self._session_ready = threading.Event()
|
||||
self._reconnect_thread: threading.Thread | None = None
|
||||
self._allow_reconnect = True
|
||||
|
||||
def _opc_debug_tags(self, session_id: str) -> dict[str, str]:
|
||||
"""
|
||||
Build Prometheus/log label tags for OPC session-scoped metrics.
|
||||
|
||||
Args:
|
||||
session_id (str): OPC UA session token string.
|
||||
|
||||
Return:
|
||||
dict[str, str]: Labels pod_id, server_name, runtime, opc_server_id, session_id.
|
||||
"""
|
||||
return {
|
||||
'pod_id': str(getattr(self, 'pod_id', 'unknown')),
|
||||
'server_name': self.server_name,
|
||||
'runtime': str(getattr(self, 'runtime', 'unknown')),
|
||||
'opc_server_id': self.id,
|
||||
'session_id': session_id,
|
||||
}
|
||||
|
||||
def _is_session_open(self) -> bool:
|
||||
"""
|
||||
Return whether the asyncua client has an open transport session.
|
||||
|
||||
Return:
|
||||
bool: True when protocol exists and is not closed.
|
||||
"""
|
||||
if self.client is None:
|
||||
return False
|
||||
try:
|
||||
proto = self.client.aio_obj.uaclient.protocol
|
||||
return proto is not None and proto.state != 'closed'
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _reconnection_window_elapsed(self) -> bool:
|
||||
"""
|
||||
Return whether enough time has passed since the last reconnect attempt.
|
||||
|
||||
Return:
|
||||
bool: True if a new reconnect is allowed.
|
||||
"""
|
||||
if self.last_reconnection_time is None:
|
||||
return True
|
||||
return (
|
||||
datetime.now() - self.last_reconnection_time
|
||||
).total_seconds() > self.reconnection_interval
|
||||
|
||||
def _not_connected_error(self) -> dict[str, Any]:
|
||||
"""
|
||||
Build the standard error payload when validate_connection finds no open protocol.
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Notification fields for OPC_CONNECTION_NOT_READY.
|
||||
"""
|
||||
return {
|
||||
'notification_id': f'OPC_CONNECTION_NOT_READY_{self.id}',
|
||||
'message': f'OPC server {self.id} is not connected',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.WARNING,
|
||||
}
|
||||
|
||||
def set_security(self) -> None:
|
||||
"""
|
||||
Configure certificates and timeouts on the sync asyncua client.
|
||||
|
||||
Raises:
|
||||
ValueError: If cert paths or client are missing.
|
||||
"""
|
||||
if self.cert_path is None or self.private_key_path is None:
|
||||
raise ValueError(
|
||||
'Certificate and private key paths must be provided for secure connection.'
|
||||
)
|
||||
|
||||
cert = Path(self.cert_path)
|
||||
private_key = Path(self.private_key_path)
|
||||
server_cert = Path(self.server_cert_path) if self.server_cert_path else None
|
||||
|
||||
if self.client is None:
|
||||
raise ValueError('Client must be initialized before setting security')
|
||||
|
||||
self.client.application_uri = self.server_uri
|
||||
self.info('Setting security...', self.metadata)
|
||||
self.client.set_security(
|
||||
security_policies.SecurityPolicyBasic256,
|
||||
str(cert),
|
||||
str(private_key),
|
||||
None,
|
||||
str(server_cert) if server_cert else None,
|
||||
)
|
||||
aio = self.client.aio_obj
|
||||
aio.secure_channel_timeout = OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS
|
||||
aio.session_timeout = OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS
|
||||
|
||||
def _create_client(self) -> None:
|
||||
"""
|
||||
Instantiate the sync Client and apply security when configured.
|
||||
|
||||
Caller must hold _connection_lock. Does not open a UA session.
|
||||
|
||||
Raises:
|
||||
OpcClientAlreadyExistsError: If self.client is already set.
|
||||
"""
|
||||
if self.client is not None:
|
||||
raise OpcClientAlreadyExistsError(
|
||||
f'OPC client already exists for server {self.id}; '
|
||||
'call disconnect() before creating a new client'
|
||||
)
|
||||
|
||||
self.client = Client(self.url, timeout=10)
|
||||
aio = self.client.aio_obj
|
||||
if hasattr(aio, 'watchdog_intervall'):
|
||||
aio.watchdog_intervall = 50
|
||||
aio.name = self.pod_id
|
||||
aio.description = self.pod_id
|
||||
pod_uri = self.pod_id.replace('-', ':')
|
||||
self.client.application_uri = pod_uri
|
||||
aio.product_uri = pod_uri
|
||||
if self.cert_path:
|
||||
self.set_security()
|
||||
|
||||
def _open_session(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Open the OPC UA session on the existing client.
|
||||
|
||||
Caller must hold _connection_lock.
|
||||
|
||||
Raises:
|
||||
OpcClientNotInitializedError: If self.client is None.
|
||||
OpcSessionAlreadyConnectedError: If a session is already open.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: Success flag and error payload on connect failure.
|
||||
"""
|
||||
if self.client is None:
|
||||
raise OpcClientNotInitializedError(
|
||||
f'OPC client is not initialized for server {self.id}; '
|
||||
'call _create_client() before opening a session'
|
||||
)
|
||||
if self._is_session_open():
|
||||
raise OpcSessionAlreadyConnectedError(
|
||||
f'OPC session already connected for server {self.id}; '
|
||||
'call disconnect() before connecting again'
|
||||
)
|
||||
|
||||
tags = {
|
||||
'pod_id': self.pod_id,
|
||||
'server_name': self.server_name,
|
||||
}
|
||||
self.emit_metric_sync(metrics.OPC_CONNECTIONS_TOTAL, tags)
|
||||
|
||||
try:
|
||||
self.client.connect()
|
||||
|
||||
aio = self.client.aio_obj
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
revised_session_timeout_ms = int(aio.session_timeout)
|
||||
revised_secure_channel_timeout_ms = int(aio.secure_channel_timeout)
|
||||
self.info(
|
||||
f'OPC new session connected opc_server_id={self.id} session_id={session_id} '
|
||||
f'revised_session_timeout_ms={revised_session_timeout_ms} '
|
||||
f'revised_secure_channel_timeout_ms={revised_secure_channel_timeout_ms}',
|
||||
self.metadata,
|
||||
)
|
||||
self.emit_metric_sync(
|
||||
metrics.OPC_SESSION_CREATED_TOTAL, self._opc_debug_tags(session_id)
|
||||
)
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.OPC_SESSION_REVISED_TIMEOUT_MS,
|
||||
method='set',
|
||||
tags=self._opc_debug_tags(session_id),
|
||||
value=revised_session_timeout_ms,
|
||||
)
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.OPC_CONNECTION_STATUS,
|
||||
method='set',
|
||||
tags={**tags, 'server_url': self.url},
|
||||
value=1,
|
||||
)
|
||||
|
||||
self._last_write_mono = None
|
||||
self._session_ready.set()
|
||||
return True, {}
|
||||
|
||||
except Exception as e:
|
||||
self._disconnect_locked()
|
||||
trace = traceback.format_exc()
|
||||
self.error(trace, self.metadata)
|
||||
self.emit_metric_sync(metrics.OPC_CONNECTIONS_FAILED, tags)
|
||||
return False, {
|
||||
'notification_id': f'OPC_CONNECTION_ERROR_{self.id}',
|
||||
'message': f'Failed to connect to OPC server: {e}',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.ERROR,
|
||||
'attachment_content': trace,
|
||||
}
|
||||
|
||||
def _connect_locked(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Create the client when absent, then open a UA session.
|
||||
|
||||
Caller must hold _connection_lock.
|
||||
|
||||
Raises:
|
||||
OpcSessionAlreadyConnectedError: If a session is already open.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: Result from _open_session on connect failure.
|
||||
"""
|
||||
if self._is_session_open():
|
||||
raise OpcSessionAlreadyConnectedError(
|
||||
f'OPC session already connected for server {self.id}; '
|
||||
'call disconnect() before connecting again'
|
||||
)
|
||||
if self.client is None:
|
||||
self._create_client()
|
||||
return self._open_session()
|
||||
|
||||
def _disconnection_fallback(self) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Try up to five times to disconnect from the OPC UA server.
|
||||
"""
|
||||
assert self.client is not None
|
||||
error_stack: list[dict[str, Any]] = []
|
||||
for i in range(5):
|
||||
try:
|
||||
self.info(
|
||||
f'Disconnecting from OPC UA server, attempt {i + 1} of 5',
|
||||
self.metadata,
|
||||
)
|
||||
self.client.disconnect()
|
||||
return []
|
||||
except Exception as e:
|
||||
self.error(
|
||||
f'Failed to disconnect from OPC UA server in attempt {i + 1} of 5: {e}',
|
||||
self.metadata,
|
||||
)
|
||||
error_stack.append(
|
||||
{
|
||||
'attempt': i + 1,
|
||||
'error': str(e),
|
||||
'traceback': traceback.format_exc(),
|
||||
}
|
||||
)
|
||||
time.sleep(self.disconnection_interval * i)
|
||||
return error_stack
|
||||
|
||||
def _disconnect_locked(self) -> None:
|
||||
"""
|
||||
Tear down the current session and client.
|
||||
|
||||
Caller must hold _connection_lock.
|
||||
"""
|
||||
self._last_write_mono = None
|
||||
self._session_ready.clear()
|
||||
|
||||
if self.client is None:
|
||||
return
|
||||
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
self.info(
|
||||
f'OPC disconnecting opc_server_id={self.id} session_id={session_id}',
|
||||
self.metadata,
|
||||
)
|
||||
self.emit_metric_sync(metrics.OPC_SESSION_CLOSED_TOTAL, self._opc_debug_tags(session_id))
|
||||
|
||||
errors = self._disconnection_fallback()
|
||||
if errors:
|
||||
self.send_notification(
|
||||
metadata=self.metadata,
|
||||
notification_id=f'OPC_DISCONNECTION_ERROR_{self.id}',
|
||||
message='Failed to disconnect from OPC server in 5 attempts.',
|
||||
block='opc_repository',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=json.dumps(errors, indent=4),
|
||||
)
|
||||
else:
|
||||
self.warning(f'Disconnected from OPC server {self.id} successfully', self.metadata)
|
||||
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.OPC_CONNECTION_STATUS,
|
||||
method='set',
|
||||
tags={
|
||||
'pod_id': self.pod_id,
|
||||
'server_name': self.server_name,
|
||||
'server_url': self.url,
|
||||
},
|
||||
value=0,
|
||||
)
|
||||
self.client = None
|
||||
|
||||
def _reconnect_locked(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Close the current session and open a new one.
|
||||
|
||||
Caller must hold _connection_lock. Records last_reconnection_time for interval gating.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: Result from _connect_locked after teardown.
|
||||
"""
|
||||
self.last_reconnection_time = datetime.now()
|
||||
self._disconnect_locked()
|
||||
return self._connect_locked()
|
||||
|
||||
def connect(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Open an OPC UA session under the connection lock (worker initialization).
|
||||
"""
|
||||
with self._connection_lock:
|
||||
self.info(
|
||||
f'Starting connection to OPC server {self.id}:{self.server_name}...',
|
||||
self.metadata,
|
||||
)
|
||||
return self._connect_locked()
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""
|
||||
Gracefully disconnect from the OPC server under the connection lock.
|
||||
|
||||
Disables background reconnect so late writes during worker shutdown do not
|
||||
respawn sessions.
|
||||
"""
|
||||
with self._connection_lock:
|
||||
self._allow_reconnect = False
|
||||
self._disconnect_locked()
|
||||
|
||||
def validate_connection(self) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Read-only check that the asyncua protocol is open.
|
||||
|
||||
Caller must ensure _session_ready before writing. Does not connect or reconnect.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: (True, {}) when open, otherwise (False, error).
|
||||
"""
|
||||
if self._is_session_open():
|
||||
return True, {}
|
||||
self.error(f'OPC server {self.id} is not connected', self.metadata)
|
||||
return False, self._not_connected_error()
|
||||
|
||||
def _reconnect_thread_in_progress(self) -> bool:
|
||||
"""
|
||||
Return whether a background reconnect thread is currently running.
|
||||
|
||||
Return:
|
||||
bool: True when a reconnect thread exists and is alive.
|
||||
"""
|
||||
return self._reconnect_thread is not None and self._reconnect_thread.is_alive()
|
||||
|
||||
def _start_reconnect(self, reason: str, session_id: str) -> None:
|
||||
"""
|
||||
Schedule a background reconnect when allowed by interval and thread state.
|
||||
|
||||
Clears _session_ready before starting the thread. No-op when _allow_reconnect is
|
||||
False, the reconnection window has not elapsed, or a reconnect is already running.
|
||||
|
||||
Args:
|
||||
reason (str): Trigger for reconnect (OPC status name or synthetic reason).
|
||||
session_id (str): Session token before failure.
|
||||
"""
|
||||
if not self._allow_reconnect:
|
||||
return
|
||||
if not self._reconnection_window_elapsed():
|
||||
self.warning(
|
||||
f'OPC reconnect skipped reason=reconnection_window opc_server_id={self.id} '
|
||||
f'reconnect_reason={reason}',
|
||||
self.metadata,
|
||||
)
|
||||
return
|
||||
if self._reconnect_thread_in_progress():
|
||||
self.warning(
|
||||
f'OPC reconnect skipped reason=in_progress opc_server_id={self.id} '
|
||||
f'reconnect_reason={reason}',
|
||||
self.metadata,
|
||||
)
|
||||
return
|
||||
|
||||
self._session_ready.clear()
|
||||
self.info(
|
||||
f'OPC reconnect scheduled reconnect_reason={reason} opc_server_id={self.id} '
|
||||
f'old_session_id={session_id}',
|
||||
self.metadata,
|
||||
)
|
||||
self._reconnect_thread = threading.Thread(
|
||||
target=self._run_reconnect,
|
||||
args=(reason, session_id),
|
||||
daemon=True,
|
||||
)
|
||||
self._reconnect_thread.start()
|
||||
|
||||
def _run_reconnect(self, reason: str, session_id: str) -> None:
|
||||
"""
|
||||
Tear down and re-establish the OPC UA session under the connection lock.
|
||||
|
||||
Args:
|
||||
reason (str): Trigger for reconnect (OPC status or ProtocolClosed).
|
||||
session_id (str): Previous session token string.
|
||||
"""
|
||||
try:
|
||||
with self._connection_lock:
|
||||
self.info(
|
||||
f'OPC reconnect started reconnect_reason={reason} opc_server_id={self.id} '
|
||||
f'old_session_id={session_id}',
|
||||
self.metadata,
|
||||
)
|
||||
success, error = self._reconnect_locked()
|
||||
if not success:
|
||||
self.error(
|
||||
f'OPC reconnect failed reconnect_reason={reason} opc_server_id={self.id}',
|
||||
self.metadata,
|
||||
)
|
||||
if error:
|
||||
self.error(error.get('message', ''), self.metadata)
|
||||
except Exception:
|
||||
self.error(
|
||||
f'OPC reconnect task failed opc_server_id={self.id} reconnect_reason={reason}',
|
||||
self.metadata,
|
||||
)
|
||||
self.error(traceback.format_exc(), self.metadata)
|
||||
|
||||
def _log_write_inter_arrival(self, session_id: str, node: str) -> None:
|
||||
"""
|
||||
Log elapsed wall time since the previous successful OPC write on this repository.
|
||||
|
||||
Args:
|
||||
session_id (str): Current OPC UA session token string.
|
||||
node (str): Node id written in this operation.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
if self._last_write_mono is not None:
|
||||
delta_s = now - self._last_write_mono
|
||||
self.info(
|
||||
f'OPC write inter-arrival_s={delta_s:.6f} opc_server_id={self.id} '
|
||||
f'session_id={session_id} node={node}',
|
||||
self.metadata,
|
||||
)
|
||||
if self.client is not None:
|
||||
session_timeout_ms = float(self.client.aio_obj.session_timeout)
|
||||
if session_timeout_ms > 0 and delta_s > (session_timeout_ms / 1000.0):
|
||||
self.emit_metric_sync(
|
||||
metrics.OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL,
|
||||
self._opc_debug_tags(session_id),
|
||||
)
|
||||
self._last_write_mono = now
|
||||
|
||||
def _emit_opc_write_metric(
|
||||
self, session_id: str, result: str, metadata: dict[str, Any] | None
|
||||
) -> None:
|
||||
self.emit_metric_sync(
|
||||
metrics.OPC_WRITE_ATTEMPTS_TOTAL,
|
||||
{
|
||||
**self._opc_debug_tags(session_id),
|
||||
**_model_labels_from_write_metadata(metadata),
|
||||
'result': result,
|
||||
},
|
||||
)
|
||||
|
||||
def _write_failure_payload(
|
||||
self,
|
||||
notification_id: str,
|
||||
message: str,
|
||||
level: NotificationLevel = NotificationLevel.ERROR,
|
||||
attachment_content: str | None = None,
|
||||
opc_error_kind: str | None = None,
|
||||
opc_status: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
'notification_id': notification_id,
|
||||
'message': message,
|
||||
'block': 'opc_repository',
|
||||
'level': level,
|
||||
}
|
||||
if attachment_content is not None:
|
||||
payload['attachment_content'] = attachment_content
|
||||
if opc_error_kind is not None:
|
||||
payload['opc_error_kind'] = opc_error_kind
|
||||
if opc_status is not None:
|
||||
payload['opc_status'] = opc_status
|
||||
return payload
|
||||
|
||||
def _handle_tier1_bad(
|
||||
self,
|
||||
exc: BaseException,
|
||||
session_id: str,
|
||||
node: str,
|
||||
metadata: dict[str, Any],
|
||||
phase: str,
|
||||
) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Record metrics/logs and schedule reconnect after a Tier-1 Bad* error.
|
||||
|
||||
Args:
|
||||
exc (BaseException): Tier-1 OPC UA error.
|
||||
session_id (str): Session token at failure time.
|
||||
node (str): Node id being written.
|
||||
metadata (dict[str, Any]): Write context.
|
||||
phase (str): get_node or write_value.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: Always (False, error payload).
|
||||
"""
|
||||
opc_status = _opc_status_from_exception(exc)
|
||||
trace = traceback.format_exc()
|
||||
self.error(trace, metadata)
|
||||
self._emit_opc_write_metric(session_id, opc_status, metadata)
|
||||
self.error(
|
||||
f'OPC write failed opc_status={opc_status} opc_server_id={self.id} '
|
||||
f'session_id={session_id} model_id={metadata.get("model_id", "unknown")} '
|
||||
f'model_name={metadata.get("model_name", "unknown")} node={node} phase={phase}',
|
||||
metadata,
|
||||
)
|
||||
self._start_reconnect(opc_status, session_id)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}',
|
||||
message=f'Failed to {phase} on OPC server: {exc} | metadata: {metadata}',
|
||||
attachment_content=trace,
|
||||
opc_error_kind='session_bad',
|
||||
opc_status=opc_status,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _write_node_value(
|
||||
node_obj: Any,
|
||||
ua_data: ua.DataValue,
|
||||
data: Any,
|
||||
variant_type: ua.VariantType,
|
||||
) -> None:
|
||||
"""
|
||||
Write a DataValue to a node, falling back to set_value when write_value is unavailable.
|
||||
|
||||
Args:
|
||||
node_obj: Sync or async node wrapper from asyncua.
|
||||
ua_data (ua.DataValue): Encoded value for write_value.
|
||||
data: Scalar converted value for set_value fallback.
|
||||
variant_type (ua.VariantType): OPC UA type for set_value fallback.
|
||||
"""
|
||||
if hasattr(node_obj, 'write_value'):
|
||||
try:
|
||||
node_obj.write_value(ua_data)
|
||||
return
|
||||
except (AttributeError, TypeError):
|
||||
pass
|
||||
node_obj.set_value(data, variant_type)
|
||||
|
||||
def _write_reconnect_in_progress(self, metadata: dict[str, Any]) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Fail a write because a background reconnect thread is already running.
|
||||
|
||||
Args:
|
||||
metadata (dict[str, Any]): Write context passed through to the activity.
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind reconnect_in_progress).
|
||||
"""
|
||||
self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata)
|
||||
self.warning(
|
||||
f'OPC write rejected reconnect_in_progress opc_server_id={self.id} '
|
||||
f'model_id={metadata.get("model_id", "unknown")} '
|
||||
f'model_name={metadata.get("model_name", "unknown")}',
|
||||
metadata,
|
||||
)
|
||||
return False, {
|
||||
'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}',
|
||||
'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}',
|
||||
'block': 'opc_repository',
|
||||
'level': NotificationLevel.WARNING,
|
||||
'opc_error_kind': 'reconnect_in_progress',
|
||||
}
|
||||
|
||||
def _write_connection_lost(
|
||||
self, metadata: dict[str, Any], opc_status: str
|
||||
) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Fail a write after scheduling reconnect for a closed or stale session.
|
||||
|
||||
Args:
|
||||
metadata (dict[str, Any]): Write context passed through to the activity.
|
||||
opc_status (str): Synthetic reason (ProtocolClosed, SessionNotReady).
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind connection_lost).
|
||||
"""
|
||||
self._emit_opc_write_metric('unknown', opc_status, metadata)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_CONNECTION_LOST_{self.id}',
|
||||
message=f'OPC write skipped: connection lost ({opc_status}) | metadata: {metadata}',
|
||||
level=NotificationLevel.WARNING,
|
||||
opc_error_kind='connection_lost',
|
||||
opc_status=opc_status,
|
||||
)
|
||||
|
||||
def write_data(
|
||||
self, node: str, value: Any, data_type: str, metadata: dict[str, Any]
|
||||
) -> tuple[bool, dict[str, Any]]:
|
||||
"""
|
||||
Write data to OPC server with a single attempt and background reconnect scheduling.
|
||||
|
||||
Reconnect is scheduled on Tier-1 Bad*, closed protocol, or stale session readiness.
|
||||
There is no retry within the same call.
|
||||
|
||||
Args:
|
||||
node (str): OPC UA node id to write.
|
||||
value (Any): Value to convert and send.
|
||||
data_type (str): Logical type key (float, int, bool, str, double).
|
||||
metadata (dict[str, Any]): Activity context (model_id, model_name, etc.).
|
||||
|
||||
Return:
|
||||
tuple[bool, dict[str, Any]]: (True, {response_time}) on success, or
|
||||
(False, structured error info) on failure.
|
||||
"""
|
||||
if self._reconnect_thread_in_progress():
|
||||
return self._write_reconnect_in_progress(metadata)
|
||||
|
||||
if not self._session_ready.is_set():
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
self._start_reconnect('SessionNotReady', session_id)
|
||||
if self._reconnect_thread_in_progress():
|
||||
return self._write_reconnect_in_progress(metadata)
|
||||
return self._write_connection_lost(metadata, 'SessionNotReady')
|
||||
|
||||
is_connected, _error = self.validate_connection()
|
||||
if not is_connected:
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
self._start_reconnect('ProtocolClosed', session_id)
|
||||
return self._write_connection_lost(metadata, 'ProtocolClosed')
|
||||
|
||||
start_time = time.time()
|
||||
session_id = _opc_authentication_token_str(self.client)
|
||||
|
||||
try:
|
||||
assert self.client is not None
|
||||
node_obj = self.client.get_node(node)
|
||||
except Exception as e:
|
||||
if is_reconnectable_opcua_bad(e):
|
||||
return self._handle_tier1_bad(e, session_id, node, metadata, 'get_node')
|
||||
trace = traceback.format_exc()
|
||||
self.error(trace, metadata)
|
||||
self._emit_opc_write_metric(session_id, f'GetNodeError:{type(e).__name__}', metadata)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_GET_NODE_ERROR_{self.id}',
|
||||
message=f'Failed to get node from OPC server: {e} | metadata: {metadata}',
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
if data_type not in data_type_map:
|
||||
self._emit_opc_write_metric(session_id, 'UnsupportedDataType', metadata)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_DATA_TYPE_ERROR_{self.id}',
|
||||
message=f'Unsupported data type: {data_type} | metadata: {metadata}',
|
||||
)
|
||||
|
||||
data = data_type_map[data_type]['converter'](value)
|
||||
self.info(f'Writing {data} - {type(data)} to {node}', metadata)
|
||||
variant_type = data_type_map[data_type]['opc_type']
|
||||
ua_data = ua.DataValue(
|
||||
ua.Variant(data, variant_type),
|
||||
)
|
||||
|
||||
try:
|
||||
self._write_node_value(node_obj, ua_data, data, variant_type)
|
||||
end_time = time.time()
|
||||
response_time = end_time - start_time
|
||||
except Exception as e:
|
||||
if is_reconnectable_opcua_bad(e):
|
||||
return self._handle_tier1_bad(e, session_id, node, metadata, 'write_value')
|
||||
trace = traceback.format_exc()
|
||||
self.error(trace, metadata)
|
||||
self._emit_opc_write_metric(session_id, type(e).__name__, metadata)
|
||||
return False, self._write_failure_payload(
|
||||
notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}',
|
||||
message=f'Failed to write data to OPC server: {e} | metadata: {metadata}',
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
self._emit_opc_write_metric(session_id, 'OK', metadata)
|
||||
self._log_write_inter_arrival(session_id, node)
|
||||
|
||||
return True, {
|
||||
'response_time': response_time,
|
||||
}
|
||||
Reference in New Issue
Block a user