gitea_admin/sientia-dataops-laborious_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Update README, requirements, and E2E tests for improved configuration and functionality

- Enhanced the README with updated model configuration examples, including the addition of an alias for production.
- Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`.
- Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity.
- Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs.
- Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.
This commit is contained in:
vitor-aignosi
2026-05-07 17:02:25 -03:00
parent aaf647efdf
commit e6018af23f
51 changed files with 4408 additions and 2660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
laborious/activities/activities.py
View File

@@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
from sientia_model.model_repository.plugin_store import PluginStore
@@ -159,7 +159,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
metrics_controller=mc,
)
async def shutdown(self):
def shutdown(self) -> None:
"""
Close database pools, sync clients, and OPC sessions in a defined order.
@@ -172,6 +172,6 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
Storage.close(self)
MLFlow.close(self)
Gates.close(self)
await OPC.close(self)
OPC.close(self)
ModelMetrics.close(self)
API.close(self)

24
laborious/activities/api.py
View File

@@ -11,7 +11,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.pi_web_api_client import PIWebAPIClient
from sientia_do.repository.pi_web_api_client_sync import PIWebAPIClient
from laborious import metrics
@@ -105,7 +105,7 @@ class API(SientiaMonitoring):
self.pi_web_api_client.close()
SientiaMonitoring.shutdown(self)
async def process_pi_web_api_response(
def process_pi_web_api_response(
self,
response_data: list[dict[str, Any]],
tags: dict[str, str],
@@ -156,7 +156,7 @@ class API(SientiaMonitoring):
self.error(
f'Error writing tag {tag_name}:{web_id} to PI Web API: {errors}', metadata
)
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT,
tags={
**core_labels,
@@ -165,7 +165,7 @@ class API(SientiaMonitoring):
)
confidence = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
else:
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_COUNT,
tags={
**core_labels,
@@ -182,7 +182,7 @@ class API(SientiaMonitoring):
metadata,
)
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
message=f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written.\nResponse:\n {json.dumps(response_data, indent=4)}\nTags:\n {json.dumps(tags, indent=4)}',
@@ -194,7 +194,7 @@ class API(SientiaMonitoring):
return confidence, message
@activity.defn(name='write_pi_web_api_data')
async def write_pi_web_api_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
def write_pi_web_api_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
"""
Write prediction and confidence data to PI Web API.
@@ -232,7 +232,7 @@ class API(SientiaMonitoring):
confidence_value = data.head(1)['prediction_confidence'].values[0]
try:
prediction_response = await self.pi_web_api_client.write_value(
prediction_response = self.pi_web_api_client.write_value(
web_ids=prediction_tags,
value={
'Timestamp': data.head(1)['timestamp'].values[0],
@@ -241,7 +241,7 @@ class API(SientiaMonitoring):
metadata=metadata,
)
confidence, message = await self.process_pi_web_api_response(
confidence, message = self.process_pi_web_api_response(
response_data=prediction_response,
tags=raw_prediction_tags,
core_labels=core_labels,
@@ -258,7 +258,7 @@ class API(SientiaMonitoring):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
message=f'Error writing prediction data to PI Web API: {e}\n Tags: {raw_prediction_tags}',
@@ -275,7 +275,7 @@ class API(SientiaMonitoring):
return data.to_dict()
try:
confidence_response = await self.pi_web_api_client.write_value(
confidence_response = self.pi_web_api_client.write_value(
web_ids=confidence_tags,
value={
'Timestamp': data.head(1)['timestamp'].values[0],
@@ -284,7 +284,7 @@ class API(SientiaMonitoring):
metadata=metadata,
)
await self.process_pi_web_api_response(
self.process_pi_web_api_response(
response_data=confidence_response,
tags=raw_confidence_tags,
core_labels=core_labels,
@@ -293,7 +293,7 @@ class API(SientiaMonitoring):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='WRITE_PI_WEB_API_CONFIDENCE_ERROR',
message=f'Error writing confidence data to PI Web API: {e}\n Tags: {raw_confidence_tags}',

70
laborious/activities/gates.py
View File

@@ -1,8 +1,5 @@
from sientia_do.repository.minio_repository import MinioRepository
from temporalio import activity, workflow
from laborious.utils.repository.minio_manager import MinioManager
with workflow.unsafe.imports_passed_through():
import traceback
from collections.abc import Callable, Mapping
@@ -13,6 +10,8 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_do.utils.formatters import create_sample_dict
from laborious import metrics
@@ -66,7 +65,7 @@ mlflow_content_path_confidence: Mapping[str, int] = {
}
class Gates(MinioManager):
class Gates(SientiaMonitoring):
"""
Data quality gates and filtering activities for the Laborious system.
@@ -106,8 +105,12 @@ class Gates(MinioManager):
Raises:
Exception: If BaseActivity initialization fails
"""
MinioManager.__init__(
self, minio_repository, logger, notification_handler, metrics_controller
self.minio_repository = minio_repository
SientiaMonitoring.__init__(
self,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
def close(self) -> None:
@@ -115,7 +118,12 @@ class Gates(MinioManager):
Close the gates activity and clean up resources.
"""
MinioManager.close(self)
if self.minio_repository is not None:
try:
self.minio_repository.close()
finally:
self.minio_repository = None
SientiaMonitoring.shutdown(self)
def __del__(self):
self.close()
@@ -155,7 +163,7 @@ class Gates(MinioManager):
return policy, filter_config
@activity.defn(name='input_gate')
async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Apply input data quality filters and validation.
@@ -194,7 +202,7 @@ class Gates(MinioManager):
filters = input_data['filters']
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
path_priority = input_data['path_priority']
filter_output = []
@@ -214,7 +222,7 @@ class Gates(MinioManager):
filter_output.append(policy)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'INTPUT_GATE_ERROR__{fil}',
message=f'Error in filter {fil}:{config}: \n {e}',
@@ -235,7 +243,7 @@ class Gates(MinioManager):
return None, 0, ''
@activity.defn(name='mlflow_response_gate')
async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Validate MLFlow API response quality and integrity.
@@ -279,7 +287,7 @@ class Gates(MinioManager):
self.debug(f'Filters: {filters}', metadata)
payload = MinioDataFramePayload.from_dict(raw_data)
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
gate_type = input_data['type']
path_priority = input_data['path_priority']
@@ -298,7 +306,7 @@ class Gates(MinioManager):
if mlflow_response_filter_functions[fil](status, filter_config):
filter_output.append(policy)
comments.append(status.get('message', 'Unknown MLFlow API error'))
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
message=status.get('message', 'Unknown MLFlow API error'),
@@ -308,7 +316,7 @@ class Gates(MinioManager):
)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'MLFLOW_GATE_RESPONSE_FILTER__{fil}',
message=f'Error in filter {fil}:{config}: \n {e}',
@@ -329,7 +337,7 @@ class Gates(MinioManager):
return None, 0, ''
@activity.defn(name='mlflow_content_gate')
async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Validate MLFlow prediction content quality and integrity.
@@ -368,7 +376,7 @@ class Gates(MinioManager):
filters = input_data['filters']
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
gate_type = input_data['type']
path_priority = input_data['path_priority']
@@ -385,7 +393,7 @@ class Gates(MinioManager):
try:
if mlflow_content_filter_functions[fil](data, filter_config):
filter_output.append(policy)
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}',
message=f'Data not passed the content filter {fil}:{config}',
@@ -395,7 +403,7 @@ class Gates(MinioManager):
)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'MLFLOW_GATE_CONTENT_FILTER__{fil}',
message=f'Error in filter {fil}:{config}: \n {e}',
@@ -471,7 +479,7 @@ class Gates(MinioManager):
return policy_type, int(policy_value)
@activity.defn(name='format_transformed_data')
async def format_transformed_data(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
def format_transformed_data(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
"""
Format transformed data for storage and export operations.
@@ -507,7 +515,7 @@ class Gates(MinioManager):
self.info('Formatting transformed data...', metadata)
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
data['timestamp'] = data.index
data = data.reset_index(drop=True)
@@ -515,7 +523,7 @@ class Gates(MinioManager):
data = data.melt(id_vars='timestamp', var_name='variable', value_name='value')
data['model_id'] = model_id
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=data,
minio_repo=self.minio_repository,
model_name=input_data['model_name'],
@@ -526,7 +534,7 @@ class Gates(MinioManager):
)
@activity.defn(name='format_prediction')
async def format_prediction(self, input_data: dict[str, Any]) -> dict:
def format_prediction(self, input_data: dict[str, Any]) -> dict:
"""
Format prediction data according to configured storage policies.
@@ -558,7 +566,7 @@ class Gates(MinioManager):
self.info('Formatting prediction...', metadata)
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
# Create timestamp column from index and reset index
data['timestamp'] = data.index
@@ -607,7 +615,7 @@ class Gates(MinioManager):
return data.to_dict()
@activity.defn(name='format_default_prediction')
async def format_default_prediction(self, input_data: dict[str, Any]) -> dict:
def format_default_prediction(self, input_data: dict[str, Any]) -> dict:
"""
Create and format default prediction data for error conditions.
@@ -652,7 +660,7 @@ class Gates(MinioManager):
return data.to_dict()
@activity.defn(name='format_retrain_report')
async def format_retrain_report(self, input_data: dict[str, Any]) -> dict:
def format_retrain_report(self, input_data: dict[str, Any]) -> dict:
"""
Format retrain report data for storage and audit trail maintenance.
@@ -722,7 +730,7 @@ class Gates(MinioManager):
return report.to_dict()
@activity.defn(name='write_metrics')
async def write_metrics(self, input_data: dict[str, Any]):
def write_metrics(self, input_data: dict[str, Any]):
"""
Write prediction performance metrics to Prometheus monitoring system.
@@ -759,19 +767,19 @@ class Gates(MinioManager):
'model_name': metadata['model_name'],
'workflow_name': metadata['workflow_name'],
}
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTIONS_WRITTEN_COUNT,
tags=core_tags,
)
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTION_CONFIDENCE_MONITOR,
method='set',
tags=core_tags,
value=prediction_confidence,
)
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTION_RESPONSE_TIME_MONITOR,
method='observe',
tags=core_tags,
@@ -781,7 +789,7 @@ class Gates(MinioManager):
for server_id, tags in opc_metrics.items():
for tag, response_time in tags.items():
if response_time is not None:
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR,
method='observe',
tags={
@@ -792,7 +800,7 @@ class Gates(MinioManager):
value=response_time,
)
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTION_OPC_WRITING_COUNT,
tags={
**core_tags,

49
laborious/activities/mlflow.py
View File

@@ -16,7 +16,8 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_do.temporal.constants import (
DATETIME_FORMAT,
DATETIME_FORMAT_MS_WITH_TZ,
@@ -29,10 +30,9 @@ with workflow.unsafe.imports_passed_through():
from laborious.utils.dataframe_debug import build_dataframe_debug_message
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
from laborious.utils.repository.minio_manager import MinioManager
class MLFlow(MinioManager):
class MLFlow(SientiaMonitoring):
"""
Temporal activities that talk to MLflow through ``SientiaMLflowRepository`` and ``SientiaModel`` wrappers.
@@ -78,8 +78,12 @@ class MLFlow(MinioManager):
None
"""
MinioManager.__init__(
self, minio_repository, logger, notification_handler, metrics_controller
self.minio_repository = minio_repository
SientiaMonitoring.__init__(
self,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
self.mlflow_repository = mlflow_repository
self.plugin_store = plugin_store
@@ -91,7 +95,12 @@ class MLFlow(MinioManager):
Return:
None
"""
MinioManager.close(self)
if self.minio_repository is not None:
try:
self.minio_repository.close()
finally:
self.minio_repository = None
SientiaMonitoring.shutdown(self)
def __del__(self):
self.close()
@@ -207,7 +216,7 @@ class MLFlow(MinioManager):
return alias or self._DEFAULT_MODEL_ALIAS
@activity.defn(name='request_transform')
async def request_transform(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
def request_transform(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
"""
Pivot long-format sensor rows, load the production wrapper, and run ``wrapper.transform``.
@@ -228,7 +237,7 @@ class MLFlow(MinioManager):
self.info('Transforming data...', metadata)
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
@@ -284,7 +293,7 @@ class MLFlow(MinioManager):
self.info('Data transformed successfully', metadata)
if not response_data.get('success', False):
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=None,
minio_repo=self.minio_repository,
model_name=model_name,
@@ -295,7 +304,7 @@ class MLFlow(MinioManager):
logger=self.logger,
)
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=response_data['content'],
minio_repo=self.minio_repository,
model_name=model_name,
@@ -309,7 +318,7 @@ class MLFlow(MinioManager):
)
@activity.defn(name='request_predict')
async def request_predict(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
def request_predict(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
"""
Load the production wrapper and call ``wrapper.predict`` on the prepared feature frame.
@@ -329,7 +338,7 @@ class MLFlow(MinioManager):
self.info('Predicting data...', metadata)
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
@@ -390,7 +399,7 @@ class MLFlow(MinioManager):
self.info('Data predicted successfully', metadata)
if not response_data.get('success', False):
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=None,
minio_repo=self.minio_repository,
model_name=model_name,
@@ -401,7 +410,7 @@ class MLFlow(MinioManager):
logger=self.logger,
)
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=response_data['content'],
minio_repo=self.minio_repository,
model_name=model_name,
@@ -415,7 +424,7 @@ class MLFlow(MinioManager):
)
@activity.defn(name='retrain_model')
async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Fit an updated wrapper from historical data, then log and register in MLflow.
@@ -441,11 +450,11 @@ class MLFlow(MinioManager):
try:
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='ERROR_LOADING_RETRAIN_DATA',
message=f'Error loading retrain data: {e}',
@@ -576,7 +585,7 @@ class MLFlow(MinioManager):
}
@activity.defn(name='update_production_model')
async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
"""
Point the ``production`` alias at the model version registered for the retrain run.
@@ -622,7 +631,7 @@ class MLFlow(MinioManager):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='UPDATE_PRODUCTION_MODEL_ERROR',
message=f'Error updating production model {model_name}: {e}',
@@ -634,7 +643,7 @@ class MLFlow(MinioManager):
raise e
@activity.defn(name='get_reference_data')
async def get_reference_data(self, input_data: dict[str, Any]) -> list[dict] | None:
def get_reference_data(self, input_data: dict[str, Any]) -> list[dict] | None:
"""
Download ``evaluation_data.csv`` from the MLflow run linked to ``production`` and parse it.

148
laborious/activities/model_metrics.py
View File

@@ -7,14 +7,15 @@ with workflow.unsafe.imports_passed_through():
from typing import Any
import numpy as np
from pandas import DataFrame, Index, to_datetime
from sientia.ModelAnalysis import ModelAnalysis
import pandas as pd
from pandas import DataFrame, Index, Series, to_datetime
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_model.analytics.model_analysis import ModelAnalysis
from laborious import metrics
from laborious.utils.dataframe_debug import build_dataframe_debug_message
@@ -27,9 +28,12 @@ warnings.filterwarnings(
class ModelMetrics(SientiaMonitoring):
"""
Metrics activities for the Laborious system.
Metrics and statistical analysis activities for the Laborious pipeline.
This class provides activities for writing metrics to the Prometheus monitoring system.
This class centralizes drift/statistical computations and model-quality
aggregates used by scheduled workflows. Besides producing tabular outputs
for persistence, it also emits operational metrics (count, lag, error)
through ``SientiaMonitoring`` so execution health is observable in runtime.
"""
_MAX_DEBUG_DATAFRAME_ROWS = 100
@@ -44,7 +48,10 @@ class ModelMetrics(SientiaMonitoring):
def close(self) -> None:
"""
Close the model metrics activity and clean up resources.
Shutdown monitoring resources associated with model metrics activities.
This is invoked during worker teardown to flush/close metric controller
internals and prevent dangling telemetry tasks.
"""
SientiaMonitoring.shutdown(self)
@@ -69,7 +76,7 @@ class ModelMetrics(SientiaMonitoring):
metadata,
)
async def get_drift_metrics(
def get_drift_metrics(
self,
reference_data: DataFrame,
target_data: DataFrame,
@@ -80,14 +87,23 @@ class ModelMetrics(SientiaMonitoring):
metadata: dict[str, Any],
) -> DataFrame:
"""
Calculate univariate drift metrics for a model.
Compute univariate and multivariate drift outputs and merge them into one dataframe.
The method orchestrates three analysis stages (univariate drift,
multivariate drift, and dataframe projection), emitting lag/count/error
metrics for each stage independently so failures are attributable.
Args:
model_analysis (ModelAnalysis): Model analysis object
reference_data (DataFrame): Reference data
target_data (DataFrame): Target data
reference_columns (list[str]): Reference columns
drift_metrics (list[str]): Drift metrics
metadata (dict[str, Any]): Workflow execution metadata
- reference_data (DataFrame): Baseline dataset representing expected behavior.
- target_data (DataFrame): Current analysis dataset to compare against reference.
- target_name (str): Target column name used by ``ModelAnalysis`` config.
- reference_columns (Index): Feature columns evaluated for drift.
- drift_metrics (list[str]): Enabled univariate methods.
- chunk_period (str): Time bucket granularity used by analysis methods.
- metadata (dict[str, Any]): Workflow metadata for logs and notifications.
Return:
DataFrame: Consolidated drift dataframe ready for downstream formatting/persistence.
"""
config = {
@@ -118,12 +134,10 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error detecting univariate drift: {e}', metadata)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
core_labels = self.get_core_labels(metadata, operation_type='detect_multivariate_drift')
start_time = time.time()
@@ -137,12 +151,10 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error detecting multivariate drift: {e}', metadata)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
start_time = time.time()
core_labels = self.get_core_labels(metadata, operation_type='get_drift_metrics_dataframe')
@@ -153,19 +165,40 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error getting drift metrics: {e}', metadata)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
return drift_df
@staticmethod
def _to_naive_utc(series: Series) -> Series:
"""
Parse ``series`` as datetime and return a TZ-naive UTC copy.
``sientia_model.analytics.model_analysis.ModelAnalysis`` preserves the
timezone of the input dataframe in its outputs, while target rows
loaded from PostgreSQL come in with ``+00:00``. Forcing both sides of
a comparison to TZ-naive UTC keeps ``isin`` / ``floor`` operations
deterministic regardless of how the analyzer (or a test double)
constructs its timestamps.
Args:
- series (Series): Input series containing datetime-parseable values.
Return:
Series: Datetime64 series with ``tz=None`` representing UTC instants.
"""
parsed = to_datetime(series)
if getattr(parsed.dt, 'tz', None) is not None:
parsed = parsed.dt.tz_convert('UTC').dt.tz_localize(None)
return parsed
@activity.defn(name='calculate_drift')
async def calculate_drift(self, input_data: dict[str, Any]) -> list[dict]:
def calculate_drift(self, input_data: dict[str, Any]) -> list[dict]:
"""
Calculate drift metrics for a model.
@@ -195,14 +228,17 @@ class ModelMetrics(SientiaMonitoring):
target_data = target_data.pivot(index='timestamp', columns='variable', values='value')
target_data['timestamp'] = target_data.index
# Keep timestamps as datetime: ModelAnalysis._chunk_dataframe relies on
# ``pd.Grouper(freq=...)`` which rejects string timestamp columns.
target_data['timestamp'] = to_datetime(target_data['timestamp'])
target_data['timestamp'] = target_data['timestamp'].dt.strftime(DATETIME_FORMAT)
target_data = target_data.reset_index(drop=True)
target_data.dropna(inplace=True)
if reference_raw_data is not None:
self.info('Using reference data', metadata)
reference_data = DataFrame(reference_raw_data)
if 'timestamp' in reference_data.columns:
reference_data['timestamp'] = to_datetime(reference_data['timestamp'])
accurate = True
else:
# Get 30% first rows of target_data
@@ -211,7 +247,7 @@ class ModelMetrics(SientiaMonitoring):
reference_data = target_data.head(int(len(target_data) * 0.3))
accurate = False
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING',
message='Using 30% first rows of target data as reference data',
@@ -225,7 +261,7 @@ class ModelMetrics(SientiaMonitoring):
).columns
try:
drift_df = await self.get_drift_metrics(
drift_df = self.get_drift_metrics(
reference_data=reference_data,
target_data=target_data,
target_name=target_name,
@@ -236,7 +272,7 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error getting drift metrics: {e}', metadata)
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MODEL_METRICS_GET_DRIFT_METRICS_ERROR',
message=f'Error getting drift metrics: {e}',
@@ -250,17 +286,13 @@ class ModelMetrics(SientiaMonitoring):
self.warning('No drift metrics found', metadata)
return []
# Drop unnecessary columns
drift_df.drop(columns=['p_value'], inplace=True)
# Extract timestamps only until minutes
if chunk_period == 'min':
target_timestamps = target_data['timestamp'].apply(lambda x: x[:16])
else:
target_timestamps = target_data['timestamp']
# Drop rows where timestamp is not in target data, to avoid save drift from reference
drift_df = drift_df[drift_df['timestamp'].isin(target_timestamps)]
# Defense-in-depth: drop chunks whose floored timestamp does not appear
# in the analysis window. ``ModelAnalysis`` already chunks only over
# ``analysis_df`` so this only excludes rows injected by upstream
# callers that pre-merge reference data into the result.
target_floor = self._to_naive_utc(target_data['timestamp']).dt.floor(chunk_period)
drift_floor = self._to_naive_utc(drift_df['timestamp']).dt.floor(chunk_period)
drift_df = drift_df[drift_floor.isin(target_floor)]
if drift_df.empty:
self.warning(
@@ -269,14 +301,21 @@ class ModelMetrics(SientiaMonitoring):
)
return []
# Rename columns to match database columns
drift_df.rename(
# Map ``sientia_model.analytics.model_analysis`` schema onto the drift
# table columns: ``metric -> method``, ``statistic -> value``,
# ``alert -> drift``, ``chunk_index -> chunk``,
# ``chunk_end_date -> timestamp_end``. ``p_value`` and
# ``chunk_start_date`` are not persisted.
drift_df = drift_df.rename(
columns={
'metric': 'method',
'statistic': 'value',
},
inplace=True,
'alert': 'drift',
'chunk_index': 'chunk',
'chunk_end_date': 'timestamp_end',
}
)
drift_df.drop(columns=['p_value', 'chunk_start_date'], inplace=True, errors='ignore')
# Drop duplicates
drift_df.drop_duplicates(
@@ -286,16 +325,23 @@ class ModelMetrics(SientiaMonitoring):
drift_df['model_id'] = model_id
drift_df['accurate'] = accurate
drift_df['timestamp'] = to_datetime(drift_df['timestamp'])
drift_df['timestamp'] = self._to_naive_utc(drift_df['timestamp'])
drift_df['timestamp'] = drift_df['timestamp'].dt.tz_localize('UTC')
drift_df['timestamp'] = drift_df['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
# ``timestamp_end`` may carry nanosecond precision (beyond
# ``timestamptz`` microseconds), so serialize as ISO text for the
# ``text`` Postgres column.
drift_df['timestamp_end'] = drift_df['timestamp_end'].apply(
lambda value: pd.Timestamp(value).isoformat() if pd.notna(value) else None
)
self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
return drift_df.to_dict(orient='records')
@activity.defn(name='calculate_simple_metrics')
async def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]:
def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]:
"""
Calculate simple metrics for a model. Metrics available are:
- rmse

210
laborious/activities/opc.py
View File

@@ -51,7 +51,7 @@ class OPC(SientiaMonitoring):
self.opc_repository: dict[str, OpcRepository] = {}
async def init_opc(self):
def init_opc(self) -> None:
"""
Initialize OPC server connections and establish communication channels.
@@ -59,58 +59,43 @@ class OPC(SientiaMonitoring):
establish secure connections using certificate-based authentication.
Each server connection is managed independently, and connection failures
are reported through the notification system.
The method performs the following operations:
1. Creates OpcRepository instances for each configured server
2. Establishes secure connections with certificate validation
3. Reports connection success/failure through notifications
4. Logs connection status for operational visibility
Raises:
Exception: If OPC repository initialization fails or connection
establishment encounters critical errors
Note:
Connection failures are logged and reported but do not prevent
the initialization of other OPC servers. Each server is handled
independently to ensure maximum availability.
"""
self.logger.info('Initializing OPC servers...')
for opc_id, server in self.opc_servers.items():
self.opc_repository[opc_id] = OpcRepository(
opc_id=server['id'],
server_name=server['server_name'],
opc_id=opc_id,
url=server['url'],
server_name=server['server_name'],
logger=self.logger,
notification_handler=self.notification_handler,
metrics_controller=self.metrics_controller,
server_uri=server['server_uri'],
cert_path=server['cert_path'],
private_key_path=server['private_key_path'],
server_cert_path=server['server_cert_path'],
notification_handler=self.notification_handler,
reconnection_interval=server['reconnection_interval'],
metrics_controller=self.metrics_controller,
)
is_connected, error_data = await self.opc_repository[opc_id].connect()
if not is_connected:
await self.send_notification_async(
ok, err = self.opc_repository[opc_id].connect()
if not ok:
self.send_notification(
metadata={
'model_id': '-',
'model_name': '-',
'workflow_name': '-',
'schedule_name': 'INITIALIZATION',
},
notification_id=error_data['notification_id'],
message=error_data['message'],
block=error_data['block'],
level=error_data.get('level', NotificationLevel.ERROR),
attachment_content=error_data.get('attachment_content', None),
notification_id=f'OPC_CONNECTION_ERROR_{server.get("id", opc_id)}',
message=err.get('message', 'Failed to connect to OPC server'),
block='opc_repository',
level=NotificationLevel.ERROR,
attachment_content=err.get('attachment_content', traceback.format_exc()),
)
else:
self.logger.info(
f'OPC server {opc_id}:{server["server_name"]} connected successfully.'
)
async def write_data(
def write_data(
self,
server_id: str,
tag: str,
@@ -122,11 +107,6 @@ class OPC(SientiaMonitoring):
"""
Write data to a specific OPC server tag with comprehensive error handling.
This method provides a secure and reliable way to write data to OPC servers
with automatic error handling, notification integration, and detailed logging.
It validates server availability before attempting write operations and
provides comprehensive error reporting for operational monitoring.
Args:
- server_id (str): The id of the OPC server.
- tag (str): The tag to write to.
@@ -135,15 +115,15 @@ class OPC(SientiaMonitoring):
- tag_type (str): The tag type.
Returns:
- bool: True if the data was written successfully, False otherwise.
- float | None: Response time in seconds if successful, None otherwise.
"""
try:
is_success, info_data = await self.opc_repository[server_id].write_data(
is_success, info_data = self.opc_repository[server_id].write_data(
tag, data, data_type, self.logger, metadata
)
if not is_success:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=info_data['notification_id'],
message=info_data['message'],
@@ -155,7 +135,7 @@ class OPC(SientiaMonitoring):
return info_data['response_time']
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'WRITE_OPC_{tag_type.upper()}_ERROR',
message=f'Error writing data to OPC server: {e}',
@@ -165,30 +145,25 @@ class OPC(SientiaMonitoring):
)
raise e
async def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool:
def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool:
"""
Validate that an OPC server is available and configured for write operations.
Validate that an OPC repository exists for the requested server identifier.
This method checks if the specified OPC server exists in the active
repository and is available for data writing operations. It provides
immediate feedback for server availability and logs validation failures
for operational monitoring.
This guard prevents write attempts against unknown/uninitialized servers.
When the server is missing, it emits an error notification with the list
of available repositories to help operators diagnose configuration drift.
Args:
server_id (str): Unique identifier for the OPC server to validate
metadata (dict[str, Any]): Context metadata for logging and notifications
- server_id (str): OPC server identifier from workflow output config.
- metadata (dict[str, Any]): Workflow metadata used for logs/alerts.
Returns:
bool: True if server is available, False otherwise
Note:
Server validation failures are automatically reported through the
notification system with detailed information about available servers.
This helps operators quickly identify configuration issues.
Return:
bool: ``True`` when the server repository is available; ``False`` otherwise.
"""
if self.opc_repository.get(server_id) is None:
message = f'OPC server {server_id} not found to perform write operation.'
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='OPC_SERVER_NOT_FOUND',
message=message,
@@ -199,7 +174,7 @@ class OPC(SientiaMonitoring):
return False
return True
async def manage_output_tags(
def manage_output_tags(
self,
server_id: str,
config: dict[str, Any],
@@ -207,37 +182,30 @@ class OPC(SientiaMonitoring):
metadata: dict[str, Any],
) -> tuple[bool, dict[str, float | None]]:
"""
Manage the writing of prediction and confidence data to OPC server tags.
Write prediction and confidence values for one OPC server configuration.
This method orchestrates the writing of multiple data types to OPC servers
based on configuration. It handles both prediction data and confidence
values independently, allowing for flexible tag configuration and
comprehensive error handling.
The method supports two main tag types:
1. Prediction tags: Write actual prediction values to configured OPC tags
2. Confidence tags: Write confidence scores to separate OPC tags
The method iterates through optional ``prediction_tags`` and
``confidence_tags``, performs synchronous writes for each tag, collects
per-tag response times, and returns an aggregate success flag
(all tags successful) with a metrics-friendly response map.
Args:
server_id (str): Unique identifier for the target OPC server
config (dict[str, Any]): OPC tag configuration containing:
- prediction_tags (dict, optional): Prediction tag configurations
- confidence_tags (dict, optional): Confidence tag configurations
data (DataFrame): DataFrame containing prediction and confidence data
metadata (dict[str, Any]): Context metadata for logging and notifications
success (bool): Current success status to maintain across operations
- server_id (str): Target OPC server id.
- config (dict[str, Any]): Server output configuration containing optional
``prediction_tags`` and ``confidence_tags`` sections.
- data (DataFrame): Prediction dataframe used as source values.
- metadata (dict[str, Any]): Workflow metadata for logging/notifications.
Returns:
tuple[bool, int]: (overall_success, total_tags_written)
- overall_success: True if all configured tags were written successfully
- total_tags_written: Count of successfully written tags
Return:
tuple[bool, dict[str, float | None]]: Global success flag and response-time
map per tag (``None`` for failed writes).
"""
response_times: dict[str, float | None] = {}
if 'prediction_tags' in config:
for tag, tag_config in config['prediction_tags'].items():
response_time = await self.write_data(
response_time = self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction'].values[0],
@@ -254,7 +222,7 @@ class OPC(SientiaMonitoring):
if 'confidence_tags' in config:
for tag, tag_config in config['confidence_tags'].items():
response_time = await self.write_data(
response_time = self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction_confidence'].values[0],
@@ -274,25 +242,25 @@ class OPC(SientiaMonitoring):
return success, response_times
@activity.defn(name='write_opc_data')
async def write_opc_data(
def write_opc_data(
self, input_data: dict[str, Any]
) -> tuple[dict[Hashable, Any], dict[str, dict[str, float | None]]]:
"""
Write prediction and confidence data to OPC servers. The two writing
operations are optional and independent of each other.
Execute OPC writes across all configured servers and collect per-tag metrics.
For each server in ``opc_output_config``, this activity validates server
availability, writes enabled prediction/confidence tags, accumulates
response-time metrics, and then normalizes confidence/comments in the
returned prediction payload when at least one write fails.
Args:
- input_data(dict[str, Any]): The input data. Contains the following keys:
- data(dict[str, Any]): The dataframe that contains the data to write
to the OPC servers.
- opc_output_config(dict[str, Any]): The OPC writing configuration.
The keys are the OPC server names and the values contain:
- prediction_tags(dict[str, Any]): The tags to write to the OPC servers.
- confidence_tags(dict[str, Any]): The tags to write to the OPC servers.
Returns:
- dict[Any, Any]: The data that was written to the OPC servers.
- input_data (dict[str, Any]): Payload containing workflow metadata, data
to write, and ``opc_output_config`` server/tag definitions.
Return:
tuple[dict[Hashable, Any], dict[str, dict[str, float | None]]]: Updated
prediction payload dict and nested metrics
``{server_id: {tag_name: response_time_or_none}}``.
"""
metadata = input_data['metadata']
self.info('Writing data to OPC servers...', metadata)
@@ -305,19 +273,21 @@ class OPC(SientiaMonitoring):
metrics: dict[str, dict[str, float | None]] = {}
for server_id, config in opc_output_config.items():
if not await self.validate_server(server_id, metadata):
if not self.validate_server(server_id, metadata):
success = False
continue
local_success, local_response_times = await self.manage_output_tags(
local_success, local_response_times = self.manage_output_tags(
server_id, config, data, metadata
)
metrics[server_id] = local_response_times
local_count = len(local_response_times)
success = success and local_success
n_pred = len(config.get('prediction_tags') or {})
n_conf = len(config.get('confidence_tags') or {})
self.info(
f'Process completed for OPC server {server_id}: {local_count} of {len(config.get("prediction_tags", []))} prediction tags and {len(config.get("confidence_tags", []))} confidence tags',
f'Process completed for OPC server {server_id}: {local_count} of {n_pred} prediction tags and {n_conf} confidence tags',
metadata,
)
@@ -327,29 +297,16 @@ class OPC(SientiaMonitoring):
self, data: DataFrame, success: bool, metadata: dict[str, Any]
) -> dict[Hashable, Any]:
"""
Process prediction confidence based on OPC write operation success.
This method updates the prediction confidence values in the DataFrame
based on the success status of OPC server write operations. If any
write operations failed, it sets the confidence to a predefined error
value to indicate data quality issues.
The method implements a confidence degradation strategy:
- Success: Maintains original confidence values
- Failure: Sets confidence to error value for operational awareness
Apply fallback confidence/comment values when OPC writes are not fully successful.
Args:
data (DataFrame): DataFrame containing prediction and confidence data
success (bool): Overall success status of OPC write operations
metadata (dict[str, Any]): Context metadata for logging and notifications
- data (DataFrame): Prediction dataframe to be returned to downstream steps.
- success (bool): Aggregate write status across all attempted OPC tags.
- metadata (dict[str, Any]): Workflow metadata used for debug logs.
Returns:
dict[Any, Any]: Processed data as a dictionary with updated confidence values
Note:
The error confidence value (OPC_WRITTING_ERROR_CONFIDENCE = 12) is
used to indicate that data was not successfully exported to OPC servers.
This allows downstream systems to handle data quality appropriately.
Return:
dict[Hashable, Any]: Serialized dataframe dict with original values on success,
or downgraded confidence/comment fields on failure.
"""
message = 'Some data could not be written to OPC servers'
@@ -367,25 +324,14 @@ class OPC(SientiaMonitoring):
return data.to_dict()
async def close(self):
def close(self) -> None:
"""
Gracefully shutdown all OPC server connections and cleanup resources.
Disconnect all tracked OPC repositories and clear in-memory references.
This method ensures proper cleanup of all active OPC server connections
by calling the disconnect method on each repository instance. It's
designed to be called during application shutdown to prevent resource
leaks and ensure clean termination.
The method performs the following cleanup operations:
1. Iterates through all active OPC repository connections
2. Calls disconnect() on each repository instance
3. Allows for graceful connection termination
4. Prevents resource leaks and connection hanging
Note:
This method should be called during application shutdown to ensure
proper cleanup. It handles all active connections regardless of
their current state and provides a clean shutdown experience.
This method should be called during worker shutdown to ensure every
synchronous OPC session is explicitly closed before process exit.
"""
for opc in self.opc_repository.values():
await opc.disconnect()
opc.disconnect()
self.opc_repository.clear()

72
laborious/activities/storage.py
View File

@@ -1,7 +1,5 @@
from temporalio import activity, workflow
from laborious.utils.repository.minio_manager import MinioManager
with workflow.unsafe.imports_passed_through():
# Extend the Temporal Postgres activities for convenient query -> MinIO export
import traceback
@@ -13,8 +11,9 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_do.temporal.activities.postgres_sync import Postgres
from sientia_do.temporal.constants import now
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
@@ -22,7 +21,7 @@ with workflow.unsafe.imports_passed_through():
_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
class Storage(Postgres, MinioManager):
class Storage(Postgres, SientiaMonitoring):
"""
Extensions for Postgres activities with a helper to export query results
directly to MinIO as Parquet and return the object name.
@@ -60,14 +59,16 @@ class Storage(Postgres, MinioManager):
metrics_controller=metrics_controller,
)
MinioManager.__init__(
self, minio_repository, logger, notification_handler, metrics_controller
self.minio_repository = minio_repository
SientiaMonitoring.__init__(
self,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
@activity.defn(name='load_query_with_minio_offload')
async def load_query_with_minio_offload(
self, input_data: dict[str, Any]
) -> MinioDataFramePayload:
def load_query_with_minio_offload(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
"""
Run the custom SQL load, then return a MinIO-aware dataframe wire dict.
@@ -88,7 +89,7 @@ class Storage(Postgres, MinioManager):
metadata: dict = input_data.get('metadata', {})
model_name = input_data['model_name']
rows = await self.load_custom_query(
rows = self.load_custom_query(
input_data,
)
if not rows:
@@ -99,7 +100,7 @@ class Storage(Postgres, MinioManager):
else:
dataframe = pd.DataFrame(rows)
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe,
minio_repo=self.minio_repository,
workflow_metadata=metadata,
@@ -109,15 +110,29 @@ class Storage(Postgres, MinioManager):
)
@activity.defn(name='export_payload_to_postgres')
async def export_payload_to_postgres(self, input_data: dict[str, Any]) -> dict:
def export_payload_to_postgres(self, input_data: dict[str, Any]) -> dict:
"""
Export a payload to PostgreSQL.
Resolve a MinIO-aware payload into a DataFrame and persist it into PostgreSQL.
This activity accepts the serialized payload produced by previous steps
(inline dict or MinIO object reference), reconstructs the tabular data,
and delegates the final write to ``export_data_to_postgres`` using the
same input contract expected by the Postgres activity mixin.
Args:
- input_data (dict[str, Any]): Activity input containing ``data`` as a
``MinioDataFramePayload``-compatible dict plus database write options
(schema/table/on_conflict/metadata and related fields).
Return:
dict: Result dictionary returned by ``export_data_to_postgres``, including
success status and optional write diagnostics.
"""
metadata = input_data.get('metadata')
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
return await self.export_data_to_postgres(
return self.export_data_to_postgres(
{
**input_data,
'data': data,
@@ -125,7 +140,7 @@ class Storage(Postgres, MinioManager):
)
@activity.defn(name='cleanup_minio_objects_expired')
async def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Delete objects under the given prefixes that are older than the retention window.
@@ -154,7 +169,7 @@ class Storage(Postgres, MinioManager):
'deleted_count': 0,
}
try:
keys = await self.minio_repository.list_objects(
keys = self.minio_repository.list_objects(
prefix=prefix,
recursive=True,
metadata=metadata,
@@ -166,7 +181,7 @@ class Storage(Postgres, MinioManager):
continue
if ts >= cutoff:
continue
await self.minio_repository.delete_file(
self.minio_repository.delete_file(
object_name=key,
metadata=metadata,
)
@@ -184,7 +199,7 @@ class Storage(Postgres, MinioManager):
report['deleted_count'] += 1
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='ERROR_CLEANUP_MINIO_OBJECTS_EXPIRED',
message=f'Error cleaning up MinIO objects: {e}',
@@ -201,9 +216,16 @@ class Storage(Postgres, MinioManager):
return report
def close(self) -> None:
"""Close Storage resources (MinIO client and Postgres engine)."""
Postgres.close(self)
MinioManager.close(self)
"""
Shutdown Storage resources in deterministic order.
def __del__(self):
self.close()
The method first closes Postgres resources via ``Postgres.close`` (engine,
sessions, and monitoring hooks), then closes the optional MinIO repository
and clears the local reference to avoid accidental reuse after shutdown.
"""
Postgres.close(self)
if self.minio_repository is not None:
try:
self.minio_repository.close()
finally:
self.minio_repository = None

30
laborious/utils/models/minio_dataframe_payload.py
View File

@@ -21,7 +21,7 @@ from typing import Any, Literal
from pandas import DataFrame, read_parquet
from sientia_do.observability.logger import Logger
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
# Keys that are part of the serialized wire format (not arbitrary metadata).
@@ -89,7 +89,7 @@ class MinioDataFramePayload:
metadata: dict[str, Any] | None = None,
) -> None:
"""
Emit debug logs only when logger is provided
Emit a debug message only when a logger instance is available.
Args:
- logger (Logger | None): Logger instance used for debug messages
@@ -182,7 +182,14 @@ class MinioDataFramePayload:
def cleanup_prefix(self) -> str | None:
"""
Return True if cleanup is enabled for this payload.
Return the MinIO prefix eligible for retention cleanup.
Cleanup is only applicable when payload data was offloaded to MinIO
(``object_key`` present and inline ``data`` absent). Inline-only payloads
return ``None`` because there is no object tree to prune.
Return:
str | None: Prefix used by cleanup listing, or ``None`` when cleanup does not apply.
"""
if self.object_key is not None and self.data is None:
return self.object_prefix
@@ -190,12 +197,19 @@ class MinioDataFramePayload:
def has_data(self) -> bool:
"""
Return True if the payload has some data internally or in MinIO.
Indicate whether the payload contains retrievable tabular content.
A payload is considered non-empty when either inline ``data`` exists
(and is not an empty dict) or an ``object_key`` is available for MinIO
download.
Return:
bool: ``True`` when data can be retrieved, ``False`` otherwise.
"""
return (self.data is not None and self.data != {}) or self.object_key is not None
@classmethod
async def from_dataframe(
def from_dataframe(
cls,
dataframe: DataFrame | None,
minio_repo: MinioRepository,
@@ -274,7 +288,7 @@ class MinioDataFramePayload:
dataframe.to_parquet(parquet_buffer, engine='pyarrow', index=True)
file_bytes = parquet_buffer.getvalue()
upload_result = await minio_repo.upload_file(
upload_result = minio_repo.upload_file(
file_bytes=file_bytes,
relative_key=object_key,
metadata=workflow_metadata,
@@ -299,7 +313,7 @@ class MinioDataFramePayload:
status=status,
)
async def retrieve(
def retrieve(
self,
minio_repo: MinioRepository,
workflow_metadata: dict[str, Any] | None = None,
@@ -336,7 +350,7 @@ class MinioDataFramePayload:
f'MinioDataFramePayload.retrieve downloading object from MinIO: {self.object_key}',
workflow_metadata,
)
file_bytes = await minio_repo.download_file(
file_bytes = minio_repo.download_file(
object_name=self.object_key, metadata=workflow_metadata
)
df = read_parquet(BytesIO(file_bytes))

32
laborious/utils/repository/minio_manager.py
View File

@@ -1,32 +0,0 @@
from sientia_do.notifications.handlers import NotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository import MinioRepository
class MinioManager(SientiaMonitoring):
minio_repository: MinioRepository | None = None
def __init__(
self,
minio_repository: MinioRepository | None = None,
logger: Logger | None = None,
notification_handler: NotificationHandler | None = None,
metrics_controller: MetricsController | None = None,
):
if self.minio_repository is None:
self.minio_repository = minio_repository
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
def close(self) -> None:
"""
Close the MinioManager and clean up resources.
"""
if self.minio_repository is not None:
try:
self.minio_repository.close()
finally:
self.minio_repository = None
SientiaMonitoring.shutdown(self)

281
laborious/utils/repository/opc_repository.py
View File

@@ -1,4 +1,10 @@
import asyncio
"""
Synchronous OPC UA client repository using python-opcua (opcua package).
Connects to OPC UA servers, optionally configures Basic256 security, validates sessions,
and writes node values with typed variants and Prometheus-compatible metrics.
"""
import json
import time
import traceback
@@ -6,9 +12,8 @@ from datetime import datetime
from pathlib import Path
from typing import Any
from asyncua import Client
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.ua import DataValue, Variant, VariantType
from opcua import Client, ua
from opcua.crypto import security_policies
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
@@ -20,28 +25,38 @@ from laborious import metrics
data_type_map = {
'float': {
'converter': float,
'opc_type': VariantType.Float,
'opc_type': ua.VariantType.Float,
},
'double': {
'converter': float,
'opc_type': VariantType.Double,
'opc_type': ua.VariantType.Double,
},
'int': {
'converter': int,
'opc_type': VariantType.Int32,
'opc_type': ua.VariantType.Int32,
},
'bool': {
'converter': bool,
'opc_type': VariantType.Boolean,
'opc_type': ua.VariantType.Boolean,
},
'str': {
'converter': str,
'opc_type': VariantType.String,
'opc_type': ua.VariantType.String,
},
}
class OpcRepository(SientiaMonitoring):
"""
Synchronous OPC UA repository for connect/disconnect and typed writes.
Attributes:
url: OPC UA endpoint URL.
id: Server identifier used in metrics and notifications.
server_name: Human-readable server name for labels.
client: Active opcua.Client instance while connected.
"""
def __init__(
self,
opc_id: str,
@@ -66,10 +81,10 @@ class OpcRepository(SientiaMonitoring):
self.logger = logger
self.error_count = 0
self.reconnection_interval = reconnection_interval
self.last_reconnection_time: None | datetime = None
self.last_reconnection_time: datetime | None = None
self.disconnection_interval = 10.0
self.notification_handler = notification_handler
self.client: None | Client = None
self.client: Client | None = None
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
@@ -80,24 +95,12 @@ class OpcRepository(SientiaMonitoring):
'schedule_name': '-',
}
async def set_security(self):
def set_security(self) -> None:
"""
Configures the security settings for the OPC UA client.
This method sets up the security policy, certificates, and timeouts
required for establishing a secure connection with the OPC UA server.
Configure Basic256 security policy, certificates, and long channel/session timeouts.
Raises:
ValueError: If either the certificate path or private key path is not provided.
Attributes:
- cert_path (str): Path to the client's certificate file.
- private_key_path (str): Path to the client's private key file.
- server_cert_path (str, optional): Path to the server's certificate file.
- server_uri (str): The URI of the server to be used as the application URI.
- client (opcua.Client): The OPC UA client instance.
- logger (logging.Logger): Logger instance for logging information.
Security Settings:
- Security Policy: Basic256
- Secure Channel Timeout: 10,000,000 ms
- Session Timeout: 10,000,000 ms
ValueError: If certificate paths are missing or client is not initialized.
"""
if self.cert_path is None or self.private_key_path is None:
@@ -114,26 +117,24 @@ class OpcRepository(SientiaMonitoring):
self.client.application_uri = self.server_uri
self.logger.custom_info('Setting security...', self.metadata)
await self.client.set_security(
SecurityPolicyBasic256,
certificate=str(cert),
private_key=str(private_key),
server_certificate=str(server_cert) if server_cert else None,
self.client.set_security(
security_policies.SecurityPolicyBasic256,
str(cert),
str(private_key),
str(server_cert) if server_cert else None,
)
self.client.secure_channel_timeout = 10000000
self.client.session_timeout = 10000000
async def connect(self) -> tuple[bool, dict[str, Any]]:
def connect(self) -> tuple[bool, dict[str, Any]]:
"""
Establishes a connection to the OPC server.
This method initializes the OPC client using the provided URL and
sets up security if a certificate path is specified. It then
attempts to connect to the server and logs the connection status.
Raises:
Exception: If the connection to the OPC server fails.
Create the synchronous client, optionally apply security, and connect to the server.
Return:
tuple[bool, dict[str, Any]]: Success flag and error payload when False.
"""
self.client = Client(self.url, timeout=10, watchdog_intervall=3600000) # type: ignore[attr-defined]
self.client = Client(self.url, timeout=10)
self.client.name = self.pod_id
self.client.application_name = self.pod_id
@@ -142,32 +143,25 @@ class OpcRepository(SientiaMonitoring):
self.client.product_uri = pod_uri
if self.cert_path:
await self.set_security()
self.set_security()
self.logger.custom_info(
f'Starting connection to OPC server {self.id}:{self.server_name}...', self.metadata
)
return await self.try_connect()
return self.try_connect()
async def try_connect(self) -> tuple[bool, dict[str, Any]]:
def try_connect(self) -> tuple[bool, dict[str, Any]]:
"""
Attempt to establish connection to the OPC server.
Perform the TCP/session handshake and emit connection metrics.
This method performs the actual connection attempt to the OPC server
and handles connection failures with comprehensive error reporting.
It updates reconnection timing and provides detailed error information
for operational monitoring and debugging.
Returns:
tuple[bool, dict[str, Any]]: Connection result
- bool: True if connection successful, False otherwise
- dict: Error information if connection failed
Return:
tuple[bool, dict[str, Any]]: Success flag and structured error when False.
"""
tags = {
'pod_id': self.pod_id,
'server_name': self.server_name,
}
await self.emit_metric(metrics.OPC_CONNECTIONS_TOTAL, tags)
self.emit_metric_sync(metrics.OPC_CONNECTIONS_TOTAL, tags)
try:
self.last_reconnection_time = datetime.now()
if self.client is None:
@@ -177,9 +171,9 @@ class OpcRepository(SientiaMonitoring):
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
}
await self.client.connect()
self.client.connect()
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.OPC_CONNECTION_STATUS,
method='set',
tags={
@@ -191,12 +185,12 @@ class OpcRepository(SientiaMonitoring):
return True, {}
except Exception as e:
await self.disconnect()
self.disconnect()
trace = traceback.format_exc()
self.logger.custom_error(trace, self.metadata)
await self.emit_metric(metrics.OPC_CONNECTIONS_FAILED, tags)
self.emit_metric_sync(metrics.OPC_CONNECTIONS_FAILED, tags)
return False, {
'notification_id': f'OPC_CONNECTION_ERROR_{self.id}',
@@ -206,21 +200,27 @@ class OpcRepository(SientiaMonitoring):
'attachment_content': trace,
}
async def disconnection_fallback(self) -> list:
def disconnection_fallback(self) -> list[dict[str, Any]]:
"""
Tries 5 times to disconnect from the OPC UA server, with a delay of 100ms x try.
Retry disconnect up to five times with linear backoff.
Return:
list[dict[str, Any]]: Empty on success, otherwise error records per attempt.
"""
assert self.client is not None
error_stack = []
for i in range(5):
try:
self.logger.info(f'Disconnecting from OPC UA server, attempt {i + 1} of 5')
await self.client.disconnect()
self.logger.custom_info(
f'Disconnecting from OPC UA server, attempt {i + 1} of 5', self.metadata
)
self.client.disconnect()
return []
except Exception as e:
self.logger.error(
f'Failed to disconnect from OPC UA server in attempt {i + 1} of 5: {e}'
self.logger.custom_error(
f'Failed to disconnect from OPC UA server in attempt {i + 1} of 5: {e}',
self.metadata,
)
error_stack.append(
{
@@ -229,23 +229,20 @@ class OpcRepository(SientiaMonitoring):
'traceback': traceback.format_exc(),
}
)
await asyncio.sleep(self.disconnection_interval * i)
time.sleep(self.disconnection_interval * i)
return error_stack
async def disconnect(self):
def disconnect(self) -> None:
"""
Tear down the UA session and reset connection metrics.
"""
Gracefully disconnect from the OPC server.
This method safely terminates the connection to the OPC server
and cleans up client resources. It handles disconnection errors
gracefully and ensures proper resource cleanup.
"""
if self.client is None:
return
errors = await self.disconnection_fallback()
errors = self.disconnection_fallback()
if errors:
await self.send_notification_async(
self.send_notification(
metadata=self.metadata,
notification_id=f'OPC_DISCONNECTION_ERROR_{self.id}',
message='Failed to disconnect from OPC server in 5 attempts.',
@@ -254,8 +251,10 @@ class OpcRepository(SientiaMonitoring):
attachment_content=json.dumps(errors, indent=4),
)
else:
self.logger.warning(f'Disconnected from OPC server {self.id} successfully')
await self.emit_metric(
self.logger.custom_warning(
f'Disconnected from OPC server {self.id} successfully', self.metadata
)
self.emit_metric_sync(
metric_object=metrics.OPC_CONNECTION_STATUS,
method='set',
tags={
@@ -268,77 +267,52 @@ class OpcRepository(SientiaMonitoring):
self.client = None
async def validate_connection(self) -> tuple[bool, dict[str, Any]]:
def _session_alive(self) -> bool:
"""
Validate and maintain OPC server connection health.
Best-effort check that the synchronous client still has a working session.
This method performs comprehensive connection validation and
implements automatic reconnection logic for production reliability.
It handles various connection states and implements intelligent
reconnection strategies with error counting and timing controls.
Connection Validation:
1. Checks client existence and connection state
2. Implements error counting with automatic disconnection
3. Enforces reconnection timing windows
4. Provides detailed error reporting and notifications
Reconnection Strategy:
- Error Count Threshold: Disconnects after 5 consecutive errors
- Reconnection Window: Enforces minimum intervals between attempts
- Automatic Recovery: Attempts reconnection when conditions allow
- State Monitoring: Continuously monitors connection health
Args:
None
Returns:
tuple[bool, dict[str, Any]]: Connection validation result
- bool: True if connection is healthy, False otherwise
- dict: Error information if validation fails
Return:
bool: True if a root browse succeeds, False otherwise.
"""
if self.client is None:
return await self.connect()
# if self.error_count > 5: # NOSONAR
# self.logger.custom_warning(
# f'OPC server {self.id} will be disconnected due to multiple errors', self.metadata
# )
# try:
# await self.disconnect()
# except Exception as e:
# trace = traceback.format_exc()
# self.logger.custom_error(
# f'Failed to disconnect from OPC server: {e}', self.metadata
# )
# self.logger.custom_error(trace, self.metadata)
# self.logger.custom_info(
# f'Attempting to reconnect to OPC server {self.id}...', self.metadata
# )
# return await self.connect()
# Check if client is connected using asyncua's connection state
return False
try:
if (
self.client.uaclient.protocol is None
or self.client.uaclient.protocol.state == 'closed'
):
# OPC server is not connected
self.client.get_root_node()
return True
except Exception:
return False
def validate_connection(self) -> tuple[bool, dict[str, Any]]:
"""
Ensure the UA session is usable; reconnect when outside the backoff window.
Return:
tuple[bool, dict[str, Any]]: Whether the session is ready and optional error payload.
"""
if self.client is None:
return self.connect()
try:
if not self._session_alive():
self.logger.custom_error(f'OPC server {self.id} is not connected', self.metadata)
if (
self.last_reconnection_time is None
or (datetime.now() - self.last_reconnection_time).total_seconds()
> self.reconnection_interval
):
await self.disconnect()
self.disconnect()
self.logger.custom_info(
f'Trying to reconnect to OPC server {self.id}...', self.metadata
)
return await self.connect()
return self.connect()
return False, {
'notification_id': f'OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}',
'message': f'OPC server {self.id} is not connected, waiting for next reconnection window...',
'message': (
f'OPC server {self.id} is not connected, waiting for next reconnection window...'
),
'block': 'opc_repository',
'level': NotificationLevel.WARNING,
}
@@ -355,39 +329,24 @@ class OpcRepository(SientiaMonitoring):
'attachment_content': trace,
}
async def write_data(
def write_data(
self, node: str, value: Any, data_type: str, logger: Logger, metadata: dict[str, Any]
) -> tuple[bool, dict[str, Any]]:
"""
Write data to OPC server with comprehensive validation and monitoring.
This method provides secure and reliable data writing to OPC servers
with automatic connection validation, data type conversion, and
comprehensive error handling. It implements performance monitoring
and metrics collection for operational visibility.
Data Writing Process:
1. Connection validation and automatic reconnection
2. Node validation and error handling
3. Data type conversion and validation
4. OPC data writing with timestamp
5. Performance metrics collection
6. Error handling and notification
Write a typed value to an OPC UA node after validating connectivity.
Args:
node (str): OPC node identifier to write data to
value (Any): Data value to write to the OPC node
data_type (str): Data type for OPC conversion
logger (Logger): Logger instance for operation logging
metadata (dict[str, Any]): Context metadata for logging and metrics
node: Node id string accepted by opcua Client.get_node.
value: Scalar value to encode.
data_type: Key into ``data_type_map`` (e.g. float, str).
logger: Caller logger for per-write traces.
metadata: Workflow metadata for error context.
Returns:
tuple[bool, dict[str, Any]]: Write operation result
- bool: True if write successful, False otherwise
- dict: Error information if write failed
Return:
tuple[bool, dict[str, Any]]: Success flag and either ``response_time`` or error fields.
"""
is_connected, error = await self.validate_connection()
is_connected, error = self.validate_connection()
if not is_connected:
return False, error
@@ -395,8 +354,8 @@ class OpcRepository(SientiaMonitoring):
start_time = time.time()
try:
# ignored because self.validate_connection is called before, so we know self.client is not None
node_obj = self.client.get_node(node) # type: ignore[union-attr]
assert self.client is not None
node_obj = self.client.get_node(node)
except Exception as e:
trace = traceback.format_exc()
logger.custom_error(trace, metadata.get('schedule_name', 'N/A'))
@@ -419,16 +378,10 @@ class OpcRepository(SientiaMonitoring):
data = data_type_map[data_type]['converter'](value)
logger.custom_info(f'Writing {data} - {type(data)} to {node}', metadata)
# now = datetime.now() # NOSONAR
ua_data = DataValue(
Variant(data, data_type_map[data_type]['opc_type']),
# SourceTimestamp=DateTime( # NOSONAR
# now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond # NOSONAR
# ), # NOSONAR
)
variant_type = data_type_map[data_type]['opc_type']
try:
await node_obj.write_value(ua_data)
node_obj.set_value(data, variant_type)
end_time = time.time()
response_time = end_time - start_time

4
laborious/worker/worker.py
View File

@@ -162,7 +162,7 @@ async def main():
)
logger.custom_info('Initializing OPC...', metadata)
await activities.init_opc()
activities.init_opc()
logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
@@ -265,7 +265,7 @@ async def main():
exit_code = 1
finally:
notification_handler.shutdown()
await activities.shutdown()
activities.shutdown()
metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN
sys.exit(exit_code)

2
laborious/workflows/sub_workflows/format_and_export_prediction.py
View File

@@ -161,7 +161,7 @@ class FormatAndExportPrediction:
write_transformed_handler = None
opc_metrics = {}
opc_metrics: dict[str, dict[str, float | None]] = {}
# write to pi web api
if pi_web_api_output_config:

2
laborious/workflows/sub_workflows/prediction_process.py
View File

@@ -250,7 +250,7 @@ class PredictionProcess:
async def path_flag_handler(
self,
data: dict[str, Any],
path_flag: str,
path_flag: str | None,
input_data: dict,
confidence: int,
last_timestamp: str,