gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1646

Browse Source 
Update README, requirements, and E2E tests for improved configuration and functionality

- Enhanced the README with updated model configuration examples, including the addition of an alias for production.
- Removed the `requirements-light.txt` file and updated `requirements-local.txt` and `requirements.txt` to replace `asyncua` with `opcua`.
- Refactored E2E test scenarios to utilize scenario input files for better maintainability and clarity.
- Improved test coverage for MinIO offload functionality and added new helper functions for loading scenario inputs.
- Updated `values.yaml` to reflect new global configurations and environment variables for the laborious worker.
This commit is contained in:
Vitor Pimentel dos Santos
2026-05-07 17:02:25 -03:00
parent aaf647efdf
commit e6018af23f
51 changed files with 4408 additions and 2660 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
laborious/activities/activities.py
View File

@@ -6,7 +6,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
from sientia_model.model_repository.plugin_store import PluginStore
@@ -159,7 +159,7 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
metrics_controller=mc,
)
async def shutdown(self):
def shutdown(self) -> None:
"""
Close database pools, sync clients, and OPC sessions in a defined order.
@@ -172,6 +172,6 @@ class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
Storage.close(self)
MLFlow.close(self)
Gates.close(self)
await OPC.close(self)
OPC.close(self)
ModelMetrics.close(self)
API.close(self)

24
laborious/activities/api.py
View File

@@ -11,7 +11,7 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.pi_web_api_client import PIWebAPIClient
from sientia_do.repository.pi_web_api_client_sync import PIWebAPIClient
from laborious import metrics
@@ -105,7 +105,7 @@ class API(SientiaMonitoring):
self.pi_web_api_client.close()
SientiaMonitoring.shutdown(self)
async def process_pi_web_api_response(
def process_pi_web_api_response(
self,
response_data: list[dict[str, Any]],
tags: dict[str, str],
@@ -156,7 +156,7 @@ class API(SientiaMonitoring):
self.error(
f'Error writing tag {tag_name}:{web_id} to PI Web API: {errors}', metadata
)
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT,
tags={
**core_labels,
@@ -165,7 +165,7 @@ class API(SientiaMonitoring):
)
confidence = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
else:
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_COUNT,
tags={
**core_labels,
@@ -182,7 +182,7 @@ class API(SientiaMonitoring):
metadata,
)
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
message=f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written.\nResponse:\n {json.dumps(response_data, indent=4)}\nTags:\n {json.dumps(tags, indent=4)}',
@@ -194,7 +194,7 @@ class API(SientiaMonitoring):
return confidence, message
@activity.defn(name='write_pi_web_api_data')
async def write_pi_web_api_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
def write_pi_web_api_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
"""
Write prediction and confidence data to PI Web API.
@@ -232,7 +232,7 @@ class API(SientiaMonitoring):
confidence_value = data.head(1)['prediction_confidence'].values[0]
try:
prediction_response = await self.pi_web_api_client.write_value(
prediction_response = self.pi_web_api_client.write_value(
web_ids=prediction_tags,
value={
'Timestamp': data.head(1)['timestamp'].values[0],
@@ -241,7 +241,7 @@ class API(SientiaMonitoring):
metadata=metadata,
)
confidence, message = await self.process_pi_web_api_response(
confidence, message = self.process_pi_web_api_response(
response_data=prediction_response,
tags=raw_prediction_tags,
core_labels=core_labels,
@@ -258,7 +258,7 @@ class API(SientiaMonitoring):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
message=f'Error writing prediction data to PI Web API: {e}\n Tags: {raw_prediction_tags}',
@@ -275,7 +275,7 @@ class API(SientiaMonitoring):
return data.to_dict()
try:
confidence_response = await self.pi_web_api_client.write_value(
confidence_response = self.pi_web_api_client.write_value(
web_ids=confidence_tags,
value={
'Timestamp': data.head(1)['timestamp'].values[0],
@@ -284,7 +284,7 @@ class API(SientiaMonitoring):
metadata=metadata,
)
await self.process_pi_web_api_response(
self.process_pi_web_api_response(
response_data=confidence_response,
tags=raw_confidence_tags,
core_labels=core_labels,
@@ -293,7 +293,7 @@ class API(SientiaMonitoring):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='WRITE_PI_WEB_API_CONFIDENCE_ERROR',
message=f'Error writing confidence data to PI Web API: {e}\n Tags: {raw_confidence_tags}',

70
laborious/activities/gates.py
View File

@@ -1,8 +1,5 @@
from sientia_do.repository.minio_repository import MinioRepository
from temporalio import activity, workflow
from laborious.utils.repository.minio_manager import MinioManager
with workflow.unsafe.imports_passed_through():
import traceback
from collections.abc import Callable, Mapping
@@ -13,6 +10,8 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_do.utils.formatters import create_sample_dict
from laborious import metrics
@@ -66,7 +65,7 @@ mlflow_content_path_confidence: Mapping[str, int] = {
}
class Gates(MinioManager):
class Gates(SientiaMonitoring):
"""
Data quality gates and filtering activities for the Laborious system.
@@ -106,8 +105,12 @@ class Gates(MinioManager):
Raises:
Exception: If BaseActivity initialization fails
"""
MinioManager.__init__(
self, minio_repository, logger, notification_handler, metrics_controller
self.minio_repository = minio_repository
SientiaMonitoring.__init__(
self,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
def close(self) -> None:
@@ -115,7 +118,12 @@ class Gates(MinioManager):
Close the gates activity and clean up resources.
"""
MinioManager.close(self)
if self.minio_repository is not None:
try:
self.minio_repository.close()
finally:
self.minio_repository = None
SientiaMonitoring.shutdown(self)
def __del__(self):
self.close()
@@ -155,7 +163,7 @@ class Gates(MinioManager):
return policy, filter_config
@activity.defn(name='input_gate')
async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Apply input data quality filters and validation.
@@ -194,7 +202,7 @@ class Gates(MinioManager):
filters = input_data['filters']
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
path_priority = input_data['path_priority']
filter_output = []
@@ -214,7 +222,7 @@ class Gates(MinioManager):
filter_output.append(policy)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'INTPUT_GATE_ERROR__{fil}',
message=f'Error in filter {fil}:{config}: \n {e}',
@@ -235,7 +243,7 @@ class Gates(MinioManager):
return None, 0, ''
@activity.defn(name='mlflow_response_gate')
async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Validate MLFlow API response quality and integrity.
@@ -279,7 +287,7 @@ class Gates(MinioManager):
self.debug(f'Filters: {filters}', metadata)
payload = MinioDataFramePayload.from_dict(raw_data)
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
gate_type = input_data['type']
path_priority = input_data['path_priority']
@@ -298,7 +306,7 @@ class Gates(MinioManager):
if mlflow_response_filter_functions[fil](status, filter_config):
filter_output.append(policy)
comments.append(status.get('message', 'Unknown MLFlow API error'))
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
message=status.get('message', 'Unknown MLFlow API error'),
@@ -308,7 +316,7 @@ class Gates(MinioManager):
)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'MLFLOW_GATE_RESPONSE_FILTER__{fil}',
message=f'Error in filter {fil}:{config}: \n {e}',
@@ -329,7 +337,7 @@ class Gates(MinioManager):
return None, 0, ''
@activity.defn(name='mlflow_content_gate')
async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Validate MLFlow prediction content quality and integrity.
@@ -368,7 +376,7 @@ class Gates(MinioManager):
filters = input_data['filters']
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
gate_type = input_data['type']
path_priority = input_data['path_priority']
@@ -385,7 +393,7 @@ class Gates(MinioManager):
try:
if mlflow_content_filter_functions[fil](data, filter_config):
filter_output.append(policy)
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}',
message=f'Data not passed the content filter {fil}:{config}',
@@ -395,7 +403,7 @@ class Gates(MinioManager):
)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'MLFLOW_GATE_CONTENT_FILTER__{fil}',
message=f'Error in filter {fil}:{config}: \n {e}',
@@ -471,7 +479,7 @@ class Gates(MinioManager):
return policy_type, int(policy_value)
@activity.defn(name='format_transformed_data')
async def format_transformed_data(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
def format_transformed_data(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
"""
Format transformed data for storage and export operations.
@@ -507,7 +515,7 @@ class Gates(MinioManager):
self.info('Formatting transformed data...', metadata)
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
data['timestamp'] = data.index
data = data.reset_index(drop=True)
@@ -515,7 +523,7 @@ class Gates(MinioManager):
data = data.melt(id_vars='timestamp', var_name='variable', value_name='value')
data['model_id'] = model_id
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=data,
minio_repo=self.minio_repository,
model_name=input_data['model_name'],
@@ -526,7 +534,7 @@ class Gates(MinioManager):
)
@activity.defn(name='format_prediction')
async def format_prediction(self, input_data: dict[str, Any]) -> dict:
def format_prediction(self, input_data: dict[str, Any]) -> dict:
"""
Format prediction data according to configured storage policies.
@@ -558,7 +566,7 @@ class Gates(MinioManager):
self.info('Formatting prediction...', metadata)
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
# Create timestamp column from index and reset index
data['timestamp'] = data.index
@@ -607,7 +615,7 @@ class Gates(MinioManager):
return data.to_dict()
@activity.defn(name='format_default_prediction')
async def format_default_prediction(self, input_data: dict[str, Any]) -> dict:
def format_default_prediction(self, input_data: dict[str, Any]) -> dict:
"""
Create and format default prediction data for error conditions.
@@ -652,7 +660,7 @@ class Gates(MinioManager):
return data.to_dict()
@activity.defn(name='format_retrain_report')
async def format_retrain_report(self, input_data: dict[str, Any]) -> dict:
def format_retrain_report(self, input_data: dict[str, Any]) -> dict:
"""
Format retrain report data for storage and audit trail maintenance.
@@ -722,7 +730,7 @@ class Gates(MinioManager):
return report.to_dict()
@activity.defn(name='write_metrics')
async def write_metrics(self, input_data: dict[str, Any]):
def write_metrics(self, input_data: dict[str, Any]):
"""
Write prediction performance metrics to Prometheus monitoring system.
@@ -759,19 +767,19 @@ class Gates(MinioManager):
'model_name': metadata['model_name'],
'workflow_name': metadata['workflow_name'],
}
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTIONS_WRITTEN_COUNT,
tags=core_tags,
)
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTION_CONFIDENCE_MONITOR,
method='set',
tags=core_tags,
value=prediction_confidence,
)
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTION_RESPONSE_TIME_MONITOR,
method='observe',
tags=core_tags,
@@ -781,7 +789,7 @@ class Gates(MinioManager):
for server_id, tags in opc_metrics.items():
for tag, response_time in tags.items():
if response_time is not None:
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR,
method='observe',
tags={
@@ -792,7 +800,7 @@ class Gates(MinioManager):
value=response_time,
)
await self.emit_metric(
self.emit_metric_sync(
metric_object=metrics.PREDICTION_OPC_WRITING_COUNT,
tags={
**core_tags,

49
laborious/activities/mlflow.py
View File

@@ -16,7 +16,8 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_do.temporal.constants import (
DATETIME_FORMAT,
DATETIME_FORMAT_MS_WITH_TZ,
@@ -29,10 +30,9 @@ with workflow.unsafe.imports_passed_through():
from laborious.utils.dataframe_debug import build_dataframe_debug_message
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
from laborious.utils.repository.minio_manager import MinioManager
class MLFlow(MinioManager):
class MLFlow(SientiaMonitoring):
"""
Temporal activities that talk to MLflow through ``SientiaMLflowRepository`` and ``SientiaModel`` wrappers.
@@ -78,8 +78,12 @@ class MLFlow(MinioManager):
None
"""
MinioManager.__init__(
self, minio_repository, logger, notification_handler, metrics_controller
self.minio_repository = minio_repository
SientiaMonitoring.__init__(
self,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
self.mlflow_repository = mlflow_repository
self.plugin_store = plugin_store
@@ -91,7 +95,12 @@ class MLFlow(MinioManager):
Return:
None
"""
MinioManager.close(self)
if self.minio_repository is not None:
try:
self.minio_repository.close()
finally:
self.minio_repository = None
SientiaMonitoring.shutdown(self)
def __del__(self):
self.close()
@@ -207,7 +216,7 @@ class MLFlow(MinioManager):
return alias or self._DEFAULT_MODEL_ALIAS
@activity.defn(name='request_transform')
async def request_transform(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
def request_transform(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
"""
Pivot long-format sensor rows, load the production wrapper, and run ``wrapper.transform``.
@@ -228,7 +237,7 @@ class MLFlow(MinioManager):
self.info('Transforming data...', metadata)
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
@@ -284,7 +293,7 @@ class MLFlow(MinioManager):
self.info('Data transformed successfully', metadata)
if not response_data.get('success', False):
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=None,
minio_repo=self.minio_repository,
model_name=model_name,
@@ -295,7 +304,7 @@ class MLFlow(MinioManager):
logger=self.logger,
)
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=response_data['content'],
minio_repo=self.minio_repository,
model_name=model_name,
@@ -309,7 +318,7 @@ class MLFlow(MinioManager):
)
@activity.defn(name='request_predict')
async def request_predict(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
def request_predict(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
"""
Load the production wrapper and call ``wrapper.predict`` on the prepared feature frame.
@@ -329,7 +338,7 @@ class MLFlow(MinioManager):
self.info('Predicting data...', metadata)
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
@@ -390,7 +399,7 @@ class MLFlow(MinioManager):
self.info('Data predicted successfully', metadata)
if not response_data.get('success', False):
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=None,
minio_repo=self.minio_repository,
model_name=model_name,
@@ -401,7 +410,7 @@ class MLFlow(MinioManager):
logger=self.logger,
)
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe=response_data['content'],
minio_repo=self.minio_repository,
model_name=model_name,
@@ -415,7 +424,7 @@ class MLFlow(MinioManager):
)
@activity.defn(name='retrain_model')
async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Fit an updated wrapper from historical data, then log and register in MLflow.
@@ -441,11 +450,11 @@ class MLFlow(MinioManager):
try:
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='ERROR_LOADING_RETRAIN_DATA',
message=f'Error loading retrain data: {e}',
@@ -576,7 +585,7 @@ class MLFlow(MinioManager):
}
@activity.defn(name='update_production_model')
async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
"""
Point the ``production`` alias at the model version registered for the retrain run.
@@ -622,7 +631,7 @@ class MLFlow(MinioManager):
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='UPDATE_PRODUCTION_MODEL_ERROR',
message=f'Error updating production model {model_name}: {e}',
@@ -634,7 +643,7 @@ class MLFlow(MinioManager):
raise e
@activity.defn(name='get_reference_data')
async def get_reference_data(self, input_data: dict[str, Any]) -> list[dict] | None:
def get_reference_data(self, input_data: dict[str, Any]) -> list[dict] | None:
"""
Download ``evaluation_data.csv`` from the MLflow run linked to ``production`` and parse it.

148
laborious/activities/model_metrics.py
View File

@@ -7,14 +7,15 @@ with workflow.unsafe.imports_passed_through():
from typing import Any
import numpy as np
from pandas import DataFrame, Index, to_datetime
from sientia.ModelAnalysis import ModelAnalysis
import pandas as pd
from pandas import DataFrame, Index, Series, to_datetime
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_model.analytics.model_analysis import ModelAnalysis
from laborious import metrics
from laborious.utils.dataframe_debug import build_dataframe_debug_message
@@ -27,9 +28,12 @@ warnings.filterwarnings(
class ModelMetrics(SientiaMonitoring):
"""
Metrics activities for the Laborious system.
Metrics and statistical analysis activities for the Laborious pipeline.
This class provides activities for writing metrics to the Prometheus monitoring system.
This class centralizes drift/statistical computations and model-quality
aggregates used by scheduled workflows. Besides producing tabular outputs
for persistence, it also emits operational metrics (count, lag, error)
through ``SientiaMonitoring`` so execution health is observable in runtime.
"""
_MAX_DEBUG_DATAFRAME_ROWS = 100
@@ -44,7 +48,10 @@ class ModelMetrics(SientiaMonitoring):
def close(self) -> None:
"""
Close the model metrics activity and clean up resources.
Shutdown monitoring resources associated with model metrics activities.
This is invoked during worker teardown to flush/close metric controller
internals and prevent dangling telemetry tasks.
"""
SientiaMonitoring.shutdown(self)
@@ -69,7 +76,7 @@ class ModelMetrics(SientiaMonitoring):
metadata,
)
async def get_drift_metrics(
def get_drift_metrics(
self,
reference_data: DataFrame,
target_data: DataFrame,
@@ -80,14 +87,23 @@ class ModelMetrics(SientiaMonitoring):
metadata: dict[str, Any],
) -> DataFrame:
"""
Calculate univariate drift metrics for a model.
Compute univariate and multivariate drift outputs and merge them into one dataframe.
The method orchestrates three analysis stages (univariate drift,
multivariate drift, and dataframe projection), emitting lag/count/error
metrics for each stage independently so failures are attributable.
Args:
model_analysis (ModelAnalysis): Model analysis object
reference_data (DataFrame): Reference data
target_data (DataFrame): Target data
reference_columns (list[str]): Reference columns
drift_metrics (list[str]): Drift metrics
metadata (dict[str, Any]): Workflow execution metadata
- reference_data (DataFrame): Baseline dataset representing expected behavior.
- target_data (DataFrame): Current analysis dataset to compare against reference.
- target_name (str): Target column name used by ``ModelAnalysis`` config.
- reference_columns (Index): Feature columns evaluated for drift.
- drift_metrics (list[str]): Enabled univariate methods.
- chunk_period (str): Time bucket granularity used by analysis methods.
- metadata (dict[str, Any]): Workflow metadata for logs and notifications.
Return:
DataFrame: Consolidated drift dataframe ready for downstream formatting/persistence.
"""
config = {
@@ -118,12 +134,10 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error detecting univariate drift: {e}', metadata)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
core_labels = self.get_core_labels(metadata, operation_type='detect_multivariate_drift')
start_time = time.time()
@@ -137,12 +151,10 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error detecting multivariate drift: {e}', metadata)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
start_time = time.time()
core_labels = self.get_core_labels(metadata, operation_type='get_drift_metrics_dataframe')
@@ -153,19 +165,40 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error getting drift metrics: {e}', metadata)
await self.emit_metric(
metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels
)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
raise e
await self.observe_lag(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
await self.emit_metric(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
return drift_df
@staticmethod
def _to_naive_utc(series: Series) -> Series:
"""
Parse ``series`` as datetime and return a TZ-naive UTC copy.
``sientia_model.analytics.model_analysis.ModelAnalysis`` preserves the
timezone of the input dataframe in its outputs, while target rows
loaded from PostgreSQL come in with ``+00:00``. Forcing both sides of
a comparison to TZ-naive UTC keeps ``isin`` / ``floor`` operations
deterministic regardless of how the analyzer (or a test double)
constructs its timestamps.
Args:
- series (Series): Input series containing datetime-parseable values.
Return:
Series: Datetime64 series with ``tz=None`` representing UTC instants.
"""
parsed = to_datetime(series)
if getattr(parsed.dt, 'tz', None) is not None:
parsed = parsed.dt.tz_convert('UTC').dt.tz_localize(None)
return parsed
@activity.defn(name='calculate_drift')
async def calculate_drift(self, input_data: dict[str, Any]) -> list[dict]:
def calculate_drift(self, input_data: dict[str, Any]) -> list[dict]:
"""
Calculate drift metrics for a model.
@@ -195,14 +228,17 @@ class ModelMetrics(SientiaMonitoring):
target_data = target_data.pivot(index='timestamp', columns='variable', values='value')
target_data['timestamp'] = target_data.index
# Keep timestamps as datetime: ModelAnalysis._chunk_dataframe relies on
# ``pd.Grouper(freq=...)`` which rejects string timestamp columns.
target_data['timestamp'] = to_datetime(target_data['timestamp'])
target_data['timestamp'] = target_data['timestamp'].dt.strftime(DATETIME_FORMAT)
target_data = target_data.reset_index(drop=True)
target_data.dropna(inplace=True)
if reference_raw_data is not None:
self.info('Using reference data', metadata)
reference_data = DataFrame(reference_raw_data)
if 'timestamp' in reference_data.columns:
reference_data['timestamp'] = to_datetime(reference_data['timestamp'])
accurate = True
else:
# Get 30% first rows of target_data
@@ -211,7 +247,7 @@ class ModelMetrics(SientiaMonitoring):
reference_data = target_data.head(int(len(target_data) * 0.3))
accurate = False
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING',
message='Using 30% first rows of target data as reference data',
@@ -225,7 +261,7 @@ class ModelMetrics(SientiaMonitoring):
).columns
try:
drift_df = await self.get_drift_metrics(
drift_df = self.get_drift_metrics(
reference_data=reference_data,
target_data=target_data,
target_name=target_name,
@@ -236,7 +272,7 @@ class ModelMetrics(SientiaMonitoring):
)
except Exception as e:
self.error(f'Error getting drift metrics: {e}', metadata)
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='MODEL_METRICS_GET_DRIFT_METRICS_ERROR',
message=f'Error getting drift metrics: {e}',
@@ -250,17 +286,13 @@ class ModelMetrics(SientiaMonitoring):
self.warning('No drift metrics found', metadata)
return []
# Drop unnecessary columns
drift_df.drop(columns=['p_value'], inplace=True)
# Extract timestamps only until minutes
if chunk_period == 'min':
target_timestamps = target_data['timestamp'].apply(lambda x: x[:16])
else:
target_timestamps = target_data['timestamp']
# Drop rows where timestamp is not in target data, to avoid save drift from reference
drift_df = drift_df[drift_df['timestamp'].isin(target_timestamps)]
# Defense-in-depth: drop chunks whose floored timestamp does not appear
# in the analysis window. ``ModelAnalysis`` already chunks only over
# ``analysis_df`` so this only excludes rows injected by upstream
# callers that pre-merge reference data into the result.
target_floor = self._to_naive_utc(target_data['timestamp']).dt.floor(chunk_period)
drift_floor = self._to_naive_utc(drift_df['timestamp']).dt.floor(chunk_period)
drift_df = drift_df[drift_floor.isin(target_floor)]
if drift_df.empty:
self.warning(
@@ -269,14 +301,21 @@ class ModelMetrics(SientiaMonitoring):
)
return []
# Rename columns to match database columns
drift_df.rename(
# Map ``sientia_model.analytics.model_analysis`` schema onto the drift
# table columns: ``metric -> method``, ``statistic -> value``,
# ``alert -> drift``, ``chunk_index -> chunk``,
# ``chunk_end_date -> timestamp_end``. ``p_value`` and
# ``chunk_start_date`` are not persisted.
drift_df = drift_df.rename(
columns={
'metric': 'method',
'statistic': 'value',
},
inplace=True,
'alert': 'drift',
'chunk_index': 'chunk',
'chunk_end_date': 'timestamp_end',
}
)
drift_df.drop(columns=['p_value', 'chunk_start_date'], inplace=True, errors='ignore')
# Drop duplicates
drift_df.drop_duplicates(
@@ -286,16 +325,23 @@ class ModelMetrics(SientiaMonitoring):
drift_df['model_id'] = model_id
drift_df['accurate'] = accurate
drift_df['timestamp'] = to_datetime(drift_df['timestamp'])
drift_df['timestamp'] = self._to_naive_utc(drift_df['timestamp'])
drift_df['timestamp'] = drift_df['timestamp'].dt.tz_localize('UTC')
drift_df['timestamp'] = drift_df['timestamp'].dt.strftime(DATETIME_FORMAT_WITH_TZ)
# ``timestamp_end`` may carry nanosecond precision (beyond
# ``timestamptz`` microseconds), so serialize as ISO text for the
# ``text`` Postgres column.
drift_df['timestamp_end'] = drift_df['timestamp_end'].apply(
lambda value: pd.Timestamp(value).isoformat() if pd.notna(value) else None
)
self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
return drift_df.to_dict(orient='records')
@activity.defn(name='calculate_simple_metrics')
async def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]:
def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]:
"""
Calculate simple metrics for a model. Metrics available are:
- rmse

210
laborious/activities/opc.py
View File

@@ -51,7 +51,7 @@ class OPC(SientiaMonitoring):
self.opc_repository: dict[str, OpcRepository] = {}
async def init_opc(self):
def init_opc(self) -> None:
"""
Initialize OPC server connections and establish communication channels.
@@ -59,58 +59,43 @@ class OPC(SientiaMonitoring):
establish secure connections using certificate-based authentication.
Each server connection is managed independently, and connection failures
are reported through the notification system.
The method performs the following operations:
1. Creates OpcRepository instances for each configured server
2. Establishes secure connections with certificate validation
3. Reports connection success/failure through notifications
4. Logs connection status for operational visibility
Raises:
Exception: If OPC repository initialization fails or connection
establishment encounters critical errors
Note:
Connection failures are logged and reported but do not prevent
the initialization of other OPC servers. Each server is handled
independently to ensure maximum availability.
"""
self.logger.info('Initializing OPC servers...')
for opc_id, server in self.opc_servers.items():
self.opc_repository[opc_id] = OpcRepository(
opc_id=server['id'],
server_name=server['server_name'],
opc_id=opc_id,
url=server['url'],
server_name=server['server_name'],
logger=self.logger,
notification_handler=self.notification_handler,
metrics_controller=self.metrics_controller,
server_uri=server['server_uri'],
cert_path=server['cert_path'],
private_key_path=server['private_key_path'],
server_cert_path=server['server_cert_path'],
notification_handler=self.notification_handler,
reconnection_interval=server['reconnection_interval'],
metrics_controller=self.metrics_controller,
)
is_connected, error_data = await self.opc_repository[opc_id].connect()
if not is_connected:
await self.send_notification_async(
ok, err = self.opc_repository[opc_id].connect()
if not ok:
self.send_notification(
metadata={
'model_id': '-',
'model_name': '-',
'workflow_name': '-',
'schedule_name': 'INITIALIZATION',
},
notification_id=error_data['notification_id'],
message=error_data['message'],
block=error_data['block'],
level=error_data.get('level', NotificationLevel.ERROR),
attachment_content=error_data.get('attachment_content', None),
notification_id=f'OPC_CONNECTION_ERROR_{server.get("id", opc_id)}',
message=err.get('message', 'Failed to connect to OPC server'),
block='opc_repository',
level=NotificationLevel.ERROR,
attachment_content=err.get('attachment_content', traceback.format_exc()),
)
else:
self.logger.info(
f'OPC server {opc_id}:{server["server_name"]} connected successfully.'
)
async def write_data(
def write_data(
self,
server_id: str,
tag: str,
@@ -122,11 +107,6 @@ class OPC(SientiaMonitoring):
"""
Write data to a specific OPC server tag with comprehensive error handling.
This method provides a secure and reliable way to write data to OPC servers
with automatic error handling, notification integration, and detailed logging.
It validates server availability before attempting write operations and
provides comprehensive error reporting for operational monitoring.
Args:
- server_id (str): The id of the OPC server.
- tag (str): The tag to write to.
@@ -135,15 +115,15 @@ class OPC(SientiaMonitoring):
- tag_type (str): The tag type.
Returns:
- bool: True if the data was written successfully, False otherwise.
- float | None: Response time in seconds if successful, None otherwise.
"""
try:
is_success, info_data = await self.opc_repository[server_id].write_data(
is_success, info_data = self.opc_repository[server_id].write_data(
tag, data, data_type, self.logger, metadata
)
if not is_success:
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=info_data['notification_id'],
message=info_data['message'],
@@ -155,7 +135,7 @@ class OPC(SientiaMonitoring):
return info_data['response_time']
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id=f'WRITE_OPC_{tag_type.upper()}_ERROR',
message=f'Error writing data to OPC server: {e}',
@@ -165,30 +145,25 @@ class OPC(SientiaMonitoring):
)
raise e
async def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool:
def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool:
"""
Validate that an OPC server is available and configured for write operations.
Validate that an OPC repository exists for the requested server identifier.
This method checks if the specified OPC server exists in the active
repository and is available for data writing operations. It provides
immediate feedback for server availability and logs validation failures
for operational monitoring.
This guard prevents write attempts against unknown/uninitialized servers.
When the server is missing, it emits an error notification with the list
of available repositories to help operators diagnose configuration drift.
Args:
server_id (str): Unique identifier for the OPC server to validate
metadata (dict[str, Any]): Context metadata for logging and notifications
- server_id (str): OPC server identifier from workflow output config.
- metadata (dict[str, Any]): Workflow metadata used for logs/alerts.
Returns:
bool: True if server is available, False otherwise
Note:
Server validation failures are automatically reported through the
notification system with detailed information about available servers.
This helps operators quickly identify configuration issues.
Return:
bool: ``True`` when the server repository is available; ``False`` otherwise.
"""
if self.opc_repository.get(server_id) is None:
message = f'OPC server {server_id} not found to perform write operation.'
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='OPC_SERVER_NOT_FOUND',
message=message,
@@ -199,7 +174,7 @@ class OPC(SientiaMonitoring):
return False
return True
async def manage_output_tags(
def manage_output_tags(
self,
server_id: str,
config: dict[str, Any],
@@ -207,37 +182,30 @@ class OPC(SientiaMonitoring):
metadata: dict[str, Any],
) -> tuple[bool, dict[str, float | None]]:
"""
Manage the writing of prediction and confidence data to OPC server tags.
Write prediction and confidence values for one OPC server configuration.
This method orchestrates the writing of multiple data types to OPC servers
based on configuration. It handles both prediction data and confidence
values independently, allowing for flexible tag configuration and
comprehensive error handling.
The method supports two main tag types:
1. Prediction tags: Write actual prediction values to configured OPC tags
2. Confidence tags: Write confidence scores to separate OPC tags
The method iterates through optional ``prediction_tags`` and
``confidence_tags``, performs synchronous writes for each tag, collects
per-tag response times, and returns an aggregate success flag
(all tags successful) with a metrics-friendly response map.
Args:
server_id (str): Unique identifier for the target OPC server
config (dict[str, Any]): OPC tag configuration containing:
- prediction_tags (dict, optional): Prediction tag configurations
- confidence_tags (dict, optional): Confidence tag configurations
data (DataFrame): DataFrame containing prediction and confidence data
metadata (dict[str, Any]): Context metadata for logging and notifications
success (bool): Current success status to maintain across operations
- server_id (str): Target OPC server id.
- config (dict[str, Any]): Server output configuration containing optional
``prediction_tags`` and ``confidence_tags`` sections.
- data (DataFrame): Prediction dataframe used as source values.
- metadata (dict[str, Any]): Workflow metadata for logging/notifications.
Returns:
tuple[bool, int]: (overall_success, total_tags_written)
- overall_success: True if all configured tags were written successfully
- total_tags_written: Count of successfully written tags
Return:
tuple[bool, dict[str, float | None]]: Global success flag and response-time
map per tag (``None`` for failed writes).
"""
response_times: dict[str, float | None] = {}
if 'prediction_tags' in config:
for tag, tag_config in config['prediction_tags'].items():
response_time = await self.write_data(
response_time = self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction'].values[0],
@@ -254,7 +222,7 @@ class OPC(SientiaMonitoring):
if 'confidence_tags' in config:
for tag, tag_config in config['confidence_tags'].items():
response_time = await self.write_data(
response_time = self.write_data(
server_id=server_id,
tag=tag,
data=data.head(1)['prediction_confidence'].values[0],
@@ -274,25 +242,25 @@ class OPC(SientiaMonitoring):
return success, response_times
@activity.defn(name='write_opc_data')
async def write_opc_data(
def write_opc_data(
self, input_data: dict[str, Any]
) -> tuple[dict[Hashable, Any], dict[str, dict[str, float | None]]]:
"""
Write prediction and confidence data to OPC servers. The two writing
operations are optional and independent of each other.
Execute OPC writes across all configured servers and collect per-tag metrics.
For each server in ``opc_output_config``, this activity validates server
availability, writes enabled prediction/confidence tags, accumulates
response-time metrics, and then normalizes confidence/comments in the
returned prediction payload when at least one write fails.
Args:
- input_data(dict[str, Any]): The input data. Contains the following keys:
- data(dict[str, Any]): The dataframe that contains the data to write
to the OPC servers.
- opc_output_config(dict[str, Any]): The OPC writing configuration.
The keys are the OPC server names and the values contain:
- prediction_tags(dict[str, Any]): The tags to write to the OPC servers.
- confidence_tags(dict[str, Any]): The tags to write to the OPC servers.
Returns:
- dict[Any, Any]: The data that was written to the OPC servers.
- input_data (dict[str, Any]): Payload containing workflow metadata, data
to write, and ``opc_output_config`` server/tag definitions.
Return:
tuple[dict[Hashable, Any], dict[str, dict[str, float | None]]]: Updated
prediction payload dict and nested metrics
``{server_id: {tag_name: response_time_or_none}}``.
"""
metadata = input_data['metadata']
self.info('Writing data to OPC servers...', metadata)
@@ -305,19 +273,21 @@ class OPC(SientiaMonitoring):
metrics: dict[str, dict[str, float | None]] = {}
for server_id, config in opc_output_config.items():
if not await self.validate_server(server_id, metadata):
if not self.validate_server(server_id, metadata):
success = False
continue
local_success, local_response_times = await self.manage_output_tags(
local_success, local_response_times = self.manage_output_tags(
server_id, config, data, metadata
)
metrics[server_id] = local_response_times
local_count = len(local_response_times)
success = success and local_success
n_pred = len(config.get('prediction_tags') or {})
n_conf = len(config.get('confidence_tags') or {})
self.info(
f'Process completed for OPC server {server_id}: {local_count} of {len(config.get("prediction_tags", []))} prediction tags and {len(config.get("confidence_tags", []))} confidence tags',
f'Process completed for OPC server {server_id}: {local_count} of {n_pred} prediction tags and {n_conf} confidence tags',
metadata,
)
@@ -327,29 +297,16 @@ class OPC(SientiaMonitoring):
self, data: DataFrame, success: bool, metadata: dict[str, Any]
) -> dict[Hashable, Any]:
"""
Process prediction confidence based on OPC write operation success.
This method updates the prediction confidence values in the DataFrame
based on the success status of OPC server write operations. If any
write operations failed, it sets the confidence to a predefined error
value to indicate data quality issues.
The method implements a confidence degradation strategy:
- Success: Maintains original confidence values
- Failure: Sets confidence to error value for operational awareness
Apply fallback confidence/comment values when OPC writes are not fully successful.
Args:
data (DataFrame): DataFrame containing prediction and confidence data
success (bool): Overall success status of OPC write operations
metadata (dict[str, Any]): Context metadata for logging and notifications
- data (DataFrame): Prediction dataframe to be returned to downstream steps.
- success (bool): Aggregate write status across all attempted OPC tags.
- metadata (dict[str, Any]): Workflow metadata used for debug logs.
Returns:
dict[Any, Any]: Processed data as a dictionary with updated confidence values
Note:
The error confidence value (OPC_WRITTING_ERROR_CONFIDENCE = 12) is
used to indicate that data was not successfully exported to OPC servers.
This allows downstream systems to handle data quality appropriately.
Return:
dict[Hashable, Any]: Serialized dataframe dict with original values on success,
or downgraded confidence/comment fields on failure.
"""
message = 'Some data could not be written to OPC servers'
@@ -367,25 +324,14 @@ class OPC(SientiaMonitoring):
return data.to_dict()
async def close(self):
def close(self) -> None:
"""
Gracefully shutdown all OPC server connections and cleanup resources.
Disconnect all tracked OPC repositories and clear in-memory references.
This method ensures proper cleanup of all active OPC server connections
by calling the disconnect method on each repository instance. It's
designed to be called during application shutdown to prevent resource
leaks and ensure clean termination.
The method performs the following cleanup operations:
1. Iterates through all active OPC repository connections
2. Calls disconnect() on each repository instance
3. Allows for graceful connection termination
4. Prevents resource leaks and connection hanging
Note:
This method should be called during application shutdown to ensure
proper cleanup. It handles all active connections regardless of
their current state and provides a clean shutdown experience.
This method should be called during worker shutdown to ensure every
synchronous OPC session is explicitly closed before process exit.
"""
for opc in self.opc_repository.values():
await opc.disconnect()
opc.disconnect()
self.opc_repository.clear()

72
laborious/activities/storage.py
View File

@@ -1,7 +1,5 @@
from temporalio import activity, workflow
from laborious.utils.repository.minio_manager import MinioManager
with workflow.unsafe.imports_passed_through():
# Extend the Temporal Postgres activities for convenient query -> MinIO export
import traceback
@@ -13,8 +11,9 @@ with workflow.unsafe.imports_passed_through():
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.observability.metrics_controller import MetricsController
from sientia_do.repository.minio_repository import MinioRepository
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
from sientia_do.repository.minio_repository_sync import MinioRepository
from sientia_do.temporal.activities.postgres_sync import Postgres
from sientia_do.temporal.constants import now
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
@@ -22,7 +21,7 @@ with workflow.unsafe.imports_passed_through():
_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
class Storage(Postgres, MinioManager):
class Storage(Postgres, SientiaMonitoring):
"""
Extensions for Postgres activities with a helper to export query results
directly to MinIO as Parquet and return the object name.
@@ -60,14 +59,16 @@ class Storage(Postgres, MinioManager):
metrics_controller=metrics_controller,
)
MinioManager.__init__(
self, minio_repository, logger, notification_handler, metrics_controller
self.minio_repository = minio_repository
SientiaMonitoring.__init__(
self,
logger=logger,
notification_handler=notification_handler,
metrics_controller=metrics_controller,
)
@activity.defn(name='load_query_with_minio_offload')
async def load_query_with_minio_offload(
self, input_data: dict[str, Any]
) -> MinioDataFramePayload:
def load_query_with_minio_offload(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
"""
Run the custom SQL load, then return a MinIO-aware dataframe wire dict.
@@ -88,7 +89,7 @@ class Storage(Postgres, MinioManager):
metadata: dict = input_data.get('metadata', {})
model_name = input_data['model_name']
rows = await self.load_custom_query(
rows = self.load_custom_query(
input_data,
)
if not rows:
@@ -99,7 +100,7 @@ class Storage(Postgres, MinioManager):
else:
dataframe = pd.DataFrame(rows)
return await MinioDataFramePayload.from_dataframe(
return MinioDataFramePayload.from_dataframe(
dataframe,
minio_repo=self.minio_repository,
workflow_metadata=metadata,
@@ -109,15 +110,29 @@ class Storage(Postgres, MinioManager):
)
@activity.defn(name='export_payload_to_postgres')
async def export_payload_to_postgres(self, input_data: dict[str, Any]) -> dict:
def export_payload_to_postgres(self, input_data: dict[str, Any]) -> dict:
"""
Export a payload to PostgreSQL.
Resolve a MinIO-aware payload into a DataFrame and persist it into PostgreSQL.
This activity accepts the serialized payload produced by previous steps
(inline dict or MinIO object reference), reconstructs the tabular data,
and delegates the final write to ``export_data_to_postgres`` using the
same input contract expected by the Postgres activity mixin.
Args:
- input_data (dict[str, Any]): Activity input containing ``data`` as a
``MinioDataFramePayload``-compatible dict plus database write options
(schema/table/on_conflict/metadata and related fields).
Return:
dict: Result dictionary returned by ``export_data_to_postgres``, including
success status and optional write diagnostics.
"""
metadata = input_data.get('metadata')
payload = MinioDataFramePayload.from_dict(input_data['data'])
data = await payload.retrieve(self.minio_repository, metadata)
data = payload.retrieve(self.minio_repository, metadata)
return await self.export_data_to_postgres(
return self.export_data_to_postgres(
{
**input_data,
'data': data,
@@ -125,7 +140,7 @@ class Storage(Postgres, MinioManager):
)
@activity.defn(name='cleanup_minio_objects_expired')
async def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Delete objects under the given prefixes that are older than the retention window.
@@ -154,7 +169,7 @@ class Storage(Postgres, MinioManager):
'deleted_count': 0,
}
try:
keys = await self.minio_repository.list_objects(
keys = self.minio_repository.list_objects(
prefix=prefix,
recursive=True,
metadata=metadata,
@@ -166,7 +181,7 @@ class Storage(Postgres, MinioManager):
continue
if ts >= cutoff:
continue
await self.minio_repository.delete_file(
self.minio_repository.delete_file(
object_name=key,
metadata=metadata,
)
@@ -184,7 +199,7 @@ class Storage(Postgres, MinioManager):
report['deleted_count'] += 1
except Exception as e:
trace = traceback.format_exc()
await self.send_notification_async(
self.send_notification(
metadata=metadata,
notification_id='ERROR_CLEANUP_MINIO_OBJECTS_EXPIRED',
message=f'Error cleaning up MinIO objects: {e}',
@@ -201,9 +216,16 @@ class Storage(Postgres, MinioManager):
return report
def close(self) -> None:
"""Close Storage resources (MinIO client and Postgres engine)."""
Postgres.close(self)
MinioManager.close(self)
"""
Shutdown Storage resources in deterministic order.
def __del__(self):
self.close()
The method first closes Postgres resources via ``Postgres.close`` (engine,
sessions, and monitoring hooks), then closes the optional MinIO repository
and clears the local reference to avoid accidental reuse after shutdown.
"""
Postgres.close(self)
if self.minio_repository is not None:
try:
self.minio_repository.close()
finally:
self.minio_repository = None