SIENTIAPDE-1646 Sync laborious/ from local release/SIENTIAPDE-1646 (272e02d)
This commit is contained in:
vitor-aignosi
@@ -1,99 +1,177 @@
|
||||
from temporalio import activity, workflow
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from sientia_do.temporal.activities.postgres import Postgres
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.observability.logger import Logger
|
||||
from laborious.activities.mlflow import MLFlow
|
||||
from laborious.activities.gates import Gates
|
||||
from laborious.activities.opc import OPC
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.repository.minio_repository_sync import MinioRepository
|
||||
from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
|
||||
from sientia_model.model_repository.plugin_store import PluginStore
|
||||
|
||||
class Activities(Postgres, MLFlow, Gates, OPC):
|
||||
from laborious.activities.api import API
|
||||
from laborious.activities.gates import Gates
|
||||
from laborious.activities.mlflow import MLFlow
|
||||
from laborious.activities.model_metrics import ModelMetrics
|
||||
from laborious.activities.opc import OPC
|
||||
from laborious.activities.storage import Storage
|
||||
from laborious.utils.connectors_config import build_mlflow_config
|
||||
|
||||
|
||||
class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API):
|
||||
"""
|
||||
Main activities orchestrator for the Laborious system.
|
||||
Central orchestrator for all Temporal activities used by Laborious workflows.
|
||||
|
||||
This class combines functionality from multiple activity classes to provide
|
||||
a unified interface for all workflow operations. It manages database connections,
|
||||
MLFlow model interactions, data quality validation, and OPC server communications.
|
||||
Composes Storage (Postgres + MinIO offload), MLFlow (wrapper-based inference and retrain
|
||||
via ``SientiaMLflowRepository``), Gates (data quality and ML response filters), OPC exports,
|
||||
drift/simple metrics, and PI Web API writes. The worker constructs one ``Activities`` instance
|
||||
per process and registers its callables on multiple workers bound to different task queues.
|
||||
|
||||
The class implements multiple inheritance to combine specialized functionality:
|
||||
- Postgres: Database operations and data persistence
|
||||
- MLFlow: Model inference and transformation operations
|
||||
- Gates: Data quality validation and filtering mechanisms
|
||||
- OPC: Real-time data export to OPC servers
|
||||
MLflow connectivity: unless ``mlflow_repository`` is injected (tests only), this class builds
|
||||
``SientiaMLflowRepository`` from ``build_mlflow_config()`` so tracking credentials and URL
|
||||
stay aligned with the rest of Laborious env-based configuration.
|
||||
|
||||
Attributes:
|
||||
postgres_config (dict): PostgreSQL connection configuration
|
||||
mlflow_config (dict): MLFlow server configuration
|
||||
opc_config (dict): OPC server configuration
|
||||
logger (Logger): Logging and observability instance
|
||||
notification_handler (NotificationHandler): Notification management instance
|
||||
Inherits and exposes behaviour from mixins; the MLFlow mixin holds ``mlflow_repository``
|
||||
and ``plugin_store`` after ``__init__``.
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
postgres_config: dict[str, Any],
|
||||
mlflow_config: dict[str, Any],
|
||||
opc_config: dict[str, Any],
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler):
|
||||
def __init__(
|
||||
self,
|
||||
postgres_config: dict[str, Any],
|
||||
plugin_store: PluginStore,
|
||||
minio_config: dict[str, Any],
|
||||
opc_config: dict[str, Any],
|
||||
pi_web_api_config: dict[str, Any],
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController | None = None,
|
||||
mlflow_repository: SientiaMLflowRepository | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize the Activities orchestrator with all required configurations.
|
||||
Wire Postgres, MinIO, MLflow, OPC, gates, metrics, and PI Web API into a single object.
|
||||
|
||||
This constructor initializes all parent classes with their respective
|
||||
configurations and sets up the foundation for all activity operations.
|
||||
A single ``MetricsController`` instance is created (or reused) and passed to MinIO,
|
||||
MLflow repository, and all mixins so Prometheus and SDK metrics stay consistent.
|
||||
|
||||
Args:
|
||||
postgres_config: PostgreSQL connection configuration dictionary
|
||||
Required keys: host, port, user, password, dbname, min_connections, max_connections
|
||||
mlflow_config: MLFlow server configuration dictionary
|
||||
Required keys: host, port, username, password
|
||||
opc_config: OPC server configuration dictionary
|
||||
Can contain multiple server configurations
|
||||
logger: Logger instance for observability and debugging
|
||||
notification_handler: Notification handler for alerts and monitoring
|
||||
- postgres_config: Host, port, credentials, db name, and pool bounds for Storage.
|
||||
- plugin_store: ``PluginStore`` instance; the worker must call ``install_runtime`` before
|
||||
activities run so wrapper code is importable.
|
||||
- minio_config: Endpoint, keys, bucket, retention, and TLS flag for object storage payloads.
|
||||
- opc_config: Map of OPC server id to connection settings for ``OPC`` mixin.
|
||||
- pi_web_api_config: Base URL and auth for ``API`` mixin.
|
||||
- logger: Structured logger used across all activities.
|
||||
- notification_handler: Handler for alerts and persisted notifications.
|
||||
- metrics_controller: Optional shared controller; if ``None``, a new one is created.
|
||||
- mlflow_repository: Optional ``SientiaMLflowRepository`` for unit/e2e tests; in production
|
||||
leave unset so the repository is built from environment via ``build_mlflow_config()``.
|
||||
|
||||
Raises:
|
||||
Exception: If any parent class initialization fails
|
||||
Exception: If any parent ``__init__`` fails (e.g. invalid config keys).
|
||||
|
||||
Return:
|
||||
None
|
||||
"""
|
||||
# Initialize parent classes
|
||||
Postgres.__init__(self, host=postgres_config['host'],
|
||||
port=postgres_config['port'],
|
||||
user=postgres_config['user'],
|
||||
password=postgres_config['password'],
|
||||
dbname=postgres_config['dbname'],
|
||||
min_connections=postgres_config['min_connections'],
|
||||
max_connections=postgres_config['max_connections'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler)
|
||||
|
||||
MLFlow.__init__(self, mlflow_host=mlflow_config['host'],
|
||||
mlflow_port=mlflow_config['port'],
|
||||
mlflow_username=mlflow_config['username'],
|
||||
mlflow_password=mlflow_config['password'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler)
|
||||
mc = metrics_controller or MetricsController(logger=logger)
|
||||
|
||||
Gates.__init__(self, logger=logger,
|
||||
notification_handler=notification_handler)
|
||||
# Production path: one shared MLflow client for all model registry / tracking calls.
|
||||
if mlflow_repository is None:
|
||||
mlflow_cfg = build_mlflow_config()
|
||||
mlflow_repository = SientiaMLflowRepository(
|
||||
host=mlflow_cfg['url'],
|
||||
username=mlflow_cfg['username'],
|
||||
password=mlflow_cfg['password'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mc,
|
||||
)
|
||||
|
||||
OPC.__init__(self,
|
||||
opc_servers=opc_config,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler)
|
||||
minio_repository = MinioRepository(
|
||||
endpoint=minio_config['endpoint_url'],
|
||||
access_key=minio_config['access_key'],
|
||||
secret_key=minio_config['secret_key'],
|
||||
bucket=minio_config['default_bucket'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mc,
|
||||
secure=minio_config['secure'],
|
||||
)
|
||||
|
||||
async def shutdown(self):
|
||||
Storage.__init__(
|
||||
self,
|
||||
host=postgres_config['host'],
|
||||
port=postgres_config['port'],
|
||||
user=postgres_config['user'],
|
||||
password=postgres_config['password'],
|
||||
dbname=postgres_config['dbname'],
|
||||
min_connections=postgres_config['min_connections'],
|
||||
max_connections=postgres_config['max_connections'],
|
||||
retention_hours=minio_config['retention_hours'],
|
||||
minio_repository=minio_repository,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mc,
|
||||
)
|
||||
|
||||
MLFlow.__init__(
|
||||
self,
|
||||
mlflow_repository=mlflow_repository,
|
||||
plugin_store=plugin_store,
|
||||
minio_repository=minio_repository,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mc,
|
||||
)
|
||||
|
||||
Gates.__init__(
|
||||
self,
|
||||
minio_repository=minio_repository,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mc,
|
||||
)
|
||||
|
||||
OPC.__init__(
|
||||
self,
|
||||
opc_servers=opc_config,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mc,
|
||||
)
|
||||
|
||||
ModelMetrics.__init__(
|
||||
self,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mc,
|
||||
)
|
||||
|
||||
API.__init__(
|
||||
self,
|
||||
base_url=pi_web_api_config['base_url'],
|
||||
auth_type=pi_web_api_config['auth_type'],
|
||||
auth_token=pi_web_api_config['auth_token'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=mc,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
Gracefully shutdown all activities and clean up resources.
|
||||
Close database pools, sync clients, and OPC sessions in a defined order.
|
||||
|
||||
This method ensures proper cleanup of all resources including:
|
||||
- PostgreSQL connection pools
|
||||
- OPC server connections
|
||||
- Any other resources that need explicit cleanup
|
||||
Should be invoked on worker exit so connection pools and OPC sessions are released
|
||||
cleanly before process termination.
|
||||
|
||||
The method should be called before the application terminates to ensure
|
||||
proper resource cleanup and prevent resource leaks.
|
||||
Return:
|
||||
None
|
||||
"""
|
||||
Postgres.close(self)
|
||||
await OPC.shutdown(self)
|
||||
Storage.close(self)
|
||||
MLFlow.close(self)
|
||||
Gates.close(self)
|
||||
OPC.close(self)
|
||||
ModelMetrics.close(self)
|
||||
API.close(self)
|
||||
|
||||
305
laborious/activities/api.py
Normal file
305
laborious/activities/api.py
Normal file
@@ -0,0 +1,305 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
import json
|
||||
import traceback
|
||||
from typing import Any
|
||||
|
||||
from pandas import DataFrame
|
||||
from sientia_do.notifications.handlers import NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.repository.pi_web_api_client_sync import PIWebAPIClient
|
||||
|
||||
from laborious import metrics
|
||||
|
||||
|
||||
PI_WEB_API_PREDICTION_ERROR_CONFIDENCE = 13
|
||||
|
||||
|
||||
class API(SientiaMonitoring):
|
||||
"""
|
||||
PI Web API operations for writing prediction data to PI Web API.
|
||||
|
||||
This class provides Temporal activities for interacting with the PI Web API
|
||||
to write prediction and confidence values to industrial systems. It handles
|
||||
error scenarios gracefully by setting error confidence values and sending
|
||||
notifications when write operations fail.
|
||||
|
||||
The class implements comprehensive error handling for both prediction and
|
||||
confidence value writes, ensuring that partial failures are properly
|
||||
reported and handled.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
auth_type: str,
|
||||
auth_token: str,
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize API activity with PI Web API client.
|
||||
|
||||
Args:
|
||||
base_url (str): Base URL of the PI Web API server
|
||||
auth_type (str): Authentication type ('basic' or 'bearer')
|
||||
auth_token (str): Authentication token
|
||||
logger (Logger): Logger instance for operation logging
|
||||
notification_handler (NotificationHandler): Handler for system notifications
|
||||
metrics_controller (MetricsController): Controller for metrics collection
|
||||
"""
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
self.pi_web_api_client = PIWebAPIClient(
|
||||
base_url=base_url,
|
||||
auth_config={
|
||||
'type': auth_type,
|
||||
'token': auth_token,
|
||||
},
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
headers_config={
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json',
|
||||
'x-requested-with': 'piwebapistreams',
|
||||
'User-Agent': 'Aig-Laborious-Agent/1.0',
|
||||
},
|
||||
)
|
||||
|
||||
def get_pi_web_api_core_labels(
|
||||
self,
|
||||
metadata: dict[str, Any],
|
||||
operation_type: str = 'write_pi_web_api_data',
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Generate core labels for PI Web API metrics.
|
||||
|
||||
PI Web API metrics in laborious use the shared ``CORE_LABELS`` from
|
||||
``sientia_do``, which includes ``operation_type``. For this reason,
|
||||
operation_type must always be present in emitted labels.
|
||||
|
||||
Args:
|
||||
- metadata (dict[str, Any]): Workflow execution metadata used to derive labels.
|
||||
- operation_type (str): Operation type label for metric cardinality.
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Core labels dictionary including operation_type.
|
||||
"""
|
||||
return super().get_core_labels(
|
||||
metadata=metadata,
|
||||
operation_type=operation_type,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Close the PI Web API client and shutdown monitoring services.
|
||||
|
||||
This method properly closes all connections and resources associated
|
||||
with the PI Web API client and monitoring services.
|
||||
"""
|
||||
self.pi_web_api_client.close()
|
||||
SientiaMonitoring.shutdown(self)
|
||||
|
||||
def process_pi_web_api_response(
|
||||
self,
|
||||
response_data: list[dict[str, Any]],
|
||||
tags: dict[str, str],
|
||||
core_labels: dict[str, str],
|
||||
metadata: dict[str, Any],
|
||||
) -> tuple[int, str]:
|
||||
"""
|
||||
Process the response data from PI Web API write operation.
|
||||
|
||||
Validates that all tags were successfully written, emits metrics for each tag
|
||||
(success or error), and returns the appropriate prediction confidence value.
|
||||
Sets error confidence if any tag write fails or if the number of written tags
|
||||
doesn't match the expected count.
|
||||
|
||||
Args:
|
||||
- response_data (dict[str, Any]): The response data from the PI Web API write operation.
|
||||
- tags (dict[str, str]): The tags that were written to the PI Web API.
|
||||
- core_labels (dict[str, str]): The core labels of the workflow execution.
|
||||
- metadata (dict[str, Any]): The metadata of the workflow execution.
|
||||
Returns:
|
||||
int: Prediction confidence value (0 for success, 13 for errors)
|
||||
"""
|
||||
|
||||
# Convert tags from name:webid to webid:name
|
||||
tags = {w: t for t, w in tags.items()}
|
||||
|
||||
tag_names = list[str](tags.values())
|
||||
|
||||
confidence = 0
|
||||
|
||||
message = ''
|
||||
|
||||
# Evaluate response for each tag
|
||||
written_tags = []
|
||||
for item in response_data:
|
||||
web_id = item.get('WebId')
|
||||
if not web_id:
|
||||
self.error('The response did not contain some WebIds', metadata)
|
||||
continue
|
||||
errors = item.get('Errors', [])
|
||||
tag_name = tags.get(web_id)
|
||||
if not tag_name:
|
||||
self.error(
|
||||
f'The response did not contain the tag name for WebId {web_id}', metadata
|
||||
)
|
||||
continue
|
||||
if errors:
|
||||
self.error(
|
||||
f'Error writing tag {tag_name}:{web_id} to PI Web API: {errors}', metadata
|
||||
)
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT,
|
||||
tags={
|
||||
**core_labels,
|
||||
'tag_name': tag_name,
|
||||
},
|
||||
)
|
||||
confidence = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
|
||||
else:
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_COUNT,
|
||||
tags={
|
||||
**core_labels,
|
||||
'tag_name': tag_name,
|
||||
},
|
||||
)
|
||||
written_tags.append(tag_name)
|
||||
|
||||
if len(written_tags) != len(tag_names):
|
||||
message = f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written.'
|
||||
|
||||
self.error(
|
||||
f'{message}\nResponse:\n {json.dumps(response_data, indent=4)}\nTags:\n {json.dumps(tags, indent=4)}',
|
||||
metadata,
|
||||
)
|
||||
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
|
||||
message=f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written.\nResponse:\n {json.dumps(response_data, indent=4)}\nTags:\n {json.dumps(tags, indent=4)}',
|
||||
block='write_pi_web_api_data',
|
||||
level=NotificationLevel.ERROR,
|
||||
)
|
||||
confidence = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
|
||||
|
||||
return confidence, message
|
||||
|
||||
@activity.defn(name='write_pi_web_api_data')
|
||||
def write_pi_web_api_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
"""
|
||||
Write prediction and confidence data to PI Web API.
|
||||
|
||||
Writes prediction values and confidence scores to PI Web API using configured
|
||||
web IDs. Processes responses to validate writes and emit metrics. Handles errors
|
||||
gracefully by setting error confidence values when writes fail and sending
|
||||
notifications for both prediction and confidence write errors.
|
||||
|
||||
Args:
|
||||
input_data (dict[str, Any]): The input data containing:
|
||||
- metadata (dict[str, Any]): Workflow execution metadata
|
||||
- pi_web_api_output_config (dict[str, Any]): PI Web API configuration with:
|
||||
- endpoint (str): PI Web API endpoint URL
|
||||
- prediction_tags (dict[str, str]): Mapping of tag names to web IDs for predictions
|
||||
- confidence_tags (dict[str, str]): Mapping of tag names to web IDs for confidence
|
||||
- data (dict[str, Any]): Prediction data, its a dataframe converted to dict.
|
||||
Returns:
|
||||
dict[Any, Any]: Data dictionary with potentially modified confidence values
|
||||
If prediction write fails, prediction_confidence is set to error value (13)
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
data = DataFrame(input_data['data'])
|
||||
pi_web_api_output_config = input_data['pi_web_api_output_config']
|
||||
|
||||
self.info(f'Writing data to PI Web API... config: {pi_web_api_output_config}', metadata)
|
||||
|
||||
raw_prediction_tags = pi_web_api_output_config['prediction_tags']
|
||||
raw_confidence_tags = pi_web_api_output_config['confidence_tags']
|
||||
prediction_tags = list[str](raw_prediction_tags.values())
|
||||
confidence_tags = list(raw_confidence_tags.values())
|
||||
|
||||
core_labels = self.get_pi_web_api_core_labels(metadata)
|
||||
|
||||
prediction_value = data.head(1)['prediction'].values[0]
|
||||
confidence_value = data.head(1)['prediction_confidence'].values[0]
|
||||
|
||||
try:
|
||||
prediction_response = self.pi_web_api_client.write_value(
|
||||
web_ids=prediction_tags,
|
||||
value={
|
||||
'Timestamp': data.head(1)['timestamp'].values[0],
|
||||
'Value': prediction_value,
|
||||
},
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
confidence, message = self.process_pi_web_api_response(
|
||||
response_data=prediction_response,
|
||||
tags=raw_prediction_tags,
|
||||
core_labels=core_labels,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
# Preserve incoming confidence/comments on successful PI writes.
|
||||
# Only downgrade confidence or override comments when PI response
|
||||
# explicitly reports a problem (e.g. partial write mismatch).
|
||||
if confidence != 0:
|
||||
data['prediction_confidence'] = confidence
|
||||
if message:
|
||||
data['comments'] = message
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR',
|
||||
message=f'Error writing prediction data to PI Web API: {e}\n Tags: {raw_prediction_tags}',
|
||||
block='write_pi_web_api_data',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
self.error(trace, metadata)
|
||||
|
||||
data['prediction_confidence'] = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE
|
||||
data['comments'] = str(e)
|
||||
|
||||
return data.to_dict()
|
||||
|
||||
try:
|
||||
confidence_response = self.pi_web_api_client.write_value(
|
||||
web_ids=confidence_tags,
|
||||
value={
|
||||
'Timestamp': data.head(1)['timestamp'].values[0],
|
||||
'Value': float(confidence_value),
|
||||
},
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
self.process_pi_web_api_response(
|
||||
response_data=confidence_response,
|
||||
tags=raw_confidence_tags,
|
||||
core_labels=core_labels,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='WRITE_PI_WEB_API_CONFIDENCE_ERROR',
|
||||
message=f'Error writing confidence data to PI Web API: {e}\n Tags: {raw_confidence_tags}',
|
||||
block='write_pi_web_api_data',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
return data.to_dict()
|
||||
@@ -1,57 +1,71 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
import traceback
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from pandas import DataFrame
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
|
||||
from sientia_do.formatters import create_sample_dict
|
||||
from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter
|
||||
from typing import Any
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.repository.minio_repository_sync import MinioRepository
|
||||
from sientia_do.utils.formatters import create_sample_dict
|
||||
|
||||
from laborious import metrics
|
||||
from laborious.utils.dataframe_debug import build_dataframe_debug_message
|
||||
from laborious.utils.filters.conditional_filters import (
|
||||
filter_empty_data,
|
||||
filter_specific_variables_null_values
|
||||
filter_specific_variables_null_values,
|
||||
)
|
||||
from pandas import DataFrame
|
||||
from laborious import metrics
|
||||
from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
|
||||
# Strongly-typed filter function signatures
|
||||
InputFilterFunc = Callable[[DataFrame, dict[str, Any]], bool]
|
||||
ResponseFilterFunc = Callable[[dict[str, Any], dict[str, Any]], bool]
|
||||
ContentFilterFunc = Callable[[DataFrame, dict[str, Any]], bool]
|
||||
|
||||
# Input filter function mappings
|
||||
input_filter_functions = {
|
||||
input_filter_functions: dict[str, InputFilterFunc] = {
|
||||
'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values,
|
||||
'EMPTY_DATA': filter_empty_data,
|
||||
'path_confidence': {
|
||||
'STOP': -1,
|
||||
'CONTINUE': 2,
|
||||
'REPEAT': -1
|
||||
}
|
||||
}
|
||||
|
||||
# Confidence mappings kept separate from function maps to avoid Union types
|
||||
input_path_confidence: Mapping[str, int] = {
|
||||
'STOP': -1,
|
||||
'CONTINUE': 2,
|
||||
'REPEAT': -1,
|
||||
}
|
||||
|
||||
# MLFlow response filter function mappings
|
||||
mlflow_response_filter_functions = {
|
||||
mlflow_response_filter_functions: dict[str, ResponseFilterFunc] = {
|
||||
'API_ERROR': api_error_filter,
|
||||
'path_confidence': {
|
||||
'STOP': -1,
|
||||
'CONTINUE': 10,
|
||||
'REPEAT': -1
|
||||
},
|
||||
}
|
||||
|
||||
mlflow_response_path_confidence: Mapping[str, int] = {
|
||||
'STOP': -1,
|
||||
'CONTINUE': 10,
|
||||
'REPEAT': -1,
|
||||
}
|
||||
|
||||
# MLFlow content filter function mappings
|
||||
mlflow_content_filter_functions = {
|
||||
mlflow_content_filter_functions: dict[str, ContentFilterFunc] = {
|
||||
'NAN_VALUES': nan_values_filter,
|
||||
'EMPTY_DATA': filter_empty_data,
|
||||
'path_confidence': {
|
||||
'STOP': -1,
|
||||
'CONTINUE': 18,
|
||||
'REPEAT': -1
|
||||
}
|
||||
}
|
||||
|
||||
mlflow_content_path_confidence: Mapping[str, int] = {
|
||||
'STOP': -1,
|
||||
'CONTINUE': 18,
|
||||
'REPEAT': -1,
|
||||
}
|
||||
|
||||
|
||||
class Gates(BaseActivity):
|
||||
class Gates(SientiaMonitoring):
|
||||
"""
|
||||
Data quality gates and filtering activities for the Laborious system.
|
||||
|
||||
@@ -71,7 +85,16 @@ class Gates(BaseActivity):
|
||||
mlflow_content_filter_functions (dict): Mapping of MLFlow content filter names to functions
|
||||
"""
|
||||
|
||||
def __init__(self, logger: Logger, notification_handler: NotificationHandler):
|
||||
minio_repository: MinioRepository | None = None
|
||||
_MAX_DEBUG_DATAFRAME_ROWS = 100
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
minio_repository: MinioRepository | None = None,
|
||||
logger: Logger | None = None,
|
||||
notification_handler: NotificationHandler | None = None,
|
||||
metrics_controller: MetricsController | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize data quality gates with logging and notification capabilities.
|
||||
|
||||
@@ -82,11 +105,65 @@ class Gates(BaseActivity):
|
||||
Raises:
|
||||
Exception: If BaseActivity initialization fails
|
||||
"""
|
||||
BaseActivity.__init__(
|
||||
self, logger, notification_handler, set_error_counter=True)
|
||||
self.minio_repository = minio_repository
|
||||
SientiaMonitoring.__init__(
|
||||
self,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
@activity.defn(name="input_gate")
|
||||
async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Close the gates activity and clean up resources.
|
||||
"""
|
||||
|
||||
if self.minio_repository is not None:
|
||||
try:
|
||||
self.minio_repository.close()
|
||||
finally:
|
||||
self.minio_repository = None
|
||||
SientiaMonitoring.shutdown(self)
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None:
|
||||
"""
|
||||
Log dataframe content only when row count is below the configured threshold
|
||||
|
||||
Args:
|
||||
- message (str): Base log message to identify the dataframe in logs
|
||||
- data (Any): Dataframe-like payload to be logged
|
||||
- metadata (dict[str, Any]): Workflow metadata for contextual logging
|
||||
"""
|
||||
self.debug(
|
||||
build_dataframe_debug_message(
|
||||
message=message,
|
||||
data=data,
|
||||
max_rows=self._MAX_DEBUG_DATAFRAME_ROWS,
|
||||
),
|
||||
metadata,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _read_filter_entry(config: dict[str, Any]) -> tuple[str, dict[str, Any]]:
|
||||
"""
|
||||
Read filter policy/config keys in a case-insensitive way.
|
||||
|
||||
Args:
|
||||
config (dict[str, Any]): Filter configuration dictionary.
|
||||
|
||||
Return:
|
||||
tuple[str, dict[str, Any]]: Parsed policy and config payload.
|
||||
"""
|
||||
normalized = {str(key).upper(): value for key, value in config.items()}
|
||||
policy = normalized['POLICY']
|
||||
filter_config = normalized.get('CONFIG', {})
|
||||
return policy, filter_config
|
||||
|
||||
@activity.defn(name='input_gate')
|
||||
def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
"""
|
||||
Apply input data quality filters and validation.
|
||||
|
||||
@@ -121,49 +198,52 @@ class Gates(BaseActivity):
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
self.info("Performing input gate...", metadata)
|
||||
self.info('Performing input gate...', metadata)
|
||||
|
||||
filters = input_data['filters']
|
||||
data = DataFrame(input_data['data'])
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
path_priority = input_data['path_priority']
|
||||
|
||||
filter_output = []
|
||||
|
||||
self.debug(f"Input data: {data.head(5).to_string()}", metadata)
|
||||
self.debug(f"Filters: {filters}", metadata)
|
||||
self._debug_dataframe('Input data:', data, metadata)
|
||||
self.debug(f'Filters: {filters}', metadata)
|
||||
|
||||
# Apply each configured filter
|
||||
for fil, config in filters.items():
|
||||
if fil not in input_filter_functions:
|
||||
self.error(f"Filter {fil} not found", metadata)
|
||||
self.error(f'Filter {fil} not found', metadata)
|
||||
continue
|
||||
policy, filter_config = self._read_filter_entry(config)
|
||||
try:
|
||||
if input_filter_functions[fil](data, config['config']):
|
||||
self.debug(
|
||||
f"Data not passed the input filter {fil}:{config}", metadata)
|
||||
filter_output.append(config['policy'])
|
||||
if input_filter_functions[fil](data, filter_config):
|
||||
self.debug(f'Data not passed the input filter {fil}:{config}', metadata)
|
||||
filter_output.append(policy)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"INTPUT_GATE_ERROR__{fil}",
|
||||
message=f"Error in filter {fil}:{config}: \n {e}",
|
||||
block="input_gate",
|
||||
notification_id=f'INTPUT_GATE_ERROR__{fil}',
|
||||
message=f'Error in filter {fil}:{config}: \n {e}',
|
||||
block='input_gate',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
for path_flag in path_priority:
|
||||
if path_flag in filter_output:
|
||||
self.info(f"Input gate result: {path_flag}", metadata)
|
||||
return path_flag, input_filter_functions['path_confidence'][path_flag], \
|
||||
"Input data with bad quality"
|
||||
self.info(f'Input gate result: {path_flag}', metadata)
|
||||
return path_flag, input_path_confidence[path_flag], 'Input data with bad quality'
|
||||
|
||||
self.info("Nothing was filtered by the input gate", metadata)
|
||||
return None, 0, ""
|
||||
self.info('Nothing was filtered by the input gate', metadata)
|
||||
|
||||
@activity.defn(name="mlflow_response_gate")
|
||||
async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
del data
|
||||
|
||||
return None, 0, ''
|
||||
|
||||
@activity.defn(name='mlflow_response_gate')
|
||||
def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
"""
|
||||
Validate MLFlow API response quality and integrity.
|
||||
|
||||
@@ -197,59 +277,67 @@ class Gates(BaseActivity):
|
||||
Exception: If response validation fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info("Performing mlflow response gate...", metadata)
|
||||
|
||||
self.info('Performing mlflow response gate...', metadata)
|
||||
raw_data = input_data['data']
|
||||
filters = input_data['filters']
|
||||
data = input_data['data']
|
||||
|
||||
self.debug(
|
||||
f'Input data: \n {create_sample_dict(raw_data, max_items=5, max_depth=5)}', metadata
|
||||
)
|
||||
self.debug(f'Filters: {filters}', metadata)
|
||||
|
||||
payload = MinioDataFramePayload.from_dict(raw_data)
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
gate_type = input_data['type']
|
||||
path_priority = input_data['path_priority']
|
||||
|
||||
filter_output = []
|
||||
|
||||
self.debug(
|
||||
f"Input data: \n {create_sample_dict(data, max_items=5, max_depth=2)}", metadata)
|
||||
self.debug(f"Filters: {filters}", metadata)
|
||||
|
||||
comments = []
|
||||
|
||||
status = payload.status or {}
|
||||
|
||||
for fil, config in filters.items():
|
||||
if fil not in mlflow_response_filter_functions:
|
||||
self.error(f"Filter {fil} not found", metadata)
|
||||
continue
|
||||
policy, filter_config = self._read_filter_entry(config)
|
||||
try:
|
||||
if mlflow_response_filter_functions[fil](data, config):
|
||||
filter_output.append(config['policy'])
|
||||
comments.append(data['content']['message'])
|
||||
if mlflow_response_filter_functions[fil](status, filter_config):
|
||||
filter_output.append(policy)
|
||||
comments.append(status.get('message', 'Unknown MLFlow API error'))
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}",
|
||||
message=data['content']['message'],
|
||||
block="mlflow_gate",
|
||||
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
|
||||
message=status.get('message', 'Unknown MLFlow API error'),
|
||||
block='mlflow_gate',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=data['content']['traceback']
|
||||
attachment_content=status.get('traceback'),
|
||||
)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"MLFLOW_GATE_RESPONSE_FILTER__{fil}",
|
||||
message=f"Error in filter {fil}:{config}: \n {e}",
|
||||
block="mlflow_gate",
|
||||
notification_id=f'MLFLOW_GATE_RESPONSE_FILTER__{fil}',
|
||||
message=f'Error in filter {fil}:{config}: \n {e}',
|
||||
block='mlflow_gate',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
for path_flag in path_priority:
|
||||
if path_flag in filter_output:
|
||||
self.info(
|
||||
f"Mlflow response gate result: {path_flag}", metadata)
|
||||
return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag], \
|
||||
", ".join(comments)
|
||||
self.info(f'Mlflow response gate result: {path_flag}', metadata)
|
||||
return path_flag, mlflow_response_path_confidence[path_flag], ', '.join(comments)
|
||||
|
||||
self.info("Nothing was filtered by the mlflow response gate", metadata)
|
||||
return None, 0, ""
|
||||
self.info('Nothing was filtered by the mlflow response gate', metadata)
|
||||
|
||||
@activity.defn(name="mlflow_content_gate")
|
||||
async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
del data
|
||||
|
||||
return None, 0, ''
|
||||
|
||||
@activity.defn(name='mlflow_content_gate')
|
||||
def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
"""
|
||||
Validate MLFlow prediction content quality and integrity.
|
||||
|
||||
@@ -283,56 +371,65 @@ class Gates(BaseActivity):
|
||||
Exception: If content validation fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info("Performing mlflow content gate...", metadata)
|
||||
self.info('Performing mlflow content gate...', metadata)
|
||||
|
||||
filters = input_data['filters']
|
||||
data = DataFrame(input_data['data'])
|
||||
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
gate_type = input_data['type']
|
||||
path_priority = input_data['path_priority']
|
||||
|
||||
filter_output = []
|
||||
|
||||
self.debug(f"Input data:\n {data.head(5).to_string()}", metadata)
|
||||
self.debug(f"Filters: \n {create_sample_dict(filters)}", metadata)
|
||||
self._debug_dataframe('Input data:', data, metadata)
|
||||
self.debug(f'Filters: \n {filters}', metadata)
|
||||
|
||||
for fil, config in filters.items():
|
||||
if fil not in mlflow_content_filter_functions:
|
||||
continue
|
||||
policy, filter_config = self._read_filter_entry(config)
|
||||
try:
|
||||
if mlflow_content_filter_functions[fil](data, config):
|
||||
filter_output.append(config['policy'])
|
||||
if mlflow_content_filter_functions[fil](data, filter_config):
|
||||
filter_output.append(policy)
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}",
|
||||
message=f"Data not passed the content filter {fil}:{config}",
|
||||
block="mlflow_gate",
|
||||
notification_id=f'{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}',
|
||||
message=f'Data not passed the content filter {fil}:{config}',
|
||||
block='mlflow_gate',
|
||||
level=NotificationLevel.WARNING,
|
||||
attachment_content=data.to_string()
|
||||
attachment_content=data.to_string(),
|
||||
)
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"MLFLOW_GATE_CONTENT_FILTER__{fil}",
|
||||
message=f"Error in filter {fil}:{config}: \n {e}",
|
||||
block="mlflow_gate",
|
||||
notification_id=f'MLFLOW_GATE_CONTENT_FILTER__{fil}',
|
||||
message=f'Error in filter {fil}:{config}: \n {e}',
|
||||
block='mlflow_gate',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
attachment_content=trace,
|
||||
)
|
||||
|
||||
for path_flag in path_priority:
|
||||
if path_flag in filter_output:
|
||||
self.info(
|
||||
f"Mlflow content gate result: {path_flag}", metadata)
|
||||
return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag], \
|
||||
"Transformed data not passed the content filter"
|
||||
self.info(f'Mlflow content gate result: {path_flag}', metadata)
|
||||
return (
|
||||
path_flag,
|
||||
mlflow_content_path_confidence[path_flag],
|
||||
'Transformed data not passed the content filter',
|
||||
)
|
||||
|
||||
self.info("Nothing was filtered by the mlflow content gate", metadata)
|
||||
return None, 0, ""
|
||||
self.info('Nothing was filtered by the mlflow content gate', metadata)
|
||||
|
||||
def get_prediction_store_policy(self,
|
||||
prediction_store_policy: str,
|
||||
metadata: dict[str, Any]) -> tuple[str, int]:
|
||||
del data
|
||||
|
||||
return None, 0, ''
|
||||
|
||||
def get_prediction_store_policy(
|
||||
self, prediction_store_policy: str, metadata: dict[str, Any]
|
||||
) -> tuple[str, int]:
|
||||
"""
|
||||
Parse and validate prediction store policy configuration.
|
||||
|
||||
@@ -358,7 +455,9 @@ class Gates(BaseActivity):
|
||||
|
||||
if len(policy_elements) < 2:
|
||||
self.error(
|
||||
f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata)
|
||||
f'Invalid prediction store policy: {prediction_store_policy}, using default policy',
|
||||
metadata,
|
||||
)
|
||||
return 'lts', 1
|
||||
|
||||
policy_type = policy_elements[0]
|
||||
@@ -366,15 +465,76 @@ class Gates(BaseActivity):
|
||||
|
||||
# If the policy_type is not lts or erl, we use the default policy
|
||||
# If the policty_value is not a number or 0, we use the default policy
|
||||
if policy_type not in ['lts', 'erl'] or not policy_value.isdigit() or int(policy_value) == 0:
|
||||
if (
|
||||
policy_type not in ['lts', 'erl']
|
||||
or not policy_value.isdigit()
|
||||
or int(policy_value) == 0
|
||||
):
|
||||
self.error(
|
||||
f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata)
|
||||
f'Invalid prediction store policy: {prediction_store_policy}, using default policy',
|
||||
metadata,
|
||||
)
|
||||
return 'lts', 1
|
||||
|
||||
return policy_type, int(policy_value)
|
||||
|
||||
@activity.defn(name="format_prediction")
|
||||
async def format_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
@activity.defn(name='format_transformed_data')
|
||||
def format_transformed_data(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
|
||||
"""
|
||||
Format transformed data for storage and export operations.
|
||||
|
||||
This method formats transformed data from MLFlow model transformations
|
||||
into a standardized format suitable for database storage. It converts
|
||||
wide-format data (columns as variables) into long-format (melted)
|
||||
with proper timestamp handling and model identification.
|
||||
|
||||
The formatting process includes:
|
||||
1. Converting input data dictionary to DataFrame
|
||||
2. Extracting timestamps from DataFrame index
|
||||
3. Resetting index to create sequential row numbers
|
||||
4. Melting data from wide format to long format (variable-value pairs)
|
||||
5. Adding model_id for data lineage tracking
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict[str, Any]): Transformed data to format (DataFrame-compatible dict)
|
||||
- model_id (str): Unique identifier for the ML model
|
||||
|
||||
Returns:
|
||||
dict: Formatted data dictionary with keys:
|
||||
- timestamp (dict): Timestamp values indexed by row number
|
||||
- variable (dict): Variable names indexed by row number
|
||||
- value (dict): Variable values indexed by row number
|
||||
- model_id (dict): Model identifiers indexed by row number
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
model_id = input_data['model_id']
|
||||
|
||||
self.info('Formatting transformed data...', metadata)
|
||||
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
data['timestamp'] = data.index
|
||||
data = data.reset_index(drop=True)
|
||||
|
||||
data = data.melt(id_vars='timestamp', var_name='variable', value_name='value')
|
||||
data['model_id'] = model_id
|
||||
|
||||
return MinioDataFramePayload.from_dataframe(
|
||||
dataframe=data,
|
||||
minio_repo=self.minio_repository,
|
||||
model_name=input_data['model_name'],
|
||||
operation='transform',
|
||||
workflow_metadata=metadata,
|
||||
last_timestamp=payload.last_timestamp,
|
||||
logger=self.logger,
|
||||
)
|
||||
|
||||
@activity.defn(name='format_prediction')
|
||||
def format_prediction(self, input_data: dict[str, Any]) -> dict:
|
||||
"""
|
||||
Format prediction data according to configured storage policies.
|
||||
|
||||
@@ -383,6 +543,8 @@ class Gates(BaseActivity):
|
||||
and ensures data consistency before persistence. The method supports
|
||||
multiple storage policies for flexible data retention strategies.
|
||||
|
||||
If only one row is present, we use the last timestamp as the timestamp
|
||||
|
||||
Storage Policies:
|
||||
- 'lts:N': Latest timestamp - retains N most recent predictions
|
||||
- 'erl:N': Earliest timestamp - retains N oldest predictions
|
||||
@@ -390,7 +552,7 @@ class Gates(BaseActivity):
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- data (dict[str, Any]): Raw prediction data to format
|
||||
- timestamp (str): Default timestamp if data lacks timestamp column
|
||||
- timestamp (str): Timestamp of the data
|
||||
- model_id (str): Unique identifier for the ML model
|
||||
- prediction_confidence (float): Confidence score for the prediction
|
||||
- prediction_store_policy (str): Storage policy in format 'type:value'
|
||||
@@ -399,58 +561,61 @@ class Gates(BaseActivity):
|
||||
dict: Formatted prediction data ready for storage and export
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
last_timestamp = input_data['timestamp']
|
||||
prediction_store_policy = input_data['prediction_store_policy']
|
||||
self.info("Formatting prediction...", metadata)
|
||||
self.info('Formatting prediction...', metadata)
|
||||
|
||||
data = DataFrame(input_data['data'])
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
# Create timestamp column from index and reset index
|
||||
data['timestamp'] = data.index
|
||||
data = data.reset_index(drop=True)
|
||||
|
||||
self.debug(
|
||||
f"Prediction store policy: {prediction_store_policy}", metadata)
|
||||
self.debug(f"Prediction data: {data.head(5).to_string()}", metadata)
|
||||
self.debug(f'Prediction store policy: {prediction_store_policy}', metadata)
|
||||
self._debug_dataframe('Prediction data:', data, metadata)
|
||||
|
||||
policy_type, policy_value = self.get_prediction_store_policy(
|
||||
prediction_store_policy, metadata)
|
||||
prediction_store_policy, metadata
|
||||
)
|
||||
|
||||
# If data has no timestamp, we use the default timestamp and not sort the data
|
||||
self.info(
|
||||
f"Sorting data by timestamp and applying policy: {policy_type}:{policy_value}", metadata)
|
||||
f'Sorting data by timestamp and applying policy: {policy_type}:{policy_value}', metadata
|
||||
)
|
||||
|
||||
# If policy_type is lts, we need to sort the data by timestamp descending and take the first policy_value rows
|
||||
if policy_type == 'lts':
|
||||
self.debug(
|
||||
"Sorting data by timestamp descending", metadata)
|
||||
self.debug('Sorting data by timestamp descending', metadata)
|
||||
data = data.sort_values(by='timestamp', ascending=False)
|
||||
# If policy_type is erl, we need to sort the data by timestamp ascending and take the first policy_value rows
|
||||
elif policy_type == 'erl':
|
||||
self.debug(
|
||||
"Sorting data by timestamp ascending", metadata)
|
||||
self.debug('Sorting data by timestamp ascending', metadata)
|
||||
data = data.sort_values(by='timestamp', ascending=True)
|
||||
else:
|
||||
self.error(
|
||||
f"Invalid policy type: {policy_type}, using default policy", metadata)
|
||||
raise ValueError(
|
||||
f"Invalid policy type: {policy_type}")
|
||||
self.error(f'Invalid policy type: {policy_type}, using default policy', metadata)
|
||||
raise ValueError(f'Invalid policy type: {policy_type}')
|
||||
|
||||
data = data.head(int(policy_value))
|
||||
int_policy_value = int(policy_value)
|
||||
|
||||
data = data.head(int_policy_value)
|
||||
if int_policy_value == 1:
|
||||
data['timestamp'] = last_timestamp
|
||||
|
||||
data['model_id'] = input_data['model_id']
|
||||
data['prediction_confidence'] = input_data['prediction_confidence']
|
||||
data['prediction_status'] = 'Good'
|
||||
data['comments'] = ""
|
||||
data['comments'] = ''
|
||||
data = data.sort_values(by='timestamp', ascending=False)
|
||||
data = data.reset_index(drop=True)
|
||||
|
||||
self.info(f"Prediction formatted: {len(data)} rows", metadata)
|
||||
self.debug(f"Prediction data: {data.head(5).to_string()}", metadata)
|
||||
self.info(f'Prediction formatted: {len(data)} rows', metadata)
|
||||
self._debug_dataframe('Prediction data:', data, metadata)
|
||||
|
||||
return data.to_dict()
|
||||
|
||||
@activity.defn(name="format_default_prediction")
|
||||
async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
@activity.defn(name='format_default_prediction')
|
||||
def format_default_prediction(self, input_data: dict[str, Any]) -> dict:
|
||||
"""
|
||||
Create and format default prediction data for error conditions.
|
||||
|
||||
@@ -477,65 +642,95 @@ class Gates(BaseActivity):
|
||||
"""
|
||||
|
||||
metadata = input_data['metadata']
|
||||
self.debug("Formatting default prediction...", metadata)
|
||||
self.debug('Formatting default prediction...', metadata)
|
||||
|
||||
data = DataFrame({
|
||||
'prediction': [0],
|
||||
'response_time': [0],
|
||||
'timestamp': [input_data['timestamp']],
|
||||
'model_id': [input_data['model_id']],
|
||||
'prediction_confidence': [input_data['prediction_confidence']],
|
||||
'prediction_status': ['Bad'],
|
||||
'comments': [input_data['comment']]
|
||||
})
|
||||
data = DataFrame(
|
||||
{
|
||||
'prediction': [0],
|
||||
'response_time': [0],
|
||||
'timestamp': [input_data['timestamp']],
|
||||
'model_id': [input_data['model_id']],
|
||||
'prediction_confidence': [input_data['prediction_confidence']],
|
||||
'prediction_status': ['Bad'],
|
||||
'comments': [input_data['comment']],
|
||||
}
|
||||
)
|
||||
|
||||
self.info(f"Default prediction formatted: {data.size} rows", metadata)
|
||||
self.info(f'Default prediction formatted: {data.size} rows', metadata)
|
||||
return data.to_dict()
|
||||
|
||||
@activity.defn(name="get_last_timestamp")
|
||||
async def get_last_timestamp(self, input_data: dict[str, Any]) -> str:
|
||||
@activity.defn(name='format_retrain_report')
|
||||
def format_retrain_report(self, input_data: dict[str, Any]) -> dict:
|
||||
"""
|
||||
Extract the most recent timestamp from prediction data.
|
||||
Format retrain report data for storage and audit trail maintenance.
|
||||
|
||||
This method analyzes prediction data to find the latest timestamp,
|
||||
enabling incremental processing and data continuity tracking.
|
||||
It handles empty datasets gracefully by returning the current time
|
||||
as a fallback timestamp.
|
||||
This method formats model retraining operation results into a standardized
|
||||
report format suitable for database storage and operational monitoring.
|
||||
It captures retraining status, timestamps, and model version information
|
||||
for comprehensive audit trails and operational visibility.
|
||||
|
||||
The method is essential for:
|
||||
1. Incremental data processing workflows
|
||||
2. Data continuity validation
|
||||
3. Timestamp-based data loading optimization
|
||||
4. Workflow execution tracking
|
||||
The formatting process includes:
|
||||
1. Extracting retraining experiment response data
|
||||
2. Capturing model update report information (version, MLflow IDs)
|
||||
3. Formatting timestamps and status information
|
||||
4. Conditionally including version information for successful retrains
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- data (dict[str, Any]): Prediction data to analyze
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- experiment_response (dict): Retraining experiment response containing:
|
||||
- success (bool): Retraining operation success status
|
||||
- timestamp (str): Timestamp of the retraining operation
|
||||
- message (str): Status message or error description
|
||||
- update_report (dict): Model update report containing:
|
||||
- version (str): New model version identifier
|
||||
- mlflow_run_id (str): MLflow run identifier
|
||||
- mlflow_experiment_id (str): MLflow experiment identifier
|
||||
- model_id (str): Unique identifier for the ML model
|
||||
- model_name (str): Name of the ML model
|
||||
|
||||
Returns:
|
||||
str: Formatted timestamp string in UTC with timezone
|
||||
dict: Formatted retrain report dictionary with keys:
|
||||
- model_id (dict): Model identifiers indexed by row number
|
||||
- model_name (dict): Model names indexed by row number
|
||||
- timestamp (dict): Retraining timestamps indexed by row number
|
||||
- status (dict): Retraining status messages indexed by row number
|
||||
- version (dict, optional): Model versions indexed by row number
|
||||
Only included if experiment_response['success'] is True
|
||||
- mlflow_run_id (dict, optional): MLflow run IDs indexed by row number
|
||||
Only included if experiment_response['success'] is True
|
||||
- mlflow_experiment_id (dict, optional): MLflow experiment IDs indexed by row number
|
||||
Only included if experiment_response['success'] is True
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info('Formatting retrain report...', metadata)
|
||||
|
||||
self.info("Getting last timestamp...", metadata)
|
||||
experiment_response = input_data['experiment_response']
|
||||
update_report = input_data['update_report']
|
||||
model_id = input_data['model_id']
|
||||
model_name = input_data['model_name']
|
||||
|
||||
data = DataFrame(input_data['data'])
|
||||
report = DataFrame(
|
||||
{
|
||||
'model_id': [model_id],
|
||||
'model_name': [model_name],
|
||||
'timestamp': [experiment_response['timestamp']],
|
||||
'status': [experiment_response['message']],
|
||||
}
|
||||
)
|
||||
|
||||
self.debug(f"Input data: {data.head(5).to_string()}", metadata)
|
||||
if experiment_response['success']:
|
||||
# Retrain was successfull
|
||||
report['version'] = update_report['version']
|
||||
report['mlflow_run_id'] = update_report['mlflow_run_id']
|
||||
report['mlflow_experiment_id'] = update_report['mlflow_experiment_id']
|
||||
|
||||
if data.empty:
|
||||
return now().strftime(DATETIME_FORMAT_WITH_TZ)
|
||||
self._debug_dataframe('Retrain report:', report, metadata)
|
||||
|
||||
max_timestamp = max(
|
||||
data['timestamp'].values.tolist())
|
||||
return report.to_dict()
|
||||
|
||||
self.info(
|
||||
f"Last timestamp: {max_timestamp}", metadata)
|
||||
|
||||
return max_timestamp
|
||||
|
||||
@activity.defn(name="write_metrics")
|
||||
async def write_metrics(self, input_data: dict[str, Any]):
|
||||
@activity.defn(name='write_metrics')
|
||||
def write_metrics(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Write prediction performance metrics to Prometheus monitoring system.
|
||||
|
||||
@@ -561,27 +756,57 @@ class Gates(BaseActivity):
|
||||
prediction = DataFrame(input_data['prediction'])
|
||||
prediction_confidence = prediction['prediction_confidence'].values[0]
|
||||
response_time = prediction['response_time'].values[0]
|
||||
opc_metrics = input_data['opc_metrics']
|
||||
|
||||
self.info(
|
||||
f"Writing metrics for model {metadata['model_name']}", metadata)
|
||||
self.info(f'Writing metrics for model {metadata["model_name"]}', metadata)
|
||||
|
||||
metrics.PREDICTIONS_WRITTEN_COUNT.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name']
|
||||
).inc()
|
||||
core_tags = {
|
||||
'pod_id': self.pod_id,
|
||||
'runtime': self.runtime,
|
||||
'operation_type': 'predict',
|
||||
'model_name': metadata['model_name'],
|
||||
'workflow_name': metadata['workflow_name'],
|
||||
}
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.PREDICTIONS_WRITTEN_COUNT,
|
||||
tags=core_tags,
|
||||
)
|
||||
|
||||
metrics.PREDICTION_CONFIDENCE_MONITOR.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name']
|
||||
).set(prediction_confidence)
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.PREDICTION_CONFIDENCE_MONITOR,
|
||||
method='set',
|
||||
tags=core_tags,
|
||||
value=prediction_confidence,
|
||||
)
|
||||
|
||||
metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels(
|
||||
pod_id=self.pod_id,
|
||||
model_name=metadata['model_name'],
|
||||
pipeline_name=metadata['workflow_name']
|
||||
).observe(response_time)
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.PREDICTION_RESPONSE_TIME_MONITOR,
|
||||
method='observe',
|
||||
tags=core_tags,
|
||||
value=response_time,
|
||||
)
|
||||
|
||||
self.info(
|
||||
f"Metrics written for model {metadata['model_name']}", metadata)
|
||||
for server_id, tags in opc_metrics.items():
|
||||
for tag, response_time in tags.items():
|
||||
if response_time is not None:
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR,
|
||||
method='observe',
|
||||
tags={
|
||||
**core_tags,
|
||||
'opc_server_id': server_id,
|
||||
'tag': tag,
|
||||
},
|
||||
value=response_time,
|
||||
)
|
||||
|
||||
self.emit_metric_sync(
|
||||
metric_object=metrics.PREDICTION_OPC_WRITING_COUNT,
|
||||
tags={
|
||||
**core_tags,
|
||||
'opc_server_id': server_id,
|
||||
'tag': tag,
|
||||
},
|
||||
)
|
||||
|
||||
self.info(f'Metrics written for model {metadata["model_name"]}', metadata)
|
||||
|
||||
@@ -1,333 +1,649 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
import tempfile
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from pandas import Timestamp, to_datetime
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from pathlib import Path
|
||||
from shutil import rmtree
|
||||
from typing import Any
|
||||
|
||||
import mlflow
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pandas import DataFrame, to_datetime
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.formatters import create_sample_dict
|
||||
from laborious.utils.repository.model_repository import MLFlowRepository
|
||||
from typing import Any
|
||||
import numpy as np
|
||||
from pandas import DataFrame
|
||||
import traceback
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.repository.minio_repository_sync import MinioRepository
|
||||
from sientia_do.temporal.constants import (
|
||||
DATETIME_FORMAT,
|
||||
DATETIME_FORMAT_MS_WITH_TZ,
|
||||
DATETIME_FORMAT_WITH_TZ,
|
||||
now,
|
||||
)
|
||||
from sientia_do.utils.formatters import create_sample_dict
|
||||
from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository
|
||||
from sientia_model.model_repository.plugin_store import PluginStore
|
||||
|
||||
from laborious.utils.dataframe_debug import build_dataframe_debug_message
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
|
||||
|
||||
class MLFlow(BaseActivity):
|
||||
class MLFlow(SientiaMonitoring):
|
||||
"""
|
||||
MLFlow integration activities for model inference operations.
|
||||
Temporal activities that talk to MLflow through ``SientiaMLflowRepository`` and ``SientiaModel`` wrappers.
|
||||
|
||||
This class provides activities for interacting with MLFlow models, including
|
||||
data transformation and prediction operations. It handles authentication,
|
||||
data preprocessing, and model management with configurable retention policies.
|
||||
Models are resolved by registered name and the ``production`` alias (not by legacy stages or
|
||||
separate transform/predict flavors). ``get_cached_model`` loads or reuses a wrapper; inference
|
||||
uses ``wrapper.transform`` / ``wrapper.predict``; retrain uses ``wrapper.retrain`` or
|
||||
``wrapper.train`` plus ``store_model`` and registry promotion via ``promote_to_alias``.
|
||||
|
||||
The class implements comprehensive error handling and logging for all
|
||||
MLFlow operations, ensuring reliable model inference in production environments.
|
||||
Large inputs and outputs flow through ``MinioDataFramePayload`` when workflows offload parquet
|
||||
to MinIO. On failure, transform/predict still return a payload with ``success: False`` and
|
||||
error details for downstream gates.
|
||||
|
||||
Attributes:
|
||||
mlflow_host (str): MLFlow server hostname
|
||||
mlflow_port (int): MLFlow server port
|
||||
mlflow_username (str): MLFlow authentication username
|
||||
mlflow_password (str): MLFlow authentication password
|
||||
model_monitoring_repository (MLFlowRepository): Repository for MLFlow operations
|
||||
mlflow_repository: Client for tracking, registry, artifact download, and run lifecycle.
|
||||
plugin_store: Reference to the store (runtime is installed on the worker; reserved for
|
||||
future store-backed helpers).
|
||||
"""
|
||||
|
||||
def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str,
|
||||
mlflow_password: str, logger: Logger, notification_handler: NotificationHandler):
|
||||
_MAX_DEBUG_DATAFRAME_ROWS = 100
|
||||
_DEFAULT_MODEL_ALIAS = 'production'
|
||||
_REFERENCE_ARTIFACT_CANDIDATES = ('evaluation_data.csv', 'test_data.csv')
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
mlflow_repository: SientiaMLflowRepository,
|
||||
plugin_store: PluginStore,
|
||||
minio_repository: MinioRepository | None = None,
|
||||
logger: Logger | None = None,
|
||||
notification_handler: NotificationHandler | None = None,
|
||||
metrics_controller: MetricsController | None = None,
|
||||
):
|
||||
"""
|
||||
Initialize MLFlow activities with server configuration.
|
||||
Attach shared MLflow and MinIO clients used by all ML activities in this mixin.
|
||||
|
||||
Args:
|
||||
mlflow_host: MLFlow server hostname or IP address
|
||||
mlflow_port: MLFlow server port number
|
||||
mlflow_username: Username for MLFlow authentication
|
||||
mlflow_password: Password for MLFlow authentication
|
||||
logger: Logger instance for observability and debugging
|
||||
notification_handler: Notification handler for alerts and monitoring
|
||||
- mlflow_repository: Repository built by ``Activities`` (or injected in tests).
|
||||
- plugin_store: Plugin store instance from worker bootstrap.
|
||||
- minio_repository: MinIO client for ``MinioDataFramePayload`` upload/download.
|
||||
- logger: Structured logger.
|
||||
- notification_handler: Notifications on hard failures where applicable.
|
||||
- metrics_controller: Shared metrics controller.
|
||||
|
||||
Raises:
|
||||
Exception: If MLFlowRepository initialization fails
|
||||
Return:
|
||||
None
|
||||
"""
|
||||
BaseActivity.__init__(
|
||||
self, logger, notification_handler, set_error_counter=True)
|
||||
self.mlflow_host = mlflow_host
|
||||
self.mlflow_port = mlflow_port
|
||||
self.mlflow_username = mlflow_username
|
||||
self.mlflow_password = mlflow_password
|
||||
|
||||
self.model_monitoring_repository = MLFlowRepository(
|
||||
f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger
|
||||
self.minio_repository = minio_repository
|
||||
SientiaMonitoring.__init__(
|
||||
self,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
self.mlflow_repository = mlflow_repository
|
||||
self.plugin_store = plugin_store
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Release MinIO manager resources held by the mixin.
|
||||
|
||||
Return:
|
||||
None
|
||||
"""
|
||||
if self.minio_repository is not None:
|
||||
try:
|
||||
self.minio_repository.close()
|
||||
finally:
|
||||
self.minio_repository = None
|
||||
SientiaMonitoring.shutdown(self)
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None:
|
||||
"""
|
||||
Log dataframe content only when row count is below the configured threshold
|
||||
|
||||
Args:
|
||||
- message (str): Base log message to identify the dataframe in logs
|
||||
- data (Any): Dataframe-like object expected to expose shape and to_csv
|
||||
- metadata (dict[str, Any]): Workflow metadata for contextual logging
|
||||
"""
|
||||
self.debug(
|
||||
build_dataframe_debug_message(
|
||||
message=message,
|
||||
data=data,
|
||||
max_rows=self._MAX_DEBUG_DATAFRAME_ROWS,
|
||||
),
|
||||
metadata,
|
||||
)
|
||||
|
||||
@activity.defn(name="request_transform")
|
||||
async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
def _detect_and_parse_datetime_index(self, data: pd.DataFrame, metadata: dict) -> pd.DataFrame:
|
||||
"""
|
||||
Transform input data using MLFlow models.
|
||||
Ensure the transform output index is homogeneous and encoded as ``DATETIME_FORMAT_WITH_TZ`` strings.
|
||||
|
||||
This activity processes input data through MLFlow model transformation,
|
||||
including data preprocessing, format conversion, and validation. It handles
|
||||
data deduplication, pivoting, and cleanup to ensure optimal model performance.
|
||||
|
||||
The transformation process includes:
|
||||
1. Data deduplication based on variable and timestamp
|
||||
2. Data pivoting for model input format
|
||||
3. Null value handling and cleanup
|
||||
4. MLFlow model transformation request
|
||||
5. Response validation and logging
|
||||
Accepts an all-string index (validated against the format), or all-``datetime`` /
|
||||
``Timestamp`` (naive timestamps are localized to UTC before formatting). Mixed element types
|
||||
or unsupported types raise ``ValueError`` with a message logged at info level.
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for transformation
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict): Input data for transformation
|
||||
- model_name (str): Name of the MLFlow model to use
|
||||
- model_retention (int): Model retention period in minutes
|
||||
- data: DataFrame whose index carries the time dimension after transform.
|
||||
- metadata: Workflow metadata for log correlation.
|
||||
|
||||
Returns:
|
||||
dict: Transformed data from MLFlow model
|
||||
Return:
|
||||
``pd.DataFrame``: Same frame with a normalized string index; empty frames are returned as-is.
|
||||
"""
|
||||
|
||||
if data.empty:
|
||||
self.info('Data is empty, skipping datetime index detection and parsing', metadata)
|
||||
return data
|
||||
|
||||
index = data.index
|
||||
index_type = type(index[0])
|
||||
|
||||
self.info(f'Index type: {index_type}', metadata)
|
||||
|
||||
message = (
|
||||
f'Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, '
|
||||
f'string in format {DATETIME_FORMAT_WITH_TZ}.'
|
||||
)
|
||||
|
||||
if not all(isinstance(i, index_type) for i in index):
|
||||
types = map(str, map(type, index))
|
||||
raise ValueError(f'{message}. Elements are {",".join(types)}')
|
||||
|
||||
if index_type is str:
|
||||
try:
|
||||
pd.to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ)
|
||||
except ValueError as e:
|
||||
raise ValueError(f'{message}. Unable to parse given date format: {e}') from e
|
||||
|
||||
elif index_type is datetime or index_type is pd.Timestamp:
|
||||
idx = data.index
|
||||
if hasattr(idx, 'tz') and idx.tz is None:
|
||||
data.index = idx.tz_localize('UTC') # type: ignore[attr-defined]
|
||||
|
||||
data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) # type: ignore[attr-defined]
|
||||
else:
|
||||
raise ValueError(f'{message}. Got {index_type}.')
|
||||
|
||||
return data
|
||||
|
||||
def _resolve_model_version_for_run(self, run_id: str) -> str:
|
||||
"""
|
||||
Map an MLflow ``run_id`` to the latest registered model version that produced that run.
|
||||
|
||||
``search_model_versions`` may return multiple versions if the model was registered more than
|
||||
once for the same run; the highest numeric ``version`` wins so promotion targets the newest
|
||||
artifact set.
|
||||
|
||||
Args:
|
||||
- run_id: Run UUID from ``retrain_model`` / experiment payload.
|
||||
|
||||
Return:
|
||||
str: Registry version string acceptable by ``promote_to_alias``.
|
||||
|
||||
Raises:
|
||||
Exception: If transformation fails or MLFlow model is unavailable
|
||||
ValueError: If the filter returns no versions (model not registered for this run).
|
||||
"""
|
||||
|
||||
versions = self.mlflow_repository._client.search_model_versions(
|
||||
filter_string=f"run_id='{run_id}'"
|
||||
)
|
||||
if not versions:
|
||||
raise ValueError(f'No registered model version found for run_id={run_id}')
|
||||
latest = max(versions, key=lambda v: int(v.version))
|
||||
return str(latest.version)
|
||||
|
||||
def _resolve_model_alias(self, model_config: dict[str, Any] | None = None) -> str:
|
||||
"""
|
||||
Resolve which MLflow alias should be used for model lookup/promotion.
|
||||
|
||||
Args:
|
||||
- model_config: Optional model configuration that may include ``alias``.
|
||||
|
||||
Return:
|
||||
str: Alias name trimmed and normalized; defaults to ``production``.
|
||||
"""
|
||||
if not model_config:
|
||||
return self._DEFAULT_MODEL_ALIAS
|
||||
alias = str(model_config.get('alias', self._DEFAULT_MODEL_ALIAS)).strip()
|
||||
return alias or self._DEFAULT_MODEL_ALIAS
|
||||
|
||||
@activity.defn(name='request_transform')
|
||||
def request_transform(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
|
||||
"""
|
||||
Pivot long-format sensor rows, load the production wrapper, and run ``wrapper.transform``.
|
||||
|
||||
Expected tabular shape after load: columns including ``variable``, ``timestamp``, ``value``,
|
||||
and ``created_at`` for deduplication. Data are sorted by ``created_at``, de-duplicated per
|
||||
``(variable, timestamp)``, pivoted wide, then passed to the model. ``model_config`` may
|
||||
include ``retention_minutes`` for wrapper cache TTL.
|
||||
|
||||
Args:
|
||||
- input_data: Dict with ``metadata``, ``model_name``, ``data`` (``MinioDataFramePayload``
|
||||
dict or inline dataframe dict), and optional ``model_config``.
|
||||
|
||||
Return:
|
||||
``MinioDataFramePayload`` with transformed frame and ``success: True``, or a payload
|
||||
with ``success: False`` and exception details in ``status`` if transform fails.
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info('Transforming data...', metadata)
|
||||
data = DataFrame(input_data['data'])
|
||||
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
model_alias = self._resolve_model_alias(model_config)
|
||||
|
||||
self.debug("Raw input data:", metadata)
|
||||
self.debug(data.head(5).to_string(), metadata)
|
||||
self._debug_dataframe('Raw input data:', data, metadata)
|
||||
|
||||
# Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
|
||||
# Long → wide: keep newest row per (variable, timestamp), then pivot for the wrapper API.
|
||||
data = data.sort_values('created_at', ascending=False).drop_duplicates(
|
||||
subset=['variable', 'timestamp'], keep='first'
|
||||
)
|
||||
|
||||
# Pivot data for model input format
|
||||
data = data.pivot(
|
||||
index='timestamp', columns='variable',
|
||||
values='value')
|
||||
data = data.pivot(index='timestamp', columns='variable', values='value')
|
||||
data.fillna(np.nan, inplace=True)
|
||||
# data.reset_index(inplace=True)
|
||||
|
||||
data.columns.name = None
|
||||
data.index.name = None
|
||||
|
||||
self.debug("Processed input data:", metadata)
|
||||
self.debug(data.head(5).to_string(), metadata)
|
||||
data['timestamp'] = data.index
|
||||
|
||||
# Request transformation from MLFlow model
|
||||
response_data = self.model_monitoring_repository.transform(
|
||||
model_name, data, model_config, metadata
|
||||
self._debug_dataframe('Processed input data:', data, metadata)
|
||||
|
||||
try:
|
||||
wrapper = self.mlflow_repository.get_cached_model(
|
||||
model_name=model_name,
|
||||
alias=model_alias,
|
||||
retention_minutes=model_config.get('retention_minutes', 0),
|
||||
metadata=metadata,
|
||||
)
|
||||
transformed_df, transform_meta = wrapper.transform(data)
|
||||
if transform_meta:
|
||||
self.info(f'Wrapper transform metadata: {transform_meta}', metadata)
|
||||
|
||||
transformed_df = self._detect_and_parse_datetime_index(transformed_df, metadata)
|
||||
|
||||
response_data: dict[str, Any] = {'success': True, 'content': transformed_df}
|
||||
except Exception as e:
|
||||
response_data = {
|
||||
'success': False,
|
||||
'content': {'message': str(e), 'traceback': traceback.format_exc()},
|
||||
}
|
||||
|
||||
self.debug(
|
||||
f'Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}',
|
||||
metadata,
|
||||
)
|
||||
|
||||
self.debug(
|
||||
f"Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata)
|
||||
f'Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}',
|
||||
metadata,
|
||||
)
|
||||
|
||||
self.debug(
|
||||
f"Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata)
|
||||
self.info('Data transformed successfully', metadata)
|
||||
|
||||
self.info("Data transformed successfully", metadata)
|
||||
if not response_data.get('success', False):
|
||||
return MinioDataFramePayload.from_dataframe(
|
||||
dataframe=None,
|
||||
minio_repo=self.minio_repository,
|
||||
model_name=model_name,
|
||||
operation='transform',
|
||||
status=response_data,
|
||||
workflow_metadata=metadata,
|
||||
last_timestamp=payload.last_timestamp,
|
||||
logger=self.logger,
|
||||
)
|
||||
|
||||
return response_data
|
||||
return MinioDataFramePayload.from_dataframe(
|
||||
dataframe=response_data['content'],
|
||||
minio_repo=self.minio_repository,
|
||||
model_name=model_name,
|
||||
operation='transform',
|
||||
workflow_metadata=metadata,
|
||||
status={
|
||||
'success': True,
|
||||
},
|
||||
last_timestamp=payload.last_timestamp,
|
||||
logger=self.logger,
|
||||
)
|
||||
|
||||
@activity.defn(name="request_predict")
|
||||
async def request_predict(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
@activity.defn(name='request_predict')
|
||||
def request_predict(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
|
||||
"""
|
||||
Execute predictions using MLFlow models.
|
||||
Load the production wrapper and call ``wrapper.predict`` on the prepared feature frame.
|
||||
|
||||
This activity performs ML model inference using MLFlow models with the
|
||||
transformed data. It handles data format conversion, null value processing,
|
||||
and model prediction requests with comprehensive error handling.
|
||||
|
||||
The prediction process includes:
|
||||
1. Data format validation and cleanup
|
||||
2. Null value handling for model compatibility
|
||||
3. MLFlow model prediction request
|
||||
4. Response validation and logging
|
||||
5. Performance monitoring and metrics
|
||||
The activity normalizes ``NaN`` to ``None`` for JSON-friendly columns, sets the row index
|
||||
the same way as ``retrain_model`` (UTC ``DatetimeIndex`` from ``DATETIME_FORMAT_WITH_TZ``),
|
||||
restores that index on the prediction frame, normalizes the prediction index to
|
||||
``DATETIME_FORMAT_WITH_TZ`` strings like ``request_transform``, and records ``response_time``.
|
||||
Non-DataFrame predictions are coerced to a single ``prediction`` column.
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for prediction
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict): Transformed data for prediction
|
||||
- model_name (str): Name of the MLFlow model to use
|
||||
- model_retention (int): Model retention period in minutes
|
||||
- input_data: Same envelope as ``request_transform`` (``metadata``, ``model_name``,
|
||||
``data``, optional ``model_config`` with ``retention_minutes``).
|
||||
|
||||
Returns:
|
||||
dict: Prediction results from MLFlow model
|
||||
|
||||
Raises:
|
||||
Exception: If prediction fails or MLFlow model is unavailable
|
||||
Return:
|
||||
``MinioDataFramePayload`` with predictions or error status mirroring transform behaviour.
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info('Predicting data...', metadata)
|
||||
data = DataFrame(input_data['data'])
|
||||
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
model_alias = self._resolve_model_alias(model_config)
|
||||
|
||||
self.debug(f"Input data for: \n {data.head(5).to_string()}", metadata)
|
||||
self._debug_dataframe('Input data for prediction:', data, metadata)
|
||||
|
||||
# Convert numpy.nan to None for model compatibility
|
||||
data.replace(np.nan, None, inplace=True)
|
||||
|
||||
data['timestamp'] = data.index
|
||||
data['timestamp'] = to_datetime(
|
||||
data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT)
|
||||
|
||||
# Request prediction from MLFlow model
|
||||
response_data = self.model_monitoring_repository.predict(
|
||||
model_name, data, model_config, metadata
|
||||
data.index = pd.DatetimeIndex(
|
||||
to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ, utc=True)
|
||||
)
|
||||
input_index = data.index
|
||||
|
||||
try:
|
||||
wrapper = self.mlflow_repository.get_cached_model(
|
||||
model_name=model_name,
|
||||
alias=model_alias,
|
||||
retention_minutes=model_config.get('retention_minutes', 0),
|
||||
metadata=metadata,
|
||||
)
|
||||
start_time = datetime.now()
|
||||
predict_data, pred_meta = wrapper.predict({}, data)
|
||||
end_time = datetime.now()
|
||||
|
||||
if pred_meta:
|
||||
self.info(f'Wrapper predict metadata: {pred_meta}', metadata)
|
||||
|
||||
if isinstance(predict_data, DataFrame):
|
||||
self._debug_dataframe(
|
||||
'Data received from model prediction:', predict_data, metadata
|
||||
)
|
||||
predict_data.columns = pd.Index(['prediction'])
|
||||
else:
|
||||
self.debug(
|
||||
f'Data received from model prediction (not a DataFrame): {predict_data}',
|
||||
metadata,
|
||||
)
|
||||
predict_data = pd.DataFrame(predict_data, columns=['prediction'])
|
||||
|
||||
predict_data.index = input_index
|
||||
predict_data['response_time'] = (end_time - start_time).total_seconds()
|
||||
predict_data = self._detect_and_parse_datetime_index(predict_data, metadata)
|
||||
|
||||
response_data: dict[str, Any] = {'success': True, 'content': predict_data}
|
||||
except Exception as e:
|
||||
response_data = {
|
||||
'success': False,
|
||||
'content': {'message': str(e), 'traceback': traceback.format_exc()},
|
||||
}
|
||||
|
||||
self.debug(
|
||||
f"Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata)
|
||||
f'Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}',
|
||||
metadata,
|
||||
)
|
||||
|
||||
self.info("Data predicted successfully", metadata)
|
||||
self.info('Data predicted successfully', metadata)
|
||||
|
||||
return response_data
|
||||
if not response_data.get('success', False):
|
||||
return MinioDataFramePayload.from_dataframe(
|
||||
dataframe=None,
|
||||
minio_repo=self.minio_repository,
|
||||
model_name=model_name,
|
||||
operation='predict',
|
||||
status=response_data,
|
||||
workflow_metadata=metadata,
|
||||
last_timestamp=payload.last_timestamp,
|
||||
logger=self.logger,
|
||||
)
|
||||
|
||||
@activity.defn(name="retrain_model")
|
||||
async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
return MinioDataFramePayload.from_dataframe(
|
||||
dataframe=response_data['content'],
|
||||
minio_repo=self.minio_repository,
|
||||
model_name=model_name,
|
||||
operation='predict',
|
||||
workflow_metadata=metadata,
|
||||
status={
|
||||
'success': True,
|
||||
},
|
||||
last_timestamp=payload.last_timestamp,
|
||||
logger=self.logger,
|
||||
)
|
||||
|
||||
@activity.defn(name='retrain_model')
|
||||
def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Retrain MLFlow models with updated training data.
|
||||
Fit an updated wrapper from historical data, then log and register in MLflow.
|
||||
|
||||
This activity orchestrates the complete model retraining process,
|
||||
including data preparation, model retraining execution, and result
|
||||
validation. It handles data preprocessing, column cleanup, and
|
||||
comprehensive error handling for production model management.
|
||||
|
||||
The retraining process includes:
|
||||
1. Data timestamp extraction and validation
|
||||
2. Column cleanup and data preparation
|
||||
3. Data pivoting for model input format
|
||||
4. MLFlow model retraining execution
|
||||
5. Result validation and error handling
|
||||
Flow: load long-format data from MinIO → dedupe/pivot like inference prep → require
|
||||
``model_config['target']`` → read current ``production`` version for ``source_run_id`` tag →
|
||||
run ``wrapper.retrain`` outside run timing → ``start_run`` with retrain tags → log input
|
||||
CSV artifact → ``store_model`` and ``log_params``. Does not promote; the workflow calls
|
||||
``update_production_model`` after validation.
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict[str, Any]): Training data for model retraining
|
||||
- model_name (str): Name of the MLFlow model to retrain
|
||||
- input_data: Must include ``metadata``, ``model_name``, ``data`` (payload), and
|
||||
``model_config`` with at least ``target``.
|
||||
|
||||
Returns:
|
||||
dict: Retraining results containing:
|
||||
- status (str): Retraining operation status
|
||||
- timestamp (str): Timestamp of the retraining operation
|
||||
- experiment (str): MLFlow experiment identifier
|
||||
|
||||
Raises:
|
||||
Exception: If retraining fails or encounters critical errors
|
||||
Return:
|
||||
On success: ``success``, ``experiment`` (``run_id``, ``experiment_id``, ``experiment_name``),
|
||||
``message``, ``timestamp``. On failure: ``success: False``, error fields, and optional trace.
|
||||
"""
|
||||
|
||||
if self.minio_repository is None:
|
||||
raise ValueError('Minio repository not initialized')
|
||||
|
||||
metadata = input_data['metadata']
|
||||
data = DataFrame(input_data['data'])
|
||||
|
||||
try:
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='ERROR_LOADING_RETRAIN_DATA',
|
||||
message=f'Error loading retrain data: {e}',
|
||||
block='retrain_model',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace,
|
||||
)
|
||||
self.error(trace, metadata)
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'Error loading retrain data: {e}',
|
||||
'traceback': trace,
|
||||
'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ),
|
||||
}
|
||||
|
||||
self.debug(f'Retrain data loaded successfully: shape {data.shape}', metadata)
|
||||
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
|
||||
self.info(f'Retraining model {model_name}...', metadata)
|
||||
|
||||
timestamp = data['timestamp'].max()
|
||||
self.debug(f'Timestamp: {timestamp}', metadata)
|
||||
|
||||
if 'created_at' in data.columns:
|
||||
data = data.sort_values('created_at', ascending=False).drop_duplicates(
|
||||
subset=['variable', 'timestamp'], keep='first'
|
||||
)
|
||||
else:
|
||||
data = data.drop_duplicates(subset=['variable', 'timestamp'], keep='first')
|
||||
|
||||
data.drop(columns=['model_id'], inplace=True, errors='ignore')
|
||||
data.drop(columns=['created_at'], inplace=True, errors='ignore')
|
||||
|
||||
data = data.pivot(index='timestamp', columns='variable',
|
||||
values='value')
|
||||
data.sort_index(inplace=True)
|
||||
data.reset_index(inplace=True)
|
||||
|
||||
data = data.dropna()
|
||||
data = data.pivot(index='timestamp', columns='variable', values='value')
|
||||
data.fillna(np.nan, inplace=True)
|
||||
data.columns.name = None
|
||||
data.index.name = None
|
||||
|
||||
data.index = pd.DatetimeIndex(
|
||||
to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ, utc=True)
|
||||
)
|
||||
|
||||
target = model_config.get('target')
|
||||
if target is None:
|
||||
msg = 'model_config must include "target" for retraining'
|
||||
self.info(msg, metadata)
|
||||
return {
|
||||
'success': False,
|
||||
'experiment': None,
|
||||
'message': msg,
|
||||
'traceback': '',
|
||||
'timestamp': str(timestamp),
|
||||
}
|
||||
|
||||
try:
|
||||
retrain_output, experiment = self.model_monitoring_repository.retrain_model(
|
||||
data=data,
|
||||
model_name=model_name
|
||||
model_alias = self._resolve_model_alias(model_config)
|
||||
mv_src = self.mlflow_repository._client.get_model_version_by_alias(
|
||||
name=model_name,
|
||||
alias=model_alias,
|
||||
)
|
||||
source_run_id = mv_src.run_id
|
||||
|
||||
wrapper = self.mlflow_repository.get_cached_model(
|
||||
model_name=model_name,
|
||||
alias=model_alias,
|
||||
retention_minutes=0,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
# Keep heavy model fitting outside MLflow run timing.
|
||||
prediction_data = wrapper.retrain(data)
|
||||
prediction_data.rename(columns={target: 'prediction'}, inplace=True)
|
||||
|
||||
# Merge prediction data with retrain data
|
||||
evaluation_data = pd.merge(
|
||||
data, prediction_data, left_index=True, right_index=True, how='left'
|
||||
)
|
||||
|
||||
# Rename target column to "target"
|
||||
evaluation_data.rename(columns={target: 'target'}, inplace=True)
|
||||
|
||||
# Reset index and put as column "timestamp"
|
||||
evaluation_data['timestamp'] = evaluation_data.index
|
||||
evaluation_data.reset_index(drop=True, inplace=True)
|
||||
evaluation_data.sort_values(by='timestamp', inplace=True, ascending=True)
|
||||
|
||||
run_name = f'{model_name}-retrain-{datetime.now().strftime("%Y%m%d%H%M%S")}'
|
||||
|
||||
with self.mlflow_repository.start_run(
|
||||
model_name=model_name,
|
||||
run_name=run_name,
|
||||
experiment_name=model_name,
|
||||
tags={'retrain': 'true', 'source_run_id': source_run_id},
|
||||
metadata=metadata,
|
||||
) as run_info:
|
||||
tmp_dir = tempfile.mkdtemp(prefix='laborious_retrain_')
|
||||
try:
|
||||
raw_csv = Path(tmp_dir) / 'retrain_input.csv'
|
||||
evaluation_csv = Path(tmp_dir) / 'evaluation_data.csv'
|
||||
data.to_csv(raw_csv, index=False)
|
||||
evaluation_data.to_csv(evaluation_csv, index=False)
|
||||
|
||||
mlflow.log_artifact(str(raw_csv))
|
||||
mlflow.log_artifact(str(evaluation_csv))
|
||||
finally:
|
||||
rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
wrapper.store_model(name=model_name)
|
||||
|
||||
self.mlflow_repository.log_params(
|
||||
{
|
||||
'retrain': 'true',
|
||||
'retrain_date': datetime.now().isoformat(),
|
||||
'source_run_id': source_run_id,
|
||||
'retrain_samples': str(data.shape),
|
||||
}
|
||||
)
|
||||
|
||||
experiment_payload = {
|
||||
'run_id': run_info.run_id,
|
||||
'experiment_id': run_info.experiment_id,
|
||||
'experiment_name': model_name,
|
||||
}
|
||||
|
||||
return {
|
||||
'status': retrain_output,
|
||||
'timestamp': timestamp,
|
||||
'experiment': experiment
|
||||
'success': True,
|
||||
'experiment': experiment_payload,
|
||||
'message': 'Model retrained successfully.',
|
||||
'timestamp': str(timestamp),
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='RETRAIN_MODEL_ERROR',
|
||||
message=f'Error retraining model {model_name}: {e}',
|
||||
block='retrain_model',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
)
|
||||
self.error(trace, metadata=metadata)
|
||||
raise e
|
||||
error_msg = f'Error retraining model {model_name}: {e}'
|
||||
self.info(error_msg, metadata)
|
||||
return {
|
||||
'success': False,
|
||||
'experiment': None,
|
||||
'message': error_msg,
|
||||
'traceback': traceback.format_exc(),
|
||||
'timestamp': str(timestamp),
|
||||
}
|
||||
|
||||
@activity.defn(name="update_production_model")
|
||||
async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
@activity.defn(name='update_production_model')
|
||||
def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
"""
|
||||
Update production model with newly trained model version.
|
||||
Point the ``production`` alias at the model version registered for the retrain run.
|
||||
|
||||
This activity manages the critical process of updating production
|
||||
models with newly trained versions. It handles model deployment,
|
||||
status tracking, and comprehensive reporting for operational
|
||||
visibility and audit trails.
|
||||
|
||||
The update process includes:
|
||||
1. Production model update execution
|
||||
2. Status and metadata tracking
|
||||
3. Comprehensive reporting and logging
|
||||
4. Error handling and notification
|
||||
5. Audit trail maintenance
|
||||
Resolves the highest numeric registry version whose ``run_id`` matches
|
||||
``experiment['run_id']``, then calls ``promote_to_alias``. On failure, sends a notification
|
||||
and re-raises so the workflow can surface the error.
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- model_name (str): Name of the MLFlow model to update
|
||||
- experiment (str): MLFlow experiment identifier
|
||||
- model_id (str): Unique identifier for the model version
|
||||
- timestamp (str): Timestamp of the update operation
|
||||
- status (str): Current status of the model update
|
||||
- input_data: ``metadata``, ``model_name``, and ``experiment`` with ``run_id`` and
|
||||
``experiment_id`` (as returned from ``retrain_model``).
|
||||
|
||||
Returns:
|
||||
dict[Any, Any]: Comprehensive update report containing:
|
||||
- model_id (str): Model version identifier
|
||||
- model_name (str): Name of the updated model
|
||||
- timestamp (str): Update operation timestamp
|
||||
- status (str): Update operation status
|
||||
- Additional MLFlow response metadata
|
||||
|
||||
Raises:
|
||||
Exception: If production model update fails
|
||||
Return:
|
||||
Dict with ``model_name``, promoted ``version``, ``mlflow_run_id``, ``mlflow_experiment_id``.
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
model_name = input_data['model_name']
|
||||
model_id = input_data['model_id']
|
||||
experiment = input_data['experiment']
|
||||
timestamp = input_data['timestamp']
|
||||
status = input_data['status']
|
||||
|
||||
self.info(
|
||||
f'Updating production model {model_name} from experiment {experiment}...', metadata)
|
||||
f'Updating production model {model_name} from experiment {experiment}...', metadata
|
||||
)
|
||||
|
||||
try:
|
||||
response = self.model_monitoring_repository.update_production_model(
|
||||
experiment=experiment,
|
||||
model_name=model_name
|
||||
run_id = experiment['run_id']
|
||||
experiment_id = experiment['experiment_id']
|
||||
|
||||
version = self._resolve_model_version_for_run(run_id)
|
||||
|
||||
promote_alias = self._resolve_model_alias(input_data.get('model_config'))
|
||||
self.mlflow_repository.promote_to_alias(
|
||||
model_name=model_name,
|
||||
version=version,
|
||||
alias=promote_alias,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
report = DataFrame([response])
|
||||
report['model_id'] = model_id
|
||||
report['model_name'] = model_name
|
||||
report['timestamp'] = timestamp
|
||||
report['status'] = status
|
||||
|
||||
self.info(
|
||||
f'Production model {model_name} updated successfully', metadata)
|
||||
return report.to_dict()
|
||||
self.info(f'Production model {model_name} updated successfully', metadata)
|
||||
return {
|
||||
'model_name': model_name,
|
||||
'version': version,
|
||||
'mlflow_run_id': run_id,
|
||||
'mlflow_experiment_id': experiment_id,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
@@ -337,7 +653,103 @@ class MLFlow(BaseActivity):
|
||||
message=f'Error updating production model {model_name}: {e}',
|
||||
block='update_production_model',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
attachment_content=trace,
|
||||
)
|
||||
self.error(trace, metadata=metadata)
|
||||
raise e
|
||||
|
||||
def _resolve_reference_artifact_name(self, run_id: str) -> str | None:
|
||||
"""
|
||||
Pick the first available reference CSV artifact path from the MLflow run.
|
||||
|
||||
Candidates are checked in priority order: ``retrain_input.csv``, then ``train_data.csv``.
|
||||
A path matches when it equals the candidate or ends with ``/<candidate>`` for nested layouts.
|
||||
|
||||
Args:
|
||||
- run_id: MLflow run UUID linked to the production model version.
|
||||
|
||||
Return:
|
||||
Artifact path string for ``download_artifacts``, or ``None`` if no candidate exists.
|
||||
"""
|
||||
listed = self.mlflow_repository._client.list_artifacts(run_id)
|
||||
paths = [file_info.path for file_info in listed]
|
||||
for candidate in self._REFERENCE_ARTIFACT_CANDIDATES:
|
||||
for path in paths:
|
||||
if path == candidate or path.endswith(f'/{candidate}'):
|
||||
return path
|
||||
return None
|
||||
|
||||
def _find_downloaded_csv(self, tmpdir: str, artifact_name: str) -> Path | None:
|
||||
"""
|
||||
Locate a downloaded reference CSV in the temp directory.
|
||||
|
||||
Args:
|
||||
- tmpdir: Directory where ``download_artifacts`` wrote files.
|
||||
- artifact_name: Basename of the resolved artifact (e.g. ``retrain_input.csv``).
|
||||
|
||||
Return:
|
||||
``Path`` to the CSV file if found, else ``None``.
|
||||
"""
|
||||
direct = Path(tmpdir) / artifact_name
|
||||
if direct.exists():
|
||||
return direct
|
||||
matches = list(Path(tmpdir).rglob(artifact_name))
|
||||
return matches[0] if matches else None
|
||||
|
||||
@activity.defn(name='get_reference_data')
|
||||
def get_reference_data(self, input_data: dict[str, Any]) -> list[dict] | None:
|
||||
"""
|
||||
Download reference training CSV from the MLflow run linked to the production alias.
|
||||
|
||||
Resolves ``retrain_input.csv`` or ``train_data.csv`` via artifact listing before download.
|
||||
``retrain_input.csv`` is preferred when both exist (most recent retrain snapshot). Used by
|
||||
drift workflows to compare live data against the reference distribution logged with the model.
|
||||
Timestamps are normalized to ``DATETIME_FORMAT`` string columns before returning records.
|
||||
|
||||
Args:
|
||||
- input_data: ``metadata``, ``model_name``, and optional ``model_config`` with ``alias``.
|
||||
|
||||
Return:
|
||||
List of row dicts with normalized timestamps, or ``None`` if resolution or load fails.
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
model_name = input_data['model_name']
|
||||
|
||||
try:
|
||||
model_alias = self._resolve_model_alias(input_data.get('model_config'))
|
||||
mv = self.mlflow_repository._client.get_model_version_by_alias(
|
||||
name=model_name,
|
||||
alias=model_alias,
|
||||
)
|
||||
run_id = mv.run_id
|
||||
|
||||
artifact_path = self._resolve_reference_artifact_name(run_id)
|
||||
if artifact_path is None:
|
||||
self.warning(f'Reference data not found for model {model_name}', metadata)
|
||||
return None
|
||||
|
||||
artifact_name = Path(artifact_path).name
|
||||
tmpdir = tempfile.mkdtemp(prefix='laborious_eval_')
|
||||
try:
|
||||
self.mlflow_repository.download_artifacts(
|
||||
run_id=run_id,
|
||||
artifact_path=artifact_path,
|
||||
dst_path=tmpdir,
|
||||
metadata=metadata,
|
||||
)
|
||||
csv_path = self._find_downloaded_csv(tmpdir, artifact_name)
|
||||
if csv_path is None:
|
||||
self.warning(f'Reference data not found for model {model_name}', metadata)
|
||||
return None
|
||||
reference_data = pd.read_csv(csv_path)
|
||||
finally:
|
||||
rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
reference_data['timestamp'] = to_datetime(reference_data['timestamp'])
|
||||
reference_data['timestamp'] = reference_data['timestamp'].dt.strftime(DATETIME_FORMAT)
|
||||
|
||||
return reference_data.to_dict(orient='records')
|
||||
|
||||
except Exception as e:
|
||||
self.warning(f'Reference data not found for model {model_name}: {e}', metadata)
|
||||
return None
|
||||
|
||||
432
laborious/activities/model_metrics.py
Normal file
432
laborious/activities/model_metrics.py
Normal file
@@ -0,0 +1,432 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
import time
|
||||
import traceback
|
||||
import warnings
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pandas import DataFrame, Index, Series, to_datetime
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_model.analytics.drift_analysis import DriftAnalysis, DriftInsufficientDataError
|
||||
|
||||
from laborious import metrics
|
||||
from laborious.utils.dataframe_debug import build_dataframe_debug_message
|
||||
|
||||
warnings.filterwarnings('ignore', category=RuntimeWarning, message='Degrees of freedom <= 0')
|
||||
warnings.filterwarnings(
|
||||
'ignore', category=RuntimeWarning, message='invalid value encountered in scalar divide'
|
||||
)
|
||||
|
||||
|
||||
class ModelMetrics(SientiaMonitoring):
|
||||
"""
|
||||
Metrics and statistical analysis activities for the Laborious pipeline.
|
||||
|
||||
This class centralizes drift/statistical computations and model-quality
|
||||
aggregates used by scheduled workflows. Besides producing tabular outputs
|
||||
for persistence, it also emits operational metrics (count, lag, error)
|
||||
through ``SientiaMonitoring`` so execution health is observable in runtime.
|
||||
"""
|
||||
|
||||
_MAX_DEBUG_DATAFRAME_ROWS = 100
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
):
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Shutdown monitoring resources associated with model metrics activities.
|
||||
|
||||
This is invoked during worker teardown to flush/close metric controller
|
||||
internals and prevent dangling telemetry tasks.
|
||||
"""
|
||||
SientiaMonitoring.shutdown(self)
|
||||
|
||||
def __del__(self):
|
||||
self.close()
|
||||
|
||||
def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None:
|
||||
"""
|
||||
Log dataframe content only when row count is below the configured threshold
|
||||
|
||||
Args:
|
||||
- message (str): Base log message to identify the dataframe in logs
|
||||
- data (Any): Dataframe-like payload to be logged
|
||||
- metadata (dict[str, Any]): Workflow metadata for contextual logging
|
||||
"""
|
||||
self.debug(
|
||||
build_dataframe_debug_message(
|
||||
message=message,
|
||||
data=data,
|
||||
max_rows=self._MAX_DEBUG_DATAFRAME_ROWS,
|
||||
),
|
||||
metadata,
|
||||
)
|
||||
|
||||
def _drift_analyze_stage_error(
|
||||
self,
|
||||
exc: Exception,
|
||||
context: str,
|
||||
metadata: dict[str, Any],
|
||||
core_labels: dict[str, Any],
|
||||
) -> None:
|
||||
"""
|
||||
Log analyzer failure for a drift stage and increment the analyze error metric.
|
||||
|
||||
Args:
|
||||
- exc (Exception): Failure raised by ``sientia_model``.
|
||||
- context (str): Short label for the log line (e.g. univariate detection).
|
||||
- metadata (dict[str, Any]): Workflow metadata for logging.
|
||||
- core_labels (dict[str, Any]): Tags from ``get_core_labels`` for metrics.
|
||||
"""
|
||||
self.error(f'{context}: {exc}', metadata)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels)
|
||||
|
||||
def get_drift_metrics(
|
||||
self,
|
||||
reference_data: DataFrame,
|
||||
target_data: DataFrame,
|
||||
target_name: str,
|
||||
reference_columns: Index,
|
||||
drift_metrics: list[str],
|
||||
chunk_period: str,
|
||||
metadata: dict[str, Any],
|
||||
) -> DataFrame:
|
||||
"""
|
||||
Compute univariate and multivariate drift outputs and merge them into one dataframe.
|
||||
|
||||
The method orchestrates three analysis stages (univariate drift,
|
||||
multivariate drift, and dataframe projection), emitting lag/count/error
|
||||
metrics for each stage independently so failures are attributable.
|
||||
|
||||
Args:
|
||||
- reference_data (DataFrame): Baseline dataset representing expected behavior.
|
||||
- target_data (DataFrame): Current analysis dataset to compare against reference.
|
||||
- target_name (str): Target column name used by ``DriftAnalysis`` config.
|
||||
- reference_columns (Index): Feature columns evaluated for drift.
|
||||
- drift_metrics (list[str]): Enabled univariate methods.
|
||||
- chunk_period (str): Time bucket granularity used by analysis methods.
|
||||
- metadata (dict[str, Any]): Workflow metadata for logs and notifications.
|
||||
|
||||
Return:
|
||||
DataFrame: Consolidated drift dataframe from ``get_drift_metrics_dataframe`` using
|
||||
``method`` / ``value`` (and optional ``threshold``, ``drift_type``), ready for
|
||||
activity-level formatting before Postgres export.
|
||||
"""
|
||||
# ``DriftAnalysis`` uses truthiness checks on ``features`` (e.g. ``if not features``);
|
||||
# a pandas ``Index`` is ambiguous in boolean context — normalize to a list.
|
||||
feature_names: list[str] = list(reference_columns)
|
||||
|
||||
config = {
|
||||
'target': target_name,
|
||||
'prediction': 'prediction',
|
||||
'timestamp': 'timestamp',
|
||||
'features': feature_names,
|
||||
}
|
||||
|
||||
drift_analysis = DriftAnalysis(config=config)
|
||||
|
||||
self._debug_dataframe(
|
||||
f'Reference data: Size {reference_data.shape}', reference_data, metadata
|
||||
)
|
||||
|
||||
self._debug_dataframe(f'Target data: Size {target_data.shape}', target_data, metadata)
|
||||
|
||||
core_labels = self.get_core_labels(metadata, operation_type='detect_univariate_drift')
|
||||
start_time = time.time()
|
||||
try:
|
||||
univariate_drift = drift_analysis.detect_univariate_drift(
|
||||
reference_df=reference_data,
|
||||
analysis_df=target_data,
|
||||
features=feature_names,
|
||||
timestamp_col=config['timestamp'],
|
||||
methods=drift_metrics,
|
||||
chunk_period=chunk_period,
|
||||
)
|
||||
except Exception as e:
|
||||
if isinstance(e, DriftInsufficientDataError):
|
||||
raise
|
||||
self._drift_analyze_stage_error(
|
||||
e, 'Error detecting univariate drift', metadata, core_labels
|
||||
)
|
||||
raise
|
||||
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
|
||||
|
||||
core_labels = self.get_core_labels(metadata, operation_type='detect_multivariate_drift')
|
||||
start_time = time.time()
|
||||
try:
|
||||
multivariate_drift = drift_analysis.detect_multivariate_drift(
|
||||
reference_df=reference_data,
|
||||
analysis_df=target_data,
|
||||
features=feature_names,
|
||||
timestamp_col=config['timestamp'],
|
||||
chunk_period=chunk_period,
|
||||
)
|
||||
except Exception as e:
|
||||
if isinstance(e, DriftInsufficientDataError):
|
||||
raise
|
||||
self._drift_analyze_stage_error(
|
||||
e, 'Error detecting multivariate drift', metadata, core_labels
|
||||
)
|
||||
raise
|
||||
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
|
||||
|
||||
start_time = time.time()
|
||||
core_labels = self.get_core_labels(metadata, operation_type='get_drift_metrics_dataframe')
|
||||
try:
|
||||
drift_df = drift_analysis.get_drift_metrics_dataframe(
|
||||
univariate_drift=univariate_drift,
|
||||
multivariate_drift=multivariate_drift,
|
||||
)
|
||||
except Exception as e:
|
||||
if isinstance(e, DriftInsufficientDataError):
|
||||
raise
|
||||
self._drift_analyze_stage_error(
|
||||
e, 'Error building drift metrics dataframe', metadata, core_labels
|
||||
)
|
||||
raise
|
||||
self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels)
|
||||
self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels)
|
||||
|
||||
return drift_df
|
||||
|
||||
@staticmethod
|
||||
def _to_naive_utc(series: Series) -> Series:
|
||||
"""
|
||||
Parse ``series`` as datetime and return a TZ-naive UTC copy.
|
||||
|
||||
``sientia_model.analytics.drift_analysis.DriftAnalysis`` preserves the
|
||||
timezone of the input dataframe in its outputs, while target rows
|
||||
loaded from PostgreSQL come in with ``+00:00``. Forcing both sides of
|
||||
a comparison to TZ-naive UTC keeps ``isin`` / ``floor`` operations
|
||||
deterministic regardless of how the analyzer constructs its
|
||||
timestamps.
|
||||
|
||||
Args:
|
||||
- series (Series): Input series containing datetime-parseable values.
|
||||
|
||||
Return:
|
||||
Series: Datetime64 series with ``tz=None`` representing UTC instants.
|
||||
"""
|
||||
parsed = to_datetime(series)
|
||||
if getattr(parsed.dt, 'tz', None) is not None:
|
||||
parsed = parsed.dt.tz_convert('UTC').dt.tz_localize(None)
|
||||
return parsed
|
||||
|
||||
@activity.defn(name='calculate_drift')
|
||||
def calculate_drift(self, input_data: dict[str, Any]) -> list[dict]:
|
||||
"""
|
||||
Calculate drift metrics for a model.
|
||||
|
||||
Args:
|
||||
input_data (dict[str, Any]): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- model_name (str): Name of the MLFlow model to calculate drift for
|
||||
- reference_data (pd.DataFrame): Reference data for the model
|
||||
- target_data (pd.DataFrame): Target data for calculating drift
|
||||
- target_name (str): Name of the target column
|
||||
- drift_metrics (list[str]): List of drift metrics to calculate
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
model_name = input_data['model_name']
|
||||
model_id = input_data['model_id']
|
||||
reference_raw_data = input_data['reference_data']
|
||||
target_data = DataFrame(input_data['target_data'])
|
||||
target_name = input_data['target_name']
|
||||
drift_metrics = input_data['drift_metrics']
|
||||
chunk_period = input_data['chunk_period']
|
||||
|
||||
if chunk_period not in ['min', 's']:
|
||||
self.error(f'Invalid chunk period: {chunk_period}', metadata)
|
||||
raise ValueError(f'Invalid chunk period: {chunk_period}, must be "min" or "s"')
|
||||
|
||||
self.info(f'Calculating drift for model {model_name}', metadata)
|
||||
|
||||
target_data = target_data.pivot(index='timestamp', columns='variable', values='value')
|
||||
target_data['timestamp'] = target_data.index
|
||||
# Keep timestamps as datetime: DriftAnalysis._chunk_dataframe relies on
|
||||
# ``pd.Grouper(freq=...)`` which rejects string timestamp columns.
|
||||
target_data['timestamp'] = to_datetime(target_data['timestamp'])
|
||||
target_data = target_data.reset_index(drop=True)
|
||||
target_data.dropna(inplace=True)
|
||||
|
||||
if reference_raw_data is not None:
|
||||
self.info('Using reference data', metadata)
|
||||
reference_data = DataFrame(reference_raw_data)
|
||||
if 'timestamp' in reference_data.columns:
|
||||
reference_data['timestamp'] = to_datetime(reference_data['timestamp'])
|
||||
accurate = True
|
||||
else:
|
||||
# Get 30% first rows of target_data
|
||||
self.warning('Using 30% first rows of target data as reference data', metadata)
|
||||
target_data.sort_values(by='timestamp', ascending=True, inplace=True)
|
||||
reference_data = target_data.head(int(len(target_data) * 0.3))
|
||||
accurate = False
|
||||
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING',
|
||||
message='Using 30% first rows of target data as reference data',
|
||||
block='model_metrics',
|
||||
level=NotificationLevel.WARNING,
|
||||
attachment_content=reference_data.to_csv(),
|
||||
)
|
||||
|
||||
reference_columns = reference_data.drop(
|
||||
columns=[target_name, 'timestamp', 'target', 'prediction'], errors='ignore'
|
||||
).columns
|
||||
|
||||
try:
|
||||
drift_df = self.get_drift_metrics(
|
||||
reference_data=reference_data,
|
||||
target_data=target_data,
|
||||
target_name=target_name,
|
||||
reference_columns=reference_columns,
|
||||
drift_metrics=drift_metrics,
|
||||
chunk_period=chunk_period,
|
||||
metadata=metadata,
|
||||
)
|
||||
except Exception as e:
|
||||
if isinstance(e, DriftInsufficientDataError):
|
||||
self.error(str(e), metadata)
|
||||
notification_id = e.notification_id
|
||||
notification_message = str(e)
|
||||
else:
|
||||
self.error(f'Error getting drift metrics: {e}', metadata)
|
||||
notification_id = 'MODEL_METRICS_GET_DRIFT_METRICS_ERROR'
|
||||
notification_message = f'Error getting drift metrics: {e}'
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=notification_id,
|
||||
message=notification_message,
|
||||
block='model_metrics',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=traceback.format_exc(),
|
||||
)
|
||||
raise
|
||||
|
||||
# Drop chunks whose floored timestamp does not appear in the analysis window.
|
||||
# ``DriftAnalysis`` chunks over ``analysis_df``; this only excludes rows that
|
||||
# do not belong to the current target window (e.g. stray merged reference rows).
|
||||
target_floor = self._to_naive_utc(target_data['timestamp']).dt.floor(chunk_period)
|
||||
drift_floor = self._to_naive_utc(drift_df['timestamp']).dt.floor(chunk_period)
|
||||
drift_df = drift_df[drift_floor.isin(target_floor)]
|
||||
|
||||
if drift_df.empty:
|
||||
self.warning(
|
||||
'No drift metrics found after dropping rows where timestamp is not in target data',
|
||||
metadata,
|
||||
)
|
||||
return []
|
||||
|
||||
# Analyzer emits diagnostic columns that are not stored in ``sientia_data.drift_metrics``.
|
||||
drift_df = drift_df.drop(columns=['threshold', 'drift_type'], errors='ignore')
|
||||
|
||||
drift_df['model_id'] = str(model_id)
|
||||
drift_df['accurate'] = accurate
|
||||
|
||||
# ``timestamp`` is overridden with the most recent target instant so
|
||||
# every persisted row shares a single business timestamp (the run's
|
||||
# logical "now"), matching what downstream consumers expect.
|
||||
latest_target_timestamp = self._to_naive_utc(target_data['timestamp']).max()
|
||||
drift_df['timestamp'] = (
|
||||
pd.Timestamp(latest_target_timestamp)
|
||||
.tz_localize('UTC')
|
||||
.strftime(DATETIME_FORMAT_WITH_TZ)
|
||||
)
|
||||
|
||||
# ``chunk_start_date`` / ``chunk_end_date`` may carry nanosecond
|
||||
# precision (beyond ``timestamptz`` microseconds), so serialize as ISO
|
||||
# text for the ``text`` Postgres columns.
|
||||
for column in ('chunk_start_date', 'chunk_end_date'):
|
||||
drift_df[column] = drift_df[column].apply(
|
||||
lambda value: pd.Timestamp(value).isoformat() if pd.notna(value) else None
|
||||
)
|
||||
|
||||
self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata)
|
||||
|
||||
return drift_df.to_dict(orient='records')
|
||||
|
||||
@activity.defn(name='calculate_simple_metrics')
|
||||
def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]:
|
||||
"""
|
||||
Calculate simple metrics for a model. Metrics available are:
|
||||
- rmse
|
||||
- mse
|
||||
- mae
|
||||
- r2
|
||||
- accuracy
|
||||
- precision
|
||||
- recall
|
||||
- f1
|
||||
Args:
|
||||
input_data (dict[str, Any]): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- model_id (str): ID of the MLFlow model
|
||||
- target_data (pd.DataFrame): Target data for calculating metrics, containing target and prediction columns
|
||||
- metrics (list[str]): List of metrics to calculate
|
||||
Returns:
|
||||
dict[Hashable, Any]: Dictionary containing the calculated metrics
|
||||
"""
|
||||
|
||||
metadata = input_data['metadata']
|
||||
model_id = input_data['model_id']
|
||||
target_data = DataFrame(input_data['target_data'])
|
||||
metric_names = input_data['metrics']
|
||||
interval_minutes = input_data['interval_minutes']
|
||||
|
||||
data_size = target_data.shape[0]
|
||||
|
||||
output_data = []
|
||||
|
||||
diff = target_data['target'] - target_data['prediction']
|
||||
diff_squared = diff**2
|
||||
|
||||
self.info(f'Calculating simple metrics for model {model_id}: {metric_names}', metadata)
|
||||
|
||||
for metric in metric_names:
|
||||
if metric == 'rmse':
|
||||
output_data.append({'metric': 'rmse', 'value': np.sqrt(np.mean(diff_squared))})
|
||||
elif metric == 'mse':
|
||||
output_data.append({'metric': 'mse', 'value': np.mean(diff_squared)})
|
||||
elif metric == 'mae':
|
||||
output_data.append({'metric': 'mae', 'value': np.mean(np.abs(diff))})
|
||||
elif metric == 'r2':
|
||||
y_true = target_data['target']
|
||||
y_mean = np.mean(y_true)
|
||||
|
||||
ss_res = np.sum(diff_squared)
|
||||
ss_tot = np.sum((y_true - y_mean) ** 2)
|
||||
|
||||
# Evita divisão por zero
|
||||
if ss_tot == 0:
|
||||
r2_score = 0.0
|
||||
else:
|
||||
r2_score = 1 - (ss_res / ss_tot)
|
||||
|
||||
output_data.append({'metric': 'r2', 'value': r2_score})
|
||||
|
||||
data = DataFrame(output_data)
|
||||
data['model_id'] = model_id
|
||||
data['timestamp'] = target_data['timestamp'].max()
|
||||
data['data_size'] = data_size
|
||||
data['interval_minutes'] = interval_minutes
|
||||
|
||||
self._debug_dataframe(f'Simple metrics dataframe: Size {data.shape}', data, metadata)
|
||||
|
||||
return data.to_dict(orient='records')
|
||||
@@ -1,20 +1,62 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
import traceback
|
||||
from collections.abc import Hashable
|
||||
from typing import Any
|
||||
|
||||
from pandas import DataFrame
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.temporal.activities.base import BaseActivity
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
|
||||
from laborious.utils.repository.opc_repository import OpcRepository
|
||||
from typing import Any
|
||||
import traceback
|
||||
from pandas import DataFrame
|
||||
|
||||
OPC_WRITTING_ERROR_CONFIDENCE = 12
|
||||
OPC_SESSION_BAD_CONFIDENCE = 14
|
||||
OPC_SESSION_BAD_COMMENT_PREFIX = 'OPC UA session/channel error:'
|
||||
OPC_WRITTING_ERROR_MESSAGE = 'Some data could not be written to OPC servers'
|
||||
OPC_RECONNECT_IN_PROGRESS_COMMENT = 'OPC UA reconnect in progress'
|
||||
OPC_COMMENT_SEPARATOR = ' | '
|
||||
|
||||
|
||||
class OPC(BaseActivity):
|
||||
def _opc_session_bad_comment(opc_status: str | None) -> str:
|
||||
status = opc_status or 'Unknown'
|
||||
return f'{OPC_SESSION_BAD_COMMENT_PREFIX} {status}'
|
||||
|
||||
|
||||
def _apply_opc_write_error(
|
||||
error_info: dict[str, Any] | None,
|
||||
session_bad_seen: bool,
|
||||
session_bad_status: str | None,
|
||||
reconnect_in_progress_seen: bool,
|
||||
) -> tuple[bool, str | None, bool]:
|
||||
"""
|
||||
Update session/reconnect flags from an OPC write error payload.
|
||||
|
||||
Args:
|
||||
error_info: Repository error details, or None when the write succeeded.
|
||||
session_bad_seen: Whether a session_bad error was seen so far.
|
||||
session_bad_status: Last known OPC status for session errors.
|
||||
reconnect_in_progress_seen: Whether reconnect_in_progress was seen so far.
|
||||
|
||||
Return:
|
||||
Updated (session_bad_seen, session_bad_status, reconnect_in_progress_seen).
|
||||
"""
|
||||
if not error_info:
|
||||
return session_bad_seen, session_bad_status, reconnect_in_progress_seen
|
||||
|
||||
kind = error_info.get('opc_error_kind')
|
||||
if kind == 'session_bad':
|
||||
return True, error_info.get('opc_status', session_bad_status), reconnect_in_progress_seen
|
||||
if kind == 'reconnect_in_progress':
|
||||
return session_bad_seen, session_bad_status, True
|
||||
return session_bad_seen, session_bad_status, reconnect_in_progress_seen
|
||||
|
||||
|
||||
class OPC(SientiaMonitoring):
|
||||
"""
|
||||
OPC server integration activities for real-time data export.
|
||||
|
||||
@@ -33,20 +75,20 @@ class OPC(BaseActivity):
|
||||
notification_handler (NotificationHandler): Notification management instance
|
||||
"""
|
||||
|
||||
def __init__(self, opc_servers: dict[str, dict[str, Any]],
|
||||
logger: Logger, notification_handler: NotificationHandler):
|
||||
|
||||
self.logger = logger
|
||||
self.notification_handler = notification_handler
|
||||
def __init__(
|
||||
self,
|
||||
opc_servers: dict[str, dict[str, Any]],
|
||||
logger: Logger,
|
||||
notification_handler: NotificationHandler,
|
||||
metrics_controller: MetricsController,
|
||||
):
|
||||
self.opc_servers = opc_servers
|
||||
|
||||
BaseActivity.__init__(
|
||||
self, logger, notification_handler, set_error_counter=True)
|
||||
SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller)
|
||||
|
||||
self.opc_repository: dict[str, OpcRepository] = {}
|
||||
self.opc_servers = opc_servers
|
||||
|
||||
async def init_opc(self):
|
||||
def init_opc(self):
|
||||
"""
|
||||
Initialize OPC server connections and establish communication channels.
|
||||
|
||||
@@ -70,10 +112,11 @@ class OPC(BaseActivity):
|
||||
the initialization of other OPC servers. Each server is handled
|
||||
independently to ensure maximum availability.
|
||||
"""
|
||||
self.logger.info("Initializing OPC servers...")
|
||||
for id, server in self.opc_servers.items():
|
||||
self.opc_repository[id] = OpcRepository(
|
||||
id=server['id'],
|
||||
self.info('Initializing OPC servers...')
|
||||
for opc_id, server in self.opc_servers.items():
|
||||
self.opc_repository[opc_id] = OpcRepository(
|
||||
opc_id=opc_id,
|
||||
server_name=server['server_name'],
|
||||
url=server['url'],
|
||||
logger=self.logger,
|
||||
server_uri=server['server_uri'],
|
||||
@@ -81,76 +124,70 @@ class OPC(BaseActivity):
|
||||
private_key_path=server['private_key_path'],
|
||||
server_cert_path=server['server_cert_path'],
|
||||
notification_handler=self.notification_handler,
|
||||
reconnection_interval=server['reconnection_interval'],
|
||||
pod_id=self.pod_id
|
||||
reconnection_interval=server.get('reconnection_interval', 60),
|
||||
metrics_controller=self.metrics_controller,
|
||||
)
|
||||
is_connected, error_data = await self.opc_repository[id].connect()
|
||||
is_connected, error_data = self.opc_repository[opc_id].connect()
|
||||
if not is_connected:
|
||||
self.send_notification(
|
||||
metadata={
|
||||
'model_id': '-',
|
||||
'model_name': '-',
|
||||
'workflow_name': '-',
|
||||
'schedule_name': 'INITIALIZATION'
|
||||
'schedule_name': 'INITIALIZATION',
|
||||
},
|
||||
notification_id=error_data['notification_id'],
|
||||
message=error_data['message'],
|
||||
block=error_data['block'],
|
||||
level=error_data.get('level', NotificationLevel.ERROR),
|
||||
attachment_content=error_data.get(
|
||||
'attachment_content', None)
|
||||
attachment_content=error_data.get('attachment_content', None),
|
||||
)
|
||||
else:
|
||||
self.logger.info(
|
||||
f"OPC server {id} connected successfully.")
|
||||
self.info(f'OPC server {opc_id}:{server["server_name"]} connected successfully.')
|
||||
|
||||
async def write_data(self, server_id: str, tag: str, data: Any,
|
||||
data_type: str, tag_type: str, metadata: dict[str, Any]) -> bool:
|
||||
def write_data(
|
||||
self,
|
||||
server_id: str,
|
||||
tag: str,
|
||||
data: Any,
|
||||
data_type: str,
|
||||
tag_type: str,
|
||||
metadata: dict[str, Any],
|
||||
) -> tuple[float | None, dict[str, Any] | None]:
|
||||
"""
|
||||
Write data to a specific OPC server tag with comprehensive error handling.
|
||||
|
||||
This method provides a secure and reliable way to write data to OPC servers
|
||||
with automatic error handling, notification integration, and detailed logging.
|
||||
It validates server availability before attempting write operations and
|
||||
provides comprehensive error reporting for operational monitoring.
|
||||
|
||||
Args:
|
||||
- server_id (str): The id of the OPC server.
|
||||
- tag (str): The tag to write to.
|
||||
- data (Any): The data to write.
|
||||
- data_type (str): The data type.
|
||||
- tag_type (str): The tag type.
|
||||
|
||||
Returns:
|
||||
- bool: True if the data was written successfully, False otherwise.
|
||||
Return:
|
||||
tuple[float | None, dict[str, Any] | None]: Response time on success, or
|
||||
(None, error info_data) on repository failure.
|
||||
"""
|
||||
|
||||
try:
|
||||
is_success, error_data = await self.opc_repository[server_id].write_data(
|
||||
tag, data, data_type, self.logger, metadata)
|
||||
is_success, info_data = self.opc_repository[server_id].write_data(
|
||||
tag, data, data_type, metadata
|
||||
)
|
||||
if not is_success:
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=error_data['notification_id'],
|
||||
message=error_data['message'],
|
||||
block=error_data['block'],
|
||||
level=error_data.get('level', NotificationLevel.ERROR),
|
||||
attachment_content=error_data.get(
|
||||
'attachment_content', None)
|
||||
notification_id=info_data['notification_id'],
|
||||
message=info_data['message'],
|
||||
block=info_data['block'],
|
||||
level=info_data.get('level', NotificationLevel.ERROR),
|
||||
attachment_content=info_data.get('attachment_content', None),
|
||||
)
|
||||
return False
|
||||
return True
|
||||
return None, info_data
|
||||
return info_data['response_time'], None
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id=f"WRITE_OPC_{tag_type.upper()}_ERROR",
|
||||
message=f"Error writing data to OPC server: {e}",
|
||||
block="write_opc_data",
|
||||
notification_id=f'WRITE_OPC_{tag_type.upper()}_ERROR',
|
||||
message=f'Error writing data to OPC server: {e}',
|
||||
block='write_opc_data',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace
|
||||
attachment_content=trace,
|
||||
)
|
||||
raise e
|
||||
raise
|
||||
|
||||
def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool:
|
||||
"""
|
||||
@@ -174,21 +211,81 @@ class OPC(BaseActivity):
|
||||
This helps operators quickly identify configuration issues.
|
||||
"""
|
||||
if self.opc_repository.get(server_id) is None:
|
||||
message = f"OPC server {server_id} not found to perform write operation."
|
||||
message = f'OPC server {server_id} not found to perform write operation.'
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id="OPC_SERVER_NOT_FOUND",
|
||||
notification_id='OPC_SERVER_NOT_FOUND',
|
||||
message=message,
|
||||
block="write_opc_data",
|
||||
block='write_opc_data',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=f"OPC servers: {list(self.opc_repository.keys())}"
|
||||
attachment_content=f'OPC servers: {list(self.opc_repository.keys())}',
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
async def manage_output_tags(
|
||||
self, server_id: str, config: dict[str, Any], data: DataFrame,
|
||||
metadata: dict[str, Any], success: bool) -> tuple[bool, int]:
|
||||
def _write_tags_from_config(
|
||||
self,
|
||||
server_id: str,
|
||||
tags_config: dict[str, dict[str, Any]],
|
||||
data: DataFrame,
|
||||
data_column: str,
|
||||
tag_type: str,
|
||||
log_label: str,
|
||||
metadata: dict[str, Any],
|
||||
) -> tuple[dict[str, float | None], bool, str | None, bool]:
|
||||
"""
|
||||
Write a group of OPC tags and collect response times and error flags.
|
||||
|
||||
Args:
|
||||
server_id: Target OPC server identifier.
|
||||
tags_config: Tag name to configuration mapping.
|
||||
data: DataFrame with prediction/confidence columns.
|
||||
data_column: Column name whose first row value is written.
|
||||
tag_type: Tag category passed to write_data ('prediction' or 'confidence').
|
||||
log_label: Human-readable label for success logs.
|
||||
metadata: Context metadata for logging and notifications.
|
||||
|
||||
Return:
|
||||
(response_times, session_bad_seen, session_bad_status, reconnect_in_progress_seen)
|
||||
"""
|
||||
response_times: dict[str, float | None] = {}
|
||||
session_bad_seen = False
|
||||
session_bad_status: str | None = None
|
||||
reconnect_in_progress_seen = False
|
||||
|
||||
for tag, tag_config in tags_config.items():
|
||||
response_time, error_info = self.write_data(
|
||||
server_id=server_id,
|
||||
tag=tag,
|
||||
data=data.head(1)[data_column].values[0],
|
||||
data_type=tag_config['data_type'],
|
||||
tag_type=tag_type,
|
||||
metadata=metadata,
|
||||
)
|
||||
session_bad_seen, session_bad_status, reconnect_in_progress_seen = (
|
||||
_apply_opc_write_error(
|
||||
error_info,
|
||||
session_bad_seen,
|
||||
session_bad_status,
|
||||
reconnect_in_progress_seen,
|
||||
)
|
||||
)
|
||||
if response_time is not None:
|
||||
self.info(
|
||||
f'{log_label} written to OPC server {server_id} for tag {tag}.',
|
||||
metadata,
|
||||
)
|
||||
response_times[tag] = response_time
|
||||
|
||||
return response_times, session_bad_seen, session_bad_status, reconnect_in_progress_seen
|
||||
|
||||
def manage_output_tags(
|
||||
self,
|
||||
server_id: str,
|
||||
config: dict[str, Any],
|
||||
data: DataFrame,
|
||||
metadata: dict[str, Any],
|
||||
) -> tuple[bool, dict[str, float | None], bool, str | None, bool]:
|
||||
"""
|
||||
Manage the writing of prediction and confidence data to OPC server tags.
|
||||
|
||||
@@ -215,44 +312,52 @@ class OPC(BaseActivity):
|
||||
- overall_success: True if all configured tags were written successfully
|
||||
- total_tags_written: Count of successfully written tags
|
||||
"""
|
||||
response_times: dict[str, float | None] = {}
|
||||
session_bad_seen = False
|
||||
session_bad_status: str | None = None
|
||||
reconnect_in_progress_seen = False
|
||||
|
||||
count = 0
|
||||
if 'prediction_tags' in config:
|
||||
for tag, tag_config in config['prediction_tags'].items():
|
||||
local_success = await self.write_data(
|
||||
server_id=server_id,
|
||||
tag=tag,
|
||||
data=data.head(1)['prediction'].values[0],
|
||||
data_type=tag_config['data_type'],
|
||||
tag_type='prediction',
|
||||
metadata=metadata
|
||||
)
|
||||
if local_success:
|
||||
self.info(
|
||||
f"Prediction data written to OPC server {server_id} for tag {tag}.", metadata)
|
||||
count += 1
|
||||
success = success and local_success
|
||||
tag_groups = (
|
||||
('prediction_tags', 'prediction', 'prediction', 'Prediction data'),
|
||||
('confidence_tags', 'prediction_confidence', 'confidence', 'Confidence data'),
|
||||
)
|
||||
for config_key, data_column, tag_type, log_label in tag_groups:
|
||||
if config_key not in config:
|
||||
continue
|
||||
(
|
||||
group_times,
|
||||
group_session_bad,
|
||||
group_status,
|
||||
group_reconnect,
|
||||
) = self._write_tags_from_config(
|
||||
server_id=server_id,
|
||||
tags_config=config[config_key],
|
||||
data=data,
|
||||
data_column=data_column,
|
||||
tag_type=tag_type,
|
||||
log_label=log_label,
|
||||
metadata=metadata,
|
||||
)
|
||||
response_times.update(group_times)
|
||||
if group_session_bad:
|
||||
session_bad_seen = True
|
||||
session_bad_status = group_status or session_bad_status
|
||||
if group_reconnect:
|
||||
reconnect_in_progress_seen = True
|
||||
|
||||
if 'confidence_tags' in config:
|
||||
for tag, tag_config in config['confidence_tags'].items():
|
||||
local_success = await self.write_data(
|
||||
server_id=server_id,
|
||||
tag=tag,
|
||||
data=data.head(1)['prediction_confidence'].values[0],
|
||||
data_type=tag_config['data_type'],
|
||||
tag_type='confidence',
|
||||
metadata=metadata
|
||||
)
|
||||
if local_success:
|
||||
self.info(
|
||||
f"Confidence data written to OPC server {server_id} for tag {tag}.", metadata)
|
||||
count += 1
|
||||
success = success and local_success
|
||||
|
||||
return success, count
|
||||
success = None not in response_times.values()
|
||||
return (
|
||||
success,
|
||||
response_times,
|
||||
session_bad_seen,
|
||||
session_bad_status,
|
||||
reconnect_in_progress_seen,
|
||||
)
|
||||
|
||||
@activity.defn(name='write_opc_data')
|
||||
async def write_opc_data(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
def write_opc_data(
|
||||
self, input_data: dict[str, Any]
|
||||
) -> tuple[dict[Hashable, Any], dict[str, dict[str, float | None]]]:
|
||||
"""
|
||||
Write prediction and confidence data to OPC servers. The two writing
|
||||
operations are optional and independent of each other.
|
||||
@@ -271,29 +376,68 @@ class OPC(BaseActivity):
|
||||
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info("Writing data to OPC servers...", metadata)
|
||||
self.info('Writing data to OPC servers...', metadata)
|
||||
data = DataFrame(input_data['data'])
|
||||
opc_output_config = input_data['opc_output_config']
|
||||
self.info(f"Data to write: {data.size} rows", metadata)
|
||||
self.info(f'Data to write: {data.size} rows', metadata)
|
||||
|
||||
success = True
|
||||
session_bad_seen = False
|
||||
session_bad_status: str | None = None
|
||||
reconnect_in_progress_seen = False
|
||||
|
||||
opc_metrics: dict[str, dict[str, float | None]] = {}
|
||||
|
||||
for server_id, config in opc_output_config.items():
|
||||
|
||||
if not self.validate_server(server_id, metadata):
|
||||
success = False
|
||||
continue
|
||||
|
||||
local_success, local_count = await self.manage_output_tags(
|
||||
server_id, config, data, metadata, success)
|
||||
(
|
||||
local_success,
|
||||
local_response_times,
|
||||
local_session_bad,
|
||||
local_status,
|
||||
local_reconnect_in_progress,
|
||||
) = self.manage_output_tags(server_id, config, data, metadata)
|
||||
opc_metrics[server_id] = local_response_times
|
||||
local_count = len(local_response_times)
|
||||
success = success and local_success
|
||||
if local_session_bad:
|
||||
session_bad_seen = True
|
||||
session_bad_status = local_status or session_bad_status
|
||||
if local_reconnect_in_progress:
|
||||
reconnect_in_progress_seen = True
|
||||
|
||||
n_pred = len(config.get('prediction_tags') or {})
|
||||
n_conf = len(config.get('confidence_tags') or {})
|
||||
self.info(
|
||||
f"Process completed for OPC server {server_id}: {local_count} of {len(config['prediction_tags'])} prediction tags and {len(config['confidence_tags'])} confidence tags", metadata)
|
||||
f'Process completed for OPC server {server_id}: {local_count} of {n_pred} prediction tags and {n_conf} confidence tags',
|
||||
metadata,
|
||||
)
|
||||
|
||||
return self.process_confidence(data, success, metadata)
|
||||
return (
|
||||
self.process_confidence(
|
||||
data,
|
||||
success,
|
||||
metadata,
|
||||
session_bad=session_bad_seen,
|
||||
opc_status=session_bad_status,
|
||||
reconnect_in_progress=reconnect_in_progress_seen,
|
||||
),
|
||||
opc_metrics,
|
||||
)
|
||||
|
||||
def process_confidence(self, data: DataFrame, success: bool, metadata: dict[str, Any]) -> dict[Any, Any]:
|
||||
def process_confidence(
|
||||
self,
|
||||
data: DataFrame,
|
||||
success: bool,
|
||||
metadata: dict[str, Any],
|
||||
*,
|
||||
session_bad: bool = False,
|
||||
opc_status: str | None = None,
|
||||
reconnect_in_progress: bool = False,
|
||||
) -> dict[Hashable, Any]:
|
||||
"""
|
||||
Process prediction confidence based on OPC write operation success.
|
||||
|
||||
@@ -321,18 +465,31 @@ class OPC(BaseActivity):
|
||||
"""
|
||||
|
||||
if not success:
|
||||
data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE
|
||||
self.debug(
|
||||
f"Some data could not be written to OPC servers, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.",
|
||||
metadata
|
||||
)
|
||||
comment_parts: list[str] = []
|
||||
confidence = OPC_WRITTING_ERROR_CONFIDENCE
|
||||
|
||||
if session_bad:
|
||||
comment_parts.append(_opc_session_bad_comment(opc_status))
|
||||
confidence = OPC_SESSION_BAD_CONFIDENCE
|
||||
if reconnect_in_progress:
|
||||
comment_parts.append(OPC_RECONNECT_IN_PROGRESS_COMMENT)
|
||||
confidence = OPC_SESSION_BAD_CONFIDENCE
|
||||
if not comment_parts:
|
||||
comment_parts.append(OPC_WRITTING_ERROR_MESSAGE)
|
||||
|
||||
comments = OPC_COMMENT_SEPARATOR.join(comment_parts)
|
||||
data['prediction_confidence'] = confidence
|
||||
data['comments'] = comments
|
||||
self.debug(
|
||||
f'OPC write issues, confidence={confidence}, comments={comments}',
|
||||
metadata,
|
||||
)
|
||||
else:
|
||||
self.debug("Data written to OPC servers successfully.", metadata)
|
||||
self.debug('Data written to OPC servers successfully.', metadata)
|
||||
|
||||
return data.to_dict()
|
||||
|
||||
async def shutdown(self):
|
||||
def close(self):
|
||||
"""
|
||||
Gracefully shutdown all OPC server connections and cleanup resources.
|
||||
|
||||
@@ -353,4 +510,5 @@ class OPC(BaseActivity):
|
||||
their current state and provides a clean shutdown experience.
|
||||
"""
|
||||
for opc in self.opc_repository.values():
|
||||
await opc.disconnect()
|
||||
opc.disconnect()
|
||||
self.opc_repository.clear()
|
||||
|
||||
231
laborious/activities/storage.py
Normal file
231
laborious/activities/storage.py
Normal file
@@ -0,0 +1,231 @@
|
||||
from temporalio import activity, workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
# Extend the Temporal Postgres activities for convenient query -> MinIO export
|
||||
import traceback
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.notifications.models import NotificationLevel
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.observability.sientia_monitoring import SientiaMonitoring
|
||||
from sientia_do.repository.minio_repository_sync import MinioRepository
|
||||
from sientia_do.temporal.activities.postgres_sync import Postgres
|
||||
from sientia_do.temporal.constants import now
|
||||
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
|
||||
_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'})
|
||||
|
||||
|
||||
class Storage(Postgres, SientiaMonitoring):
|
||||
"""
|
||||
Extensions for Postgres activities with a helper to export query results
|
||||
directly to MinIO as Parquet and return the object name.
|
||||
"""
|
||||
|
||||
minio_repository: MinioRepository | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
port: int,
|
||||
user: str,
|
||||
password: str,
|
||||
dbname: str,
|
||||
min_connections: int,
|
||||
max_connections: int,
|
||||
retention_hours: int = 24,
|
||||
minio_repository: MinioRepository | None = None,
|
||||
logger: Logger | None = None,
|
||||
notification_handler: NotificationHandler | None = None,
|
||||
metrics_controller: MetricsController | None = None,
|
||||
):
|
||||
self.retention_hours = retention_hours
|
||||
Postgres.__init__(
|
||||
self,
|
||||
host=host,
|
||||
port=port,
|
||||
user=user,
|
||||
password=password,
|
||||
dbname=dbname,
|
||||
min_connections=min_connections,
|
||||
max_connections=max_connections,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
self.minio_repository = minio_repository
|
||||
SientiaMonitoring.__init__(
|
||||
self,
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
@activity.defn(name='load_query_with_minio_offload')
|
||||
def load_query_with_minio_offload(self, input_data: dict[str, Any]) -> MinioDataFramePayload:
|
||||
"""
|
||||
Run the custom SQL load, then return a MinIO-aware dataframe wire dict.
|
||||
|
||||
Args (input_data):
|
||||
metadata (dict): Workflow metadata (same as load_custom_query).
|
||||
query (str): SQL query.
|
||||
datetime_columns (list[str], optional): Datetime column names.
|
||||
model_name (str): Model name for object key basename.
|
||||
key_prefix (str, optional): Directory prefix inside the bucket.
|
||||
size_threshold_bytes (int, optional): Override env offload threshold.
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: Flat ``MinioDataFramePayload`` dict or ``success: False`` on failure.
|
||||
"""
|
||||
if self.minio_repository is None:
|
||||
raise ValueError('Minio repository not initialized')
|
||||
|
||||
metadata: dict = input_data.get('metadata', {})
|
||||
model_name = input_data['model_name']
|
||||
|
||||
rows = self.load_custom_query(
|
||||
input_data,
|
||||
)
|
||||
if not rows:
|
||||
self.error(
|
||||
'load_query_with_minio_offload failed: No data returned from query', metadata
|
||||
)
|
||||
dataframe = None
|
||||
else:
|
||||
dataframe = pd.DataFrame(rows)
|
||||
|
||||
return MinioDataFramePayload.from_dataframe(
|
||||
dataframe,
|
||||
minio_repo=self.minio_repository,
|
||||
workflow_metadata=metadata,
|
||||
model_name=model_name,
|
||||
operation='initial',
|
||||
logger=self.logger,
|
||||
)
|
||||
|
||||
@activity.defn(name='export_payload_to_postgres')
|
||||
def export_payload_to_postgres(self, input_data: dict[str, Any]) -> dict:
|
||||
"""
|
||||
Resolve a MinIO-aware payload into a DataFrame and persist it into PostgreSQL.
|
||||
|
||||
This activity accepts the serialized payload produced by previous steps
|
||||
(inline dict or MinIO object reference), reconstructs the tabular data,
|
||||
and delegates the final write to ``export_data_to_postgres`` using the
|
||||
same input contract expected by the Postgres activity mixin.
|
||||
|
||||
Args:
|
||||
- input_data (dict[str, Any]): Activity input containing ``data`` as a
|
||||
``MinioDataFramePayload``-compatible dict plus database write options
|
||||
(schema/table/on_conflict/metadata and related fields).
|
||||
|
||||
Return:
|
||||
dict: Result dictionary returned by ``export_data_to_postgres``, including
|
||||
success status and optional write diagnostics.
|
||||
"""
|
||||
metadata = input_data.get('metadata')
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
data = payload.retrieve(self.minio_repository, metadata)
|
||||
|
||||
return self.export_data_to_postgres(
|
||||
{
|
||||
**input_data,
|
||||
'data': data,
|
||||
}
|
||||
)
|
||||
|
||||
@activity.defn(name='cleanup_minio_objects_expired')
|
||||
def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Delete objects under the given prefixes that are older than the retention window.
|
||||
|
||||
Args (input_data):
|
||||
metadata (dict): Workflow metadata for logging and metrics.
|
||||
prefixes (list[str]): Key prefixes to scan (one level or subtree per prefix).
|
||||
|
||||
Returns:
|
||||
dict[str, Any]: ``success``, ``deleted_count``, and optional ``message``.
|
||||
"""
|
||||
if self.minio_repository is None:
|
||||
raise ValueError('Minio repository not initialized')
|
||||
|
||||
metadata = input_data.get('metadata', {})
|
||||
payload = MinioDataFramePayload.from_dict(input_data['data'])
|
||||
prefix = payload.cleanup_prefix()
|
||||
base = now()
|
||||
cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta(
|
||||
hours=self.retention_hours
|
||||
)
|
||||
|
||||
report: dict[str, Any] = {
|
||||
'failed': {},
|
||||
'deleted': {},
|
||||
'failed_count': 0,
|
||||
'deleted_count': 0,
|
||||
}
|
||||
try:
|
||||
keys = self.minio_repository.list_objects(
|
||||
prefix=prefix,
|
||||
recursive=True,
|
||||
metadata=metadata,
|
||||
)
|
||||
for key in keys:
|
||||
try:
|
||||
ts = MinioDataFramePayload.parse_object_timestamp(key)
|
||||
if ts is None:
|
||||
continue
|
||||
if ts >= cutoff:
|
||||
continue
|
||||
self.minio_repository.delete_file(
|
||||
object_name=key,
|
||||
metadata=metadata,
|
||||
)
|
||||
except Exception as e:
|
||||
report['failed'][key] = {
|
||||
'success': False,
|
||||
'message': str(e),
|
||||
}
|
||||
report['failed_count'] += 1
|
||||
continue
|
||||
report['deleted'][key] = {
|
||||
'success': True,
|
||||
'message': 'Deleted',
|
||||
}
|
||||
report['deleted_count'] += 1
|
||||
except Exception as e:
|
||||
trace = traceback.format_exc()
|
||||
self.send_notification(
|
||||
metadata=metadata,
|
||||
notification_id='ERROR_CLEANUP_MINIO_OBJECTS_EXPIRED',
|
||||
message=f'Error cleaning up MinIO objects: {e}',
|
||||
block='cleanup_minio_objects_expired',
|
||||
level=NotificationLevel.ERROR,
|
||||
attachment_content=trace,
|
||||
)
|
||||
self.error(trace, metadata)
|
||||
else:
|
||||
# Cleanup success is expected in normal flow; avoid noisy INFO notifications
|
||||
# that do not impact behavior and can flood observability in test runs.
|
||||
self.info('MinIO objects cleaned up successfully', metadata)
|
||||
|
||||
return report
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
Shutdown Storage resources in deterministic order.
|
||||
|
||||
The method first closes Postgres resources via ``Postgres.close`` (engine,
|
||||
sessions, and monitoring hooks), then closes the optional MinIO repository
|
||||
and clears the local reference to avoid accidental reuse after shutdown.
|
||||
"""
|
||||
Postgres.close(self)
|
||||
if self.minio_repository is not None:
|
||||
try:
|
||||
self.minio_repository.close()
|
||||
finally:
|
||||
self.minio_repository = None
|
||||
@@ -18,55 +18,183 @@ Key Metric Categories:
|
||||
|
||||
Metric Labels:
|
||||
- pod_id: Kubernetes pod identifier for multi-instance deployments
|
||||
- runtime: Runtime / environment identifier (matches ``RUNTIME`` env, see ``SientiaMonitoring``)
|
||||
- model_name: Name of the ML model being used
|
||||
- pipeline_name: Name of the prediction pipeline
|
||||
- workflow_name: Name of the prediction pipeline
|
||||
- opc_server_id: Identifier for OPC server operations
|
||||
"""
|
||||
|
||||
from prometheus_client import Gauge, Counter, Histogram
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
from sientia_do.observability.metrics import CORE_LABELS
|
||||
|
||||
# Application health metric
|
||||
APP_UP = Gauge(
|
||||
"app_up",
|
||||
"Indicates if the application is running (1) or shutting down (0)",
|
||||
["pod_id"],
|
||||
'app_up',
|
||||
'Indicates if the application is running (1) or shutting down (0)',
|
||||
['pod_id'],
|
||||
)
|
||||
|
||||
# Core labels used across multiple metrics
|
||||
CORE_LABELS = ["pod_id", "model_name", "pipeline_name"]
|
||||
|
||||
# Prediction operation metrics
|
||||
PREDICTIONS_WRITTEN_COUNT = Counter(
|
||||
"laborious_predictions_written_count",
|
||||
"Number of predictions written to the database table predictions",
|
||||
'laborious_predictions_written_count',
|
||||
'Number of predictions written to the database table predictions',
|
||||
CORE_LABELS,
|
||||
)
|
||||
|
||||
# Prediction quality metrics
|
||||
PREDICTION_CONFIDENCE_MONITOR = Gauge(
|
||||
"laborious_prediction_confidence_monitor",
|
||||
"Current confidence of each prediction",
|
||||
'laborious_prediction_confidence_monitor',
|
||||
'Current confidence of each prediction',
|
||||
CORE_LABELS,
|
||||
)
|
||||
|
||||
# Performance monitoring metrics
|
||||
# Prediction total response time
|
||||
PREDICTION_RESPONSE_TIME_MONITOR = Histogram(
|
||||
"laborious_prediction_response_time_monitor",
|
||||
"Current response time of each prediction",
|
||||
'laborious_prediction_response_time_monitor',
|
||||
'Current response time of each prediction',
|
||||
CORE_LABELS,
|
||||
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0]
|
||||
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
||||
)
|
||||
|
||||
# OPC export metrics
|
||||
|
||||
# ================== OPC metrics ==================
|
||||
|
||||
|
||||
PREDICTION_OPC_WRITING_COUNT = Counter(
|
||||
"laborious_prediction_opc_writing_count",
|
||||
"Number of predictions written to the OPC server",
|
||||
[*CORE_LABELS, "opc_server_id"],
|
||||
'laborious_prediction_opc_writing_count',
|
||||
'Number of predictions written to the OPC server',
|
||||
[*CORE_LABELS, 'opc_server_id', 'tag'],
|
||||
)
|
||||
|
||||
PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram(
|
||||
"laborious_prediction_opc_writing_response_time_monitor",
|
||||
"Current response time of each prediction written to the OPC server",
|
||||
[*CORE_LABELS, "opc_server_id"],
|
||||
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0]
|
||||
'laborious_prediction_opc_writing_response_time_monitor',
|
||||
'Current response time of each prediction written to the OPC server',
|
||||
[*CORE_LABELS, 'opc_server_id', 'tag'],
|
||||
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
||||
)
|
||||
|
||||
OPC_CONNECTIONS_TOTAL = Counter(
|
||||
'opc_connections_initiated_total',
|
||||
'Total connection attempts to OPC servers',
|
||||
['pod_id', 'server_name'],
|
||||
)
|
||||
OPC_CONNECTIONS_FAILED = Counter(
|
||||
'opc_connections_failed_total',
|
||||
'Total failed connection attempts to OPC servers',
|
||||
['pod_id', 'server_name'],
|
||||
)
|
||||
OPC_CONNECTION_STATUS = Gauge(
|
||||
'opc_connection_status',
|
||||
'Connection status with the OPC server (1=connected, 0=disconnected)',
|
||||
['pod_id', 'server_name', 'server_url'],
|
||||
)
|
||||
|
||||
_OPC_SESSION_DEBUG_LABELS = ['pod_id', 'server_name', 'runtime', 'opc_server_id', 'session_id']
|
||||
|
||||
OPC_SESSION_CREATED_TOTAL = Counter(
|
||||
'opc_session_created_total',
|
||||
'OPC UA sessions established (after successful connect)',
|
||||
_OPC_SESSION_DEBUG_LABELS,
|
||||
)
|
||||
|
||||
OPC_SESSION_CLOSED_TOTAL = Counter(
|
||||
'opc_session_closed_total',
|
||||
'OPC UA client disconnects completed (session tear-down initiated)',
|
||||
_OPC_SESSION_DEBUG_LABELS,
|
||||
)
|
||||
|
||||
OPC_SESSION_REVISED_TIMEOUT_MS = Gauge(
|
||||
'opc_session_revised_timeout_milliseconds',
|
||||
'Server-revised OPC UA session timeout (RevisedSessionTimeout) in ms after connect',
|
||||
_OPC_SESSION_DEBUG_LABELS,
|
||||
)
|
||||
|
||||
OPC_WRITE_ATTEMPT_LABELS = [*_OPC_SESSION_DEBUG_LABELS, 'model_id', 'model_name', 'result']
|
||||
|
||||
OPC_WRITE_ATTEMPTS_TOTAL = Counter(
|
||||
'opc_write_attempts_total',
|
||||
'OPC UA write attempts with session and outcome (result=OK or exception class name)',
|
||||
OPC_WRITE_ATTEMPT_LABELS,
|
||||
)
|
||||
|
||||
OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL = Counter(
|
||||
'opc_write_inter_arrival_over_session_timeout_total',
|
||||
'Successful writes where seconds since the previous successful write exceeded RevisedSessionTimeout (ms)',
|
||||
_OPC_SESSION_DEBUG_LABELS,
|
||||
)
|
||||
|
||||
# ================== Model metrics ==================
|
||||
|
||||
MODEL_READ_LAG = Histogram(
|
||||
'laborious_model_read_lag',
|
||||
'Lag between the start and read of read operations',
|
||||
CORE_LABELS,
|
||||
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
||||
)
|
||||
|
||||
MODEL_WRITE_LAG = Histogram(
|
||||
'laborious_model_write_lag',
|
||||
'Lag between the start and end of write operations',
|
||||
CORE_LABELS,
|
||||
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
||||
)
|
||||
|
||||
MODEL_READ_COUNT = Counter(
|
||||
'laborious_model_read_count',
|
||||
'Number of reads from the model',
|
||||
CORE_LABELS,
|
||||
)
|
||||
|
||||
MODEL_WRITE_COUNT = Counter(
|
||||
'laborious_model_write_count',
|
||||
'Number of writes to the model',
|
||||
CORE_LABELS,
|
||||
)
|
||||
|
||||
MODEL_READ_ERROR_COUNT = Counter(
|
||||
'laborious_model_read_error_count',
|
||||
'Number of errors reading from the model',
|
||||
CORE_LABELS,
|
||||
)
|
||||
|
||||
MODEL_WRITE_ERROR_COUNT = Counter(
|
||||
'laborious_model_write_error_count',
|
||||
'Number of errors writing to the model',
|
||||
CORE_LABELS,
|
||||
)
|
||||
|
||||
MODEL_ANALYZE_LAG = Histogram(
|
||||
'laborious_model_analyze_lag',
|
||||
'Lag between the start and end of analyze operations',
|
||||
CORE_LABELS,
|
||||
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
|
||||
)
|
||||
|
||||
MODEL_ANALYZE_COUNT = Counter(
|
||||
'laborious_model_analyze_count',
|
||||
'Number of analyze operations',
|
||||
CORE_LABELS,
|
||||
)
|
||||
|
||||
MODEL_ANALYZE_ERROR_COUNT = Counter(
|
||||
'laborious_model_analyze_error_count',
|
||||
'Number of errors during analyze operations',
|
||||
CORE_LABELS,
|
||||
)
|
||||
|
||||
|
||||
# ================== PI Web API metrics ==================
|
||||
|
||||
PI_WEB_API_LABELS = [*CORE_LABELS, 'tag_name']
|
||||
|
||||
PI_WEB_API_PREDICTION_WRITTEN_COUNT = Counter(
|
||||
'laborious_pi_web_api_prediction_written_count',
|
||||
'Number of predictions written to the PI Web API',
|
||||
PI_WEB_API_LABELS,
|
||||
)
|
||||
|
||||
PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT = Counter(
|
||||
'laborious_pi_web_api_prediction_written_error_count',
|
||||
'Number of errors writing predictions to the PI Web API',
|
||||
PI_WEB_API_LABELS,
|
||||
)
|
||||
|
||||
@@ -1,65 +1,68 @@
|
||||
from os import getenv
|
||||
import json
|
||||
from typing import Dict, Any
|
||||
from os import getenv
|
||||
from typing import Any
|
||||
|
||||
|
||||
def build_postgres_config() -> Dict[str, Any]:
|
||||
def build_mlflow_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build PostgreSQL database configuration from environment variables.
|
||||
Read MLflow tracking and registry credentials from the environment.
|
||||
|
||||
This function constructs a PostgreSQL configuration dictionary from
|
||||
environment variables with sensible defaults for local development.
|
||||
It handles connection pool configuration and security parameters.
|
||||
Used by ``Activities`` when constructing ``SientiaMLflowRepository``. The ``url`` value is the
|
||||
same string workers and notebooks should use for ``MLFLOW_TRACKING_URI``-style clients.
|
||||
|
||||
Environment Variables:
|
||||
POSTGRES_HOST: Database hostname (default: localhost)
|
||||
POSTGRES_PORT: Database port (default: 5432)
|
||||
POSTGRES_USER: Database username (default: sientia)
|
||||
POSTGRES_PASSWORD: Database password (default: sientia)
|
||||
POSTGRES_DBNAME: Database name (default: sientia)
|
||||
POSTGRES_MIN_CONNECTIONS: Minimum connection pool size (default: 5)
|
||||
POSTGRES_MAX_CONNECTIONS: Maximum connection pool size (default: 20)
|
||||
MLFLOW_URL: Host with scheme
|
||||
MLFLOW_USERNAME: Basic-auth or service user (default: aignosi)
|
||||
MLFLOW_PASSWORD: Password or token (default: aignosi)
|
||||
|
||||
Returns:
|
||||
dict: PostgreSQL configuration dictionary with all required parameters
|
||||
Return:
|
||||
dict[str, Any]: ``url``, ``username``, ``password``.
|
||||
"""
|
||||
return {
|
||||
'host': getenv('POSTGRES_HOST', 'localhost'),
|
||||
'port': int(getenv('POSTGRES_PORT', '5432')),
|
||||
'user': getenv('POSTGRES_USER', 'sientia'),
|
||||
'password': getenv('POSTGRES_PASSWORD', 'sientia'),
|
||||
'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
|
||||
'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
|
||||
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20'))
|
||||
}
|
||||
|
||||
|
||||
def build_mlflow_config() -> Dict[str, Any]:
|
||||
"""
|
||||
Build MLFlow server configuration from environment variables.
|
||||
|
||||
This function constructs an MLFlow configuration dictionary from
|
||||
environment variables with sensible defaults for local development.
|
||||
It handles server connection and authentication parameters.
|
||||
|
||||
Environment Variables:
|
||||
MLFLOW_HOST: MLFlow server hostname (default: http://localhost)
|
||||
MLFLOW_PORT: MLFlow server port (default: 5080)
|
||||
MLFLOW_USERNAME: MLFlow username (default: aignosi)
|
||||
MLFLOW_PASSWORD: MLFlow password (default: aignosi)
|
||||
|
||||
Returns:
|
||||
dict: MLFlow configuration dictionary with all required parameters
|
||||
"""
|
||||
return {
|
||||
'host': getenv('MLFLOW_HOST', 'http://localhost'),
|
||||
'port': int(getenv('MLFLOW_PORT', '5080')),
|
||||
'url': getenv('MLFLOW_URL', 'http://localhost:5080'),
|
||||
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
|
||||
'password': getenv('MLFLOW_PASSWORD', 'aignosi')
|
||||
'password': getenv('MLFLOW_PASSWORD', 'aignosi'),
|
||||
}
|
||||
|
||||
|
||||
def build_opc_config() -> Dict[str, Any]:
|
||||
def build_plugin_store_config() -> dict[str, Any]:
|
||||
"""
|
||||
Collect settings for ``PluginStore`` (Git-backed catalog + runtime install via pip).
|
||||
|
||||
Mirrors the model-manager service: the worker passes these kwargs into ``PluginStore`` after
|
||||
``install_runtime`` resolves wheels from the configured PyPI index. Missing optional env vars
|
||||
become ``None`` so the store can run without auth in local dev.
|
||||
|
||||
Environment Variables:
|
||||
STORE_BASE_URL: Git HTTP(S) server (e.g. Gitea) base URL (default: http://localhost:3000)
|
||||
STORE_OWNER: Namespace or org owning the store repo (default: sientia)
|
||||
STORE_REPO: Repository name (default: model-library-store)
|
||||
STORE_BRANCH: Checkout branch; unset lets the client use default
|
||||
STORE_USERNAME / STORE_PASSWORD: HTTP basic credentials for Git fetch
|
||||
STORE_CACHE_TTL_SECONDS: Optional integer seconds for metadata cache TTL
|
||||
PYPI_SERVER: Index URL for ``pip install`` during runtime install (default: http://localhost:5000)
|
||||
PYPI_USERNAME / PYPI_PASSWORD: Optional index authentication
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Keys aligned with ``PluginStore`` constructor parameter names.
|
||||
"""
|
||||
|
||||
cache_ttl_seconds = getenv('STORE_CACHE_TTL_SECONDS')
|
||||
return {
|
||||
'base_url': getenv('STORE_BASE_URL', 'http://localhost:3000'),
|
||||
'owner': getenv('STORE_OWNER', 'sientia'),
|
||||
'repo': getenv('STORE_REPO', 'model-library-store'),
|
||||
'username': getenv('STORE_USERNAME'),
|
||||
'password': getenv('STORE_PASSWORD'),
|
||||
'branch': getenv('STORE_BRANCH'),
|
||||
'cache_ttl_seconds': int(cache_ttl_seconds) if cache_ttl_seconds else None,
|
||||
'pypi_index_url': getenv('PYPI_SERVER', 'http://localhost:5000'),
|
||||
'pypi_username': getenv('PYPI_USERNAME'),
|
||||
'pypi_password': getenv('PYPI_PASSWORD'),
|
||||
}
|
||||
|
||||
|
||||
def build_opc_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build OPC server configuration from environment variables.
|
||||
|
||||
@@ -75,7 +78,7 @@ def build_opc_config() -> Dict[str, Any]:
|
||||
OPC_CERT_PATH: Client certificate path (fallback, default: None)
|
||||
OPC_PRIVATE_KEY_PATH: Client private key path (fallback, default: None)
|
||||
OPC_SERVER_CERT_PATH: Server certificate path (fallback, default: None)
|
||||
OPC_RECONNECTION_INTERVAL: Reconnection interval in milliseconds (fallback, default: 120)
|
||||
OPC_RECONNECTION_INTERVAL: Reconnection interval in seconds (fallback, default: 120)
|
||||
|
||||
Returns:
|
||||
dict: OPC server configuration dictionary
|
||||
@@ -88,42 +91,37 @@ def build_opc_config() -> Dict[str, Any]:
|
||||
return {
|
||||
getenv('OPC_ID', '1'): {
|
||||
'id': getenv('OPC_ID', '1'),
|
||||
'server_name': getenv('OPC_SERVER_NAME', 'default_server'),
|
||||
'url': getenv('OPC_URL', 'opc.tcp://localhost:4840'),
|
||||
'server_uri': getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'),
|
||||
'cert_path': getenv('OPC_CERT_PATH', None),
|
||||
'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None),
|
||||
'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None),
|
||||
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120'))
|
||||
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120')),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def build_mongodb_config() -> Dict[str, Any]:
|
||||
def build_minio_config() -> dict[str, Any]:
|
||||
"""
|
||||
Build MongoDB configuration from environment variables.
|
||||
|
||||
This function constructs a MongoDB configuration dictionary from
|
||||
environment variables with sensible defaults for local development.
|
||||
It handles connection string and database name configuration.
|
||||
Build MinIO (S3-compatible) configuration from environment variables.
|
||||
|
||||
Environment Variables:
|
||||
MONGODB_USERNAME: MongoDB username (default: root)
|
||||
MONGODB_PASSWORD: MongoDB password (default: wKZDbMNU1c)
|
||||
MONGODB_URL: MongoDB connection URI (default: localhost:27018)
|
||||
MONGODB_DATABASE_NAME: MongoDB database name (default: sientia)
|
||||
MONGODB_TTL_INDEX_HOURS: TTL index duration in hours (default: 1)
|
||||
MINIO_ENDPOINT_URL: Host:port or URL for the S3 API (default: http://localhost:9000)
|
||||
MINIO_ACCESS_KEY: Access key (default: minioadmin)
|
||||
MINIO_SECRET_KEY: Secret key (default: minioadmin)
|
||||
MINIO_DEFAULT_BUCKET: Default bucket for Laborious payloads (default: laborious)
|
||||
MINIO_RETENTION_HOURS: Offloaded object retention window (default: 24)
|
||||
MINIO_SECURE: If ``true``, use HTTPS (default: false)
|
||||
|
||||
Returns:
|
||||
dict: MongoDB configuration dictionary with connection parameters
|
||||
Return:
|
||||
dict[str, Any]: Keys consumed by ``Activities`` / ``MinioRepository``.
|
||||
"""
|
||||
username = getenv('MONGODB_USERNAME', 'root')
|
||||
password = getenv('MONGODB_PASSWORD', 'wKZDbMNU1c')
|
||||
uri = getenv('MONGODB_URL', 'localhost:27018')
|
||||
|
||||
connection_string = f'mongodb://{username}:{password}@{uri}'
|
||||
|
||||
return {
|
||||
'connection_string': connection_string,
|
||||
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
|
||||
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600
|
||||
'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'),
|
||||
'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'),
|
||||
'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'),
|
||||
'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious'),
|
||||
'retention_hours': int(getenv('MINIO_RETENTION_HOURS', '24')),
|
||||
'secure': getenv('MINIO_SECURE', 'false') == 'true',
|
||||
}
|
||||
|
||||
34
laborious/utils/dataframe_debug.py
Normal file
34
laborious/utils/dataframe_debug.py
Normal file
@@ -0,0 +1,34 @@
|
||||
from typing import Any
|
||||
|
||||
from pandas import DataFrame
|
||||
|
||||
DEFAULT_MAX_DEBUG_DATAFRAME_ROWS = 100
|
||||
|
||||
|
||||
def build_dataframe_debug_message(
|
||||
message: str,
|
||||
data: Any,
|
||||
max_rows: int = DEFAULT_MAX_DEBUG_DATAFRAME_ROWS,
|
||||
) -> str:
|
||||
"""
|
||||
Build a safe debug message for dataframe payloads
|
||||
|
||||
Args:
|
||||
- message (str): Base message to identify the logged payload
|
||||
- data (Any): Payload to evaluate for dataframe-aware logging
|
||||
- max_rows (int): Maximum dataframe row count allowed for full payload logging
|
||||
|
||||
Return:
|
||||
Formatted debug message with full dataframe content or compact summary
|
||||
"""
|
||||
if not isinstance(data, DataFrame):
|
||||
return f'{message} {data}'
|
||||
|
||||
rows = data.shape[0]
|
||||
if rows <= max_rows:
|
||||
return f'{message}\n{data.to_csv()}'
|
||||
|
||||
return (
|
||||
f'{message} skipped because dataframe has {rows} rows '
|
||||
f'(max: {max_rows}). Shape: {data.shape}'
|
||||
)
|
||||
@@ -20,8 +20,11 @@ def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool
|
||||
False if none of the specified variables contain null values.
|
||||
|
||||
"""
|
||||
return not data[
|
||||
data['variable'].isin(config['variables']) & data['value'].isna()].empty
|
||||
|
||||
if data.empty:
|
||||
return False
|
||||
|
||||
return not data[data['variable'].isin(config['variables']) & data['value'].isna()].empty
|
||||
|
||||
|
||||
def filter_empty_data(data: DataFrame, _config: dict) -> bool:
|
||||
|
||||
@@ -52,8 +52,11 @@ def nan_values_filter(predictions: DataFrame, _config: dict) -> bool:
|
||||
bool: True if data should be filtered (too many NaN values), False otherwise
|
||||
|
||||
"""
|
||||
data = predictions.replace({None: np.nan}).drop(
|
||||
columns=['timestamp'], errors='ignore').infer_objects()
|
||||
data = (
|
||||
predictions.replace({None: np.nan})
|
||||
.drop(columns=['timestamp'], errors='ignore')
|
||||
.infer_objects()
|
||||
)
|
||||
|
||||
if data.isna().all().all():
|
||||
return True
|
||||
|
||||
0
laborious/utils/models/__init__.py
Normal file
0
laborious/utils/models/__init__.py
Normal file
362
laborious/utils/models/minio_dataframe_payload.py
Normal file
362
laborious/utils/models/minio_dataframe_payload.py
Normal file
@@ -0,0 +1,362 @@
|
||||
"""
|
||||
MinIO-backed DataFrame payload for Temporal workflows.
|
||||
|
||||
Data is never stored as a pandas ``DataFrame`` field on the dataclass.
|
||||
Instead, the DataFrame is only provided as an input to:
|
||||
`from_dataframe` / `from_dataframe_to_dict`.
|
||||
|
||||
At build time, the DataFrame is evaluated for its serialized size; if it exceeds
|
||||
the configured threshold, it is serialized to parquet bytes and uploaded to MinIO.
|
||||
Otherwise, it is inlined as a Temporal-friendly ``dict``.
|
||||
"""
|
||||
|
||||
import pickle
|
||||
import re
|
||||
from collections.abc import Hashable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from os import getenv
|
||||
from typing import Any, Literal
|
||||
|
||||
from pandas import DataFrame, read_parquet
|
||||
from sientia_do.observability.logger import Logger
|
||||
from sientia_do.repository.minio_repository_sync import MinioRepository
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now
|
||||
|
||||
# Keys that are part of the serialized wire format (not arbitrary metadata).
|
||||
_SERIALIZED_FIELD_KEYS = frozenset({'data', 'bucket', 'object_key', 'object_prefix', 'uri'})
|
||||
|
||||
_OBJECT_TIMESTAMP_PATTERN = re.compile(
|
||||
r'-(?:initial|transform)-(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.parquet$'
|
||||
)
|
||||
|
||||
OFFLOAD_THRESHOLD_BYTES = int(
|
||||
float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1.5')) * 1024 * 1024
|
||||
)
|
||||
|
||||
# Relative prefix used for storing offloaded prediction datasets in MinIO.
|
||||
# It is also the root directory for retention cleanup listing.
|
||||
PREDICTION_DATASETS_PREFIX = 'prediction_datasets'
|
||||
|
||||
OperationKind = Literal['initial', 'transform', 'predict']
|
||||
|
||||
|
||||
def _build_object_key(
|
||||
model_name: str, operation: OperationKind, timestamp: str
|
||||
) -> tuple[str, str | None]:
|
||||
"""
|
||||
Build the MinIO object key and the directory prefix used for retention listing.
|
||||
|
||||
Args:
|
||||
model_name: Registered model name used in the pipeline.
|
||||
operation: Either initial (pre-transform load) or transform (post-MLFlow transform).
|
||||
timestamp: Filename timestamp segment from DATETIME_FORMAT_FILENAME.
|
||||
|
||||
Return:
|
||||
tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root).
|
||||
"""
|
||||
# Naming convention:
|
||||
# - Directory is always `prediction_datasets/<model_name>`
|
||||
# - Filename follows the retention-parsing pattern
|
||||
basename = f'{model_name}-{operation}-{timestamp}.parquet'
|
||||
model_dir = model_name.strip().strip('/')
|
||||
prefix = f'{PREDICTION_DATASETS_PREFIX}/{model_dir}'
|
||||
return f'{prefix}/{basename}', prefix
|
||||
|
||||
|
||||
@dataclass
|
||||
class MinioDataFramePayload:
|
||||
"""
|
||||
Serializable payload after a DataFrame was evaluated: inline tabular dict and/or MinIO keys.
|
||||
|
||||
Build from a live DataFrame only via `from_dataframe` / `from_dataframe_to_dict`.
|
||||
Rehydrate from Temporal via `from_dict`. The DataFrame is not a field on this class.
|
||||
"""
|
||||
|
||||
last_timestamp: str
|
||||
status: dict[str, Any] | None = None
|
||||
data: dict[Hashable, Any] | None = None
|
||||
bucket: str | None = None
|
||||
object_key: str | None = None
|
||||
object_prefix: str | None = None
|
||||
uri: str | None = None
|
||||
|
||||
@staticmethod
|
||||
def _debug(
|
||||
logger: Logger | None,
|
||||
message: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Emit a debug message only when a logger instance is available.
|
||||
|
||||
Args:
|
||||
- logger (Logger | None): Logger instance used for debug messages
|
||||
- message (str): Message to be logged
|
||||
- metadata (dict[str, Any] | None): Optional workflow metadata context
|
||||
"""
|
||||
if logger is None:
|
||||
return
|
||||
logger.custom_debug(message, metadata)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, raw: 'dict[str, Any] | MinioDataFramePayload') -> 'MinioDataFramePayload':
|
||||
"""
|
||||
Reconstruct a MinioDataFramePayload from a plain dict produced by Temporal serialization.
|
||||
|
||||
Temporal converts dataclass return values into plain dicts when crossing
|
||||
workflow/activity boundaries. This method rebuilds the typed instance so
|
||||
that methods like ``retrieve``, ``cleanup_prefix`` and ``has_data`` are
|
||||
available on the receiving side.
|
||||
|
||||
If the argument is already a MinioDataFramePayload, it is returned as-is.
|
||||
|
||||
Args:
|
||||
raw: Dict with keys matching the dataclass fields
|
||||
(last_timestamp, status, data, bucket, object_key, object_prefix, uri),
|
||||
or an existing MinioDataFramePayload instance.
|
||||
|
||||
Return:
|
||||
MinioDataFramePayload: Reconstructed (or original) instance.
|
||||
"""
|
||||
if isinstance(raw, MinioDataFramePayload):
|
||||
return raw
|
||||
return cls(
|
||||
last_timestamp=raw['last_timestamp'],
|
||||
status=raw.get('status'),
|
||||
data=raw.get('data'),
|
||||
bucket=raw.get('bucket'),
|
||||
object_key=raw.get('object_key'),
|
||||
object_prefix=raw.get('object_prefix'),
|
||||
uri=raw.get('uri'),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def estimate_size_bytes(
|
||||
df: DataFrame,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
logger: Logger | None = None,
|
||||
) -> int:
|
||||
"""
|
||||
Approximate serialized size of the DataFrame as the default-orient dict.
|
||||
|
||||
Args:
|
||||
df: DataFrame whose tabular content size is estimated.
|
||||
|
||||
Return:
|
||||
int: Estimated size in bytes (pickle of dict representation).
|
||||
"""
|
||||
try:
|
||||
size = len(pickle.dumps(df.to_dict()))
|
||||
except Exception:
|
||||
size = len(pickle.dumps(df))
|
||||
|
||||
MinioDataFramePayload._debug(
|
||||
logger,
|
||||
f'DataFrame size: {size} bytes',
|
||||
metadata,
|
||||
)
|
||||
return size
|
||||
|
||||
@staticmethod
|
||||
def parse_object_timestamp(object_key: str) -> datetime | None:
|
||||
"""
|
||||
Parse the timestamp embedded in the object key basename (before .parquet).
|
||||
|
||||
Args:
|
||||
object_key: S3/MinIO object key whose basename follows
|
||||
``{model}-{initial|transform}-{DATETIME_FORMAT_FILENAME}.parquet``.
|
||||
|
||||
Return:
|
||||
datetime | None: Parsed UTC-naive datetime from the key, or None if not matched.
|
||||
"""
|
||||
basename = object_key.rsplit('/', 1)[-1]
|
||||
match = _OBJECT_TIMESTAMP_PATTERN.search(basename)
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(match.group(1), DATETIME_FORMAT_FILENAME)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def cleanup_prefix(self) -> str | None:
|
||||
"""
|
||||
Return the MinIO prefix eligible for retention cleanup.
|
||||
|
||||
Cleanup is only applicable when payload data was offloaded to MinIO
|
||||
(``object_key`` present and inline ``data`` absent). Inline-only payloads
|
||||
return ``None`` because there is no object tree to prune.
|
||||
|
||||
Return:
|
||||
str | None: Prefix used by cleanup listing, or ``None`` when cleanup does not apply.
|
||||
"""
|
||||
if self.object_key is not None and self.data is None:
|
||||
return self.object_prefix
|
||||
return None
|
||||
|
||||
def has_data(self) -> bool:
|
||||
"""
|
||||
Indicate whether the payload contains retrievable tabular content.
|
||||
|
||||
A payload is considered non-empty when either inline ``data`` exists
|
||||
(and is not an empty dict) or an ``object_key`` is available for MinIO
|
||||
download.
|
||||
|
||||
Return:
|
||||
bool: ``True`` when data can be retrieved, ``False`` otherwise.
|
||||
"""
|
||||
return (self.data is not None and self.data != {}) or self.object_key is not None
|
||||
|
||||
@classmethod
|
||||
def from_dataframe(
|
||||
cls,
|
||||
dataframe: DataFrame | None,
|
||||
minio_repo: MinioRepository,
|
||||
model_name: str,
|
||||
operation: OperationKind,
|
||||
status: dict[str, Any] | None = None,
|
||||
workflow_metadata: dict | None = None,
|
||||
last_timestamp: str | None = None,
|
||||
logger: Logger | None = None,
|
||||
) -> 'MinioDataFramePayload':
|
||||
"""
|
||||
Evaluate the DataFrame size, then either inline dict or upload parquet to MinIO.
|
||||
|
||||
The DataFrame is not stored on the returned instance.
|
||||
|
||||
Args:
|
||||
dataframe: Tabular data to evaluate and persist (inline or MinIO).
|
||||
metadata: Small metadata dict merged into the payload (e.g. success, message).
|
||||
minio_repo: sientia_do MinioRepository (or compatible) with `upload_file()`.
|
||||
workflow_metadata: Metadata passed to MinIO store for logging/metrics.
|
||||
model_name: Registered model name used in the object basename.
|
||||
operation: Either ``initial`` (query load) or ``transform`` (post-transform).
|
||||
key_prefix: Backward-compatible parameter (currently ignored for object naming).
|
||||
size_threshold_bytes: Byte limit before offload. When None, the module-level
|
||||
environment-derived default is used.
|
||||
|
||||
Return:
|
||||
MinioDataFramePayload: Instance with data and/or MinIO fields set.
|
||||
"""
|
||||
|
||||
if dataframe is None or dataframe.empty:
|
||||
cls._debug(
|
||||
logger,
|
||||
'MinioDataFramePayload.from_dataframe received empty dataframe, returning empty payload',
|
||||
workflow_metadata,
|
||||
)
|
||||
return cls(
|
||||
data=None, last_timestamp=now().strftime(DATETIME_FORMAT_WITH_TZ), status=status
|
||||
)
|
||||
|
||||
if last_timestamp is None:
|
||||
last_timestamp = max(dataframe['timestamp'].values.tolist())
|
||||
|
||||
dataframe_size = cls.estimate_size_bytes(dataframe, workflow_metadata, logger)
|
||||
cls._debug(
|
||||
logger,
|
||||
(
|
||||
f'MinioDataFramePayload.from_dataframe estimated size: {dataframe_size} bytes '
|
||||
f'(threshold: {OFFLOAD_THRESHOLD_BYTES} bytes)'
|
||||
),
|
||||
workflow_metadata,
|
||||
)
|
||||
|
||||
if dataframe_size <= OFFLOAD_THRESHOLD_BYTES:
|
||||
cls._debug(
|
||||
logger,
|
||||
'MinioDataFramePayload.from_dataframe using inline payload',
|
||||
workflow_metadata,
|
||||
)
|
||||
return cls(data=dataframe.to_dict(), last_timestamp=last_timestamp, status=status)
|
||||
|
||||
timestamp = now().strftime(DATETIME_FORMAT_FILENAME)
|
||||
object_key, object_prefix = _build_object_key(model_name, operation, timestamp)
|
||||
cls._debug(
|
||||
logger,
|
||||
(
|
||||
'MinioDataFramePayload.from_dataframe offloading payload to MinIO '
|
||||
f'with key {object_key}'
|
||||
),
|
||||
workflow_metadata,
|
||||
)
|
||||
|
||||
# Upload using the relative object key. The upstream repository will
|
||||
# prefix it internally under its MinIO namespace.
|
||||
parquet_buffer = BytesIO()
|
||||
dataframe.to_parquet(parquet_buffer, engine='pyarrow', index=True)
|
||||
file_bytes = parquet_buffer.getvalue()
|
||||
|
||||
upload_result = minio_repo.upload_file(
|
||||
file_bytes=file_bytes,
|
||||
relative_key=object_key,
|
||||
metadata=workflow_metadata,
|
||||
)
|
||||
|
||||
bucket = minio_repo.bucket
|
||||
object_key_full = upload_result.get('minio_object_name', object_key)
|
||||
uri = f's3://{bucket}/{object_key_full}' if bucket else None
|
||||
cls._debug(
|
||||
logger,
|
||||
f'MinioDataFramePayload.from_dataframe upload completed: {uri}',
|
||||
workflow_metadata,
|
||||
)
|
||||
|
||||
return cls(
|
||||
data=None,
|
||||
bucket=bucket,
|
||||
object_key=object_key_full,
|
||||
object_prefix=object_prefix,
|
||||
uri=uri,
|
||||
last_timestamp=last_timestamp,
|
||||
status=status,
|
||||
)
|
||||
|
||||
def retrieve(
|
||||
self,
|
||||
minio_repo: MinioRepository,
|
||||
workflow_metadata: dict[str, Any] | None = None,
|
||||
logger: Logger | None = None,
|
||||
) -> DataFrame:
|
||||
"""
|
||||
Load parquet from MinIO when object_key is set and populate inline data.
|
||||
|
||||
Args:
|
||||
minio_repo: sientia_do MinioRepository (or compatible) with download_file().
|
||||
workflow_metadata: Metadata passed to MinIO read for logging/metrics.
|
||||
|
||||
Return:
|
||||
dict[str, Any]: Flat dict with data filled (same keys as to_dict after load).
|
||||
"""
|
||||
if self.data is not None:
|
||||
self._debug(
|
||||
logger,
|
||||
'MinioDataFramePayload.retrieve using inline payload data',
|
||||
workflow_metadata,
|
||||
)
|
||||
return DataFrame(self.data)
|
||||
|
||||
if not self.has_data():
|
||||
self._debug(
|
||||
logger,
|
||||
'MinioDataFramePayload.retrieve found no payload data, returning empty dataframe',
|
||||
workflow_metadata,
|
||||
)
|
||||
return DataFrame()
|
||||
|
||||
self._debug(
|
||||
logger,
|
||||
f'MinioDataFramePayload.retrieve downloading object from MinIO: {self.object_key}',
|
||||
workflow_metadata,
|
||||
)
|
||||
file_bytes = minio_repo.download_file(
|
||||
object_name=self.object_key, metadata=workflow_metadata
|
||||
)
|
||||
df = read_parquet(BytesIO(file_bytes))
|
||||
self._debug(
|
||||
logger,
|
||||
f'MinioDataFramePayload.retrieve loaded dataframe from MinIO with shape {df.shape}',
|
||||
workflow_metadata,
|
||||
)
|
||||
return df
|
||||
@@ -1,481 +0,0 @@
|
||||
"""
|
||||
Model Monitoring Repository
|
||||
|
||||
This module contains the ModelMonitoringRepository class,
|
||||
which is responsible for handling the communication with the Model Monitoring API.
|
||||
|
||||
It includes the methods that are used to answer ModelMonitoringService
|
||||
requests using the Model Monitoring API functions.
|
||||
|
||||
By Monitoring we mean the evaluation of the performance of models, the generation of reports.
|
||||
|
||||
"""
|
||||
from datetime import datetime
|
||||
import traceback
|
||||
import pandas as pd
|
||||
import mlflow
|
||||
from os import makedirs, path, remove
|
||||
from sientia.ModelServing import ModelServing
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.observability.logger import Logger
|
||||
|
||||
|
||||
class MLFlowRepository():
|
||||
def __init__(self, host, username, password, logger: Logger):
|
||||
|
||||
self.model_serving = ModelServing(tracking_uri=host,
|
||||
username=username, password=password,
|
||||
logger=logger)
|
||||
self.logger = logger
|
||||
|
||||
def detect_and_parse_datetime_index(self, data: pd.DataFrame, metadata: dict) -> pd.DataFrame:
|
||||
"""
|
||||
Detect and parse datetime index from data. index must be a timestamp like column.
|
||||
This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ.
|
||||
If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ.
|
||||
If another type or format, must raise an error.
|
||||
"""
|
||||
index = data.index
|
||||
|
||||
# Get type of first element of index
|
||||
index_type = type(index[0])
|
||||
|
||||
self.logger.custom_info(f"Index type: {index_type}", metadata)
|
||||
|
||||
message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}"
|
||||
|
||||
# Check if all in index are of the same type
|
||||
if not all(isinstance(i, index_type) for i in index):
|
||||
raise ValueError(
|
||||
f"{message}")
|
||||
|
||||
# Check type and converts to DATETIME_FORMAT_WITH_TZ
|
||||
if index_type == str:
|
||||
# Validate format of string and return error if not valid
|
||||
try:
|
||||
pd.to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ)
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f"{message}")
|
||||
|
||||
elif index_type == datetime or index_type == pd.Timestamp:
|
||||
data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"{message}")
|
||||
|
||||
return data
|
||||
|
||||
def transform(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict) -> dict:
|
||||
"""
|
||||
Transform data using a model.
|
||||
|
||||
Parameters:
|
||||
- model_name (str): The name of the model to use for transformation.
|
||||
- data (pandas.DataFrame): The data to transform.
|
||||
- model_retention (int): The number of minutes to keep the model.
|
||||
|
||||
Returns:
|
||||
- dict: A dictionary containing the transformed data.
|
||||
"""
|
||||
|
||||
try:
|
||||
self.logger.custom_debug(
|
||||
f"Data received for model transformation: {data.to_csv()}", metadata)
|
||||
|
||||
model_retention = model_config.get('retention_minutes', 0)
|
||||
flavor = model_config.get('transform_flavor', 'sklearn')
|
||||
compressed = model_config.get('is_compressed', False)
|
||||
retention_target = model_config.get('retention_target', 'model')
|
||||
transform_keyword = model_config.get(
|
||||
'transform_function_keyword', 'predict')
|
||||
|
||||
transformed_data = self.model_serving.get_cached_transform(
|
||||
model_name, data, model_retention, flavor,
|
||||
compressed, retention_target, transform_keyword
|
||||
)
|
||||
|
||||
self.logger.custom_debug(
|
||||
f"Data received from model transformation: {transformed_data.to_csv()}", metadata)
|
||||
|
||||
transformed_data = self.detect_and_parse_datetime_index(
|
||||
transformed_data, metadata)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'content': transformed_data.to_dict()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
'success': False,
|
||||
'content': {
|
||||
'message': str(e),
|
||||
'traceback': traceback.format_exc()
|
||||
}
|
||||
}
|
||||
|
||||
def predict(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict) -> dict:
|
||||
"""
|
||||
Predict data using a model.
|
||||
|
||||
Parameters:
|
||||
- model_name (str): The name of the model to use for prediction.
|
||||
- data (pandas.DataFrame): The data to predict.
|
||||
- model_retention (int): The number of minutes to keep the model.
|
||||
|
||||
Returns:
|
||||
- dict: A dictionary containing the predicted data.
|
||||
"""
|
||||
try:
|
||||
model_retention = model_config.get('retention_minutes', 0)
|
||||
flavor = model_config.get('predict_flavor', 'pyfunc')
|
||||
compressed = model_config.get('is_compressed', False)
|
||||
retention_target = model_config.get('retention_target', 'model')
|
||||
|
||||
input_index = data.index
|
||||
start_time = datetime.now()
|
||||
|
||||
self.logger.custom_debug(
|
||||
f"Data received for model prediction: {data.to_csv()}", metadata)
|
||||
data = self.model_serving.get_cached_predict(
|
||||
model_name, data, model_retention, flavor,
|
||||
compressed, retention_target
|
||||
)
|
||||
|
||||
end_time = datetime.now()
|
||||
data = pd.DataFrame(data, columns=['prediction'])
|
||||
self.logger.custom_debug(
|
||||
f"Data received from model prediction: {data.to_csv()}", metadata)
|
||||
data.index = input_index
|
||||
data['response_time'] = (end_time - start_time).total_seconds()
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'content': data.to_dict()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
'success': False,
|
||||
'content': {
|
||||
'message': str(e),
|
||||
'traceback': traceback.format_exc()
|
||||
}
|
||||
}
|
||||
|
||||
def get_experiment_by_run_id(self, run_id: str) -> dict:
|
||||
# Get the run information using the run_id
|
||||
run = mlflow.get_run(run_id)
|
||||
|
||||
# Extract the experiment ID from the run
|
||||
experiment_id = run.info.experiment_id
|
||||
|
||||
# Get the experiment details using the experiment ID
|
||||
experiment = mlflow.get_experiment(experiment_id)
|
||||
experiment_name = experiment.name
|
||||
return experiment_name
|
||||
|
||||
def get_next_run_name(self, model_name: str) -> str:
|
||||
"""
|
||||
Generate the next run name for a specific MLFlow model.
|
||||
|
||||
This method calculates the next sequential run number for a model
|
||||
by searching existing runs and incrementing the count. It ensures
|
||||
unique run names for model training and retraining operations.
|
||||
|
||||
Args:
|
||||
model_name (str): The name of the MLFlow model
|
||||
|
||||
Returns:
|
||||
str: The next run name in format 'model_name-run_number'
|
||||
"""
|
||||
runs = mlflow.search_runs(
|
||||
experiment_names=[model_name], order_by=["start_time desc"])
|
||||
next_run_number = len(runs) + 1
|
||||
return f"{model_name}-{next_run_number}"
|
||||
|
||||
def create_model_experiment(self, model_name: str, data: pd.DataFrame) -> tuple:
|
||||
"""
|
||||
Create a new MLFlow experiment for model retraining.
|
||||
|
||||
This method sets up the complete environment for model retraining by:
|
||||
1. Loading the current production prediction model
|
||||
2. Loading the current production transformation model
|
||||
3. Fitting the transformation model with new data
|
||||
4. Preparing data for prediction model retraining
|
||||
5. Setting up the MLFlow experiment context
|
||||
|
||||
Args:
|
||||
model_name (str): Name of the MLFlow model to retrain
|
||||
data (pd.DataFrame): Training data for model retraining
|
||||
|
||||
Returns:
|
||||
tuple: (prediction_model, data_model, experiment)
|
||||
- prediction_model: Loaded prediction model for retraining
|
||||
- data_model: Fitted transformation model
|
||||
- experiment: MLFlow experiment name
|
||||
"""
|
||||
# load predictor model
|
||||
predictor_uri = f"models:/{model_name}/production"
|
||||
# load transform model
|
||||
latest_production_id = self.model_serving.get_model_run_id(
|
||||
model_name, stage="Production"
|
||||
)
|
||||
transform_uri = self.model_serving.get_model_uri(
|
||||
latest_production_id, prediction=False
|
||||
)
|
||||
# load
|
||||
data_model = mlflow.sklearn.load_model(transform_uri)
|
||||
prediction_model = mlflow.sklearn.load_model(predictor_uri)
|
||||
data_model = data_model.fit(data)
|
||||
treated_data = data_model.predict(data)
|
||||
|
||||
target_name = data_model.target_variable
|
||||
y = data[target_name]
|
||||
treated_data = pd.merge(
|
||||
treated_data, y, left_index=True, right_index=True)
|
||||
prediction_model = prediction_model.fit(treated_data)
|
||||
experiment = self.get_experiment_by_run_id(latest_production_id)
|
||||
mlflow.set_experiment(experiment)
|
||||
|
||||
return prediction_model, data_model, experiment
|
||||
|
||||
def perform_model_retrain(self,
|
||||
prediction_model,
|
||||
data_model,
|
||||
experiment: str,
|
||||
model_name: str,
|
||||
data: pd.DataFrame):
|
||||
"""
|
||||
Execute the complete model retraining process in MLFlow.
|
||||
|
||||
This method performs the actual model retraining by:
|
||||
1. Starting a new MLFlow run with descriptive metadata
|
||||
2. Logging model parameters and hyperparameters
|
||||
3. Retraining both prediction and transformation models
|
||||
4. Logging training data as artifacts
|
||||
5. Saving retrained models to MLFlow registry
|
||||
|
||||
Args:
|
||||
prediction_model: MLFlow prediction model to retrain
|
||||
data_model: MLFlow transformation model to retrain
|
||||
experiment (str): MLFlow experiment name for the retraining
|
||||
model_name (str): Name of the model being retrained
|
||||
data (pd.DataFrame): Training data used for retraining
|
||||
|
||||
Returns:
|
||||
tuple: (status_message, experiment_name)
|
||||
- status_message (str): Success confirmation message
|
||||
- experiment_name (str): Name of the experiment
|
||||
"""
|
||||
pred_model_atributes = vars(prediction_model) # load class attributes
|
||||
data_model_atributes = vars(data_model) # load class attributes
|
||||
experiment_description = f"Retrain model {model_name} with new data"
|
||||
current_run_name = self.get_next_run_name(experiment)
|
||||
with mlflow.start_run(
|
||||
run_name=current_run_name, description=experiment_description
|
||||
) as _run:
|
||||
# update transfomation model
|
||||
# fixed parameters
|
||||
for name_atribute, val_atribute in pred_model_atributes.items():
|
||||
if name_atribute != "model":
|
||||
mlflow.log_param(name_atribute, val_atribute)
|
||||
# update prediction model
|
||||
for name_atribute, val_atribute in data_model_atributes.items():
|
||||
if name_atribute != "model":
|
||||
mlflow.log_param(name_atribute, val_atribute)
|
||||
# dynamic parameters, including model itself
|
||||
mlflow.sklearn.log_model(data_model, "data_model")
|
||||
|
||||
makedirs("temp", exist_ok=True)
|
||||
|
||||
file_path = f"temp/raw_data_{model_name}.csv"
|
||||
data.to_csv(file_path, index=True)
|
||||
|
||||
# log the data raw
|
||||
mlflow.log_artifact(file_path)
|
||||
|
||||
# dynamic parameters, including model itself
|
||||
mlflow.sklearn.log_model(prediction_model, "prediction_model")
|
||||
mlflow.log_param("retrain", True)
|
||||
|
||||
# clear temp file
|
||||
if path.exists(file_path):
|
||||
remove(file_path)
|
||||
|
||||
return "Model retrained successfully", experiment
|
||||
|
||||
def retrain_model(self, data: pd.DataFrame, model_name: str) -> tuple:
|
||||
"""
|
||||
Orchestrate the complete model retraining workflow.
|
||||
|
||||
This method coordinates the entire model retraining process by:
|
||||
1. Creating the MLFlow experiment environment
|
||||
2. Loading existing production models
|
||||
3. Executing the retraining process
|
||||
4. Returning comprehensive retraining results
|
||||
|
||||
Args:
|
||||
data (pd.DataFrame): Training data for model retraining
|
||||
model_name (str): Name of the MLFlow model to retrain
|
||||
|
||||
Returns:
|
||||
tuple: (status_message, experiment_name)
|
||||
- status_message (str): Retraining operation status
|
||||
- experiment_name (str): MLFlow experiment identifier
|
||||
"""
|
||||
prediction_model, data_model, experiment = self.create_model_experiment(
|
||||
model_name, data)
|
||||
retrain_result = self.perform_model_retrain(
|
||||
prediction_model, data_model, experiment, model_name, data)
|
||||
return retrain_result
|
||||
|
||||
def get_experiment(self, experiment_name: str) -> int:
|
||||
"""
|
||||
Retrieve MLFlow experiment ID by experiment name.
|
||||
|
||||
This method searches for an MLFlow experiment by name and
|
||||
returns its unique identifier. It provides error handling
|
||||
for non-existent experiments.
|
||||
|
||||
Args:
|
||||
experiment_name (str): Name of the MLFlow experiment
|
||||
|
||||
Returns:
|
||||
int: MLFlow experiment ID
|
||||
|
||||
Raises:
|
||||
ValueError: If the experiment name is not found
|
||||
"""
|
||||
experiment = mlflow.get_experiment_by_name(experiment_name)
|
||||
|
||||
if experiment is None:
|
||||
raise ValueError(f'Experiment {experiment_name} not found')
|
||||
|
||||
return int(experiment.experiment_id)
|
||||
|
||||
def get_experiment_last_run(self, experiment_id: int) -> str:
|
||||
"""
|
||||
Retrieve the most recent retraining run ID for an experiment.
|
||||
|
||||
This method searches for the latest run in an MLFlow experiment
|
||||
that has been marked as a retraining run. It filters runs by
|
||||
the 'retrain' parameter and orders them by completion time.
|
||||
|
||||
Args:
|
||||
experiment_id (int): MLFlow experiment ID
|
||||
|
||||
Returns:
|
||||
str: MLFlow run ID of the most recent retraining run
|
||||
|
||||
Raises:
|
||||
ValueError: If runs data is not in expected DataFrame format
|
||||
"""
|
||||
runs = mlflow.search_runs(
|
||||
experiment_ids=[experiment_id],
|
||||
filter_string="", # Sem filtro no MLflow ainda
|
||||
output_format="pandas"
|
||||
)
|
||||
|
||||
if not isinstance(runs, pd.DataFrame):
|
||||
raise ValueError('Runs is not a pandas DataFrame')
|
||||
|
||||
# Filtrar apenas as runs onde params.retrain == True
|
||||
filtered_runs = runs[runs["params.retrain"] == 'True']
|
||||
|
||||
# Converter a coluna 'end_time' para datetime
|
||||
filtered_runs['end_time'] = pd.to_datetime(filtered_runs['end_time'])
|
||||
|
||||
# Ordenar o DataFrame de forma descendente pela coluna 'end_time'
|
||||
filtered_runs = filtered_runs.sort_values(
|
||||
by='end_time', ascending=False)
|
||||
|
||||
# Pegar a última run_id do DataFrame filtrado e ordenado
|
||||
latest_run_id = filtered_runs.iloc[0]['run_id']
|
||||
|
||||
return latest_run_id
|
||||
|
||||
def update_production_model_by_run_id(self, run_id: str, model_name: str) -> dict:
|
||||
"""
|
||||
Update production model with a specific MLFlow run.
|
||||
|
||||
This method promotes a model from a specific MLFlow run to
|
||||
production stage. It handles model registration, versioning,
|
||||
and stage transitions with proper error handling.
|
||||
|
||||
Args:
|
||||
run_id (str): MLFlow run ID containing the model to promote
|
||||
model_name (str): Name of the MLFlow model
|
||||
|
||||
Returns:
|
||||
dict: Model update metadata containing:
|
||||
- model_name (str): Name of the updated model
|
||||
- version (str): New model version number
|
||||
- mlflow_run_id (str): Source run ID
|
||||
|
||||
Update Process:
|
||||
1. Registers the model from the specified run
|
||||
2. Retrieves the latest model version
|
||||
3. Transitions the model to 'Production' stage
|
||||
4. Archives existing production versions
|
||||
"""
|
||||
# Registrar o modelo
|
||||
# Aqui estamos assumindo que você já tem um modelo salvo, caso contrário você precisará treiná-lo e salvá-lo primeiro.
|
||||
# Se o modelo já está registrado, você pode usar o método register_model() ou pyfunc.load_model() para isso.
|
||||
mlflow.register_model(
|
||||
f"runs:/{run_id}/prediction_model", model_name)
|
||||
|
||||
# Colocar a versão do modelo em produção
|
||||
# Depois de registrar o modelo, precisamos pegar a versão mais recente do modelo e movê-lo para o estágio 'Production'
|
||||
client = mlflow.tracking.MlflowClient()
|
||||
|
||||
# Obter a versão mais recente registrada do modelo
|
||||
model_versions = client.get_registered_model(
|
||||
model_name).latest_versions
|
||||
|
||||
if not isinstance(model_versions, list):
|
||||
raise ValueError('Model versions is not a list')
|
||||
|
||||
max_version = max(model_versions, key=lambda x: int(x.version)).version
|
||||
|
||||
# Mover a versão mais recente do modelo para o estágio de 'Production'
|
||||
client.transition_model_version_stage(
|
||||
name=model_name,
|
||||
version=max_version,
|
||||
stage="Production",
|
||||
archive_existing_versions=True
|
||||
)
|
||||
|
||||
return {
|
||||
'model_name': model_name,
|
||||
'version': max_version,
|
||||
'mlflow_run_id': run_id
|
||||
}
|
||||
|
||||
def update_production_model(self, experiment: str, model_name: str) -> dict:
|
||||
"""
|
||||
Update production model using the latest retraining run.
|
||||
|
||||
This method orchestrates the complete production model update
|
||||
process by identifying the most recent retraining run and
|
||||
promoting it to production stage.
|
||||
|
||||
Args:
|
||||
experiment (str): MLFlow experiment name
|
||||
model_name (str): Name of the MLFlow model
|
||||
|
||||
Returns:
|
||||
dict: Complete model update metadata containing:
|
||||
- model_name (str): Name of the updated model
|
||||
- version (str): New model version number
|
||||
- mlflow_run_id (str): Source run ID
|
||||
- mlflow_experiment_id (int): Experiment ID
|
||||
"""
|
||||
experiment_id = self.get_experiment(experiment)
|
||||
run_id = self.get_experiment_last_run(experiment_id)
|
||||
metadata = self.update_production_model_by_run_id(run_id, model_name)
|
||||
|
||||
metadata['mlflow_experiment_id'] = experiment_id
|
||||
|
||||
return metadata
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,78 +1,88 @@
|
||||
"""
|
||||
Laborious Worker Module
|
||||
|
||||
This module provides the main worker implementation for the Sientia DataOps Laborious system.
|
||||
It orchestrates Temporal workers, manages task queues, and handles the lifecycle of
|
||||
prediction and retraining workflows.
|
||||
Entry process that connects to Temporal, registers Laborious activities, and runs four workers in
|
||||
parallel. Each worker shares the same ``Activities`` instance (single Postgres pool, single MLflow
|
||||
repository, single PluginStore handle) but polls a different task queue.
|
||||
|
||||
The worker supports two main task queues:
|
||||
- predictions_batch-queue: Handles batch prediction workflows
|
||||
- minimal_retrain-queue: Handles model retraining workflows
|
||||
Task queues (see ``sientia_do.temporal.worker.prepare_worker``):
|
||||
- ``predictions_batch-{runtime}-queue`` + sub-workflows on the same queue (ML-heavy path).
|
||||
- ``minimal_retrain-{runtime}-queue`` (retrain + promote + export).
|
||||
- ``drift-queue`` and ``simple_metrics-queue`` without a runtime suffix so existing schedulers
|
||||
keep stable queue names.
|
||||
|
||||
Key Features:
|
||||
- Automatic scaling with PollerBehaviorAutoscaling
|
||||
- Prometheus metrics integration
|
||||
- Comprehensive error handling and logging
|
||||
- Graceful shutdown with cleanup
|
||||
- Multiple worker instances for different workflow types
|
||||
Bootstrap order:
|
||||
1. Prometheus app metrics and Mongo-backed notification handler.
|
||||
2. ``RUNTIME`` validation and ``PluginStore.install_runtime`` so ``SientiaModel`` code is importable.
|
||||
3. ``Activities`` construction (builds ``SientiaMLflowRepository`` internally from env).
|
||||
4. OPC client initialization inside activities.
|
||||
5. Temporal ``Runtime`` with SDK Prometheus bind, client connect, then ``prepare_worker`` per workflow.
|
||||
|
||||
Shutdown closes workers, notifications, activities (pools + OPC), and clears ``app_up``.
|
||||
|
||||
Environment Variables:
|
||||
- TEMPORAL_HOST: Temporal server address (default: localhost:7233)
|
||||
- TEMPORAL_NAMESPACE: Temporal namespace (default: laborious)
|
||||
- POD_ID: Kubernetes pod identifier for metrics
|
||||
- HTTP_METRICS_PORT: Prometheus metrics server port (default: 9090)
|
||||
- HTTP_SDK_METRICS_PORT: Temporal SDK metrics port (default: 9091)
|
||||
- PROJECT_NAME: Project name for notifications (default: laborious)
|
||||
- RUNTIME: Required non-empty string passed to ``install_runtime``.
|
||||
- STORE_* / PYPI_*: Plugin store and private index (see ``build_plugin_store_config``).
|
||||
- TEMPORAL_HOST, TEMPORAL_NAMESPACE: Cluster connection.
|
||||
- POD_ID, HTTP_METRICS_PORT, HTTP_SDK_METRICS_PORT: Observability.
|
||||
- PROJECT_NAME, MONGODB_*: Notifications (via ``build_mongodb_config`` in handler).
|
||||
- POSTGRES_*, MINIO_*, OPC_*, PI_WEB_API_*, MLFLOW_*: Passed through ``Activities`` helpers.
|
||||
"""
|
||||
|
||||
from temporalio import workflow, client
|
||||
from temporalio.worker import Worker, PollerBehaviorAutoscaling
|
||||
from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig
|
||||
from temporalio import client, workflow
|
||||
from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
from laborious.workflows.minimal_retrain import MinimalRetrain
|
||||
from laborious.workflows.predictions_batch import PredictionsBatch
|
||||
from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
|
||||
from laborious.workflows.sub_workflows.format_and_export_prediction import \
|
||||
FormatAndExportPrediction
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.utils.connectors_config import (
|
||||
build_postgres_config,
|
||||
build_mlflow_config,
|
||||
build_opc_config,
|
||||
build_mongodb_config
|
||||
)
|
||||
|
||||
from prometheus_client import start_http_server
|
||||
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
|
||||
from sientia_do.observability.logger import get_logger
|
||||
from laborious import metrics
|
||||
from prometheus_client import start_http_server
|
||||
from sientia_do.observability.metrics_controller import MetricsController
|
||||
from sientia_do.temporal.worker.prepare_worker import prepare_worker
|
||||
from sientia_do.utils.connectors_config import (
|
||||
build_api_config,
|
||||
build_mongodb_config,
|
||||
build_postgres_config,
|
||||
)
|
||||
from sientia_model.model_repository.plugin_store import PluginStore
|
||||
|
||||
POD_ID = os.getenv('POD_ID')
|
||||
SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091"))
|
||||
from laborious import metrics
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.utils.connectors_config import (
|
||||
build_minio_config,
|
||||
build_opc_config,
|
||||
build_plugin_store_config,
|
||||
)
|
||||
from laborious.workflows.drift import Drift
|
||||
from laborious.workflows.minimal_retrain import MinimalRetrain
|
||||
from laborious.workflows.predictions_batch import PredictionsBatch
|
||||
from laborious.workflows.simple_metrics import SimpleMetrics
|
||||
from laborious.workflows.sub_workflows.format_and_export_prediction import (
|
||||
FormatAndExportPrediction,
|
||||
)
|
||||
from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
|
||||
|
||||
POD_ID = os.getenv('HOSTNAME')
|
||||
SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091'))
|
||||
|
||||
|
||||
async def main():
|
||||
"""
|
||||
Main entry point for the Laborious worker application.
|
||||
Run the full worker lifecycle: metrics, notifications, runtime install, workers, gather.
|
||||
|
||||
This function initializes and starts all components of the worker:
|
||||
1. Sets up logging and metadata
|
||||
2. Starts Prometheus metrics server
|
||||
3. Initializes notification handler
|
||||
4. Creates and configures activities
|
||||
5. Initializes OPC connections
|
||||
6. Starts Temporal client and workers
|
||||
7. Manages worker lifecycle and graceful shutdown
|
||||
|
||||
The function runs indefinitely until interrupted or an error occurs.
|
||||
On error, it performs cleanup and exits with a non-zero status code.
|
||||
Exits the process with code 0 on normal completion of all worker tasks, or 1 after logging
|
||||
if any worker raises. ``finally`` always shuts down notifications and activities and sets
|
||||
``app_up`` to 0 before ``sys.exit``.
|
||||
|
||||
Raises:
|
||||
Exception: Any unhandled exception during worker execution
|
||||
SystemExit: On graceful shutdown or error conditions
|
||||
Exception: Propagated from ``asyncio.gather`` only before ``finally`` handling; typically
|
||||
workers run until cancelled.
|
||||
|
||||
Return:
|
||||
None (process terminates via ``sys.exit`` from the ``finally`` block).
|
||||
"""
|
||||
host = os.getenv('TEMPORAL_HOST', 'localhost:7233')
|
||||
logger = get_logger(__name__)
|
||||
@@ -87,7 +97,7 @@ async def main():
|
||||
|
||||
logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata)
|
||||
|
||||
logger.custom_info("Starting prometheus client...", metadata)
|
||||
logger.custom_info('Starting prometheus client...', metadata)
|
||||
start_prometheus_server()
|
||||
|
||||
logger.custom_info('Starting Notification Handler...', metadata)
|
||||
@@ -97,29 +107,77 @@ async def main():
|
||||
connection_string=mongo_config['connection_string'],
|
||||
database=mongo_config['database_name'],
|
||||
logger=logger,
|
||||
project_name=os.getenv('PROJECT_NAME', 'laborious')
|
||||
project_name=os.getenv('PROJECT_NAME', 'laborious'),
|
||||
)
|
||||
|
||||
metrics_controller = MetricsController(logger=logger)
|
||||
|
||||
runtime = os.getenv('RUNTIME', '').strip()
|
||||
if not runtime:
|
||||
logger.custom_critical(
|
||||
'RUNTIME environment variable is required and must be non-empty',
|
||||
metadata,
|
||||
)
|
||||
metrics.APP_UP.labels(pod_id=POD_ID).set(0)
|
||||
sys.exit(1)
|
||||
|
||||
metadata_runtime = {**metadata, 'runtime': runtime}
|
||||
logger.custom_info(f'Installing PluginStore runtime: {runtime}', metadata_runtime)
|
||||
|
||||
ps_cfg = build_plugin_store_config()
|
||||
plugin_store = PluginStore(
|
||||
base_url=ps_cfg['base_url'],
|
||||
owner=ps_cfg['owner'],
|
||||
repo=ps_cfg['repo'],
|
||||
username=ps_cfg['username'],
|
||||
password=ps_cfg['password'],
|
||||
branch=ps_cfg['branch'],
|
||||
cache_ttl_seconds=ps_cfg['cache_ttl_seconds'],
|
||||
pypi_index_url=ps_cfg['pypi_index_url'],
|
||||
pypi_username=ps_cfg['pypi_username'],
|
||||
pypi_password=ps_cfg['pypi_password'],
|
||||
logger=logger,
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
if runtime == 'legacy':
|
||||
to_install_runtime = 'single'
|
||||
else:
|
||||
to_install_runtime = runtime
|
||||
|
||||
try:
|
||||
await plugin_store.install_runtime(
|
||||
runtime_name=to_install_runtime, metadata=metadata_runtime
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.custom_critical(
|
||||
f'Failed to install runtime {to_install_runtime}: {exc}', metadata_runtime
|
||||
)
|
||||
metrics.APP_UP.labels(pod_id=POD_ID).set(0)
|
||||
sys.exit(1)
|
||||
|
||||
logger.custom_info('Starting Activities...', metadata)
|
||||
|
||||
activities = Activities(
|
||||
postgres_config=build_postgres_config(),
|
||||
mlflow_config=build_mlflow_config(),
|
||||
plugin_store=plugin_store,
|
||||
minio_config=build_minio_config(),
|
||||
opc_config=build_opc_config(),
|
||||
pi_web_api_config=build_api_config(),
|
||||
logger=logger,
|
||||
notification_handler=notification_handler
|
||||
notification_handler=notification_handler,
|
||||
metrics_controller=metrics_controller,
|
||||
)
|
||||
|
||||
logger.custom_info('Initializing OPC...', metadata)
|
||||
await activities.init_opc()
|
||||
activities.init_opc()
|
||||
|
||||
logger.custom_info(
|
||||
f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
|
||||
logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
|
||||
|
||||
new_runtime = Runtime(
|
||||
telemetry=TelemetryConfig(
|
||||
metrics=PrometheusConfig(
|
||||
bind_address=f"0.0.0.0:{SDK_METRICS_PORT}")
|
||||
metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}')
|
||||
)
|
||||
)
|
||||
|
||||
@@ -128,34 +186,55 @@ async def main():
|
||||
temporal_client = await client.Client.connect(
|
||||
target_host=host,
|
||||
namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'),
|
||||
runtime=new_runtime
|
||||
runtime=new_runtime,
|
||||
)
|
||||
|
||||
logger.custom_info('Starting Workers...', metadata)
|
||||
|
||||
workers = [
|
||||
Worker(
|
||||
temporal_client,
|
||||
task_queue='minimal_retrain-queue',
|
||||
workflows=[MinimalRetrain],
|
||||
prepare_worker(
|
||||
temporal_client=temporal_client,
|
||||
main_workflow=MinimalRetrain,
|
||||
other_workflows=[],
|
||||
activities=[
|
||||
activities.load_custom_query,
|
||||
activities.load_query_with_minio_offload,
|
||||
activities.retrain_model,
|
||||
activities.update_production_model,
|
||||
activities.export_data_to_postgres
|
||||
activities.format_retrain_report,
|
||||
activities.export_data_to_postgres,
|
||||
],
|
||||
max_concurrent_workflow_tasks=50,
|
||||
max_concurrent_activities=50,
|
||||
max_concurrent_local_activities=50,
|
||||
max_cached_workflows=200,
|
||||
workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
|
||||
activity_task_poller_behavior=PollerBehaviorAutoscaling()
|
||||
logger=logger,
|
||||
runtime=runtime,
|
||||
),
|
||||
Worker(
|
||||
temporal_client,
|
||||
task_queue='predictions_batch-queue',
|
||||
workflows=[PredictionsBatch, PredictionProcess,
|
||||
FormatAndExportPrediction],
|
||||
prepare_worker(
|
||||
temporal_client=temporal_client,
|
||||
main_workflow=SimpleMetrics,
|
||||
other_workflows=[],
|
||||
activities=[
|
||||
activities.load_custom_query,
|
||||
activities.calculate_simple_metrics,
|
||||
activities.export_data_to_postgres,
|
||||
],
|
||||
logger=logger,
|
||||
runtime='core',
|
||||
),
|
||||
prepare_worker(
|
||||
temporal_client=temporal_client,
|
||||
main_workflow=Drift,
|
||||
other_workflows=[],
|
||||
activities=[
|
||||
activities.load_custom_query,
|
||||
activities.get_reference_data,
|
||||
activities.calculate_drift,
|
||||
activities.export_data_to_postgres,
|
||||
],
|
||||
logger=logger,
|
||||
runtime='core',
|
||||
),
|
||||
prepare_worker(
|
||||
temporal_client=temporal_client,
|
||||
main_workflow=PredictionsBatch,
|
||||
other_workflows=[PredictionProcess, FormatAndExportPrediction],
|
||||
activities=[
|
||||
# MLFlow
|
||||
activities.request_predict,
|
||||
@@ -164,24 +243,24 @@ async def main():
|
||||
activities.input_gate,
|
||||
activities.mlflow_response_gate,
|
||||
activities.mlflow_content_gate,
|
||||
activities.format_transformed_data,
|
||||
activities.format_prediction,
|
||||
activities.format_default_prediction,
|
||||
activities.get_last_timestamp,
|
||||
# OPC
|
||||
activities.write_opc_data,
|
||||
# Postgres
|
||||
activities.load_custom_query,
|
||||
# Postgres / MinIO offload
|
||||
activities.load_query_with_minio_offload,
|
||||
activities.cleanup_minio_objects_expired,
|
||||
activities.repeat_last_prediction,
|
||||
activities.export_data_to_postgres,
|
||||
activities.write_metrics
|
||||
activities.export_payload_to_postgres,
|
||||
activities.write_metrics,
|
||||
# Pi Web API
|
||||
activities.write_pi_web_api_data,
|
||||
],
|
||||
max_concurrent_workflow_tasks=50,
|
||||
max_concurrent_activities=50,
|
||||
max_concurrent_local_activities=50,
|
||||
max_cached_workflows=200,
|
||||
workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
|
||||
activity_task_poller_behavior=PollerBehaviorAutoscaling()
|
||||
)
|
||||
logger=logger,
|
||||
runtime=runtime,
|
||||
),
|
||||
]
|
||||
|
||||
handlers = []
|
||||
@@ -190,20 +269,17 @@ async def main():
|
||||
|
||||
logger.custom_info('Workers started successfully', metadata)
|
||||
|
||||
exit_code = 0
|
||||
try:
|
||||
# This will run the workers and wait for them to complete.
|
||||
# If an exception occurs in any of the worker handlers, it will be propagated here.
|
||||
await asyncio.gather(*handlers)
|
||||
except BaseException as e: # NOSONAR
|
||||
logger.custom_error(f"An unhandled exception occurred: {e}", metadata)
|
||||
logger.custom_error(f'An unhandled exception occurred: {e}', metadata)
|
||||
exit_code = 1
|
||||
finally:
|
||||
if notification_handler:
|
||||
notification_handler.shutdown()
|
||||
if activities:
|
||||
await activities.shutdown()
|
||||
# Exit with a non-zero status code to indicate failure to Kubernetes
|
||||
notification_handler.shutdown()
|
||||
activities.shutdown()
|
||||
metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN
|
||||
sys.exit(1)
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
def start_prometheus_server():
|
||||
@@ -224,12 +300,12 @@ def start_prometheus_server():
|
||||
SystemExit: If the metrics server fails to start
|
||||
"""
|
||||
try:
|
||||
port = int(os.getenv("HTTP_METRICS_PORT", 9090))
|
||||
port = int(os.getenv('HTTP_METRICS_PORT', 9090))
|
||||
start_http_server(port)
|
||||
print(f"Prometheus server started on port {port}.")
|
||||
print(f'Prometheus server started on port {port}.')
|
||||
metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP
|
||||
except Exception as e:
|
||||
print(f"Failed to start Prometheus server: {e}")
|
||||
print(f'Failed to start Prometheus server: {e}')
|
||||
os._exit(1)
|
||||
|
||||
|
||||
|
||||
107
laborious/workflows/drift.py
Normal file
107
laborious/workflows/drift.py
Normal file
@@ -0,0 +1,107 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name='drift')
|
||||
class Drift:
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the drift workflow.
|
||||
|
||||
This method orchestrates the complete drift process by:
|
||||
1. Loading data using the provided custom SQL query
|
||||
2. Preparing prediction configuration and filters
|
||||
3. Delegating to the PredictionProcess workflow for ML operations
|
||||
"""
|
||||
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'workflow_name': 'drift',
|
||||
}
|
||||
}
|
||||
|
||||
print(f'Input data: {input_data}', metadata)
|
||||
|
||||
model_config = input_data['model_config']
|
||||
target_name = model_config['target']
|
||||
|
||||
gathering_query = f"""
|
||||
SELECT *
|
||||
FROM "{input_data['schema']}"."{input_data['source_table_name']}"
|
||||
WHERE
|
||||
model_id = '{input_data['model_id']}' AND
|
||||
timestamp > NOW() - INTERVAL '{input_data['interval']} minutes'
|
||||
ORDER BY timestamp ASC
|
||||
""" # nosec B608 - values come from internal Temporal workflow config, not user input
|
||||
|
||||
target_data_handler = workflow.start_activity_method(
|
||||
Activities.load_custom_query,
|
||||
{
|
||||
**metadata,
|
||||
'query': gathering_query,
|
||||
'datetime_columns': ['timestamp', 'created_at'],
|
||||
'orient': 'records',
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
reference_data_handler = workflow.start_activity_method(
|
||||
Activities.get_reference_data,
|
||||
{**metadata, 'model_name': input_data['model_name']},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
target_data = await target_data_handler
|
||||
reference_data = await reference_data_handler
|
||||
|
||||
if not target_data:
|
||||
return
|
||||
|
||||
drift_data = await workflow.execute_local_activity_method(
|
||||
Activities.calculate_drift,
|
||||
{
|
||||
**metadata,
|
||||
'target_data': target_data,
|
||||
'reference_data': reference_data,
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'target_name': target_name,
|
||||
'drift_metrics': input_data.get(
|
||||
'drift_metrics', ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']
|
||||
),
|
||||
'chunk_period': input_data.get('chunk_period', 'min'),
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
if drift_data:
|
||||
await workflow.execute_activity_method(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'data': drift_data,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['target_table_name'],
|
||||
'timestamp_conversion': {
|
||||
'column': 'timestamp',
|
||||
'format': DATETIME_FORMAT_WITH_TZ,
|
||||
},
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
@@ -1,14 +1,17 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from laborious.activities.activities import Activities
|
||||
from typing import Any
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
|
||||
|
||||
@workflow.defn(name="minimal_retrain")
|
||||
class MinimalRetrain():
|
||||
@workflow.defn(name='minimal_retrain')
|
||||
class MinimalRetrain:
|
||||
"""
|
||||
Automated model retraining workflow for the Laborious system.
|
||||
|
||||
@@ -63,44 +66,62 @@ class MinimalRetrain():
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'workflow_name': 'minimal_retrain'
|
||||
'workflow_name': 'minimal_retrain',
|
||||
}
|
||||
}
|
||||
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
|
||||
data = await workflow.execute_local_activity_method(
|
||||
Activities.load_custom_query,
|
||||
storage_result = await workflow.execute_activity_method(
|
||||
Activities.load_query_with_minio_offload,
|
||||
{
|
||||
**metadata,
|
||||
'query': input_data['query'],
|
||||
'datetime_columns': input_data.get('datetime_columns', [])
|
||||
'datetime_columns': input_data.get('datetime_columns', []),
|
||||
'model_name': model_name,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
start_to_close_timeout=timedelta(seconds=600),
|
||||
)
|
||||
|
||||
storage_payload = MinioDataFramePayload.from_dict(storage_result)
|
||||
if not storage_payload.has_data():
|
||||
raise ValueError('No data returned from query')
|
||||
|
||||
experiment_response = await workflow.execute_activity_method(
|
||||
Activities.retrain_model,
|
||||
{
|
||||
**metadata,
|
||||
'data': data,
|
||||
'model_name': model_name
|
||||
'data': storage_result,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
start_to_close_timeout=timedelta(hours=1),
|
||||
)
|
||||
|
||||
report = await workflow.execute_activity_method(
|
||||
Activities.update_production_model,
|
||||
if experiment_response['success']:
|
||||
update_report = await workflow.execute_activity_method(
|
||||
Activities.update_production_model,
|
||||
{**metadata, 'model_name': model_name, **experiment_response},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
else:
|
||||
update_report = {}
|
||||
|
||||
report = await workflow.execute_local_activity_method(
|
||||
Activities.format_retrain_report,
|
||||
{
|
||||
**metadata,
|
||||
'experiment_response': experiment_response,
|
||||
'model_name': model_name,
|
||||
'model_id': input_data['model_id'],
|
||||
**experiment_response
|
||||
'update_report': update_report,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
await workflow.execute_activity_method(
|
||||
@@ -109,8 +130,8 @@ class MinimalRetrain():
|
||||
**metadata,
|
||||
'data': report,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name']
|
||||
'table_name': input_data['table_name'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
start_to_close_timeout=timedelta(seconds=600),
|
||||
)
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from laborious.activities.activities import Activities
|
||||
from typing import Any
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name="predictions_batch")
|
||||
class PredictionsBatch():
|
||||
@workflow.defn(name='predictions_batch')
|
||||
class PredictionsBatch:
|
||||
"""
|
||||
Main batch prediction workflow for the Laborious system.
|
||||
|
||||
@@ -59,7 +61,10 @@ class PredictionsBatch():
|
||||
- model_retention (int, optional): Model retention period in minutes
|
||||
- path_priority (list[str]): Decision path priority configuration
|
||||
- opc_output_config (dict, optional): OPC server export configuration
|
||||
- pi_web_api_output_config (dict, optional): PI Web API export configuration
|
||||
- datetime_columns (list[str], optional): Columns to treat as datetime
|
||||
- save_transform (bool, optional): Whether to save transformed data (default: True)
|
||||
- prediction_store_policy (str, optional): Data retention policy (default: 'lts:1')
|
||||
|
||||
Returns:
|
||||
None: The workflow completes successfully when the child workflow finishes
|
||||
@@ -74,20 +79,21 @@ class PredictionsBatch():
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'workflow_name': 'predictions_batch'
|
||||
'workflow_name': 'predictions_batch',
|
||||
}
|
||||
}
|
||||
|
||||
# Load data using custom query
|
||||
data = await workflow.execute_local_activity_method(
|
||||
Activities.load_custom_query,
|
||||
# Load data using custom query with optional MinIO offload for large frames
|
||||
data = await workflow.execute_activity_method(
|
||||
Activities.load_query_with_minio_offload,
|
||||
{
|
||||
**metadata,
|
||||
'query': input_data['query'],
|
||||
'datetime_columns': input_data.get('datetime_columns', [])
|
||||
'datetime_columns': input_data.get('datetime_columns', []),
|
||||
'model_name': input_data['model_name'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300)
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
# Prepare input for prediction_process workflow
|
||||
@@ -96,30 +102,26 @@ class PredictionsBatch():
|
||||
'data': data,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'transform_table_name': input_data['transform_table_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'model_name': input_data['model_name'],
|
||||
'input_filters': input_data.get('input_filters', {
|
||||
'EMPTY_DATA': {
|
||||
'POLICY': 'STOP'
|
||||
}
|
||||
}),
|
||||
'mlflow_transform_filters': input_data.get('mlflow_transform_filters', {
|
||||
'API_ERROR': {
|
||||
'POLICY': 'STOP'
|
||||
}
|
||||
}),
|
||||
'mlflow_predict_filters': input_data.get('mlflow_predict_filters', {
|
||||
'API_ERROR': {
|
||||
'POLICY': 'STOP'
|
||||
}
|
||||
}),
|
||||
'input_filters': input_data.get(
|
||||
'input_filters', {'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}}}
|
||||
),
|
||||
'mlflow_transform_filters': input_data.get(
|
||||
'mlflow_transform_filters', {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}}
|
||||
),
|
||||
'mlflow_predict_filters': input_data.get(
|
||||
'mlflow_predict_filters', {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}}
|
||||
),
|
||||
'model_config': input_data.get('model_config', {}),
|
||||
'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']),
|
||||
'opc_output_config': input_data.get('opc_output_config', {}),
|
||||
'prediction_store_policy': input_data.get(
|
||||
'prediction_store_policy', 'lts:1')
|
||||
'on_conflict': input_data.get('on_conflict', 'error'),
|
||||
'pi_web_api_output_config': input_data.get('pi_web_api_output_config', {}),
|
||||
'prediction_store_policy': input_data.get('prediction_store_policy', 'lts:1'),
|
||||
'save_transform': input_data.get('save_transform', True),
|
||||
}
|
||||
|
||||
# Execute prediction process workflow
|
||||
await workflow.execute_child_workflow(
|
||||
'prediction_process', prediction_input)
|
||||
await workflow.execute_child_workflow('subworkflow.prediction_process', prediction_input)
|
||||
|
||||
95
laborious/workflows/simple_metrics.py
Normal file
95
laborious/workflows/simple_metrics.py
Normal file
@@ -0,0 +1,95 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name='simple_metrics')
|
||||
class SimpleMetrics:
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the simple metrics workflow.
|
||||
"""
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'model_id': input_data['model_id'],
|
||||
'model_name': input_data['model_name'],
|
||||
'workflow_name': 'simple_metrics',
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
}
|
||||
}
|
||||
|
||||
model_id = input_data['model_id']
|
||||
interval_minutes = input_data['interval_minutes']
|
||||
|
||||
model_config = input_data['model_config']
|
||||
target_name = model_config['target']
|
||||
|
||||
query = f"""
|
||||
select p."timestamp", p.prediction, ld.value as "target"
|
||||
from "{input_data['schema']}"."{input_data['predictions_table_name']}" p
|
||||
inner join "{input_data['schema']}"."{input_data['data_table_name']}" ld
|
||||
on p."timestamp" = ld."timestamp"
|
||||
where
|
||||
p.model_id = '{model_id}' and
|
||||
p.prediction is not null and
|
||||
ld.variable = '{target_name}' and
|
||||
ld.value is not null and
|
||||
p."timestamp" >= NOW() - INTERVAL '{interval_minutes} minutes'
|
||||
order by
|
||||
p."timestamp" desc;
|
||||
""" # nosec B608 - values come from internal Temporal workflow config, not user input
|
||||
|
||||
target_data = await workflow.execute_activity_method(
|
||||
Activities.load_custom_query,
|
||||
{
|
||||
**metadata,
|
||||
'query': query,
|
||||
'datetime_columns': ['timestamp'],
|
||||
'orient': 'records',
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
if not target_data:
|
||||
return
|
||||
|
||||
simple_metrics = await workflow.execute_local_activity_method(
|
||||
Activities.calculate_simple_metrics,
|
||||
{
|
||||
**metadata,
|
||||
'model_id': model_id,
|
||||
'target_data': target_data,
|
||||
'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']),
|
||||
'interval_minutes': interval_minutes,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
if not simple_metrics:
|
||||
return
|
||||
|
||||
await workflow.execute_activity_method(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'data': simple_metrics,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['target_table_name'],
|
||||
'timestamp_conversion': {
|
||||
'column': 'timestamp',
|
||||
'format': DATETIME_FORMAT_WITH_TZ,
|
||||
},
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
@@ -1,15 +1,17 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from laborious.activities.activities import Activities
|
||||
from typing import Any
|
||||
from datetime import timedelta
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name="format_and_export_prediction")
|
||||
class FormatAndExportPrediction():
|
||||
@workflow.defn(name='subworkflow.format_and_export_prediction')
|
||||
class FormatAndExportPrediction:
|
||||
"""
|
||||
Data formatting and export workflow for prediction results.
|
||||
|
||||
@@ -24,6 +26,7 @@ class FormatAndExportPrediction():
|
||||
|
||||
Export Destinations:
|
||||
- PostgreSQL Database: Persistent storage with timestamp conversion
|
||||
- PI Web API: Real-time industrial system integration for prediction and confidence values
|
||||
- OPC Servers: Real-time industrial system integration
|
||||
- Prometheus Metrics: Performance monitoring and operational visibility
|
||||
"""
|
||||
@@ -36,9 +39,10 @@ class FormatAndExportPrediction():
|
||||
This method orchestrates the complete data export process by:
|
||||
1. Determining the appropriate formatting strategy based on path_flag
|
||||
2. Formatting prediction data according to quality and requirements
|
||||
3. Exporting data to OPC servers for real-time industrial access
|
||||
4. Persisting data to PostgreSQL database with comprehensive metadata
|
||||
5. Recording performance metrics for operational monitoring
|
||||
3. Exporting data to PI Web API for real-time industrial access (if configured)
|
||||
4. Exporting data to OPC servers for real-time industrial access (if configured)
|
||||
5. Persisting data to PostgreSQL database with comprehensive metadata
|
||||
6. Recording performance metrics for operational monitoring
|
||||
|
||||
The method implements flexible formatting strategies:
|
||||
- Normal predictions: Full data formatting with confidence scores
|
||||
@@ -48,29 +52,50 @@ class FormatAndExportPrediction():
|
||||
Args:
|
||||
input_data: Complete configuration for the export workflow
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- path_flag (str | None): Decision path flag for formatting strategy
|
||||
- None: Normal prediction path with full formatting
|
||||
- Any other value: Default prediction path for error conditions
|
||||
- data (dict[str, Any]): Prediction data to format and export
|
||||
- prediction_confidence (float): Confidence score for the prediction
|
||||
- timestamp (str): ISO-formatted timestamp for the prediction
|
||||
- model_id (int): Unique identifier for the ML model
|
||||
- model_name (str): Name of the ML model
|
||||
- model_retention (str): Model retention policy configuration
|
||||
- comment (str): Operational comment or error description
|
||||
- schema (str): Database schema for data storage
|
||||
- table_name (str): Target table for data persistence
|
||||
Optional keys:
|
||||
- opc_output_config (dict[str, Any]): OPC server export configuration
|
||||
- prediction_store_policy (str, optional): Data retention policy
|
||||
- pi_web_api_output_config (dict[str, Any]): PI Web API export configuration
|
||||
Contains endpoint, prediction_tags, and confidence_tags mappings
|
||||
- transformed_data (dict[str, Any]): Transformed data to export separately
|
||||
Only processed when path_flag is None
|
||||
- transform_table_name (str): Target table for transformed data export
|
||||
Required if transformed_data is provided
|
||||
- prediction_store_policy (str): Data retention policy (e.g., 'lts:1', 'erl:2')
|
||||
Required when path_flag is None
|
||||
- comment (str): Operational comment or error description
|
||||
Required when path_flag is not None
|
||||
|
||||
Returns:
|
||||
bool: True if the workflow completes successfully, False otherwise
|
||||
None: The workflow completes successfully when all export operations finish
|
||||
|
||||
Note:
|
||||
When transformed_data is provided and path_flag is None, the workflow will:
|
||||
1. Format the transformed data using format_transformed_data
|
||||
2. Export it to a separate table (transform_table_name) asynchronously
|
||||
3. Wait for both prediction and transformed data exports to complete
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
path_flag = input_data['path_flag']
|
||||
data = input_data['data']
|
||||
transformed_data = input_data.get('transformed_data', None)
|
||||
prediction_confidence = input_data['prediction_confidence']
|
||||
|
||||
opc_output_config = input_data.get('opc_output_config', None)
|
||||
pi_web_api_output_config = input_data.get('pi_web_api_output_config', None)
|
||||
|
||||
if path_flag is None:
|
||||
# proceed with formatting and exporting
|
||||
# Normal prediction path: format prediction data with full metadata
|
||||
prediction = await workflow.execute_local_activity_method(
|
||||
Activities.format_prediction,
|
||||
{
|
||||
@@ -79,14 +104,48 @@ class FormatAndExportPrediction():
|
||||
'timestamp': input_data['timestamp'],
|
||||
'model_id': input_data['model_id'],
|
||||
'prediction_confidence': prediction_confidence,
|
||||
'prediction_store_policy': input_data['prediction_store_policy']
|
||||
'prediction_store_policy': input_data['prediction_store_policy'],
|
||||
'model_name': input_data['model_name'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
# Optionally format and export transformed data to separate table
|
||||
if transformed_data is not None:
|
||||
transformed = await workflow.execute_local_activity_method(
|
||||
Activities.format_transformed_data,
|
||||
{
|
||||
**metadata,
|
||||
'data': transformed_data,
|
||||
'model_id': input_data['model_id'],
|
||||
'model_name': input_data['model_name'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
write_transformed_handler = workflow.start_activity_method(
|
||||
Activities.export_payload_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['transform_table_name'],
|
||||
'data': transformed,
|
||||
'timestamp_conversion': {
|
||||
'column': 'timestamp',
|
||||
'format': DATETIME_FORMAT_WITH_TZ,
|
||||
},
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
else:
|
||||
write_transformed_handler = None
|
||||
|
||||
else:
|
||||
# create default prediction
|
||||
# Error path: create default prediction with error indicators
|
||||
prediction = await workflow.execute_local_activity_method(
|
||||
Activities.format_default_prediction,
|
||||
{
|
||||
@@ -94,23 +153,41 @@ class FormatAndExportPrediction():
|
||||
'timestamp': input_data['timestamp'],
|
||||
'model_id': input_data['model_id'],
|
||||
'prediction_confidence': prediction_confidence,
|
||||
'comment': input_data['comment']
|
||||
'comment': input_data['comment'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
write_transformed_handler = None
|
||||
|
||||
opc_metrics: dict[str, dict[str, float | None]] = {}
|
||||
|
||||
# write to pi web api
|
||||
if pi_web_api_output_config:
|
||||
prediction = await workflow.execute_activity_method(
|
||||
Activities.write_pi_web_api_data,
|
||||
{
|
||||
'pi_web_api_output_config': pi_web_api_output_config,
|
||||
'data': prediction,
|
||||
**metadata,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
# write to opc
|
||||
prediction = await workflow.execute_activity_method(
|
||||
Activities.write_opc_data,
|
||||
{
|
||||
**metadata,
|
||||
'opc_output_config': input_data['opc_output_config'],
|
||||
'data': prediction
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
if opc_output_config:
|
||||
prediction, opc_metrics = await workflow.execute_activity_method(
|
||||
Activities.write_opc_data,
|
||||
{
|
||||
'opc_output_config': opc_output_config,
|
||||
'data': prediction,
|
||||
**metadata,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
# write to postgres
|
||||
await workflow.execute_activity_method(
|
||||
@@ -120,21 +197,24 @@ class FormatAndExportPrediction():
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'data': prediction,
|
||||
'timestamp_conversion': {
|
||||
'column': 'timestamp',
|
||||
'format': DATETIME_FORMAT_WITH_TZ
|
||||
}
|
||||
'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ},
|
||||
'on_conflict': input_data.get('on_conflict', 'error'),
|
||||
'unique_columns': ['model_id', 'timestamp'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
start_to_close_timeout=timedelta(seconds=180),
|
||||
)
|
||||
|
||||
if write_transformed_handler is not None:
|
||||
await write_transformed_handler
|
||||
|
||||
await workflow.execute_activity_method(
|
||||
Activities.write_metrics,
|
||||
{
|
||||
**metadata,
|
||||
'prediction': prediction
|
||||
'prediction': prediction,
|
||||
'opc_metrics': opc_metrics,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from laborious.activities.activities import Activities
|
||||
from typing import Any
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name="prediction_process")
|
||||
class PredictionProcess():
|
||||
@workflow.defn(name='subworkflow.prediction_process')
|
||||
class PredictionProcess:
|
||||
"""
|
||||
Core prediction processing workflow for the Laborious system.
|
||||
|
||||
@@ -64,7 +66,10 @@ class PredictionProcess():
|
||||
- mlflow_predict_filters (dict): MLFlow prediction filters
|
||||
- model_retention (int): Model retention period in minutes
|
||||
- path_priority (list[str]): Decision path priority configuration
|
||||
- opc_output_config (dict): OPC server export configuration
|
||||
- opc_output_config (dict, optional): OPC server export configuration
|
||||
- pi_web_api_output_config (dict, optional): PI Web API export configuration
|
||||
- save_transform (bool, optional): Whether to save transformed data (default: True)
|
||||
- prediction_store_policy (str, optional): Data retention policy (default: 'lts:1')
|
||||
|
||||
Returns:
|
||||
None: The workflow completes successfully when export workflow finishes
|
||||
@@ -80,24 +85,51 @@ class PredictionProcess():
|
||||
model_id = input_data['model_id']
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
save_transform = input_data.get('save_transform', True)
|
||||
|
||||
# Get last timestamp for incremental processing
|
||||
last_timestamp = await workflow.execute_local_activity_method(
|
||||
Activities.get_last_timestamp,
|
||||
{
|
||||
**metadata,
|
||||
'data': data
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
try:
|
||||
await self._run_prediction_pipeline(
|
||||
input_data,
|
||||
metadata,
|
||||
data,
|
||||
model_id,
|
||||
model_name,
|
||||
model_config,
|
||||
save_transform,
|
||||
)
|
||||
await workflow.execute_activity_method(
|
||||
Activities.cleanup_minio_objects_expired,
|
||||
{**metadata, 'data': data},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
except Exception as e:
|
||||
await workflow.execute_activity_method(
|
||||
Activities.cleanup_minio_objects_expired,
|
||||
{**metadata, 'data': data},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
raise e
|
||||
|
||||
async def _run_prediction_pipeline(
|
||||
self,
|
||||
input_data: dict[str, Any],
|
||||
metadata: dict[str, Any],
|
||||
data: dict[str, Any],
|
||||
model_id: str,
|
||||
model_name: str,
|
||||
model_config: dict[str, Any],
|
||||
save_transform: bool,
|
||||
) -> None:
|
||||
last_timestamp = data['last_timestamp']
|
||||
|
||||
# Apply input data quality gates
|
||||
gate_input = {
|
||||
**metadata,
|
||||
'filters': input_data['input_filters'],
|
||||
'data': data,
|
||||
'path_priority': input_data['path_priority']
|
||||
'path_priority': input_data['path_priority'],
|
||||
}
|
||||
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
@@ -114,14 +146,9 @@ class PredictionProcess():
|
||||
return
|
||||
|
||||
# Request MLFlow model transformation
|
||||
response_data = await workflow.execute_local_activity_method(
|
||||
transformed_data = await workflow.execute_activity_method(
|
||||
Activities.request_transform,
|
||||
{
|
||||
**metadata,
|
||||
'data': data,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config
|
||||
},
|
||||
{**metadata, 'data': data, 'model_name': model_name, 'model_config': model_config},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
@@ -132,9 +159,9 @@ class PredictionProcess():
|
||||
{
|
||||
**metadata,
|
||||
'filters': input_data['mlflow_transform_filters'],
|
||||
'data': response_data,
|
||||
'data': transformed_data,
|
||||
'type': 'transform',
|
||||
'path_priority': input_data['path_priority']
|
||||
'path_priority': input_data['path_priority'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
@@ -146,8 +173,6 @@ class PredictionProcess():
|
||||
):
|
||||
return
|
||||
|
||||
transformed_data = response_data['content']
|
||||
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_content_gate,
|
||||
{
|
||||
@@ -155,7 +180,7 @@ class PredictionProcess():
|
||||
'filters': input_data['mlflow_transform_filters'],
|
||||
'data': transformed_data,
|
||||
'type': 'transform',
|
||||
'path_priority': input_data['path_priority']
|
||||
'path_priority': input_data['path_priority'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
@@ -166,13 +191,13 @@ class PredictionProcess():
|
||||
):
|
||||
return
|
||||
|
||||
response_data = await workflow.execute_local_activity_method(
|
||||
predicted_data = await workflow.execute_activity_method(
|
||||
Activities.request_predict,
|
||||
{
|
||||
**metadata,
|
||||
'data': transformed_data,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config
|
||||
'model_config': model_config,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
@@ -184,9 +209,9 @@ class PredictionProcess():
|
||||
{
|
||||
**metadata,
|
||||
'filters': input_data['mlflow_predict_filters'],
|
||||
'data': response_data,
|
||||
'data': predicted_data,
|
||||
'type': 'predict',
|
||||
'path_priority': input_data['path_priority']
|
||||
'path_priority': input_data['path_priority'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
@@ -200,26 +225,37 @@ class PredictionProcess():
|
||||
|
||||
# Delegate to export workflow for data persistence
|
||||
await workflow.execute_child_workflow(
|
||||
'format_and_export_prediction',
|
||||
'subworkflow.format_and_export_prediction',
|
||||
{
|
||||
'metadata': metadata,
|
||||
'on_conflict': input_data.get('on_conflict', 'error'),
|
||||
'path_flag': path_flag,
|
||||
'data': response_data['content'],
|
||||
'data': predicted_data,
|
||||
'transformed_data': transformed_data if save_transform else None,
|
||||
'prediction_confidence': confidence,
|
||||
'timestamp': last_timestamp,
|
||||
'model_id': model_id,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config,
|
||||
'opc_output_config': input_data['opc_output_config'],
|
||||
'pi_web_api_output_config': input_data['pi_web_api_output_config'],
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'transform_table_name': input_data['transform_table_name'],
|
||||
'comment': comment,
|
||||
'prediction_store_policy': input_data['prediction_store_policy']
|
||||
}
|
||||
'prediction_store_policy': input_data['prediction_store_policy'],
|
||||
},
|
||||
)
|
||||
|
||||
async def path_flag_handler(self, data: dict, path_flag: str, input_data: dict,
|
||||
confidence: int, last_timestamp: str, comment: str) -> bool:
|
||||
async def path_flag_handler(
|
||||
self,
|
||||
data: dict[str, Any],
|
||||
path_flag: str | None,
|
||||
input_data: dict,
|
||||
confidence: int,
|
||||
last_timestamp: str,
|
||||
comment: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Handle path decisions based on filter results and confidence levels.
|
||||
|
||||
@@ -230,7 +266,17 @@ class PredictionProcess():
|
||||
Args:
|
||||
data: Input data for processing
|
||||
path_flag: Path decision from filter (STOP, CONTINUE, REPEAT)
|
||||
input_data: Complete workflow input configuration
|
||||
input_data: Complete workflow input configuration including:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- schema (str): Database schema
|
||||
- table_name (str): Target table for predictions
|
||||
- transform_table_name (str): Target table for transformed data
|
||||
- model_id (str): ML model identifier
|
||||
- model_name (str): ML model name
|
||||
- model_config (dict, optional): Model configuration
|
||||
- opc_output_config (dict, optional): OPC server export configuration
|
||||
- pi_web_api_output_config (dict, optional): PI Web API export configuration
|
||||
- prediction_store_policy (str, optional): Data retention policy
|
||||
confidence: Confidence level from filter validation
|
||||
last_timestamp: Last processed timestamp
|
||||
comment: Additional information about the filter result
|
||||
@@ -240,13 +286,14 @@ class PredictionProcess():
|
||||
|
||||
Path Handling:
|
||||
- STOP: Terminates workflow execution
|
||||
- CONTINUE: Proceeds with normal processing
|
||||
- CONTINUE: Delegates to FormatAndExportPrediction workflow with current data
|
||||
- REPEAT: Repeats last prediction if available
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
schema = input_data['schema']
|
||||
table_name = input_data['table_name']
|
||||
transform_table_name = input_data['transform_table_name']
|
||||
model_id = input_data['model_id']
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
@@ -265,7 +312,7 @@ class PredictionProcess():
|
||||
'schema': schema,
|
||||
'table_name': table_name,
|
||||
'model': model_id,
|
||||
'last_timestamp': last_timestamp
|
||||
'last_timestamp': last_timestamp,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
@@ -274,7 +321,7 @@ class PredictionProcess():
|
||||
elif path_flag == 'CONTINUE':
|
||||
# call write workflow
|
||||
await workflow.execute_child_workflow(
|
||||
'format_and_export_prediction',
|
||||
'subworkflow.format_and_export_prediction',
|
||||
{
|
||||
'metadata': metadata,
|
||||
'path_flag': path_flag,
|
||||
@@ -286,10 +333,13 @@ class PredictionProcess():
|
||||
'model_config': model_config,
|
||||
'schema': schema,
|
||||
'table_name': table_name,
|
||||
'transform_table_name': transform_table_name,
|
||||
'comment': comment,
|
||||
'opc_output_config': input_data['opc_output_config'],
|
||||
'prediction_store_policy': input_data['prediction_store_policy']
|
||||
}
|
||||
'pi_web_api_output_config': input_data['pi_web_api_output_config'],
|
||||
'prediction_store_policy': input_data['prediction_store_policy'],
|
||||
'on_conflict': input_data.get('on_conflict', 'error'),
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user