diff --git a/laborious/activities/activities.py b/laborious/activities/activities.py index ec5ae46..45e9541 100644 --- a/laborious/activities/activities.py +++ b/laborious/activities/activities.py @@ -1,99 +1,177 @@ -from temporalio import activity, workflow +from sientia_do.observability.metrics_controller import MetricsController +from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from sientia_do.temporal.activities.postgres import Postgres - from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler - from sientia_do.observability.logger import Logger - from laborious.activities.mlflow import MLFlow - from laborious.activities.gates import Gates - from laborious.activities.opc import OPC from typing import Any + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.observability.logger import Logger + from sientia_do.repository.minio_repository_sync import MinioRepository + from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository + from sientia_model.model_repository.plugin_store import PluginStore -class Activities(Postgres, MLFlow, Gates, OPC): + from laborious.activities.api import API + from laborious.activities.gates import Gates + from laborious.activities.mlflow import MLFlow + from laborious.activities.model_metrics import ModelMetrics + from laborious.activities.opc import OPC + from laborious.activities.storage import Storage + from laborious.utils.connectors_config import build_mlflow_config + + +class Activities(Storage, MLFlow, Gates, OPC, ModelMetrics, API): """ - Main activities orchestrator for the Laborious system. + Central orchestrator for all Temporal activities used by Laborious workflows. - This class combines functionality from multiple activity classes to provide - a unified interface for all workflow operations. It manages database connections, - MLFlow model interactions, data quality validation, and OPC server communications. + Composes Storage (Postgres + MinIO offload), MLFlow (wrapper-based inference and retrain + via ``SientiaMLflowRepository``), Gates (data quality and ML response filters), OPC exports, + drift/simple metrics, and PI Web API writes. The worker constructs one ``Activities`` instance + per process and registers its callables on multiple workers bound to different task queues. - The class implements multiple inheritance to combine specialized functionality: - - Postgres: Database operations and data persistence - - MLFlow: Model inference and transformation operations - - Gates: Data quality validation and filtering mechanisms - - OPC: Real-time data export to OPC servers + MLflow connectivity: unless ``mlflow_repository`` is injected (tests only), this class builds + ``SientiaMLflowRepository`` from ``build_mlflow_config()`` so tracking credentials and URL + stay aligned with the rest of Laborious env-based configuration. Attributes: - postgres_config (dict): PostgreSQL connection configuration - mlflow_config (dict): MLFlow server configuration - opc_config (dict): OPC server configuration - logger (Logger): Logging and observability instance - notification_handler (NotificationHandler): Notification management instance + Inherits and exposes behaviour from mixins; the MLFlow mixin holds ``mlflow_repository`` + and ``plugin_store`` after ``__init__``. """ - def __init__(self, - postgres_config: dict[str, Any], - mlflow_config: dict[str, Any], - opc_config: dict[str, Any], - logger: Logger, - notification_handler: NotificationHandler): + def __init__( + self, + postgres_config: dict[str, Any], + plugin_store: PluginStore, + minio_config: dict[str, Any], + opc_config: dict[str, Any], + pi_web_api_config: dict[str, Any], + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController | None = None, + mlflow_repository: SientiaMLflowRepository | None = None, + ): """ - Initialize the Activities orchestrator with all required configurations. + Wire Postgres, MinIO, MLflow, OPC, gates, metrics, and PI Web API into a single object. - This constructor initializes all parent classes with their respective - configurations and sets up the foundation for all activity operations. + A single ``MetricsController`` instance is created (or reused) and passed to MinIO, + MLflow repository, and all mixins so Prometheus and SDK metrics stay consistent. Args: - postgres_config: PostgreSQL connection configuration dictionary - Required keys: host, port, user, password, dbname, min_connections, max_connections - mlflow_config: MLFlow server configuration dictionary - Required keys: host, port, username, password - opc_config: OPC server configuration dictionary - Can contain multiple server configurations - logger: Logger instance for observability and debugging - notification_handler: Notification handler for alerts and monitoring + - postgres_config: Host, port, credentials, db name, and pool bounds for Storage. + - plugin_store: ``PluginStore`` instance; the worker must call ``install_runtime`` before + activities run so wrapper code is importable. + - minio_config: Endpoint, keys, bucket, retention, and TLS flag for object storage payloads. + - opc_config: Map of OPC server id to connection settings for ``OPC`` mixin. + - pi_web_api_config: Base URL and auth for ``API`` mixin. + - logger: Structured logger used across all activities. + - notification_handler: Handler for alerts and persisted notifications. + - metrics_controller: Optional shared controller; if ``None``, a new one is created. + - mlflow_repository: Optional ``SientiaMLflowRepository`` for unit/e2e tests; in production + leave unset so the repository is built from environment via ``build_mlflow_config()``. Raises: - Exception: If any parent class initialization fails + Exception: If any parent ``__init__`` fails (e.g. invalid config keys). + + Return: + None """ - # Initialize parent classes - Postgres.__init__(self, host=postgres_config['host'], - port=postgres_config['port'], - user=postgres_config['user'], - password=postgres_config['password'], - dbname=postgres_config['dbname'], - min_connections=postgres_config['min_connections'], - max_connections=postgres_config['max_connections'], - logger=logger, - notification_handler=notification_handler) - MLFlow.__init__(self, mlflow_host=mlflow_config['host'], - mlflow_port=mlflow_config['port'], - mlflow_username=mlflow_config['username'], - mlflow_password=mlflow_config['password'], - logger=logger, - notification_handler=notification_handler) + mc = metrics_controller or MetricsController(logger=logger) - Gates.__init__(self, logger=logger, - notification_handler=notification_handler) + # Production path: one shared MLflow client for all model registry / tracking calls. + if mlflow_repository is None: + mlflow_cfg = build_mlflow_config() + mlflow_repository = SientiaMLflowRepository( + host=mlflow_cfg['url'], + username=mlflow_cfg['username'], + password=mlflow_cfg['password'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=mc, + ) - OPC.__init__(self, - opc_servers=opc_config, - logger=logger, - notification_handler=notification_handler) + minio_repository = MinioRepository( + endpoint=minio_config['endpoint_url'], + access_key=minio_config['access_key'], + secret_key=minio_config['secret_key'], + bucket=minio_config['default_bucket'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=mc, + secure=minio_config['secure'], + ) - async def shutdown(self): + Storage.__init__( + self, + host=postgres_config['host'], + port=postgres_config['port'], + user=postgres_config['user'], + password=postgres_config['password'], + dbname=postgres_config['dbname'], + min_connections=postgres_config['min_connections'], + max_connections=postgres_config['max_connections'], + retention_hours=minio_config['retention_hours'], + minio_repository=minio_repository, + logger=logger, + notification_handler=notification_handler, + metrics_controller=mc, + ) + + MLFlow.__init__( + self, + mlflow_repository=mlflow_repository, + plugin_store=plugin_store, + minio_repository=minio_repository, + logger=logger, + notification_handler=notification_handler, + metrics_controller=mc, + ) + + Gates.__init__( + self, + minio_repository=minio_repository, + logger=logger, + notification_handler=notification_handler, + metrics_controller=mc, + ) + + OPC.__init__( + self, + opc_servers=opc_config, + logger=logger, + notification_handler=notification_handler, + metrics_controller=mc, + ) + + ModelMetrics.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=mc, + ) + + API.__init__( + self, + base_url=pi_web_api_config['base_url'], + auth_type=pi_web_api_config['auth_type'], + auth_token=pi_web_api_config['auth_token'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=mc, + ) + + def shutdown(self) -> None: """ - Gracefully shutdown all activities and clean up resources. + Close database pools, sync clients, and OPC sessions in a defined order. - This method ensures proper cleanup of all resources including: - - PostgreSQL connection pools - - OPC server connections - - Any other resources that need explicit cleanup + Should be invoked on worker exit so connection pools and OPC sessions are released + cleanly before process termination. - The method should be called before the application terminates to ensure - proper resource cleanup and prevent resource leaks. + Return: + None """ - Postgres.close(self) - await OPC.shutdown(self) + Storage.close(self) + MLFlow.close(self) + Gates.close(self) + OPC.close(self) + ModelMetrics.close(self) + API.close(self) diff --git a/laborious/activities/api.py b/laborious/activities/api.py new file mode 100644 index 0000000..8b651d9 --- /dev/null +++ b/laborious/activities/api.py @@ -0,0 +1,305 @@ +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import json + import traceback + from typing import Any + + from pandas import DataFrame + from sientia_do.notifications.handlers import NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.pi_web_api_client_sync import PIWebAPIClient + + from laborious import metrics + + +PI_WEB_API_PREDICTION_ERROR_CONFIDENCE = 13 + + +class API(SientiaMonitoring): + """ + PI Web API operations for writing prediction data to PI Web API. + + This class provides Temporal activities for interacting with the PI Web API + to write prediction and confidence values to industrial systems. It handles + error scenarios gracefully by setting error confidence values and sending + notifications when write operations fail. + + The class implements comprehensive error handling for both prediction and + confidence value writes, ensuring that partial failures are properly + reported and handled. + """ + + def __init__( + self, + base_url: str, + auth_type: str, + auth_token: str, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ) -> None: + """ + Initialize API activity with PI Web API client. + + Args: + base_url (str): Base URL of the PI Web API server + auth_type (str): Authentication type ('basic' or 'bearer') + auth_token (str): Authentication token + logger (Logger): Logger instance for operation logging + notification_handler (NotificationHandler): Handler for system notifications + metrics_controller (MetricsController): Controller for metrics collection + """ + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) + self.pi_web_api_client = PIWebAPIClient( + base_url=base_url, + auth_config={ + 'type': auth_type, + 'token': auth_token, + }, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + headers_config={ + 'Content-Type': 'application/json', + 'Accept': 'application/json', + 'x-requested-with': 'piwebapistreams', + 'User-Agent': 'Aig-Laborious-Agent/1.0', + }, + ) + + def get_pi_web_api_core_labels( + self, + metadata: dict[str, Any], + operation_type: str = 'write_pi_web_api_data', + ) -> dict[str, Any]: + """ + Generate core labels for PI Web API metrics. + + PI Web API metrics in laborious use the shared ``CORE_LABELS`` from + ``sientia_do``, which includes ``operation_type``. For this reason, + operation_type must always be present in emitted labels. + + Args: + - metadata (dict[str, Any]): Workflow execution metadata used to derive labels. + - operation_type (str): Operation type label for metric cardinality. + + Return: + dict[str, Any]: Core labels dictionary including operation_type. + """ + return super().get_core_labels( + metadata=metadata, + operation_type=operation_type, + ) + + def close(self) -> None: + """ + Close the PI Web API client and shutdown monitoring services. + + This method properly closes all connections and resources associated + with the PI Web API client and monitoring services. + """ + self.pi_web_api_client.close() + SientiaMonitoring.shutdown(self) + + def process_pi_web_api_response( + self, + response_data: list[dict[str, Any]], + tags: dict[str, str], + core_labels: dict[str, str], + metadata: dict[str, Any], + ) -> tuple[int, str]: + """ + Process the response data from PI Web API write operation. + + Validates that all tags were successfully written, emits metrics for each tag + (success or error), and returns the appropriate prediction confidence value. + Sets error confidence if any tag write fails or if the number of written tags + doesn't match the expected count. + + Args: + - response_data (dict[str, Any]): The response data from the PI Web API write operation. + - tags (dict[str, str]): The tags that were written to the PI Web API. + - core_labels (dict[str, str]): The core labels of the workflow execution. + - metadata (dict[str, Any]): The metadata of the workflow execution. + Returns: + int: Prediction confidence value (0 for success, 13 for errors) + """ + + # Convert tags from name:webid to webid:name + tags = {w: t for t, w in tags.items()} + + tag_names = list[str](tags.values()) + + confidence = 0 + + message = '' + + # Evaluate response for each tag + written_tags = [] + for item in response_data: + web_id = item.get('WebId') + if not web_id: + self.error('The response did not contain some WebIds', metadata) + continue + errors = item.get('Errors', []) + tag_name = tags.get(web_id) + if not tag_name: + self.error( + f'The response did not contain the tag name for WebId {web_id}', metadata + ) + continue + if errors: + self.error( + f'Error writing tag {tag_name}:{web_id} to PI Web API: {errors}', metadata + ) + self.emit_metric_sync( + metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT, + tags={ + **core_labels, + 'tag_name': tag_name, + }, + ) + confidence = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE + else: + self.emit_metric_sync( + metric_object=metrics.PI_WEB_API_PREDICTION_WRITTEN_COUNT, + tags={ + **core_labels, + 'tag_name': tag_name, + }, + ) + written_tags.append(tag_name) + + if len(written_tags) != len(tag_names): + message = f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written.' + + self.error( + f'{message}\nResponse:\n {json.dumps(response_data, indent=4)}\nTags:\n {json.dumps(tags, indent=4)}', + metadata, + ) + + self.send_notification( + metadata=metadata, + notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR', + message=f'The number of written tags does not match the number of tag names: Expected {tag_names} tags, but {written_tags} tags were written.\nResponse:\n {json.dumps(response_data, indent=4)}\nTags:\n {json.dumps(tags, indent=4)}', + block='write_pi_web_api_data', + level=NotificationLevel.ERROR, + ) + confidence = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE + + return confidence, message + + @activity.defn(name='write_pi_web_api_data') + def write_pi_web_api_data(self, input_data: dict[str, Any]) -> dict[Any, Any]: + """ + Write prediction and confidence data to PI Web API. + + Writes prediction values and confidence scores to PI Web API using configured + web IDs. Processes responses to validate writes and emit metrics. Handles errors + gracefully by setting error confidence values when writes fail and sending + notifications for both prediction and confidence write errors. + + Args: + input_data (dict[str, Any]): The input data containing: + - metadata (dict[str, Any]): Workflow execution metadata + - pi_web_api_output_config (dict[str, Any]): PI Web API configuration with: + - endpoint (str): PI Web API endpoint URL + - prediction_tags (dict[str, str]): Mapping of tag names to web IDs for predictions + - confidence_tags (dict[str, str]): Mapping of tag names to web IDs for confidence + - data (dict[str, Any]): Prediction data, its a dataframe converted to dict. + Returns: + dict[Any, Any]: Data dictionary with potentially modified confidence values + If prediction write fails, prediction_confidence is set to error value (13) + """ + metadata = input_data['metadata'] + data = DataFrame(input_data['data']) + pi_web_api_output_config = input_data['pi_web_api_output_config'] + + self.info(f'Writing data to PI Web API... config: {pi_web_api_output_config}', metadata) + + raw_prediction_tags = pi_web_api_output_config['prediction_tags'] + raw_confidence_tags = pi_web_api_output_config['confidence_tags'] + prediction_tags = list[str](raw_prediction_tags.values()) + confidence_tags = list(raw_confidence_tags.values()) + + core_labels = self.get_pi_web_api_core_labels(metadata) + + prediction_value = data.head(1)['prediction'].values[0] + confidence_value = data.head(1)['prediction_confidence'].values[0] + + try: + prediction_response = self.pi_web_api_client.write_value( + web_ids=prediction_tags, + value={ + 'Timestamp': data.head(1)['timestamp'].values[0], + 'Value': prediction_value, + }, + metadata=metadata, + ) + + confidence, message = self.process_pi_web_api_response( + response_data=prediction_response, + tags=raw_prediction_tags, + core_labels=core_labels, + metadata=metadata, + ) + + # Preserve incoming confidence/comments on successful PI writes. + # Only downgrade confidence or override comments when PI response + # explicitly reports a problem (e.g. partial write mismatch). + if confidence != 0: + data['prediction_confidence'] = confidence + if message: + data['comments'] = message + + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='WRITE_PI_WEB_API_PREDICTION_ERROR', + message=f'Error writing prediction data to PI Web API: {e}\n Tags: {raw_prediction_tags}', + block='write_pi_web_api_data', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + + self.error(trace, metadata) + + data['prediction_confidence'] = PI_WEB_API_PREDICTION_ERROR_CONFIDENCE + data['comments'] = str(e) + + return data.to_dict() + + try: + confidence_response = self.pi_web_api_client.write_value( + web_ids=confidence_tags, + value={ + 'Timestamp': data.head(1)['timestamp'].values[0], + 'Value': float(confidence_value), + }, + metadata=metadata, + ) + + self.process_pi_web_api_response( + response_data=confidence_response, + tags=raw_confidence_tags, + core_labels=core_labels, + metadata=metadata, + ) + + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='WRITE_PI_WEB_API_CONFIDENCE_ERROR', + message=f'Error writing confidence data to PI Web API: {e}\n Tags: {raw_confidence_tags}', + block='write_pi_web_api_data', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + + return data.to_dict() diff --git a/laborious/activities/gates.py b/laborious/activities/gates.py index 4a71455..f72bd2e 100644 --- a/laborious/activities/gates.py +++ b/laborious/activities/gates.py @@ -1,57 +1,71 @@ from temporalio import activity, workflow - with workflow.unsafe.imports_passed_through(): import traceback + from collections.abc import Callable, Mapping + from typing import Any + + from pandas import DataFrame from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel - from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger - from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now - from sientia_do.formatters import create_sample_dict - from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter - from typing import Any + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.minio_repository_sync import MinioRepository + from sientia_do.utils.formatters import create_sample_dict + + from laborious import metrics + from laborious.utils.dataframe_debug import build_dataframe_debug_message from laborious.utils.filters.conditional_filters import ( filter_empty_data, - filter_specific_variables_null_values + filter_specific_variables_null_values, ) - from pandas import DataFrame - from laborious import metrics + from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter + from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload + +# Strongly-typed filter function signatures +InputFilterFunc = Callable[[DataFrame, dict[str, Any]], bool] +ResponseFilterFunc = Callable[[dict[str, Any], dict[str, Any]], bool] +ContentFilterFunc = Callable[[DataFrame, dict[str, Any]], bool] # Input filter function mappings -input_filter_functions = { +input_filter_functions: dict[str, InputFilterFunc] = { 'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values, 'EMPTY_DATA': filter_empty_data, - 'path_confidence': { - 'STOP': -1, - 'CONTINUE': 2, - 'REPEAT': -1 - } +} + +# Confidence mappings kept separate from function maps to avoid Union types +input_path_confidence: Mapping[str, int] = { + 'STOP': -1, + 'CONTINUE': 2, + 'REPEAT': -1, } # MLFlow response filter function mappings -mlflow_response_filter_functions = { +mlflow_response_filter_functions: dict[str, ResponseFilterFunc] = { 'API_ERROR': api_error_filter, - 'path_confidence': { - 'STOP': -1, - 'CONTINUE': 10, - 'REPEAT': -1 - }, +} + +mlflow_response_path_confidence: Mapping[str, int] = { + 'STOP': -1, + 'CONTINUE': 10, + 'REPEAT': -1, } # MLFlow content filter function mappings -mlflow_content_filter_functions = { +mlflow_content_filter_functions: dict[str, ContentFilterFunc] = { 'NAN_VALUES': nan_values_filter, 'EMPTY_DATA': filter_empty_data, - 'path_confidence': { - 'STOP': -1, - 'CONTINUE': 18, - 'REPEAT': -1 - } +} + +mlflow_content_path_confidence: Mapping[str, int] = { + 'STOP': -1, + 'CONTINUE': 18, + 'REPEAT': -1, } -class Gates(BaseActivity): +class Gates(SientiaMonitoring): """ Data quality gates and filtering activities for the Laborious system. @@ -71,7 +85,16 @@ class Gates(BaseActivity): mlflow_content_filter_functions (dict): Mapping of MLFlow content filter names to functions """ - def __init__(self, logger: Logger, notification_handler: NotificationHandler): + minio_repository: MinioRepository | None = None + _MAX_DEBUG_DATAFRAME_ROWS = 100 + + def __init__( + self, + minio_repository: MinioRepository | None = None, + logger: Logger | None = None, + notification_handler: NotificationHandler | None = None, + metrics_controller: MetricsController | None = None, + ): """ Initialize data quality gates with logging and notification capabilities. @@ -82,11 +105,65 @@ class Gates(BaseActivity): Raises: Exception: If BaseActivity initialization fails """ - BaseActivity.__init__( - self, logger, notification_handler, set_error_counter=True) + self.minio_repository = minio_repository + SientiaMonitoring.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) - @activity.defn(name="input_gate") - async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: + def close(self) -> None: + """ + Close the gates activity and clean up resources. + """ + + if self.minio_repository is not None: + try: + self.minio_repository.close() + finally: + self.minio_repository = None + SientiaMonitoring.shutdown(self) + + def __del__(self): + self.close() + + def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None: + """ + Log dataframe content only when row count is below the configured threshold + + Args: + - message (str): Base log message to identify the dataframe in logs + - data (Any): Dataframe-like payload to be logged + - metadata (dict[str, Any]): Workflow metadata for contextual logging + """ + self.debug( + build_dataframe_debug_message( + message=message, + data=data, + max_rows=self._MAX_DEBUG_DATAFRAME_ROWS, + ), + metadata, + ) + + @staticmethod + def _read_filter_entry(config: dict[str, Any]) -> tuple[str, dict[str, Any]]: + """ + Read filter policy/config keys in a case-insensitive way. + + Args: + config (dict[str, Any]): Filter configuration dictionary. + + Return: + tuple[str, dict[str, Any]]: Parsed policy and config payload. + """ + normalized = {str(key).upper(): value for key, value in config.items()} + policy = normalized['POLICY'] + filter_config = normalized.get('CONFIG', {}) + return policy, filter_config + + @activity.defn(name='input_gate') + def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: """ Apply input data quality filters and validation. @@ -121,49 +198,52 @@ class Gates(BaseActivity): """ metadata = input_data['metadata'] - self.info("Performing input gate...", metadata) + self.info('Performing input gate...', metadata) filters = input_data['filters'] - data = DataFrame(input_data['data']) + payload = MinioDataFramePayload.from_dict(input_data['data']) + data = payload.retrieve(self.minio_repository, metadata) path_priority = input_data['path_priority'] filter_output = [] - self.debug(f"Input data: {data.head(5).to_string()}", metadata) - self.debug(f"Filters: {filters}", metadata) + self._debug_dataframe('Input data:', data, metadata) + self.debug(f'Filters: {filters}', metadata) # Apply each configured filter for fil, config in filters.items(): if fil not in input_filter_functions: - self.error(f"Filter {fil} not found", metadata) + self.error(f'Filter {fil} not found', metadata) continue + policy, filter_config = self._read_filter_entry(config) try: - if input_filter_functions[fil](data, config['config']): - self.debug( - f"Data not passed the input filter {fil}:{config}", metadata) - filter_output.append(config['policy']) + if input_filter_functions[fil](data, filter_config): + self.debug(f'Data not passed the input filter {fil}:{config}', metadata) + filter_output.append(policy) except Exception as e: trace = traceback.format_exc() self.send_notification( metadata=metadata, - notification_id=f"INTPUT_GATE_ERROR__{fil}", - message=f"Error in filter {fil}:{config}: \n {e}", - block="input_gate", + notification_id=f'INTPUT_GATE_ERROR__{fil}', + message=f'Error in filter {fil}:{config}: \n {e}', + block='input_gate', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) for path_flag in path_priority: if path_flag in filter_output: - self.info(f"Input gate result: {path_flag}", metadata) - return path_flag, input_filter_functions['path_confidence'][path_flag], \ - "Input data with bad quality" + self.info(f'Input gate result: {path_flag}', metadata) + return path_flag, input_path_confidence[path_flag], 'Input data with bad quality' - self.info("Nothing was filtered by the input gate", metadata) - return None, 0, "" + self.info('Nothing was filtered by the input gate', metadata) - @activity.defn(name="mlflow_response_gate") - async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: + del data + + return None, 0, '' + + @activity.defn(name='mlflow_response_gate') + def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: """ Validate MLFlow API response quality and integrity. @@ -197,59 +277,67 @@ class Gates(BaseActivity): Exception: If response validation fails or configuration is invalid """ metadata = input_data['metadata'] - self.info("Performing mlflow response gate...", metadata) - + self.info('Performing mlflow response gate...', metadata) + raw_data = input_data['data'] filters = input_data['filters'] - data = input_data['data'] + + self.debug( + f'Input data: \n {create_sample_dict(raw_data, max_items=5, max_depth=5)}', metadata + ) + self.debug(f'Filters: {filters}', metadata) + + payload = MinioDataFramePayload.from_dict(raw_data) + data = payload.retrieve(self.minio_repository, metadata) + gate_type = input_data['type'] path_priority = input_data['path_priority'] filter_output = [] - self.debug( - f"Input data: \n {create_sample_dict(data, max_items=5, max_depth=2)}", metadata) - self.debug(f"Filters: {filters}", metadata) - comments = [] + + status = payload.status or {} + for fil, config in filters.items(): if fil not in mlflow_response_filter_functions: - self.error(f"Filter {fil} not found", metadata) continue + policy, filter_config = self._read_filter_entry(config) try: - if mlflow_response_filter_functions[fil](data, config): - filter_output.append(config['policy']) - comments.append(data['content']['message']) + if mlflow_response_filter_functions[fil](status, filter_config): + filter_output.append(policy) + comments.append(status.get('message', 'Unknown MLFlow API error')) self.send_notification( metadata=metadata, - notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}", - message=data['content']['message'], - block="mlflow_gate", + notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}', + message=status.get('message', 'Unknown MLFlow API error'), + block='mlflow_gate', level=NotificationLevel.ERROR, - attachment_content=data['content']['traceback'] + attachment_content=status.get('traceback'), ) except Exception as e: trace = traceback.format_exc() self.send_notification( metadata=metadata, - notification_id=f"MLFLOW_GATE_RESPONSE_FILTER__{fil}", - message=f"Error in filter {fil}:{config}: \n {e}", - block="mlflow_gate", + notification_id=f'MLFLOW_GATE_RESPONSE_FILTER__{fil}', + message=f'Error in filter {fil}:{config}: \n {e}', + block='mlflow_gate', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) for path_flag in path_priority: if path_flag in filter_output: - self.info( - f"Mlflow response gate result: {path_flag}", metadata) - return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag], \ - ", ".join(comments) + self.info(f'Mlflow response gate result: {path_flag}', metadata) + return path_flag, mlflow_response_path_confidence[path_flag], ', '.join(comments) - self.info("Nothing was filtered by the mlflow response gate", metadata) - return None, 0, "" + self.info('Nothing was filtered by the mlflow response gate', metadata) - @activity.defn(name="mlflow_content_gate") - async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: + del data + + return None, 0, '' + + @activity.defn(name='mlflow_content_gate') + def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]: """ Validate MLFlow prediction content quality and integrity. @@ -283,56 +371,65 @@ class Gates(BaseActivity): Exception: If content validation fails or configuration is invalid """ metadata = input_data['metadata'] - self.info("Performing mlflow content gate...", metadata) + self.info('Performing mlflow content gate...', metadata) filters = input_data['filters'] - data = DataFrame(input_data['data']) + + payload = MinioDataFramePayload.from_dict(input_data['data']) + data = payload.retrieve(self.minio_repository, metadata) + gate_type = input_data['type'] path_priority = input_data['path_priority'] filter_output = [] - self.debug(f"Input data:\n {data.head(5).to_string()}", metadata) - self.debug(f"Filters: \n {create_sample_dict(filters)}", metadata) + self._debug_dataframe('Input data:', data, metadata) + self.debug(f'Filters: \n {filters}', metadata) for fil, config in filters.items(): if fil not in mlflow_content_filter_functions: continue + policy, filter_config = self._read_filter_entry(config) try: - if mlflow_content_filter_functions[fil](data, config): - filter_output.append(config['policy']) + if mlflow_content_filter_functions[fil](data, filter_config): + filter_output.append(policy) self.send_notification( metadata=metadata, - notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}", - message=f"Data not passed the content filter {fil}:{config}", - block="mlflow_gate", + notification_id=f'{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}', + message=f'Data not passed the content filter {fil}:{config}', + block='mlflow_gate', level=NotificationLevel.WARNING, - attachment_content=data.to_string() + attachment_content=data.to_string(), ) except Exception as e: trace = traceback.format_exc() self.send_notification( metadata=metadata, - notification_id=f"MLFLOW_GATE_CONTENT_FILTER__{fil}", - message=f"Error in filter {fil}:{config}: \n {e}", - block="mlflow_gate", + notification_id=f'MLFLOW_GATE_CONTENT_FILTER__{fil}', + message=f'Error in filter {fil}:{config}: \n {e}', + block='mlflow_gate', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) for path_flag in path_priority: if path_flag in filter_output: - self.info( - f"Mlflow content gate result: {path_flag}", metadata) - return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag], \ - "Transformed data not passed the content filter" + self.info(f'Mlflow content gate result: {path_flag}', metadata) + return ( + path_flag, + mlflow_content_path_confidence[path_flag], + 'Transformed data not passed the content filter', + ) - self.info("Nothing was filtered by the mlflow content gate", metadata) - return None, 0, "" + self.info('Nothing was filtered by the mlflow content gate', metadata) - def get_prediction_store_policy(self, - prediction_store_policy: str, - metadata: dict[str, Any]) -> tuple[str, int]: + del data + + return None, 0, '' + + def get_prediction_store_policy( + self, prediction_store_policy: str, metadata: dict[str, Any] + ) -> tuple[str, int]: """ Parse and validate prediction store policy configuration. @@ -358,7 +455,9 @@ class Gates(BaseActivity): if len(policy_elements) < 2: self.error( - f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata) + f'Invalid prediction store policy: {prediction_store_policy}, using default policy', + metadata, + ) return 'lts', 1 policy_type = policy_elements[0] @@ -366,15 +465,76 @@ class Gates(BaseActivity): # If the policy_type is not lts or erl, we use the default policy # If the policty_value is not a number or 0, we use the default policy - if policy_type not in ['lts', 'erl'] or not policy_value.isdigit() or int(policy_value) == 0: + if ( + policy_type not in ['lts', 'erl'] + or not policy_value.isdigit() + or int(policy_value) == 0 + ): self.error( - f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata) + f'Invalid prediction store policy: {prediction_store_policy}, using default policy', + metadata, + ) return 'lts', 1 return policy_type, int(policy_value) - @activity.defn(name="format_prediction") - async def format_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]: + @activity.defn(name='format_transformed_data') + def format_transformed_data(self, input_data: dict[str, Any]) -> MinioDataFramePayload: + """ + Format transformed data for storage and export operations. + + This method formats transformed data from MLFlow model transformations + into a standardized format suitable for database storage. It converts + wide-format data (columns as variables) into long-format (melted) + with proper timestamp handling and model identification. + + The formatting process includes: + 1. Converting input data dictionary to DataFrame + 2. Extracting timestamps from DataFrame index + 3. Resetting index to create sequential row numbers + 4. Melting data from wide format to long format (variable-value pairs) + 5. Adding model_id for data lineage tracking + + Args: + input_data (dict): Input data containing: + - metadata (dict): Workflow execution metadata + - data (dict[str, Any]): Transformed data to format (DataFrame-compatible dict) + - model_id (str): Unique identifier for the ML model + + Returns: + dict: Formatted data dictionary with keys: + - timestamp (dict): Timestamp values indexed by row number + - variable (dict): Variable names indexed by row number + - value (dict): Variable values indexed by row number + - model_id (dict): Model identifiers indexed by row number + """ + metadata = input_data['metadata'] + + model_id = input_data['model_id'] + + self.info('Formatting transformed data...', metadata) + + payload = MinioDataFramePayload.from_dict(input_data['data']) + data = payload.retrieve(self.minio_repository, metadata) + + data['timestamp'] = data.index + data = data.reset_index(drop=True) + + data = data.melt(id_vars='timestamp', var_name='variable', value_name='value') + data['model_id'] = model_id + + return MinioDataFramePayload.from_dataframe( + dataframe=data, + minio_repo=self.minio_repository, + model_name=input_data['model_name'], + operation='transform', + workflow_metadata=metadata, + last_timestamp=payload.last_timestamp, + logger=self.logger, + ) + + @activity.defn(name='format_prediction') + def format_prediction(self, input_data: dict[str, Any]) -> dict: """ Format prediction data according to configured storage policies. @@ -383,6 +543,8 @@ class Gates(BaseActivity): and ensures data consistency before persistence. The method supports multiple storage policies for flexible data retention strategies. + If only one row is present, we use the last timestamp as the timestamp + Storage Policies: - 'lts:N': Latest timestamp - retains N most recent predictions - 'erl:N': Earliest timestamp - retains N oldest predictions @@ -390,7 +552,7 @@ class Gates(BaseActivity): Args: input_data (dict): Input data containing: - data (dict[str, Any]): Raw prediction data to format - - timestamp (str): Default timestamp if data lacks timestamp column + - timestamp (str): Timestamp of the data - model_id (str): Unique identifier for the ML model - prediction_confidence (float): Confidence score for the prediction - prediction_store_policy (str): Storage policy in format 'type:value' @@ -399,58 +561,61 @@ class Gates(BaseActivity): dict: Formatted prediction data ready for storage and export """ metadata = input_data['metadata'] + last_timestamp = input_data['timestamp'] prediction_store_policy = input_data['prediction_store_policy'] - self.info("Formatting prediction...", metadata) + self.info('Formatting prediction...', metadata) - data = DataFrame(input_data['data']) + payload = MinioDataFramePayload.from_dict(input_data['data']) + data = payload.retrieve(self.minio_repository, metadata) # Create timestamp column from index and reset index data['timestamp'] = data.index data = data.reset_index(drop=True) - self.debug( - f"Prediction store policy: {prediction_store_policy}", metadata) - self.debug(f"Prediction data: {data.head(5).to_string()}", metadata) + self.debug(f'Prediction store policy: {prediction_store_policy}', metadata) + self._debug_dataframe('Prediction data:', data, metadata) policy_type, policy_value = self.get_prediction_store_policy( - prediction_store_policy, metadata) + prediction_store_policy, metadata + ) # If data has no timestamp, we use the default timestamp and not sort the data self.info( - f"Sorting data by timestamp and applying policy: {policy_type}:{policy_value}", metadata) + f'Sorting data by timestamp and applying policy: {policy_type}:{policy_value}', metadata + ) # If policy_type is lts, we need to sort the data by timestamp descending and take the first policy_value rows if policy_type == 'lts': - self.debug( - "Sorting data by timestamp descending", metadata) + self.debug('Sorting data by timestamp descending', metadata) data = data.sort_values(by='timestamp', ascending=False) # If policy_type is erl, we need to sort the data by timestamp ascending and take the first policy_value rows elif policy_type == 'erl': - self.debug( - "Sorting data by timestamp ascending", metadata) + self.debug('Sorting data by timestamp ascending', metadata) data = data.sort_values(by='timestamp', ascending=True) else: - self.error( - f"Invalid policy type: {policy_type}, using default policy", metadata) - raise ValueError( - f"Invalid policy type: {policy_type}") + self.error(f'Invalid policy type: {policy_type}, using default policy', metadata) + raise ValueError(f'Invalid policy type: {policy_type}') - data = data.head(int(policy_value)) + int_policy_value = int(policy_value) + + data = data.head(int_policy_value) + if int_policy_value == 1: + data['timestamp'] = last_timestamp data['model_id'] = input_data['model_id'] data['prediction_confidence'] = input_data['prediction_confidence'] data['prediction_status'] = 'Good' - data['comments'] = "" + data['comments'] = '' data = data.sort_values(by='timestamp', ascending=False) data = data.reset_index(drop=True) - self.info(f"Prediction formatted: {len(data)} rows", metadata) - self.debug(f"Prediction data: {data.head(5).to_string()}", metadata) + self.info(f'Prediction formatted: {len(data)} rows', metadata) + self._debug_dataframe('Prediction data:', data, metadata) return data.to_dict() - @activity.defn(name="format_default_prediction") - async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]: + @activity.defn(name='format_default_prediction') + def format_default_prediction(self, input_data: dict[str, Any]) -> dict: """ Create and format default prediction data for error conditions. @@ -477,65 +642,95 @@ class Gates(BaseActivity): """ metadata = input_data['metadata'] - self.debug("Formatting default prediction...", metadata) + self.debug('Formatting default prediction...', metadata) - data = DataFrame({ - 'prediction': [0], - 'response_time': [0], - 'timestamp': [input_data['timestamp']], - 'model_id': [input_data['model_id']], - 'prediction_confidence': [input_data['prediction_confidence']], - 'prediction_status': ['Bad'], - 'comments': [input_data['comment']] - }) + data = DataFrame( + { + 'prediction': [0], + 'response_time': [0], + 'timestamp': [input_data['timestamp']], + 'model_id': [input_data['model_id']], + 'prediction_confidence': [input_data['prediction_confidence']], + 'prediction_status': ['Bad'], + 'comments': [input_data['comment']], + } + ) - self.info(f"Default prediction formatted: {data.size} rows", metadata) + self.info(f'Default prediction formatted: {data.size} rows', metadata) return data.to_dict() - @activity.defn(name="get_last_timestamp") - async def get_last_timestamp(self, input_data: dict[str, Any]) -> str: + @activity.defn(name='format_retrain_report') + def format_retrain_report(self, input_data: dict[str, Any]) -> dict: """ - Extract the most recent timestamp from prediction data. + Format retrain report data for storage and audit trail maintenance. - This method analyzes prediction data to find the latest timestamp, - enabling incremental processing and data continuity tracking. - It handles empty datasets gracefully by returning the current time - as a fallback timestamp. + This method formats model retraining operation results into a standardized + report format suitable for database storage and operational monitoring. + It captures retraining status, timestamps, and model version information + for comprehensive audit trails and operational visibility. - The method is essential for: - 1. Incremental data processing workflows - 2. Data continuity validation - 3. Timestamp-based data loading optimization - 4. Workflow execution tracking + The formatting process includes: + 1. Extracting retraining experiment response data + 2. Capturing model update report information (version, MLflow IDs) + 3. Formatting timestamps and status information + 4. Conditionally including version information for successful retrains Args: input_data (dict): Input data containing: - - data (dict[str, Any]): Prediction data to analyze + - metadata (dict): Workflow execution metadata + - experiment_response (dict): Retraining experiment response containing: + - success (bool): Retraining operation success status + - timestamp (str): Timestamp of the retraining operation + - message (str): Status message or error description + - update_report (dict): Model update report containing: + - version (str): New model version identifier + - mlflow_run_id (str): MLflow run identifier + - mlflow_experiment_id (str): MLflow experiment identifier + - model_id (str): Unique identifier for the ML model + - model_name (str): Name of the ML model Returns: - str: Formatted timestamp string in UTC with timezone + dict: Formatted retrain report dictionary with keys: + - model_id (dict): Model identifiers indexed by row number + - model_name (dict): Model names indexed by row number + - timestamp (dict): Retraining timestamps indexed by row number + - status (dict): Retraining status messages indexed by row number + - version (dict, optional): Model versions indexed by row number + Only included if experiment_response['success'] is True + - mlflow_run_id (dict, optional): MLflow run IDs indexed by row number + Only included if experiment_response['success'] is True + - mlflow_experiment_id (dict, optional): MLflow experiment IDs indexed by row number + Only included if experiment_response['success'] is True """ metadata = input_data['metadata'] + self.info('Formatting retrain report...', metadata) - self.info("Getting last timestamp...", metadata) + experiment_response = input_data['experiment_response'] + update_report = input_data['update_report'] + model_id = input_data['model_id'] + model_name = input_data['model_name'] - data = DataFrame(input_data['data']) + report = DataFrame( + { + 'model_id': [model_id], + 'model_name': [model_name], + 'timestamp': [experiment_response['timestamp']], + 'status': [experiment_response['message']], + } + ) - self.debug(f"Input data: {data.head(5).to_string()}", metadata) + if experiment_response['success']: + # Retrain was successfull + report['version'] = update_report['version'] + report['mlflow_run_id'] = update_report['mlflow_run_id'] + report['mlflow_experiment_id'] = update_report['mlflow_experiment_id'] - if data.empty: - return now().strftime(DATETIME_FORMAT_WITH_TZ) + self._debug_dataframe('Retrain report:', report, metadata) - max_timestamp = max( - data['timestamp'].values.tolist()) + return report.to_dict() - self.info( - f"Last timestamp: {max_timestamp}", metadata) - - return max_timestamp - - @activity.defn(name="write_metrics") - async def write_metrics(self, input_data: dict[str, Any]): + @activity.defn(name='write_metrics') + def write_metrics(self, input_data: dict[str, Any]): """ Write prediction performance metrics to Prometheus monitoring system. @@ -561,27 +756,57 @@ class Gates(BaseActivity): prediction = DataFrame(input_data['prediction']) prediction_confidence = prediction['prediction_confidence'].values[0] response_time = prediction['response_time'].values[0] + opc_metrics = input_data['opc_metrics'] - self.info( - f"Writing metrics for model {metadata['model_name']}", metadata) + self.info(f'Writing metrics for model {metadata["model_name"]}', metadata) - metrics.PREDICTIONS_WRITTEN_COUNT.labels( - pod_id=self.pod_id, - model_name=metadata['model_name'], - pipeline_name=metadata['workflow_name'] - ).inc() + core_tags = { + 'pod_id': self.pod_id, + 'runtime': self.runtime, + 'operation_type': 'predict', + 'model_name': metadata['model_name'], + 'workflow_name': metadata['workflow_name'], + } + self.emit_metric_sync( + metric_object=metrics.PREDICTIONS_WRITTEN_COUNT, + tags=core_tags, + ) - metrics.PREDICTION_CONFIDENCE_MONITOR.labels( - pod_id=self.pod_id, - model_name=metadata['model_name'], - pipeline_name=metadata['workflow_name'] - ).set(prediction_confidence) + self.emit_metric_sync( + metric_object=metrics.PREDICTION_CONFIDENCE_MONITOR, + method='set', + tags=core_tags, + value=prediction_confidence, + ) - metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels( - pod_id=self.pod_id, - model_name=metadata['model_name'], - pipeline_name=metadata['workflow_name'] - ).observe(response_time) + self.emit_metric_sync( + metric_object=metrics.PREDICTION_RESPONSE_TIME_MONITOR, + method='observe', + tags=core_tags, + value=response_time, + ) - self.info( - f"Metrics written for model {metadata['model_name']}", metadata) + for server_id, tags in opc_metrics.items(): + for tag, response_time in tags.items(): + if response_time is not None: + self.emit_metric_sync( + metric_object=metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR, + method='observe', + tags={ + **core_tags, + 'opc_server_id': server_id, + 'tag': tag, + }, + value=response_time, + ) + + self.emit_metric_sync( + metric_object=metrics.PREDICTION_OPC_WRITING_COUNT, + tags={ + **core_tags, + 'opc_server_id': server_id, + 'tag': tag, + }, + ) + + self.info(f'Metrics written for model {metadata["model_name"]}', metadata) diff --git a/laborious/activities/mlflow.py b/laborious/activities/mlflow.py index bd06283..bbc3e45 100644 --- a/laborious/activities/mlflow.py +++ b/laborious/activities/mlflow.py @@ -1,333 +1,649 @@ from temporalio import activity, workflow - with workflow.unsafe.imports_passed_through(): + import tempfile + import traceback from datetime import datetime - from pandas import Timestamp, to_datetime - from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ - from sientia_do.temporal.activities.base import BaseActivity + from pathlib import Path + from shutil import rmtree + from typing import Any + + import mlflow + import numpy as np + import pandas as pd + from pandas import DataFrame, to_datetime from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger - from sientia_do.formatters import create_sample_dict - from laborious.utils.repository.model_repository import MLFlowRepository - from typing import Any - import numpy as np - from pandas import DataFrame - import traceback + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.minio_repository_sync import MinioRepository + from sientia_do.temporal.constants import ( + DATETIME_FORMAT, + DATETIME_FORMAT_MS_WITH_TZ, + DATETIME_FORMAT_WITH_TZ, + now, + ) + from sientia_do.utils.formatters import create_sample_dict + from sientia_model.model_repository.mlflow_repository import SientiaMLflowRepository + from sientia_model.model_repository.plugin_store import PluginStore + + from laborious.utils.dataframe_debug import build_dataframe_debug_message + from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload -class MLFlow(BaseActivity): +class MLFlow(SientiaMonitoring): """ - MLFlow integration activities for model inference operations. + Temporal activities that talk to MLflow through ``SientiaMLflowRepository`` and ``SientiaModel`` wrappers. - This class provides activities for interacting with MLFlow models, including - data transformation and prediction operations. It handles authentication, - data preprocessing, and model management with configurable retention policies. + Models are resolved by registered name and the ``production`` alias (not by legacy stages or + separate transform/predict flavors). ``get_cached_model`` loads or reuses a wrapper; inference + uses ``wrapper.transform`` / ``wrapper.predict``; retrain uses ``wrapper.retrain`` or + ``wrapper.train`` plus ``store_model`` and registry promotion via ``promote_to_alias``. - The class implements comprehensive error handling and logging for all - MLFlow operations, ensuring reliable model inference in production environments. + Large inputs and outputs flow through ``MinioDataFramePayload`` when workflows offload parquet + to MinIO. On failure, transform/predict still return a payload with ``success: False`` and + error details for downstream gates. Attributes: - mlflow_host (str): MLFlow server hostname - mlflow_port (int): MLFlow server port - mlflow_username (str): MLFlow authentication username - mlflow_password (str): MLFlow authentication password - model_monitoring_repository (MLFlowRepository): Repository for MLFlow operations + mlflow_repository: Client for tracking, registry, artifact download, and run lifecycle. + plugin_store: Reference to the store (runtime is installed on the worker; reserved for + future store-backed helpers). """ - def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str, - mlflow_password: str, logger: Logger, notification_handler: NotificationHandler): + _MAX_DEBUG_DATAFRAME_ROWS = 100 + _DEFAULT_MODEL_ALIAS = 'production' + _REFERENCE_ARTIFACT_CANDIDATES = ('evaluation_data.csv', 'test_data.csv') + + def __init__( + self, + mlflow_repository: SientiaMLflowRepository, + plugin_store: PluginStore, + minio_repository: MinioRepository | None = None, + logger: Logger | None = None, + notification_handler: NotificationHandler | None = None, + metrics_controller: MetricsController | None = None, + ): """ - Initialize MLFlow activities with server configuration. + Attach shared MLflow and MinIO clients used by all ML activities in this mixin. Args: - mlflow_host: MLFlow server hostname or IP address - mlflow_port: MLFlow server port number - mlflow_username: Username for MLFlow authentication - mlflow_password: Password for MLFlow authentication - logger: Logger instance for observability and debugging - notification_handler: Notification handler for alerts and monitoring + - mlflow_repository: Repository built by ``Activities`` (or injected in tests). + - plugin_store: Plugin store instance from worker bootstrap. + - minio_repository: MinIO client for ``MinioDataFramePayload`` upload/download. + - logger: Structured logger. + - notification_handler: Notifications on hard failures where applicable. + - metrics_controller: Shared metrics controller. - Raises: - Exception: If MLFlowRepository initialization fails + Return: + None """ - BaseActivity.__init__( - self, logger, notification_handler, set_error_counter=True) - self.mlflow_host = mlflow_host - self.mlflow_port = mlflow_port - self.mlflow_username = mlflow_username - self.mlflow_password = mlflow_password - self.model_monitoring_repository = MLFlowRepository( - f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger + self.minio_repository = minio_repository + SientiaMonitoring.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + self.mlflow_repository = mlflow_repository + self.plugin_store = plugin_store + + def close(self) -> None: + """ + Release MinIO manager resources held by the mixin. + + Return: + None + """ + if self.minio_repository is not None: + try: + self.minio_repository.close() + finally: + self.minio_repository = None + SientiaMonitoring.shutdown(self) + + def __del__(self): + self.close() + + def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None: + """ + Log dataframe content only when row count is below the configured threshold + + Args: + - message (str): Base log message to identify the dataframe in logs + - data (Any): Dataframe-like object expected to expose shape and to_csv + - metadata (dict[str, Any]): Workflow metadata for contextual logging + """ + self.debug( + build_dataframe_debug_message( + message=message, + data=data, + max_rows=self._MAX_DEBUG_DATAFRAME_ROWS, + ), + metadata, ) - @activity.defn(name="request_transform") - async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]: + def _detect_and_parse_datetime_index(self, data: pd.DataFrame, metadata: dict) -> pd.DataFrame: """ - Transform input data using MLFlow models. + Ensure the transform output index is homogeneous and encoded as ``DATETIME_FORMAT_WITH_TZ`` strings. - This activity processes input data through MLFlow model transformation, - including data preprocessing, format conversion, and validation. It handles - data deduplication, pivoting, and cleanup to ensure optimal model performance. - - The transformation process includes: - 1. Data deduplication based on variable and timestamp - 2. Data pivoting for model input format - 3. Null value handling and cleanup - 4. MLFlow model transformation request - 5. Response validation and logging + Accepts an all-string index (validated against the format), or all-``datetime`` / + ``Timestamp`` (naive timestamps are localized to UTC before formatting). Mixed element types + or unsupported types raise ``ValueError`` with a message logged at info level. Args: - input_data: Configuration and data for transformation - Required keys: - - metadata (dict): Workflow execution metadata - - data (dict): Input data for transformation - - model_name (str): Name of the MLFlow model to use - - model_retention (int): Model retention period in minutes + - data: DataFrame whose index carries the time dimension after transform. + - metadata: Workflow metadata for log correlation. - Returns: - dict: Transformed data from MLFlow model + Return: + ``pd.DataFrame``: Same frame with a normalized string index; empty frames are returned as-is. + """ + + if data.empty: + self.info('Data is empty, skipping datetime index detection and parsing', metadata) + return data + + index = data.index + index_type = type(index[0]) + + self.info(f'Index type: {index_type}', metadata) + + message = ( + f'Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, ' + f'string in format {DATETIME_FORMAT_WITH_TZ}.' + ) + + if not all(isinstance(i, index_type) for i in index): + types = map(str, map(type, index)) + raise ValueError(f'{message}. Elements are {",".join(types)}') + + if index_type is str: + try: + pd.to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ) + except ValueError as e: + raise ValueError(f'{message}. Unable to parse given date format: {e}') from e + + elif index_type is datetime or index_type is pd.Timestamp: + idx = data.index + if hasattr(idx, 'tz') and idx.tz is None: + data.index = idx.tz_localize('UTC') # type: ignore[attr-defined] + + data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) # type: ignore[attr-defined] + else: + raise ValueError(f'{message}. Got {index_type}.') + + return data + + def _resolve_model_version_for_run(self, run_id: str) -> str: + """ + Map an MLflow ``run_id`` to the latest registered model version that produced that run. + + ``search_model_versions`` may return multiple versions if the model was registered more than + once for the same run; the highest numeric ``version`` wins so promotion targets the newest + artifact set. + + Args: + - run_id: Run UUID from ``retrain_model`` / experiment payload. + + Return: + str: Registry version string acceptable by ``promote_to_alias``. Raises: - Exception: If transformation fails or MLFlow model is unavailable + ValueError: If the filter returns no versions (model not registered for this run). + """ + + versions = self.mlflow_repository._client.search_model_versions( + filter_string=f"run_id='{run_id}'" + ) + if not versions: + raise ValueError(f'No registered model version found for run_id={run_id}') + latest = max(versions, key=lambda v: int(v.version)) + return str(latest.version) + + def _resolve_model_alias(self, model_config: dict[str, Any] | None = None) -> str: + """ + Resolve which MLflow alias should be used for model lookup/promotion. + + Args: + - model_config: Optional model configuration that may include ``alias``. + + Return: + str: Alias name trimmed and normalized; defaults to ``production``. + """ + if not model_config: + return self._DEFAULT_MODEL_ALIAS + alias = str(model_config.get('alias', self._DEFAULT_MODEL_ALIAS)).strip() + return alias or self._DEFAULT_MODEL_ALIAS + + @activity.defn(name='request_transform') + def request_transform(self, input_data: dict[str, Any]) -> MinioDataFramePayload: + """ + Pivot long-format sensor rows, load the production wrapper, and run ``wrapper.transform``. + + Expected tabular shape after load: columns including ``variable``, ``timestamp``, ``value``, + and ``created_at`` for deduplication. Data are sorted by ``created_at``, de-duplicated per + ``(variable, timestamp)``, pivoted wide, then passed to the model. ``model_config`` may + include ``retention_minutes`` for wrapper cache TTL. + + Args: + - input_data: Dict with ``metadata``, ``model_name``, ``data`` (``MinioDataFramePayload`` + dict or inline dataframe dict), and optional ``model_config``. + + Return: + ``MinioDataFramePayload`` with transformed frame and ``success: True``, or a payload + with ``success: False`` and exception details in ``status`` if transform fails. """ metadata = input_data['metadata'] self.info('Transforming data...', metadata) - data = DataFrame(input_data['data']) + + payload = MinioDataFramePayload.from_dict(input_data['data']) + data = payload.retrieve(self.minio_repository, metadata) + model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) + model_alias = self._resolve_model_alias(model_config) - self.debug("Raw input data:", metadata) - self.debug(data.head(5).to_string(), metadata) + self._debug_dataframe('Raw input data:', data, metadata) - # Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair + # Long → wide: keep newest row per (variable, timestamp), then pivot for the wrapper API. data = data.sort_values('created_at', ascending=False).drop_duplicates( subset=['variable', 'timestamp'], keep='first' ) - # Pivot data for model input format - data = data.pivot( - index='timestamp', columns='variable', - values='value') + data = data.pivot(index='timestamp', columns='variable', values='value') data.fillna(np.nan, inplace=True) - # data.reset_index(inplace=True) + data.columns.name = None + data.index.name = None - self.debug("Processed input data:", metadata) - self.debug(data.head(5).to_string(), metadata) + data['timestamp'] = data.index - # Request transformation from MLFlow model - response_data = self.model_monitoring_repository.transform( - model_name, data, model_config, metadata + self._debug_dataframe('Processed input data:', data, metadata) + + try: + wrapper = self.mlflow_repository.get_cached_model( + model_name=model_name, + alias=model_alias, + retention_minutes=model_config.get('retention_minutes', 0), + metadata=metadata, + ) + transformed_df, transform_meta = wrapper.transform(data) + if transform_meta: + self.info(f'Wrapper transform metadata: {transform_meta}', metadata) + + transformed_df = self._detect_and_parse_datetime_index(transformed_df, metadata) + + response_data: dict[str, Any] = {'success': True, 'content': transformed_df} + except Exception as e: + response_data = { + 'success': False, + 'content': {'message': str(e), 'traceback': traceback.format_exc()}, + } + + self.debug( + f'Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}', + metadata, ) self.debug( - f"Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) + f'Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}', + metadata, + ) - self.debug( - f"Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) + self.info('Data transformed successfully', metadata) - self.info("Data transformed successfully", metadata) + if not response_data.get('success', False): + return MinioDataFramePayload.from_dataframe( + dataframe=None, + minio_repo=self.minio_repository, + model_name=model_name, + operation='transform', + status=response_data, + workflow_metadata=metadata, + last_timestamp=payload.last_timestamp, + logger=self.logger, + ) - return response_data + return MinioDataFramePayload.from_dataframe( + dataframe=response_data['content'], + minio_repo=self.minio_repository, + model_name=model_name, + operation='transform', + workflow_metadata=metadata, + status={ + 'success': True, + }, + last_timestamp=payload.last_timestamp, + logger=self.logger, + ) - @activity.defn(name="request_predict") - async def request_predict(self, input_data: dict[str, Any]) -> dict[str, Any]: + @activity.defn(name='request_predict') + def request_predict(self, input_data: dict[str, Any]) -> MinioDataFramePayload: """ - Execute predictions using MLFlow models. + Load the production wrapper and call ``wrapper.predict`` on the prepared feature frame. - This activity performs ML model inference using MLFlow models with the - transformed data. It handles data format conversion, null value processing, - and model prediction requests with comprehensive error handling. - - The prediction process includes: - 1. Data format validation and cleanup - 2. Null value handling for model compatibility - 3. MLFlow model prediction request - 4. Response validation and logging - 5. Performance monitoring and metrics + The activity normalizes ``NaN`` to ``None`` for JSON-friendly columns, sets the row index + the same way as ``retrain_model`` (UTC ``DatetimeIndex`` from ``DATETIME_FORMAT_WITH_TZ``), + restores that index on the prediction frame, normalizes the prediction index to + ``DATETIME_FORMAT_WITH_TZ`` strings like ``request_transform``, and records ``response_time``. + Non-DataFrame predictions are coerced to a single ``prediction`` column. Args: - input_data: Configuration and data for prediction - Required keys: - - metadata (dict): Workflow execution metadata - - data (dict): Transformed data for prediction - - model_name (str): Name of the MLFlow model to use - - model_retention (int): Model retention period in minutes + - input_data: Same envelope as ``request_transform`` (``metadata``, ``model_name``, + ``data``, optional ``model_config`` with ``retention_minutes``). - Returns: - dict: Prediction results from MLFlow model - - Raises: - Exception: If prediction fails or MLFlow model is unavailable + Return: + ``MinioDataFramePayload`` with predictions or error status mirroring transform behaviour. """ metadata = input_data['metadata'] self.info('Predicting data...', metadata) - data = DataFrame(input_data['data']) + + payload = MinioDataFramePayload.from_dict(input_data['data']) + data = payload.retrieve(self.minio_repository, metadata) + model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) + model_alias = self._resolve_model_alias(model_config) - self.debug(f"Input data for: \n {data.head(5).to_string()}", metadata) + self._debug_dataframe('Input data for prediction:', data, metadata) - # Convert numpy.nan to None for model compatibility data.replace(np.nan, None, inplace=True) - data['timestamp'] = data.index - data['timestamp'] = to_datetime( - data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT) - - # Request prediction from MLFlow model - response_data = self.model_monitoring_repository.predict( - model_name, data, model_config, metadata + data.index = pd.DatetimeIndex( + to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ, utc=True) ) + input_index = data.index + + try: + wrapper = self.mlflow_repository.get_cached_model( + model_name=model_name, + alias=model_alias, + retention_minutes=model_config.get('retention_minutes', 0), + metadata=metadata, + ) + start_time = datetime.now() + predict_data, pred_meta = wrapper.predict({}, data) + end_time = datetime.now() + + if pred_meta: + self.info(f'Wrapper predict metadata: {pred_meta}', metadata) + + if isinstance(predict_data, DataFrame): + self._debug_dataframe( + 'Data received from model prediction:', predict_data, metadata + ) + predict_data.columns = pd.Index(['prediction']) + else: + self.debug( + f'Data received from model prediction (not a DataFrame): {predict_data}', + metadata, + ) + predict_data = pd.DataFrame(predict_data, columns=['prediction']) + + predict_data.index = input_index + predict_data['response_time'] = (end_time - start_time).total_seconds() + predict_data = self._detect_and_parse_datetime_index(predict_data, metadata) + + response_data: dict[str, Any] = {'success': True, 'content': predict_data} + except Exception as e: + response_data = { + 'success': False, + 'content': {'message': str(e), 'traceback': traceback.format_exc()}, + } self.debug( - f"Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata) + f'Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}', + metadata, + ) - self.info("Data predicted successfully", metadata) + self.info('Data predicted successfully', metadata) - return response_data + if not response_data.get('success', False): + return MinioDataFramePayload.from_dataframe( + dataframe=None, + minio_repo=self.minio_repository, + model_name=model_name, + operation='predict', + status=response_data, + workflow_metadata=metadata, + last_timestamp=payload.last_timestamp, + logger=self.logger, + ) - @activity.defn(name="retrain_model") - async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]: + return MinioDataFramePayload.from_dataframe( + dataframe=response_data['content'], + minio_repo=self.minio_repository, + model_name=model_name, + operation='predict', + workflow_metadata=metadata, + status={ + 'success': True, + }, + last_timestamp=payload.last_timestamp, + logger=self.logger, + ) + + @activity.defn(name='retrain_model') + def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]: """ - Retrain MLFlow models with updated training data. + Fit an updated wrapper from historical data, then log and register in MLflow. - This activity orchestrates the complete model retraining process, - including data preparation, model retraining execution, and result - validation. It handles data preprocessing, column cleanup, and - comprehensive error handling for production model management. - - The retraining process includes: - 1. Data timestamp extraction and validation - 2. Column cleanup and data preparation - 3. Data pivoting for model input format - 4. MLFlow model retraining execution - 5. Result validation and error handling + Flow: load long-format data from MinIO → dedupe/pivot like inference prep → require + ``model_config['target']`` → read current ``production`` version for ``source_run_id`` tag → + run ``wrapper.retrain`` outside run timing → ``start_run`` with retrain tags → log input + CSV artifact → ``store_model`` and ``log_params``. Does not promote; the workflow calls + ``update_production_model`` after validation. Args: - input_data (dict): Input data containing: - - metadata (dict): Workflow execution metadata - - data (dict[str, Any]): Training data for model retraining - - model_name (str): Name of the MLFlow model to retrain + - input_data: Must include ``metadata``, ``model_name``, ``data`` (payload), and + ``model_config`` with at least ``target``. - Returns: - dict: Retraining results containing: - - status (str): Retraining operation status - - timestamp (str): Timestamp of the retraining operation - - experiment (str): MLFlow experiment identifier - - Raises: - Exception: If retraining fails or encounters critical errors + Return: + On success: ``success``, ``experiment`` (``run_id``, ``experiment_id``, ``experiment_name``), + ``message``, ``timestamp``. On failure: ``success: False``, error fields, and optional trace. """ + + if self.minio_repository is None: + raise ValueError('Minio repository not initialized') + metadata = input_data['metadata'] - data = DataFrame(input_data['data']) + + try: + payload = MinioDataFramePayload.from_dict(input_data['data']) + data = payload.retrieve(self.minio_repository, metadata) + + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='ERROR_LOADING_RETRAIN_DATA', + message=f'Error loading retrain data: {e}', + block='retrain_model', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + self.error(trace, metadata) + return { + 'success': False, + 'message': f'Error loading retrain data: {e}', + 'traceback': trace, + 'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ), + } + + self.debug(f'Retrain data loaded successfully: shape {data.shape}', metadata) + model_name = input_data['model_name'] + model_config = input_data.get('model_config', {}) self.info(f'Retraining model {model_name}...', metadata) timestamp = data['timestamp'].max() self.debug(f'Timestamp: {timestamp}', metadata) + if 'created_at' in data.columns: + data = data.sort_values('created_at', ascending=False).drop_duplicates( + subset=['variable', 'timestamp'], keep='first' + ) + else: + data = data.drop_duplicates(subset=['variable', 'timestamp'], keep='first') + data.drop(columns=['model_id'], inplace=True, errors='ignore') data.drop(columns=['created_at'], inplace=True, errors='ignore') - data = data.pivot(index='timestamp', columns='variable', - values='value') - data.sort_index(inplace=True) - data.reset_index(inplace=True) - - data = data.dropna() + data = data.pivot(index='timestamp', columns='variable', values='value') + data.fillna(np.nan, inplace=True) data.columns.name = None + data.index.name = None + + data.index = pd.DatetimeIndex( + to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ, utc=True) + ) + + target = model_config.get('target') + if target is None: + msg = 'model_config must include "target" for retraining' + self.info(msg, metadata) + return { + 'success': False, + 'experiment': None, + 'message': msg, + 'traceback': '', + 'timestamp': str(timestamp), + } try: - retrain_output, experiment = self.model_monitoring_repository.retrain_model( - data=data, - model_name=model_name + model_alias = self._resolve_model_alias(model_config) + mv_src = self.mlflow_repository._client.get_model_version_by_alias( + name=model_name, + alias=model_alias, ) + source_run_id = mv_src.run_id + + wrapper = self.mlflow_repository.get_cached_model( + model_name=model_name, + alias=model_alias, + retention_minutes=0, + metadata=metadata, + ) + + # Keep heavy model fitting outside MLflow run timing. + prediction_data = wrapper.retrain(data) + prediction_data.rename(columns={target: 'prediction'}, inplace=True) + + # Merge prediction data with retrain data + evaluation_data = pd.merge( + data, prediction_data, left_index=True, right_index=True, how='left' + ) + + # Rename target column to "target" + evaluation_data.rename(columns={target: 'target'}, inplace=True) + + # Reset index and put as column "timestamp" + evaluation_data['timestamp'] = evaluation_data.index + evaluation_data.reset_index(drop=True, inplace=True) + evaluation_data.sort_values(by='timestamp', inplace=True, ascending=True) + + run_name = f'{model_name}-retrain-{datetime.now().strftime("%Y%m%d%H%M%S")}' + + with self.mlflow_repository.start_run( + model_name=model_name, + run_name=run_name, + experiment_name=model_name, + tags={'retrain': 'true', 'source_run_id': source_run_id}, + metadata=metadata, + ) as run_info: + tmp_dir = tempfile.mkdtemp(prefix='laborious_retrain_') + try: + raw_csv = Path(tmp_dir) / 'retrain_input.csv' + evaluation_csv = Path(tmp_dir) / 'evaluation_data.csv' + data.to_csv(raw_csv, index=False) + evaluation_data.to_csv(evaluation_csv, index=False) + + mlflow.log_artifact(str(raw_csv)) + mlflow.log_artifact(str(evaluation_csv)) + finally: + rmtree(tmp_dir, ignore_errors=True) + + wrapper.store_model(name=model_name) + + self.mlflow_repository.log_params( + { + 'retrain': 'true', + 'retrain_date': datetime.now().isoformat(), + 'source_run_id': source_run_id, + 'retrain_samples': str(data.shape), + } + ) + + experiment_payload = { + 'run_id': run_info.run_id, + 'experiment_id': run_info.experiment_id, + 'experiment_name': model_name, + } return { - 'status': retrain_output, - 'timestamp': timestamp, - 'experiment': experiment + 'success': True, + 'experiment': experiment_payload, + 'message': 'Model retrained successfully.', + 'timestamp': str(timestamp), } + except Exception as e: - trace = traceback.format_exc() - self.send_notification( - metadata=metadata, - notification_id='RETRAIN_MODEL_ERROR', - message=f'Error retraining model {model_name}: {e}', - block='retrain_model', - level=NotificationLevel.ERROR, - attachment_content=trace - ) - self.error(trace, metadata=metadata) - raise e + error_msg = f'Error retraining model {model_name}: {e}' + self.info(error_msg, metadata) + return { + 'success': False, + 'experiment': None, + 'message': error_msg, + 'traceback': traceback.format_exc(), + 'timestamp': str(timestamp), + } - @activity.defn(name="update_production_model") - async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]: + @activity.defn(name='update_production_model') + def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]: """ - Update production model with newly trained model version. + Point the ``production`` alias at the model version registered for the retrain run. - This activity manages the critical process of updating production - models with newly trained versions. It handles model deployment, - status tracking, and comprehensive reporting for operational - visibility and audit trails. - - The update process includes: - 1. Production model update execution - 2. Status and metadata tracking - 3. Comprehensive reporting and logging - 4. Error handling and notification - 5. Audit trail maintenance + Resolves the highest numeric registry version whose ``run_id`` matches + ``experiment['run_id']``, then calls ``promote_to_alias``. On failure, sends a notification + and re-raises so the workflow can surface the error. Args: - input_data (dict): Input data containing: - - metadata (dict): Workflow execution metadata - - model_name (str): Name of the MLFlow model to update - - experiment (str): MLFlow experiment identifier - - model_id (str): Unique identifier for the model version - - timestamp (str): Timestamp of the update operation - - status (str): Current status of the model update + - input_data: ``metadata``, ``model_name``, and ``experiment`` with ``run_id`` and + ``experiment_id`` (as returned from ``retrain_model``). - Returns: - dict[Any, Any]: Comprehensive update report containing: - - model_id (str): Model version identifier - - model_name (str): Name of the updated model - - timestamp (str): Update operation timestamp - - status (str): Update operation status - - Additional MLFlow response metadata - - Raises: - Exception: If production model update fails + Return: + Dict with ``model_name``, promoted ``version``, ``mlflow_run_id``, ``mlflow_experiment_id``. """ metadata = input_data['metadata'] model_name = input_data['model_name'] - model_id = input_data['model_id'] experiment = input_data['experiment'] - timestamp = input_data['timestamp'] - status = input_data['status'] - self.info( - f'Updating production model {model_name} from experiment {experiment}...', metadata) + f'Updating production model {model_name} from experiment {experiment}...', metadata + ) try: - response = self.model_monitoring_repository.update_production_model( - experiment=experiment, - model_name=model_name + run_id = experiment['run_id'] + experiment_id = experiment['experiment_id'] + + version = self._resolve_model_version_for_run(run_id) + + promote_alias = self._resolve_model_alias(input_data.get('model_config')) + self.mlflow_repository.promote_to_alias( + model_name=model_name, + version=version, + alias=promote_alias, + metadata=metadata, ) - report = DataFrame([response]) - report['model_id'] = model_id - report['model_name'] = model_name - report['timestamp'] = timestamp - report['status'] = status - - self.info( - f'Production model {model_name} updated successfully', metadata) - return report.to_dict() + self.info(f'Production model {model_name} updated successfully', metadata) + return { + 'model_name': model_name, + 'version': version, + 'mlflow_run_id': run_id, + 'mlflow_experiment_id': experiment_id, + } except Exception as e: trace = traceback.format_exc() @@ -337,7 +653,103 @@ class MLFlow(BaseActivity): message=f'Error updating production model {model_name}: {e}', block='update_production_model', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) self.error(trace, metadata=metadata) raise e + + def _resolve_reference_artifact_name(self, run_id: str) -> str | None: + """ + Pick the first available reference CSV artifact path from the MLflow run. + + Candidates are checked in priority order: ``retrain_input.csv``, then ``train_data.csv``. + A path matches when it equals the candidate or ends with ``/`` for nested layouts. + + Args: + - run_id: MLflow run UUID linked to the production model version. + + Return: + Artifact path string for ``download_artifacts``, or ``None`` if no candidate exists. + """ + listed = self.mlflow_repository._client.list_artifacts(run_id) + paths = [file_info.path for file_info in listed] + for candidate in self._REFERENCE_ARTIFACT_CANDIDATES: + for path in paths: + if path == candidate or path.endswith(f'/{candidate}'): + return path + return None + + def _find_downloaded_csv(self, tmpdir: str, artifact_name: str) -> Path | None: + """ + Locate a downloaded reference CSV in the temp directory. + + Args: + - tmpdir: Directory where ``download_artifacts`` wrote files. + - artifact_name: Basename of the resolved artifact (e.g. ``retrain_input.csv``). + + Return: + ``Path`` to the CSV file if found, else ``None``. + """ + direct = Path(tmpdir) / artifact_name + if direct.exists(): + return direct + matches = list(Path(tmpdir).rglob(artifact_name)) + return matches[0] if matches else None + + @activity.defn(name='get_reference_data') + def get_reference_data(self, input_data: dict[str, Any]) -> list[dict] | None: + """ + Download reference training CSV from the MLflow run linked to the production alias. + + Resolves ``retrain_input.csv`` or ``train_data.csv`` via artifact listing before download. + ``retrain_input.csv`` is preferred when both exist (most recent retrain snapshot). Used by + drift workflows to compare live data against the reference distribution logged with the model. + Timestamps are normalized to ``DATETIME_FORMAT`` string columns before returning records. + + Args: + - input_data: ``metadata``, ``model_name``, and optional ``model_config`` with ``alias``. + + Return: + List of row dicts with normalized timestamps, or ``None`` if resolution or load fails. + """ + metadata = input_data['metadata'] + model_name = input_data['model_name'] + + try: + model_alias = self._resolve_model_alias(input_data.get('model_config')) + mv = self.mlflow_repository._client.get_model_version_by_alias( + name=model_name, + alias=model_alias, + ) + run_id = mv.run_id + + artifact_path = self._resolve_reference_artifact_name(run_id) + if artifact_path is None: + self.warning(f'Reference data not found for model {model_name}', metadata) + return None + + artifact_name = Path(artifact_path).name + tmpdir = tempfile.mkdtemp(prefix='laborious_eval_') + try: + self.mlflow_repository.download_artifacts( + run_id=run_id, + artifact_path=artifact_path, + dst_path=tmpdir, + metadata=metadata, + ) + csv_path = self._find_downloaded_csv(tmpdir, artifact_name) + if csv_path is None: + self.warning(f'Reference data not found for model {model_name}', metadata) + return None + reference_data = pd.read_csv(csv_path) + finally: + rmtree(tmpdir, ignore_errors=True) + + reference_data['timestamp'] = to_datetime(reference_data['timestamp']) + reference_data['timestamp'] = reference_data['timestamp'].dt.strftime(DATETIME_FORMAT) + + return reference_data.to_dict(orient='records') + + except Exception as e: + self.warning(f'Reference data not found for model {model_name}: {e}', metadata) + return None diff --git a/laborious/activities/model_metrics.py b/laborious/activities/model_metrics.py new file mode 100644 index 0000000..cb44805 --- /dev/null +++ b/laborious/activities/model_metrics.py @@ -0,0 +1,432 @@ +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + import time + import traceback + import warnings + from typing import Any + + import numpy as np + import pandas as pd + from pandas import DataFrame, Index, Series, to_datetime + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + from sientia_model.analytics.drift_analysis import DriftAnalysis, DriftInsufficientDataError + + from laborious import metrics + from laborious.utils.dataframe_debug import build_dataframe_debug_message + +warnings.filterwarnings('ignore', category=RuntimeWarning, message='Degrees of freedom <= 0') +warnings.filterwarnings( + 'ignore', category=RuntimeWarning, message='invalid value encountered in scalar divide' +) + + +class ModelMetrics(SientiaMonitoring): + """ + Metrics and statistical analysis activities for the Laborious pipeline. + + This class centralizes drift/statistical computations and model-quality + aggregates used by scheduled workflows. Besides producing tabular outputs + for persistence, it also emits operational metrics (count, lag, error) + through ``SientiaMonitoring`` so execution health is observable in runtime. + """ + + _MAX_DEBUG_DATAFRAME_ROWS = 100 + + def __init__( + self, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) + + def close(self) -> None: + """ + Shutdown monitoring resources associated with model metrics activities. + + This is invoked during worker teardown to flush/close metric controller + internals and prevent dangling telemetry tasks. + """ + SientiaMonitoring.shutdown(self) + + def __del__(self): + self.close() + + def _debug_dataframe(self, message: str, data: Any, metadata: dict[str, Any]) -> None: + """ + Log dataframe content only when row count is below the configured threshold + + Args: + - message (str): Base log message to identify the dataframe in logs + - data (Any): Dataframe-like payload to be logged + - metadata (dict[str, Any]): Workflow metadata for contextual logging + """ + self.debug( + build_dataframe_debug_message( + message=message, + data=data, + max_rows=self._MAX_DEBUG_DATAFRAME_ROWS, + ), + metadata, + ) + + def _drift_analyze_stage_error( + self, + exc: Exception, + context: str, + metadata: dict[str, Any], + core_labels: dict[str, Any], + ) -> None: + """ + Log analyzer failure for a drift stage and increment the analyze error metric. + + Args: + - exc (Exception): Failure raised by ``sientia_model``. + - context (str): Short label for the log line (e.g. univariate detection). + - metadata (dict[str, Any]): Workflow metadata for logging. + - core_labels (dict[str, Any]): Tags from ``get_core_labels`` for metrics. + """ + self.error(f'{context}: {exc}', metadata) + self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_ERROR_COUNT, tags=core_labels) + + def get_drift_metrics( + self, + reference_data: DataFrame, + target_data: DataFrame, + target_name: str, + reference_columns: Index, + drift_metrics: list[str], + chunk_period: str, + metadata: dict[str, Any], + ) -> DataFrame: + """ + Compute univariate and multivariate drift outputs and merge them into one dataframe. + + The method orchestrates three analysis stages (univariate drift, + multivariate drift, and dataframe projection), emitting lag/count/error + metrics for each stage independently so failures are attributable. + + Args: + - reference_data (DataFrame): Baseline dataset representing expected behavior. + - target_data (DataFrame): Current analysis dataset to compare against reference. + - target_name (str): Target column name used by ``DriftAnalysis`` config. + - reference_columns (Index): Feature columns evaluated for drift. + - drift_metrics (list[str]): Enabled univariate methods. + - chunk_period (str): Time bucket granularity used by analysis methods. + - metadata (dict[str, Any]): Workflow metadata for logs and notifications. + + Return: + DataFrame: Consolidated drift dataframe from ``get_drift_metrics_dataframe`` using + ``method`` / ``value`` (and optional ``threshold``, ``drift_type``), ready for + activity-level formatting before Postgres export. + """ + # ``DriftAnalysis`` uses truthiness checks on ``features`` (e.g. ``if not features``); + # a pandas ``Index`` is ambiguous in boolean context — normalize to a list. + feature_names: list[str] = list(reference_columns) + + config = { + 'target': target_name, + 'prediction': 'prediction', + 'timestamp': 'timestamp', + 'features': feature_names, + } + + drift_analysis = DriftAnalysis(config=config) + + self._debug_dataframe( + f'Reference data: Size {reference_data.shape}', reference_data, metadata + ) + + self._debug_dataframe(f'Target data: Size {target_data.shape}', target_data, metadata) + + core_labels = self.get_core_labels(metadata, operation_type='detect_univariate_drift') + start_time = time.time() + try: + univariate_drift = drift_analysis.detect_univariate_drift( + reference_df=reference_data, + analysis_df=target_data, + features=feature_names, + timestamp_col=config['timestamp'], + methods=drift_metrics, + chunk_period=chunk_period, + ) + except Exception as e: + if isinstance(e, DriftInsufficientDataError): + raise + self._drift_analyze_stage_error( + e, 'Error detecting univariate drift', metadata, core_labels + ) + raise + self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels) + self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels) + + core_labels = self.get_core_labels(metadata, operation_type='detect_multivariate_drift') + start_time = time.time() + try: + multivariate_drift = drift_analysis.detect_multivariate_drift( + reference_df=reference_data, + analysis_df=target_data, + features=feature_names, + timestamp_col=config['timestamp'], + chunk_period=chunk_period, + ) + except Exception as e: + if isinstance(e, DriftInsufficientDataError): + raise + self._drift_analyze_stage_error( + e, 'Error detecting multivariate drift', metadata, core_labels + ) + raise + self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels) + self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels) + + start_time = time.time() + core_labels = self.get_core_labels(metadata, operation_type='get_drift_metrics_dataframe') + try: + drift_df = drift_analysis.get_drift_metrics_dataframe( + univariate_drift=univariate_drift, + multivariate_drift=multivariate_drift, + ) + except Exception as e: + if isinstance(e, DriftInsufficientDataError): + raise + self._drift_analyze_stage_error( + e, 'Error building drift metrics dataframe', metadata, core_labels + ) + raise + self.observe_lag_sync(start_time, metrics.MODEL_ANALYZE_LAG, core_labels) + self.emit_metric_sync(metric_object=metrics.MODEL_ANALYZE_COUNT, tags=core_labels) + + return drift_df + + @staticmethod + def _to_naive_utc(series: Series) -> Series: + """ + Parse ``series`` as datetime and return a TZ-naive UTC copy. + + ``sientia_model.analytics.drift_analysis.DriftAnalysis`` preserves the + timezone of the input dataframe in its outputs, while target rows + loaded from PostgreSQL come in with ``+00:00``. Forcing both sides of + a comparison to TZ-naive UTC keeps ``isin`` / ``floor`` operations + deterministic regardless of how the analyzer constructs its + timestamps. + + Args: + - series (Series): Input series containing datetime-parseable values. + + Return: + Series: Datetime64 series with ``tz=None`` representing UTC instants. + """ + parsed = to_datetime(series) + if getattr(parsed.dt, 'tz', None) is not None: + parsed = parsed.dt.tz_convert('UTC').dt.tz_localize(None) + return parsed + + @activity.defn(name='calculate_drift') + def calculate_drift(self, input_data: dict[str, Any]) -> list[dict]: + """ + Calculate drift metrics for a model. + + Args: + input_data (dict[str, Any]): Input data containing: + - metadata (dict): Workflow execution metadata + - model_name (str): Name of the MLFlow model to calculate drift for + - reference_data (pd.DataFrame): Reference data for the model + - target_data (pd.DataFrame): Target data for calculating drift + - target_name (str): Name of the target column + - drift_metrics (list[str]): List of drift metrics to calculate + """ + metadata = input_data['metadata'] + model_name = input_data['model_name'] + model_id = input_data['model_id'] + reference_raw_data = input_data['reference_data'] + target_data = DataFrame(input_data['target_data']) + target_name = input_data['target_name'] + drift_metrics = input_data['drift_metrics'] + chunk_period = input_data['chunk_period'] + + if chunk_period not in ['min', 's']: + self.error(f'Invalid chunk period: {chunk_period}', metadata) + raise ValueError(f'Invalid chunk period: {chunk_period}, must be "min" or "s"') + + self.info(f'Calculating drift for model {model_name}', metadata) + + target_data = target_data.pivot(index='timestamp', columns='variable', values='value') + target_data['timestamp'] = target_data.index + # Keep timestamps as datetime: DriftAnalysis._chunk_dataframe relies on + # ``pd.Grouper(freq=...)`` which rejects string timestamp columns. + target_data['timestamp'] = to_datetime(target_data['timestamp']) + target_data = target_data.reset_index(drop=True) + target_data.dropna(inplace=True) + + if reference_raw_data is not None: + self.info('Using reference data', metadata) + reference_data = DataFrame(reference_raw_data) + if 'timestamp' in reference_data.columns: + reference_data['timestamp'] = to_datetime(reference_data['timestamp']) + accurate = True + else: + # Get 30% first rows of target_data + self.warning('Using 30% first rows of target data as reference data', metadata) + target_data.sort_values(by='timestamp', ascending=True, inplace=True) + reference_data = target_data.head(int(len(target_data) * 0.3)) + accurate = False + + self.send_notification( + metadata=metadata, + notification_id='MODEL_METRICS_REFERENCE_DATA_WARNING', + message='Using 30% first rows of target data as reference data', + block='model_metrics', + level=NotificationLevel.WARNING, + attachment_content=reference_data.to_csv(), + ) + + reference_columns = reference_data.drop( + columns=[target_name, 'timestamp', 'target', 'prediction'], errors='ignore' + ).columns + + try: + drift_df = self.get_drift_metrics( + reference_data=reference_data, + target_data=target_data, + target_name=target_name, + reference_columns=reference_columns, + drift_metrics=drift_metrics, + chunk_period=chunk_period, + metadata=metadata, + ) + except Exception as e: + if isinstance(e, DriftInsufficientDataError): + self.error(str(e), metadata) + notification_id = e.notification_id + notification_message = str(e) + else: + self.error(f'Error getting drift metrics: {e}', metadata) + notification_id = 'MODEL_METRICS_GET_DRIFT_METRICS_ERROR' + notification_message = f'Error getting drift metrics: {e}' + self.send_notification( + metadata=metadata, + notification_id=notification_id, + message=notification_message, + block='model_metrics', + level=NotificationLevel.ERROR, + attachment_content=traceback.format_exc(), + ) + raise + + # Drop chunks whose floored timestamp does not appear in the analysis window. + # ``DriftAnalysis`` chunks over ``analysis_df``; this only excludes rows that + # do not belong to the current target window (e.g. stray merged reference rows). + target_floor = self._to_naive_utc(target_data['timestamp']).dt.floor(chunk_period) + drift_floor = self._to_naive_utc(drift_df['timestamp']).dt.floor(chunk_period) + drift_df = drift_df[drift_floor.isin(target_floor)] + + if drift_df.empty: + self.warning( + 'No drift metrics found after dropping rows where timestamp is not in target data', + metadata, + ) + return [] + + # Analyzer emits diagnostic columns that are not stored in ``sientia_data.drift_metrics``. + drift_df = drift_df.drop(columns=['threshold', 'drift_type'], errors='ignore') + + drift_df['model_id'] = str(model_id) + drift_df['accurate'] = accurate + + # ``timestamp`` is overridden with the most recent target instant so + # every persisted row shares a single business timestamp (the run's + # logical "now"), matching what downstream consumers expect. + latest_target_timestamp = self._to_naive_utc(target_data['timestamp']).max() + drift_df['timestamp'] = ( + pd.Timestamp(latest_target_timestamp) + .tz_localize('UTC') + .strftime(DATETIME_FORMAT_WITH_TZ) + ) + + # ``chunk_start_date`` / ``chunk_end_date`` may carry nanosecond + # precision (beyond ``timestamptz`` microseconds), so serialize as ISO + # text for the ``text`` Postgres columns. + for column in ('chunk_start_date', 'chunk_end_date'): + drift_df[column] = drift_df[column].apply( + lambda value: pd.Timestamp(value).isoformat() if pd.notna(value) else None + ) + + self._debug_dataframe(f'Drift dataframe: Size {drift_df.shape}', drift_df, metadata) + + return drift_df.to_dict(orient='records') + + @activity.defn(name='calculate_simple_metrics') + def calculate_simple_metrics(self, input_data: dict[str, Any]) -> list[dict]: + """ + Calculate simple metrics for a model. Metrics available are: + - rmse + - mse + - mae + - r2 + - accuracy + - precision + - recall + - f1 + Args: + input_data (dict[str, Any]): Input data containing: + - metadata (dict): Workflow execution metadata + - model_id (str): ID of the MLFlow model + - target_data (pd.DataFrame): Target data for calculating metrics, containing target and prediction columns + - metrics (list[str]): List of metrics to calculate + Returns: + dict[Hashable, Any]: Dictionary containing the calculated metrics + """ + + metadata = input_data['metadata'] + model_id = input_data['model_id'] + target_data = DataFrame(input_data['target_data']) + metric_names = input_data['metrics'] + interval_minutes = input_data['interval_minutes'] + + data_size = target_data.shape[0] + + output_data = [] + + diff = target_data['target'] - target_data['prediction'] + diff_squared = diff**2 + + self.info(f'Calculating simple metrics for model {model_id}: {metric_names}', metadata) + + for metric in metric_names: + if metric == 'rmse': + output_data.append({'metric': 'rmse', 'value': np.sqrt(np.mean(diff_squared))}) + elif metric == 'mse': + output_data.append({'metric': 'mse', 'value': np.mean(diff_squared)}) + elif metric == 'mae': + output_data.append({'metric': 'mae', 'value': np.mean(np.abs(diff))}) + elif metric == 'r2': + y_true = target_data['target'] + y_mean = np.mean(y_true) + + ss_res = np.sum(diff_squared) + ss_tot = np.sum((y_true - y_mean) ** 2) + + # Evita divisão por zero + if ss_tot == 0: + r2_score = 0.0 + else: + r2_score = 1 - (ss_res / ss_tot) + + output_data.append({'metric': 'r2', 'value': r2_score}) + + data = DataFrame(output_data) + data['model_id'] = model_id + data['timestamp'] = target_data['timestamp'].max() + data['data_size'] = data_size + data['interval_minutes'] = interval_minutes + + self._debug_dataframe(f'Simple metrics dataframe: Size {data.shape}', data, metadata) + + return data.to_dict(orient='records') diff --git a/laborious/activities/opc.py b/laborious/activities/opc.py index 0b4ec9f..ebe285c 100644 --- a/laborious/activities/opc.py +++ b/laborious/activities/opc.py @@ -1,20 +1,62 @@ from temporalio import activity, workflow - with workflow.unsafe.imports_passed_through(): + import traceback + from collections.abc import Hashable + from typing import Any + + from pandas import DataFrame from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel - from sientia_do.temporal.activities.base import BaseActivity from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from laborious.utils.repository.opc_repository import OpcRepository - from typing import Any - import traceback - from pandas import DataFrame OPC_WRITTING_ERROR_CONFIDENCE = 12 +OPC_SESSION_BAD_CONFIDENCE = 14 +OPC_SESSION_BAD_COMMENT_PREFIX = 'OPC UA session/channel error:' +OPC_WRITTING_ERROR_MESSAGE = 'Some data could not be written to OPC servers' +OPC_RECONNECT_IN_PROGRESS_COMMENT = 'OPC UA reconnect in progress' +OPC_COMMENT_SEPARATOR = ' | ' -class OPC(BaseActivity): +def _opc_session_bad_comment(opc_status: str | None) -> str: + status = opc_status or 'Unknown' + return f'{OPC_SESSION_BAD_COMMENT_PREFIX} {status}' + + +def _apply_opc_write_error( + error_info: dict[str, Any] | None, + session_bad_seen: bool, + session_bad_status: str | None, + reconnect_in_progress_seen: bool, +) -> tuple[bool, str | None, bool]: + """ + Update session/reconnect flags from an OPC write error payload. + + Args: + error_info: Repository error details, or None when the write succeeded. + session_bad_seen: Whether a session_bad error was seen so far. + session_bad_status: Last known OPC status for session errors. + reconnect_in_progress_seen: Whether reconnect_in_progress was seen so far. + + Return: + Updated (session_bad_seen, session_bad_status, reconnect_in_progress_seen). + """ + if not error_info: + return session_bad_seen, session_bad_status, reconnect_in_progress_seen + + kind = error_info.get('opc_error_kind') + if kind == 'session_bad': + return True, error_info.get('opc_status', session_bad_status), reconnect_in_progress_seen + if kind == 'reconnect_in_progress': + return session_bad_seen, session_bad_status, True + return session_bad_seen, session_bad_status, reconnect_in_progress_seen + + +class OPC(SientiaMonitoring): """ OPC server integration activities for real-time data export. @@ -33,20 +75,20 @@ class OPC(BaseActivity): notification_handler (NotificationHandler): Notification management instance """ - def __init__(self, opc_servers: dict[str, dict[str, Any]], - logger: Logger, notification_handler: NotificationHandler): - - self.logger = logger - self.notification_handler = notification_handler + def __init__( + self, + opc_servers: dict[str, dict[str, Any]], + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + ): self.opc_servers = opc_servers - BaseActivity.__init__( - self, logger, notification_handler, set_error_counter=True) + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) self.opc_repository: dict[str, OpcRepository] = {} - self.opc_servers = opc_servers - async def init_opc(self): + def init_opc(self): """ Initialize OPC server connections and establish communication channels. @@ -70,10 +112,11 @@ class OPC(BaseActivity): the initialization of other OPC servers. Each server is handled independently to ensure maximum availability. """ - self.logger.info("Initializing OPC servers...") - for id, server in self.opc_servers.items(): - self.opc_repository[id] = OpcRepository( - id=server['id'], + self.info('Initializing OPC servers...') + for opc_id, server in self.opc_servers.items(): + self.opc_repository[opc_id] = OpcRepository( + opc_id=opc_id, + server_name=server['server_name'], url=server['url'], logger=self.logger, server_uri=server['server_uri'], @@ -81,76 +124,70 @@ class OPC(BaseActivity): private_key_path=server['private_key_path'], server_cert_path=server['server_cert_path'], notification_handler=self.notification_handler, - reconnection_interval=server['reconnection_interval'], - pod_id=self.pod_id + reconnection_interval=server.get('reconnection_interval', 60), + metrics_controller=self.metrics_controller, ) - is_connected, error_data = await self.opc_repository[id].connect() + is_connected, error_data = self.opc_repository[opc_id].connect() if not is_connected: self.send_notification( metadata={ 'model_id': '-', 'model_name': '-', 'workflow_name': '-', - 'schedule_name': 'INITIALIZATION' + 'schedule_name': 'INITIALIZATION', }, notification_id=error_data['notification_id'], message=error_data['message'], block=error_data['block'], level=error_data.get('level', NotificationLevel.ERROR), - attachment_content=error_data.get( - 'attachment_content', None) + attachment_content=error_data.get('attachment_content', None), ) else: - self.logger.info( - f"OPC server {id} connected successfully.") + self.info(f'OPC server {opc_id}:{server["server_name"]} connected successfully.') - async def write_data(self, server_id: str, tag: str, data: Any, - data_type: str, tag_type: str, metadata: dict[str, Any]) -> bool: + def write_data( + self, + server_id: str, + tag: str, + data: Any, + data_type: str, + tag_type: str, + metadata: dict[str, Any], + ) -> tuple[float | None, dict[str, Any] | None]: """ Write data to a specific OPC server tag with comprehensive error handling. - This method provides a secure and reliable way to write data to OPC servers - with automatic error handling, notification integration, and detailed logging. - It validates server availability before attempting write operations and - provides comprehensive error reporting for operational monitoring. - - Args: - - server_id (str): The id of the OPC server. - - tag (str): The tag to write to. - - data (Any): The data to write. - - data_type (str): The data type. - - tag_type (str): The tag type. - - Returns: - - bool: True if the data was written successfully, False otherwise. + Return: + tuple[float | None, dict[str, Any] | None]: Response time on success, or + (None, error info_data) on repository failure. """ try: - is_success, error_data = await self.opc_repository[server_id].write_data( - tag, data, data_type, self.logger, metadata) + is_success, info_data = self.opc_repository[server_id].write_data( + tag, data, data_type, metadata + ) if not is_success: self.send_notification( metadata=metadata, - notification_id=error_data['notification_id'], - message=error_data['message'], - block=error_data['block'], - level=error_data.get('level', NotificationLevel.ERROR), - attachment_content=error_data.get( - 'attachment_content', None) + notification_id=info_data['notification_id'], + message=info_data['message'], + block=info_data['block'], + level=info_data.get('level', NotificationLevel.ERROR), + attachment_content=info_data.get('attachment_content', None), ) - return False - return True + return None, info_data + return info_data['response_time'], None except Exception as e: trace = traceback.format_exc() self.send_notification( metadata=metadata, - notification_id=f"WRITE_OPC_{tag_type.upper()}_ERROR", - message=f"Error writing data to OPC server: {e}", - block="write_opc_data", + notification_id=f'WRITE_OPC_{tag_type.upper()}_ERROR', + message=f'Error writing data to OPC server: {e}', + block='write_opc_data', level=NotificationLevel.ERROR, - attachment_content=trace + attachment_content=trace, ) - raise e + raise def validate_server(self, server_id: str, metadata: dict[str, Any]) -> bool: """ @@ -174,21 +211,81 @@ class OPC(BaseActivity): This helps operators quickly identify configuration issues. """ if self.opc_repository.get(server_id) is None: - message = f"OPC server {server_id} not found to perform write operation." + message = f'OPC server {server_id} not found to perform write operation.' self.send_notification( metadata=metadata, - notification_id="OPC_SERVER_NOT_FOUND", + notification_id='OPC_SERVER_NOT_FOUND', message=message, - block="write_opc_data", + block='write_opc_data', level=NotificationLevel.ERROR, - attachment_content=f"OPC servers: {list(self.opc_repository.keys())}" + attachment_content=f'OPC servers: {list(self.opc_repository.keys())}', ) return False return True - async def manage_output_tags( - self, server_id: str, config: dict[str, Any], data: DataFrame, - metadata: dict[str, Any], success: bool) -> tuple[bool, int]: + def _write_tags_from_config( + self, + server_id: str, + tags_config: dict[str, dict[str, Any]], + data: DataFrame, + data_column: str, + tag_type: str, + log_label: str, + metadata: dict[str, Any], + ) -> tuple[dict[str, float | None], bool, str | None, bool]: + """ + Write a group of OPC tags and collect response times and error flags. + + Args: + server_id: Target OPC server identifier. + tags_config: Tag name to configuration mapping. + data: DataFrame with prediction/confidence columns. + data_column: Column name whose first row value is written. + tag_type: Tag category passed to write_data ('prediction' or 'confidence'). + log_label: Human-readable label for success logs. + metadata: Context metadata for logging and notifications. + + Return: + (response_times, session_bad_seen, session_bad_status, reconnect_in_progress_seen) + """ + response_times: dict[str, float | None] = {} + session_bad_seen = False + session_bad_status: str | None = None + reconnect_in_progress_seen = False + + for tag, tag_config in tags_config.items(): + response_time, error_info = self.write_data( + server_id=server_id, + tag=tag, + data=data.head(1)[data_column].values[0], + data_type=tag_config['data_type'], + tag_type=tag_type, + metadata=metadata, + ) + session_bad_seen, session_bad_status, reconnect_in_progress_seen = ( + _apply_opc_write_error( + error_info, + session_bad_seen, + session_bad_status, + reconnect_in_progress_seen, + ) + ) + if response_time is not None: + self.info( + f'{log_label} written to OPC server {server_id} for tag {tag}.', + metadata, + ) + response_times[tag] = response_time + + return response_times, session_bad_seen, session_bad_status, reconnect_in_progress_seen + + def manage_output_tags( + self, + server_id: str, + config: dict[str, Any], + data: DataFrame, + metadata: dict[str, Any], + ) -> tuple[bool, dict[str, float | None], bool, str | None, bool]: """ Manage the writing of prediction and confidence data to OPC server tags. @@ -215,44 +312,52 @@ class OPC(BaseActivity): - overall_success: True if all configured tags were written successfully - total_tags_written: Count of successfully written tags """ + response_times: dict[str, float | None] = {} + session_bad_seen = False + session_bad_status: str | None = None + reconnect_in_progress_seen = False - count = 0 - if 'prediction_tags' in config: - for tag, tag_config in config['prediction_tags'].items(): - local_success = await self.write_data( - server_id=server_id, - tag=tag, - data=data.head(1)['prediction'].values[0], - data_type=tag_config['data_type'], - tag_type='prediction', - metadata=metadata - ) - if local_success: - self.info( - f"Prediction data written to OPC server {server_id} for tag {tag}.", metadata) - count += 1 - success = success and local_success + tag_groups = ( + ('prediction_tags', 'prediction', 'prediction', 'Prediction data'), + ('confidence_tags', 'prediction_confidence', 'confidence', 'Confidence data'), + ) + for config_key, data_column, tag_type, log_label in tag_groups: + if config_key not in config: + continue + ( + group_times, + group_session_bad, + group_status, + group_reconnect, + ) = self._write_tags_from_config( + server_id=server_id, + tags_config=config[config_key], + data=data, + data_column=data_column, + tag_type=tag_type, + log_label=log_label, + metadata=metadata, + ) + response_times.update(group_times) + if group_session_bad: + session_bad_seen = True + session_bad_status = group_status or session_bad_status + if group_reconnect: + reconnect_in_progress_seen = True - if 'confidence_tags' in config: - for tag, tag_config in config['confidence_tags'].items(): - local_success = await self.write_data( - server_id=server_id, - tag=tag, - data=data.head(1)['prediction_confidence'].values[0], - data_type=tag_config['data_type'], - tag_type='confidence', - metadata=metadata - ) - if local_success: - self.info( - f"Confidence data written to OPC server {server_id} for tag {tag}.", metadata) - count += 1 - success = success and local_success - - return success, count + success = None not in response_times.values() + return ( + success, + response_times, + session_bad_seen, + session_bad_status, + reconnect_in_progress_seen, + ) @activity.defn(name='write_opc_data') - async def write_opc_data(self, input_data: dict[str, Any]) -> dict[Any, Any]: + def write_opc_data( + self, input_data: dict[str, Any] + ) -> tuple[dict[Hashable, Any], dict[str, dict[str, float | None]]]: """ Write prediction and confidence data to OPC servers. The two writing operations are optional and independent of each other. @@ -271,29 +376,68 @@ class OPC(BaseActivity): """ metadata = input_data['metadata'] - self.info("Writing data to OPC servers...", metadata) + self.info('Writing data to OPC servers...', metadata) data = DataFrame(input_data['data']) opc_output_config = input_data['opc_output_config'] - self.info(f"Data to write: {data.size} rows", metadata) + self.info(f'Data to write: {data.size} rows', metadata) success = True + session_bad_seen = False + session_bad_status: str | None = None + reconnect_in_progress_seen = False + + opc_metrics: dict[str, dict[str, float | None]] = {} for server_id, config in opc_output_config.items(): - if not self.validate_server(server_id, metadata): success = False continue - local_success, local_count = await self.manage_output_tags( - server_id, config, data, metadata, success) + ( + local_success, + local_response_times, + local_session_bad, + local_status, + local_reconnect_in_progress, + ) = self.manage_output_tags(server_id, config, data, metadata) + opc_metrics[server_id] = local_response_times + local_count = len(local_response_times) success = success and local_success + if local_session_bad: + session_bad_seen = True + session_bad_status = local_status or session_bad_status + if local_reconnect_in_progress: + reconnect_in_progress_seen = True + n_pred = len(config.get('prediction_tags') or {}) + n_conf = len(config.get('confidence_tags') or {}) self.info( - f"Process completed for OPC server {server_id}: {local_count} of {len(config['prediction_tags'])} prediction tags and {len(config['confidence_tags'])} confidence tags", metadata) + f'Process completed for OPC server {server_id}: {local_count} of {n_pred} prediction tags and {n_conf} confidence tags', + metadata, + ) - return self.process_confidence(data, success, metadata) + return ( + self.process_confidence( + data, + success, + metadata, + session_bad=session_bad_seen, + opc_status=session_bad_status, + reconnect_in_progress=reconnect_in_progress_seen, + ), + opc_metrics, + ) - def process_confidence(self, data: DataFrame, success: bool, metadata: dict[str, Any]) -> dict[Any, Any]: + def process_confidence( + self, + data: DataFrame, + success: bool, + metadata: dict[str, Any], + *, + session_bad: bool = False, + opc_status: str | None = None, + reconnect_in_progress: bool = False, + ) -> dict[Hashable, Any]: """ Process prediction confidence based on OPC write operation success. @@ -321,18 +465,31 @@ class OPC(BaseActivity): """ if not success: - data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE - self.debug( - f"Some data could not be written to OPC servers, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.", - metadata - ) + comment_parts: list[str] = [] + confidence = OPC_WRITTING_ERROR_CONFIDENCE + if session_bad: + comment_parts.append(_opc_session_bad_comment(opc_status)) + confidence = OPC_SESSION_BAD_CONFIDENCE + if reconnect_in_progress: + comment_parts.append(OPC_RECONNECT_IN_PROGRESS_COMMENT) + confidence = OPC_SESSION_BAD_CONFIDENCE + if not comment_parts: + comment_parts.append(OPC_WRITTING_ERROR_MESSAGE) + + comments = OPC_COMMENT_SEPARATOR.join(comment_parts) + data['prediction_confidence'] = confidence + data['comments'] = comments + self.debug( + f'OPC write issues, confidence={confidence}, comments={comments}', + metadata, + ) else: - self.debug("Data written to OPC servers successfully.", metadata) + self.debug('Data written to OPC servers successfully.', metadata) return data.to_dict() - async def shutdown(self): + def close(self): """ Gracefully shutdown all OPC server connections and cleanup resources. @@ -353,4 +510,5 @@ class OPC(BaseActivity): their current state and provides a clean shutdown experience. """ for opc in self.opc_repository.values(): - await opc.disconnect() + opc.disconnect() + self.opc_repository.clear() diff --git a/laborious/activities/storage.py b/laborious/activities/storage.py new file mode 100644 index 0000000..6a996a5 --- /dev/null +++ b/laborious/activities/storage.py @@ -0,0 +1,231 @@ +from temporalio import activity, workflow + +with workflow.unsafe.imports_passed_through(): + # Extend the Temporal Postgres activities for convenient query -> MinIO export + import traceback + from datetime import timedelta + from typing import Any + + import pandas as pd + from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler + from sientia_do.notifications.models import NotificationLevel + from sientia_do.observability.logger import Logger + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from sientia_do.repository.minio_repository_sync import MinioRepository + from sientia_do.temporal.activities.postgres_sync import Postgres + from sientia_do.temporal.constants import now + + from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload + +_LOAD_QUERY_OFFLOAD_SKIP_KEYS = frozenset({'model_name', 'key_prefix', 'size_threshold_bytes'}) + + +class Storage(Postgres, SientiaMonitoring): + """ + Extensions for Postgres activities with a helper to export query results + directly to MinIO as Parquet and return the object name. + """ + + minio_repository: MinioRepository | None = None + + def __init__( + self, + host: str, + port: int, + user: str, + password: str, + dbname: str, + min_connections: int, + max_connections: int, + retention_hours: int = 24, + minio_repository: MinioRepository | None = None, + logger: Logger | None = None, + notification_handler: NotificationHandler | None = None, + metrics_controller: MetricsController | None = None, + ): + self.retention_hours = retention_hours + Postgres.__init__( + self, + host=host, + port=port, + user=user, + password=password, + dbname=dbname, + min_connections=min_connections, + max_connections=max_connections, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + self.minio_repository = minio_repository + SientiaMonitoring.__init__( + self, + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + @activity.defn(name='load_query_with_minio_offload') + def load_query_with_minio_offload(self, input_data: dict[str, Any]) -> MinioDataFramePayload: + """ + Run the custom SQL load, then return a MinIO-aware dataframe wire dict. + + Args (input_data): + metadata (dict): Workflow metadata (same as load_custom_query). + query (str): SQL query. + datetime_columns (list[str], optional): Datetime column names. + model_name (str): Model name for object key basename. + key_prefix (str, optional): Directory prefix inside the bucket. + size_threshold_bytes (int, optional): Override env offload threshold. + + Returns: + dict[str, Any]: Flat ``MinioDataFramePayload`` dict or ``success: False`` on failure. + """ + if self.minio_repository is None: + raise ValueError('Minio repository not initialized') + + metadata: dict = input_data.get('metadata', {}) + model_name = input_data['model_name'] + + rows = self.load_custom_query( + input_data, + ) + if not rows: + self.error( + 'load_query_with_minio_offload failed: No data returned from query', metadata + ) + dataframe = None + else: + dataframe = pd.DataFrame(rows) + + return MinioDataFramePayload.from_dataframe( + dataframe, + minio_repo=self.minio_repository, + workflow_metadata=metadata, + model_name=model_name, + operation='initial', + logger=self.logger, + ) + + @activity.defn(name='export_payload_to_postgres') + def export_payload_to_postgres(self, input_data: dict[str, Any]) -> dict: + """ + Resolve a MinIO-aware payload into a DataFrame and persist it into PostgreSQL. + + This activity accepts the serialized payload produced by previous steps + (inline dict or MinIO object reference), reconstructs the tabular data, + and delegates the final write to ``export_data_to_postgres`` using the + same input contract expected by the Postgres activity mixin. + + Args: + - input_data (dict[str, Any]): Activity input containing ``data`` as a + ``MinioDataFramePayload``-compatible dict plus database write options + (schema/table/on_conflict/metadata and related fields). + + Return: + dict: Result dictionary returned by ``export_data_to_postgres``, including + success status and optional write diagnostics. + """ + metadata = input_data.get('metadata') + payload = MinioDataFramePayload.from_dict(input_data['data']) + data = payload.retrieve(self.minio_repository, metadata) + + return self.export_data_to_postgres( + { + **input_data, + 'data': data, + } + ) + + @activity.defn(name='cleanup_minio_objects_expired') + def cleanup_minio_objects_expired(self, input_data: dict[str, Any]) -> dict[str, Any]: + """ + Delete objects under the given prefixes that are older than the retention window. + + Args (input_data): + metadata (dict): Workflow metadata for logging and metrics. + prefixes (list[str]): Key prefixes to scan (one level or subtree per prefix). + + Returns: + dict[str, Any]: ``success``, ``deleted_count``, and optional ``message``. + """ + if self.minio_repository is None: + raise ValueError('Minio repository not initialized') + + metadata = input_data.get('metadata', {}) + payload = MinioDataFramePayload.from_dict(input_data['data']) + prefix = payload.cleanup_prefix() + base = now() + cutoff = (base.replace(tzinfo=None) if base.tzinfo else base) - timedelta( + hours=self.retention_hours + ) + + report: dict[str, Any] = { + 'failed': {}, + 'deleted': {}, + 'failed_count': 0, + 'deleted_count': 0, + } + try: + keys = self.minio_repository.list_objects( + prefix=prefix, + recursive=True, + metadata=metadata, + ) + for key in keys: + try: + ts = MinioDataFramePayload.parse_object_timestamp(key) + if ts is None: + continue + if ts >= cutoff: + continue + self.minio_repository.delete_file( + object_name=key, + metadata=metadata, + ) + except Exception as e: + report['failed'][key] = { + 'success': False, + 'message': str(e), + } + report['failed_count'] += 1 + continue + report['deleted'][key] = { + 'success': True, + 'message': 'Deleted', + } + report['deleted_count'] += 1 + except Exception as e: + trace = traceback.format_exc() + self.send_notification( + metadata=metadata, + notification_id='ERROR_CLEANUP_MINIO_OBJECTS_EXPIRED', + message=f'Error cleaning up MinIO objects: {e}', + block='cleanup_minio_objects_expired', + level=NotificationLevel.ERROR, + attachment_content=trace, + ) + self.error(trace, metadata) + else: + # Cleanup success is expected in normal flow; avoid noisy INFO notifications + # that do not impact behavior and can flood observability in test runs. + self.info('MinIO objects cleaned up successfully', metadata) + + return report + + def close(self) -> None: + """ + Shutdown Storage resources in deterministic order. + + The method first closes Postgres resources via ``Postgres.close`` (engine, + sessions, and monitoring hooks), then closes the optional MinIO repository + and clears the local reference to avoid accidental reuse after shutdown. + """ + Postgres.close(self) + if self.minio_repository is not None: + try: + self.minio_repository.close() + finally: + self.minio_repository = None diff --git a/laborious/metrics.py b/laborious/metrics.py index 97f7bb9..74a9189 100644 --- a/laborious/metrics.py +++ b/laborious/metrics.py @@ -18,55 +18,183 @@ Key Metric Categories: Metric Labels: - pod_id: Kubernetes pod identifier for multi-instance deployments +- runtime: Runtime / environment identifier (matches ``RUNTIME`` env, see ``SientiaMonitoring``) - model_name: Name of the ML model being used -- pipeline_name: Name of the prediction pipeline +- workflow_name: Name of the prediction pipeline - opc_server_id: Identifier for OPC server operations """ -from prometheus_client import Gauge, Counter, Histogram +from prometheus_client import Counter, Gauge, Histogram +from sientia_do.observability.metrics import CORE_LABELS # Application health metric APP_UP = Gauge( - "app_up", - "Indicates if the application is running (1) or shutting down (0)", - ["pod_id"], + 'app_up', + 'Indicates if the application is running (1) or shutting down (0)', + ['pod_id'], ) -# Core labels used across multiple metrics -CORE_LABELS = ["pod_id", "model_name", "pipeline_name"] - # Prediction operation metrics PREDICTIONS_WRITTEN_COUNT = Counter( - "laborious_predictions_written_count", - "Number of predictions written to the database table predictions", + 'laborious_predictions_written_count', + 'Number of predictions written to the database table predictions', CORE_LABELS, ) # Prediction quality metrics PREDICTION_CONFIDENCE_MONITOR = Gauge( - "laborious_prediction_confidence_monitor", - "Current confidence of each prediction", + 'laborious_prediction_confidence_monitor', + 'Current confidence of each prediction', CORE_LABELS, ) -# Performance monitoring metrics +# Prediction total response time PREDICTION_RESPONSE_TIME_MONITOR = Histogram( - "laborious_prediction_response_time_monitor", - "Current response time of each prediction", + 'laborious_prediction_response_time_monitor', + 'Current response time of each prediction', CORE_LABELS, - buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] + buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], ) -# OPC export metrics + +# ================== OPC metrics ================== + + PREDICTION_OPC_WRITING_COUNT = Counter( - "laborious_prediction_opc_writing_count", - "Number of predictions written to the OPC server", - [*CORE_LABELS, "opc_server_id"], + 'laborious_prediction_opc_writing_count', + 'Number of predictions written to the OPC server', + [*CORE_LABELS, 'opc_server_id', 'tag'], ) PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram( - "laborious_prediction_opc_writing_response_time_monitor", - "Current response time of each prediction written to the OPC server", - [*CORE_LABELS, "opc_server_id"], - buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0] + 'laborious_prediction_opc_writing_response_time_monitor', + 'Current response time of each prediction written to the OPC server', + [*CORE_LABELS, 'opc_server_id', 'tag'], + buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], +) + +OPC_CONNECTIONS_TOTAL = Counter( + 'opc_connections_initiated_total', + 'Total connection attempts to OPC servers', + ['pod_id', 'server_name'], +) +OPC_CONNECTIONS_FAILED = Counter( + 'opc_connections_failed_total', + 'Total failed connection attempts to OPC servers', + ['pod_id', 'server_name'], +) +OPC_CONNECTION_STATUS = Gauge( + 'opc_connection_status', + 'Connection status with the OPC server (1=connected, 0=disconnected)', + ['pod_id', 'server_name', 'server_url'], +) + +_OPC_SESSION_DEBUG_LABELS = ['pod_id', 'server_name', 'runtime', 'opc_server_id', 'session_id'] + +OPC_SESSION_CREATED_TOTAL = Counter( + 'opc_session_created_total', + 'OPC UA sessions established (after successful connect)', + _OPC_SESSION_DEBUG_LABELS, +) + +OPC_SESSION_CLOSED_TOTAL = Counter( + 'opc_session_closed_total', + 'OPC UA client disconnects completed (session tear-down initiated)', + _OPC_SESSION_DEBUG_LABELS, +) + +OPC_SESSION_REVISED_TIMEOUT_MS = Gauge( + 'opc_session_revised_timeout_milliseconds', + 'Server-revised OPC UA session timeout (RevisedSessionTimeout) in ms after connect', + _OPC_SESSION_DEBUG_LABELS, +) + +OPC_WRITE_ATTEMPT_LABELS = [*_OPC_SESSION_DEBUG_LABELS, 'model_id', 'model_name', 'result'] + +OPC_WRITE_ATTEMPTS_TOTAL = Counter( + 'opc_write_attempts_total', + 'OPC UA write attempts with session and outcome (result=OK or exception class name)', + OPC_WRITE_ATTEMPT_LABELS, +) + +OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL = Counter( + 'opc_write_inter_arrival_over_session_timeout_total', + 'Successful writes where seconds since the previous successful write exceeded RevisedSessionTimeout (ms)', + _OPC_SESSION_DEBUG_LABELS, +) + +# ================== Model metrics ================== + +MODEL_READ_LAG = Histogram( + 'laborious_model_read_lag', + 'Lag between the start and read of read operations', + CORE_LABELS, + buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], +) + +MODEL_WRITE_LAG = Histogram( + 'laborious_model_write_lag', + 'Lag between the start and end of write operations', + CORE_LABELS, + buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], +) + +MODEL_READ_COUNT = Counter( + 'laborious_model_read_count', + 'Number of reads from the model', + CORE_LABELS, +) + +MODEL_WRITE_COUNT = Counter( + 'laborious_model_write_count', + 'Number of writes to the model', + CORE_LABELS, +) + +MODEL_READ_ERROR_COUNT = Counter( + 'laborious_model_read_error_count', + 'Number of errors reading from the model', + CORE_LABELS, +) + +MODEL_WRITE_ERROR_COUNT = Counter( + 'laborious_model_write_error_count', + 'Number of errors writing to the model', + CORE_LABELS, +) + +MODEL_ANALYZE_LAG = Histogram( + 'laborious_model_analyze_lag', + 'Lag between the start and end of analyze operations', + CORE_LABELS, + buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0], +) + +MODEL_ANALYZE_COUNT = Counter( + 'laborious_model_analyze_count', + 'Number of analyze operations', + CORE_LABELS, +) + +MODEL_ANALYZE_ERROR_COUNT = Counter( + 'laborious_model_analyze_error_count', + 'Number of errors during analyze operations', + CORE_LABELS, +) + + +# ================== PI Web API metrics ================== + +PI_WEB_API_LABELS = [*CORE_LABELS, 'tag_name'] + +PI_WEB_API_PREDICTION_WRITTEN_COUNT = Counter( + 'laborious_pi_web_api_prediction_written_count', + 'Number of predictions written to the PI Web API', + PI_WEB_API_LABELS, +) + +PI_WEB_API_PREDICTION_WRITTEN_ERROR_COUNT = Counter( + 'laborious_pi_web_api_prediction_written_error_count', + 'Number of errors writing predictions to the PI Web API', + PI_WEB_API_LABELS, ) diff --git a/laborious/utils/connectors_config.py b/laborious/utils/connectors_config.py index 145f248..7e5e7f9 100644 --- a/laborious/utils/connectors_config.py +++ b/laborious/utils/connectors_config.py @@ -1,65 +1,68 @@ -from os import getenv import json -from typing import Dict, Any +from os import getenv +from typing import Any -def build_postgres_config() -> Dict[str, Any]: +def build_mlflow_config() -> dict[str, Any]: """ - Build PostgreSQL database configuration from environment variables. + Read MLflow tracking and registry credentials from the environment. - This function constructs a PostgreSQL configuration dictionary from - environment variables with sensible defaults for local development. - It handles connection pool configuration and security parameters. + Used by ``Activities`` when constructing ``SientiaMLflowRepository``. The ``url`` value is the + same string workers and notebooks should use for ``MLFLOW_TRACKING_URI``-style clients. Environment Variables: - POSTGRES_HOST: Database hostname (default: localhost) - POSTGRES_PORT: Database port (default: 5432) - POSTGRES_USER: Database username (default: sientia) - POSTGRES_PASSWORD: Database password (default: sientia) - POSTGRES_DBNAME: Database name (default: sientia) - POSTGRES_MIN_CONNECTIONS: Minimum connection pool size (default: 5) - POSTGRES_MAX_CONNECTIONS: Maximum connection pool size (default: 20) + MLFLOW_URL: Host with scheme + MLFLOW_USERNAME: Basic-auth or service user (default: aignosi) + MLFLOW_PASSWORD: Password or token (default: aignosi) - Returns: - dict: PostgreSQL configuration dictionary with all required parameters + Return: + dict[str, Any]: ``url``, ``username``, ``password``. """ return { - 'host': getenv('POSTGRES_HOST', 'localhost'), - 'port': int(getenv('POSTGRES_PORT', '5432')), - 'user': getenv('POSTGRES_USER', 'sientia'), - 'password': getenv('POSTGRES_PASSWORD', 'sientia'), - 'dbname': getenv('POSTGRES_DBNAME', 'sientia'), - 'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')), - 'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')) - } - - -def build_mlflow_config() -> Dict[str, Any]: - """ - Build MLFlow server configuration from environment variables. - - This function constructs an MLFlow configuration dictionary from - environment variables with sensible defaults for local development. - It handles server connection and authentication parameters. - - Environment Variables: - MLFLOW_HOST: MLFlow server hostname (default: http://localhost) - MLFLOW_PORT: MLFlow server port (default: 5080) - MLFLOW_USERNAME: MLFlow username (default: aignosi) - MLFLOW_PASSWORD: MLFlow password (default: aignosi) - - Returns: - dict: MLFlow configuration dictionary with all required parameters - """ - return { - 'host': getenv('MLFLOW_HOST', 'http://localhost'), - 'port': int(getenv('MLFLOW_PORT', '5080')), + 'url': getenv('MLFLOW_URL', 'http://localhost:5080'), 'username': getenv('MLFLOW_USERNAME', 'aignosi'), - 'password': getenv('MLFLOW_PASSWORD', 'aignosi') + 'password': getenv('MLFLOW_PASSWORD', 'aignosi'), } -def build_opc_config() -> Dict[str, Any]: +def build_plugin_store_config() -> dict[str, Any]: + """ + Collect settings for ``PluginStore`` (Git-backed catalog + runtime install via pip). + + Mirrors the model-manager service: the worker passes these kwargs into ``PluginStore`` after + ``install_runtime`` resolves wheels from the configured PyPI index. Missing optional env vars + become ``None`` so the store can run without auth in local dev. + + Environment Variables: + STORE_BASE_URL: Git HTTP(S) server (e.g. Gitea) base URL (default: http://localhost:3000) + STORE_OWNER: Namespace or org owning the store repo (default: sientia) + STORE_REPO: Repository name (default: model-library-store) + STORE_BRANCH: Checkout branch; unset lets the client use default + STORE_USERNAME / STORE_PASSWORD: HTTP basic credentials for Git fetch + STORE_CACHE_TTL_SECONDS: Optional integer seconds for metadata cache TTL + PYPI_SERVER: Index URL for ``pip install`` during runtime install (default: http://localhost:5000) + PYPI_USERNAME / PYPI_PASSWORD: Optional index authentication + + Return: + dict[str, Any]: Keys aligned with ``PluginStore`` constructor parameter names. + """ + + cache_ttl_seconds = getenv('STORE_CACHE_TTL_SECONDS') + return { + 'base_url': getenv('STORE_BASE_URL', 'http://localhost:3000'), + 'owner': getenv('STORE_OWNER', 'sientia'), + 'repo': getenv('STORE_REPO', 'model-library-store'), + 'username': getenv('STORE_USERNAME'), + 'password': getenv('STORE_PASSWORD'), + 'branch': getenv('STORE_BRANCH'), + 'cache_ttl_seconds': int(cache_ttl_seconds) if cache_ttl_seconds else None, + 'pypi_index_url': getenv('PYPI_SERVER', 'http://localhost:5000'), + 'pypi_username': getenv('PYPI_USERNAME'), + 'pypi_password': getenv('PYPI_PASSWORD'), + } + + +def build_opc_config() -> dict[str, Any]: """ Build OPC server configuration from environment variables. @@ -75,7 +78,7 @@ def build_opc_config() -> Dict[str, Any]: OPC_CERT_PATH: Client certificate path (fallback, default: None) OPC_PRIVATE_KEY_PATH: Client private key path (fallback, default: None) OPC_SERVER_CERT_PATH: Server certificate path (fallback, default: None) - OPC_RECONNECTION_INTERVAL: Reconnection interval in milliseconds (fallback, default: 120) + OPC_RECONNECTION_INTERVAL: Reconnection interval in seconds (fallback, default: 120) Returns: dict: OPC server configuration dictionary @@ -88,42 +91,37 @@ def build_opc_config() -> Dict[str, Any]: return { getenv('OPC_ID', '1'): { 'id': getenv('OPC_ID', '1'), + 'server_name': getenv('OPC_SERVER_NAME', 'default_server'), 'url': getenv('OPC_URL', 'opc.tcp://localhost:4840'), 'server_uri': getenv('OPC_SERVER_URI', 'opc.tcp://localhost:4840'), 'cert_path': getenv('OPC_CERT_PATH', None), 'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None), 'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None), - 'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120')) + 'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120')), } } -def build_mongodb_config() -> Dict[str, Any]: +def build_minio_config() -> dict[str, Any]: """ - Build MongoDB configuration from environment variables. - - This function constructs a MongoDB configuration dictionary from - environment variables with sensible defaults for local development. - It handles connection string and database name configuration. + Build MinIO (S3-compatible) configuration from environment variables. Environment Variables: - MONGODB_USERNAME: MongoDB username (default: root) - MONGODB_PASSWORD: MongoDB password (default: wKZDbMNU1c) - MONGODB_URL: MongoDB connection URI (default: localhost:27018) - MONGODB_DATABASE_NAME: MongoDB database name (default: sientia) - MONGODB_TTL_INDEX_HOURS: TTL index duration in hours (default: 1) + MINIO_ENDPOINT_URL: Host:port or URL for the S3 API (default: http://localhost:9000) + MINIO_ACCESS_KEY: Access key (default: minioadmin) + MINIO_SECRET_KEY: Secret key (default: minioadmin) + MINIO_DEFAULT_BUCKET: Default bucket for Laborious payloads (default: laborious) + MINIO_RETENTION_HOURS: Offloaded object retention window (default: 24) + MINIO_SECURE: If ``true``, use HTTPS (default: false) - Returns: - dict: MongoDB configuration dictionary with connection parameters + Return: + dict[str, Any]: Keys consumed by ``Activities`` / ``MinioRepository``. """ - username = getenv('MONGODB_USERNAME', 'root') - password = getenv('MONGODB_PASSWORD', 'wKZDbMNU1c') - uri = getenv('MONGODB_URL', 'localhost:27018') - - connection_string = f'mongodb://{username}:{password}@{uri}' - return { - 'connection_string': connection_string, - 'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'), - 'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600 + 'endpoint_url': getenv('MINIO_ENDPOINT_URL', 'http://localhost:9000'), + 'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'), + 'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'), + 'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious'), + 'retention_hours': int(getenv('MINIO_RETENTION_HOURS', '24')), + 'secure': getenv('MINIO_SECURE', 'false') == 'true', } diff --git a/laborious/utils/dataframe_debug.py b/laborious/utils/dataframe_debug.py new file mode 100644 index 0000000..995b606 --- /dev/null +++ b/laborious/utils/dataframe_debug.py @@ -0,0 +1,34 @@ +from typing import Any + +from pandas import DataFrame + +DEFAULT_MAX_DEBUG_DATAFRAME_ROWS = 100 + + +def build_dataframe_debug_message( + message: str, + data: Any, + max_rows: int = DEFAULT_MAX_DEBUG_DATAFRAME_ROWS, +) -> str: + """ + Build a safe debug message for dataframe payloads + + Args: + - message (str): Base message to identify the logged payload + - data (Any): Payload to evaluate for dataframe-aware logging + - max_rows (int): Maximum dataframe row count allowed for full payload logging + + Return: + Formatted debug message with full dataframe content or compact summary + """ + if not isinstance(data, DataFrame): + return f'{message} {data}' + + rows = data.shape[0] + if rows <= max_rows: + return f'{message}\n{data.to_csv()}' + + return ( + f'{message} skipped because dataframe has {rows} rows ' + f'(max: {max_rows}). Shape: {data.shape}' + ) diff --git a/laborious/utils/filters/conditional_filters.py b/laborious/utils/filters/conditional_filters.py index 2c51805..967e52c 100644 --- a/laborious/utils/filters/conditional_filters.py +++ b/laborious/utils/filters/conditional_filters.py @@ -20,8 +20,11 @@ def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool False if none of the specified variables contain null values. """ - return not data[ - data['variable'].isin(config['variables']) & data['value'].isna()].empty + + if data.empty: + return False + + return not data[data['variable'].isin(config['variables']) & data['value'].isna()].empty def filter_empty_data(data: DataFrame, _config: dict) -> bool: diff --git a/laborious/utils/filters/mlflow_filters.py b/laborious/utils/filters/mlflow_filters.py index f792a0e..82970e8 100644 --- a/laborious/utils/filters/mlflow_filters.py +++ b/laborious/utils/filters/mlflow_filters.py @@ -52,8 +52,11 @@ def nan_values_filter(predictions: DataFrame, _config: dict) -> bool: bool: True if data should be filtered (too many NaN values), False otherwise """ - data = predictions.replace({None: np.nan}).drop( - columns=['timestamp'], errors='ignore').infer_objects() + data = ( + predictions.replace({None: np.nan}) + .drop(columns=['timestamp'], errors='ignore') + .infer_objects() + ) if data.isna().all().all(): return True diff --git a/laborious/utils/models/__init__.py b/laborious/utils/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/laborious/utils/models/minio_dataframe_payload.py b/laborious/utils/models/minio_dataframe_payload.py new file mode 100644 index 0000000..9b75f8f --- /dev/null +++ b/laborious/utils/models/minio_dataframe_payload.py @@ -0,0 +1,362 @@ +""" +MinIO-backed DataFrame payload for Temporal workflows. + +Data is never stored as a pandas ``DataFrame`` field on the dataclass. +Instead, the DataFrame is only provided as an input to: +`from_dataframe` / `from_dataframe_to_dict`. + +At build time, the DataFrame is evaluated for its serialized size; if it exceeds +the configured threshold, it is serialized to parquet bytes and uploaded to MinIO. +Otherwise, it is inlined as a Temporal-friendly ``dict``. +""" + +import pickle +import re +from collections.abc import Hashable +from dataclasses import dataclass +from datetime import datetime +from io import BytesIO +from os import getenv +from typing import Any, Literal + +from pandas import DataFrame, read_parquet +from sientia_do.observability.logger import Logger +from sientia_do.repository.minio_repository_sync import MinioRepository +from sientia_do.temporal.constants import DATETIME_FORMAT_FILENAME, DATETIME_FORMAT_WITH_TZ, now + +# Keys that are part of the serialized wire format (not arbitrary metadata). +_SERIALIZED_FIELD_KEYS = frozenset({'data', 'bucket', 'object_key', 'object_prefix', 'uri'}) + +_OBJECT_TIMESTAMP_PATTERN = re.compile( + r'-(?:initial|transform)-(\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2})\.parquet$' +) + +OFFLOAD_THRESHOLD_BYTES = int( + float(getenv('SIENTIA_MINIO_OFFLOAD_THRESHOLD_MEGABYTES', '1.5')) * 1024 * 1024 +) + +# Relative prefix used for storing offloaded prediction datasets in MinIO. +# It is also the root directory for retention cleanup listing. +PREDICTION_DATASETS_PREFIX = 'prediction_datasets' + +OperationKind = Literal['initial', 'transform', 'predict'] + + +def _build_object_key( + model_name: str, operation: OperationKind, timestamp: str +) -> tuple[str, str | None]: + """ + Build the MinIO object key and the directory prefix used for retention listing. + + Args: + model_name: Registered model name used in the pipeline. + operation: Either initial (pre-transform load) or transform (post-MLFlow transform). + timestamp: Filename timestamp segment from DATETIME_FORMAT_FILENAME. + + Return: + tuple[str, str | None]: Full object key and normalized prefix (or None if at bucket root). + """ + # Naming convention: + # - Directory is always `prediction_datasets/` + # - Filename follows the retention-parsing pattern + basename = f'{model_name}-{operation}-{timestamp}.parquet' + model_dir = model_name.strip().strip('/') + prefix = f'{PREDICTION_DATASETS_PREFIX}/{model_dir}' + return f'{prefix}/{basename}', prefix + + +@dataclass +class MinioDataFramePayload: + """ + Serializable payload after a DataFrame was evaluated: inline tabular dict and/or MinIO keys. + + Build from a live DataFrame only via `from_dataframe` / `from_dataframe_to_dict`. + Rehydrate from Temporal via `from_dict`. The DataFrame is not a field on this class. + """ + + last_timestamp: str + status: dict[str, Any] | None = None + data: dict[Hashable, Any] | None = None + bucket: str | None = None + object_key: str | None = None + object_prefix: str | None = None + uri: str | None = None + + @staticmethod + def _debug( + logger: Logger | None, + message: str, + metadata: dict[str, Any] | None = None, + ) -> None: + """ + Emit a debug message only when a logger instance is available. + + Args: + - logger (Logger | None): Logger instance used for debug messages + - message (str): Message to be logged + - metadata (dict[str, Any] | None): Optional workflow metadata context + """ + if logger is None: + return + logger.custom_debug(message, metadata) + + @classmethod + def from_dict(cls, raw: 'dict[str, Any] | MinioDataFramePayload') -> 'MinioDataFramePayload': + """ + Reconstruct a MinioDataFramePayload from a plain dict produced by Temporal serialization. + + Temporal converts dataclass return values into plain dicts when crossing + workflow/activity boundaries. This method rebuilds the typed instance so + that methods like ``retrieve``, ``cleanup_prefix`` and ``has_data`` are + available on the receiving side. + + If the argument is already a MinioDataFramePayload, it is returned as-is. + + Args: + raw: Dict with keys matching the dataclass fields + (last_timestamp, status, data, bucket, object_key, object_prefix, uri), + or an existing MinioDataFramePayload instance. + + Return: + MinioDataFramePayload: Reconstructed (or original) instance. + """ + if isinstance(raw, MinioDataFramePayload): + return raw + return cls( + last_timestamp=raw['last_timestamp'], + status=raw.get('status'), + data=raw.get('data'), + bucket=raw.get('bucket'), + object_key=raw.get('object_key'), + object_prefix=raw.get('object_prefix'), + uri=raw.get('uri'), + ) + + @staticmethod + def estimate_size_bytes( + df: DataFrame, + metadata: dict[str, Any] | None = None, + logger: Logger | None = None, + ) -> int: + """ + Approximate serialized size of the DataFrame as the default-orient dict. + + Args: + df: DataFrame whose tabular content size is estimated. + + Return: + int: Estimated size in bytes (pickle of dict representation). + """ + try: + size = len(pickle.dumps(df.to_dict())) + except Exception: + size = len(pickle.dumps(df)) + + MinioDataFramePayload._debug( + logger, + f'DataFrame size: {size} bytes', + metadata, + ) + return size + + @staticmethod + def parse_object_timestamp(object_key: str) -> datetime | None: + """ + Parse the timestamp embedded in the object key basename (before .parquet). + + Args: + object_key: S3/MinIO object key whose basename follows + ``{model}-{initial|transform}-{DATETIME_FORMAT_FILENAME}.parquet``. + + Return: + datetime | None: Parsed UTC-naive datetime from the key, or None if not matched. + """ + basename = object_key.rsplit('/', 1)[-1] + match = _OBJECT_TIMESTAMP_PATTERN.search(basename) + if not match: + return None + try: + return datetime.strptime(match.group(1), DATETIME_FORMAT_FILENAME) + except ValueError: + return None + + def cleanup_prefix(self) -> str | None: + """ + Return the MinIO prefix eligible for retention cleanup. + + Cleanup is only applicable when payload data was offloaded to MinIO + (``object_key`` present and inline ``data`` absent). Inline-only payloads + return ``None`` because there is no object tree to prune. + + Return: + str | None: Prefix used by cleanup listing, or ``None`` when cleanup does not apply. + """ + if self.object_key is not None and self.data is None: + return self.object_prefix + return None + + def has_data(self) -> bool: + """ + Indicate whether the payload contains retrievable tabular content. + + A payload is considered non-empty when either inline ``data`` exists + (and is not an empty dict) or an ``object_key`` is available for MinIO + download. + + Return: + bool: ``True`` when data can be retrieved, ``False`` otherwise. + """ + return (self.data is not None and self.data != {}) or self.object_key is not None + + @classmethod + def from_dataframe( + cls, + dataframe: DataFrame | None, + minio_repo: MinioRepository, + model_name: str, + operation: OperationKind, + status: dict[str, Any] | None = None, + workflow_metadata: dict | None = None, + last_timestamp: str | None = None, + logger: Logger | None = None, + ) -> 'MinioDataFramePayload': + """ + Evaluate the DataFrame size, then either inline dict or upload parquet to MinIO. + + The DataFrame is not stored on the returned instance. + + Args: + dataframe: Tabular data to evaluate and persist (inline or MinIO). + metadata: Small metadata dict merged into the payload (e.g. success, message). + minio_repo: sientia_do MinioRepository (or compatible) with `upload_file()`. + workflow_metadata: Metadata passed to MinIO store for logging/metrics. + model_name: Registered model name used in the object basename. + operation: Either ``initial`` (query load) or ``transform`` (post-transform). + key_prefix: Backward-compatible parameter (currently ignored for object naming). + size_threshold_bytes: Byte limit before offload. When None, the module-level + environment-derived default is used. + + Return: + MinioDataFramePayload: Instance with data and/or MinIO fields set. + """ + + if dataframe is None or dataframe.empty: + cls._debug( + logger, + 'MinioDataFramePayload.from_dataframe received empty dataframe, returning empty payload', + workflow_metadata, + ) + return cls( + data=None, last_timestamp=now().strftime(DATETIME_FORMAT_WITH_TZ), status=status + ) + + if last_timestamp is None: + last_timestamp = max(dataframe['timestamp'].values.tolist()) + + dataframe_size = cls.estimate_size_bytes(dataframe, workflow_metadata, logger) + cls._debug( + logger, + ( + f'MinioDataFramePayload.from_dataframe estimated size: {dataframe_size} bytes ' + f'(threshold: {OFFLOAD_THRESHOLD_BYTES} bytes)' + ), + workflow_metadata, + ) + + if dataframe_size <= OFFLOAD_THRESHOLD_BYTES: + cls._debug( + logger, + 'MinioDataFramePayload.from_dataframe using inline payload', + workflow_metadata, + ) + return cls(data=dataframe.to_dict(), last_timestamp=last_timestamp, status=status) + + timestamp = now().strftime(DATETIME_FORMAT_FILENAME) + object_key, object_prefix = _build_object_key(model_name, operation, timestamp) + cls._debug( + logger, + ( + 'MinioDataFramePayload.from_dataframe offloading payload to MinIO ' + f'with key {object_key}' + ), + workflow_metadata, + ) + + # Upload using the relative object key. The upstream repository will + # prefix it internally under its MinIO namespace. + parquet_buffer = BytesIO() + dataframe.to_parquet(parquet_buffer, engine='pyarrow', index=True) + file_bytes = parquet_buffer.getvalue() + + upload_result = minio_repo.upload_file( + file_bytes=file_bytes, + relative_key=object_key, + metadata=workflow_metadata, + ) + + bucket = minio_repo.bucket + object_key_full = upload_result.get('minio_object_name', object_key) + uri = f's3://{bucket}/{object_key_full}' if bucket else None + cls._debug( + logger, + f'MinioDataFramePayload.from_dataframe upload completed: {uri}', + workflow_metadata, + ) + + return cls( + data=None, + bucket=bucket, + object_key=object_key_full, + object_prefix=object_prefix, + uri=uri, + last_timestamp=last_timestamp, + status=status, + ) + + def retrieve( + self, + minio_repo: MinioRepository, + workflow_metadata: dict[str, Any] | None = None, + logger: Logger | None = None, + ) -> DataFrame: + """ + Load parquet from MinIO when object_key is set and populate inline data. + + Args: + minio_repo: sientia_do MinioRepository (or compatible) with download_file(). + workflow_metadata: Metadata passed to MinIO read for logging/metrics. + + Return: + dict[str, Any]: Flat dict with data filled (same keys as to_dict after load). + """ + if self.data is not None: + self._debug( + logger, + 'MinioDataFramePayload.retrieve using inline payload data', + workflow_metadata, + ) + return DataFrame(self.data) + + if not self.has_data(): + self._debug( + logger, + 'MinioDataFramePayload.retrieve found no payload data, returning empty dataframe', + workflow_metadata, + ) + return DataFrame() + + self._debug( + logger, + f'MinioDataFramePayload.retrieve downloading object from MinIO: {self.object_key}', + workflow_metadata, + ) + file_bytes = minio_repo.download_file( + object_name=self.object_key, metadata=workflow_metadata + ) + df = read_parquet(BytesIO(file_bytes)) + self._debug( + logger, + f'MinioDataFramePayload.retrieve loaded dataframe from MinIO with shape {df.shape}', + workflow_metadata, + ) + return df diff --git a/laborious/utils/repository/model_repository.py b/laborious/utils/repository/model_repository.py deleted file mode 100644 index 3107d1c..0000000 --- a/laborious/utils/repository/model_repository.py +++ /dev/null @@ -1,481 +0,0 @@ -""" -Model Monitoring Repository - -This module contains the ModelMonitoringRepository class, -which is responsible for handling the communication with the Model Monitoring API. - -It includes the methods that are used to answer ModelMonitoringService -requests using the Model Monitoring API functions. - -By Monitoring we mean the evaluation of the performance of models, the generation of reports. - -""" -from datetime import datetime -import traceback -import pandas as pd -import mlflow -from os import makedirs, path, remove -from sientia.ModelServing import ModelServing -from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ -from sientia_do.observability.logger import Logger - - -class MLFlowRepository(): - def __init__(self, host, username, password, logger: Logger): - - self.model_serving = ModelServing(tracking_uri=host, - username=username, password=password, - logger=logger) - self.logger = logger - - def detect_and_parse_datetime_index(self, data: pd.DataFrame, metadata: dict) -> pd.DataFrame: - """ - Detect and parse datetime index from data. index must be a timestamp like column. - This function must detect the timestamp type (pandas Timestamp or datetime) and convert it to DATETIME_FORMAT_WITH_TZ. - If the index is a string, must be in format DATETIME_FORMAT_WITH_TZ. - If another type or format, must raise an error. - """ - index = data.index - - # Get type of first element of index - index_type = type(index[0]) - - self.logger.custom_info(f"Index type: {index_type}", metadata) - - message = f"Index must be all timestamp like column. Valid formats are: pandas Timestamp, datetime, string in format {DATETIME_FORMAT_WITH_TZ}" - - # Check if all in index are of the same type - if not all(isinstance(i, index_type) for i in index): - raise ValueError( - f"{message}") - - # Check type and converts to DATETIME_FORMAT_WITH_TZ - if index_type == str: - # Validate format of string and return error if not valid - try: - pd.to_datetime(data.index, format=DATETIME_FORMAT_WITH_TZ) - except ValueError: - raise ValueError( - f"{message}") - - elif index_type == datetime or index_type == pd.Timestamp: - data.index = data.index.strftime(DATETIME_FORMAT_WITH_TZ) - else: - raise ValueError( - f"{message}") - - return data - - def transform(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict) -> dict: - """ - Transform data using a model. - - Parameters: - - model_name (str): The name of the model to use for transformation. - - data (pandas.DataFrame): The data to transform. - - model_retention (int): The number of minutes to keep the model. - - Returns: - - dict: A dictionary containing the transformed data. - """ - - try: - self.logger.custom_debug( - f"Data received for model transformation: {data.to_csv()}", metadata) - - model_retention = model_config.get('retention_minutes', 0) - flavor = model_config.get('transform_flavor', 'sklearn') - compressed = model_config.get('is_compressed', False) - retention_target = model_config.get('retention_target', 'model') - transform_keyword = model_config.get( - 'transform_function_keyword', 'predict') - - transformed_data = self.model_serving.get_cached_transform( - model_name, data, model_retention, flavor, - compressed, retention_target, transform_keyword - ) - - self.logger.custom_debug( - f"Data received from model transformation: {transformed_data.to_csv()}", metadata) - - transformed_data = self.detect_and_parse_datetime_index( - transformed_data, metadata) - - return { - 'success': True, - 'content': transformed_data.to_dict() - } - - except Exception as e: - return { - 'success': False, - 'content': { - 'message': str(e), - 'traceback': traceback.format_exc() - } - } - - def predict(self, model_name: str, data: pd.DataFrame, model_config: dict, metadata: dict) -> dict: - """ - Predict data using a model. - - Parameters: - - model_name (str): The name of the model to use for prediction. - - data (pandas.DataFrame): The data to predict. - - model_retention (int): The number of minutes to keep the model. - - Returns: - - dict: A dictionary containing the predicted data. - """ - try: - model_retention = model_config.get('retention_minutes', 0) - flavor = model_config.get('predict_flavor', 'pyfunc') - compressed = model_config.get('is_compressed', False) - retention_target = model_config.get('retention_target', 'model') - - input_index = data.index - start_time = datetime.now() - - self.logger.custom_debug( - f"Data received for model prediction: {data.to_csv()}", metadata) - data = self.model_serving.get_cached_predict( - model_name, data, model_retention, flavor, - compressed, retention_target - ) - - end_time = datetime.now() - data = pd.DataFrame(data, columns=['prediction']) - self.logger.custom_debug( - f"Data received from model prediction: {data.to_csv()}", metadata) - data.index = input_index - data['response_time'] = (end_time - start_time).total_seconds() - - return { - 'success': True, - 'content': data.to_dict() - } - - except Exception as e: - return { - 'success': False, - 'content': { - 'message': str(e), - 'traceback': traceback.format_exc() - } - } - - def get_experiment_by_run_id(self, run_id: str) -> dict: - # Get the run information using the run_id - run = mlflow.get_run(run_id) - - # Extract the experiment ID from the run - experiment_id = run.info.experiment_id - - # Get the experiment details using the experiment ID - experiment = mlflow.get_experiment(experiment_id) - experiment_name = experiment.name - return experiment_name - - def get_next_run_name(self, model_name: str) -> str: - """ - Generate the next run name for a specific MLFlow model. - - This method calculates the next sequential run number for a model - by searching existing runs and incrementing the count. It ensures - unique run names for model training and retraining operations. - - Args: - model_name (str): The name of the MLFlow model - - Returns: - str: The next run name in format 'model_name-run_number' - """ - runs = mlflow.search_runs( - experiment_names=[model_name], order_by=["start_time desc"]) - next_run_number = len(runs) + 1 - return f"{model_name}-{next_run_number}" - - def create_model_experiment(self, model_name: str, data: pd.DataFrame) -> tuple: - """ - Create a new MLFlow experiment for model retraining. - - This method sets up the complete environment for model retraining by: - 1. Loading the current production prediction model - 2. Loading the current production transformation model - 3. Fitting the transformation model with new data - 4. Preparing data for prediction model retraining - 5. Setting up the MLFlow experiment context - - Args: - model_name (str): Name of the MLFlow model to retrain - data (pd.DataFrame): Training data for model retraining - - Returns: - tuple: (prediction_model, data_model, experiment) - - prediction_model: Loaded prediction model for retraining - - data_model: Fitted transformation model - - experiment: MLFlow experiment name - """ - # load predictor model - predictor_uri = f"models:/{model_name}/production" - # load transform model - latest_production_id = self.model_serving.get_model_run_id( - model_name, stage="Production" - ) - transform_uri = self.model_serving.get_model_uri( - latest_production_id, prediction=False - ) - # load - data_model = mlflow.sklearn.load_model(transform_uri) - prediction_model = mlflow.sklearn.load_model(predictor_uri) - data_model = data_model.fit(data) - treated_data = data_model.predict(data) - - target_name = data_model.target_variable - y = data[target_name] - treated_data = pd.merge( - treated_data, y, left_index=True, right_index=True) - prediction_model = prediction_model.fit(treated_data) - experiment = self.get_experiment_by_run_id(latest_production_id) - mlflow.set_experiment(experiment) - - return prediction_model, data_model, experiment - - def perform_model_retrain(self, - prediction_model, - data_model, - experiment: str, - model_name: str, - data: pd.DataFrame): - """ - Execute the complete model retraining process in MLFlow. - - This method performs the actual model retraining by: - 1. Starting a new MLFlow run with descriptive metadata - 2. Logging model parameters and hyperparameters - 3. Retraining both prediction and transformation models - 4. Logging training data as artifacts - 5. Saving retrained models to MLFlow registry - - Args: - prediction_model: MLFlow prediction model to retrain - data_model: MLFlow transformation model to retrain - experiment (str): MLFlow experiment name for the retraining - model_name (str): Name of the model being retrained - data (pd.DataFrame): Training data used for retraining - - Returns: - tuple: (status_message, experiment_name) - - status_message (str): Success confirmation message - - experiment_name (str): Name of the experiment - """ - pred_model_atributes = vars(prediction_model) # load class attributes - data_model_atributes = vars(data_model) # load class attributes - experiment_description = f"Retrain model {model_name} with new data" - current_run_name = self.get_next_run_name(experiment) - with mlflow.start_run( - run_name=current_run_name, description=experiment_description - ) as _run: - # update transfomation model - # fixed parameters - for name_atribute, val_atribute in pred_model_atributes.items(): - if name_atribute != "model": - mlflow.log_param(name_atribute, val_atribute) - # update prediction model - for name_atribute, val_atribute in data_model_atributes.items(): - if name_atribute != "model": - mlflow.log_param(name_atribute, val_atribute) - # dynamic parameters, including model itself - mlflow.sklearn.log_model(data_model, "data_model") - - makedirs("temp", exist_ok=True) - - file_path = f"temp/raw_data_{model_name}.csv" - data.to_csv(file_path, index=True) - - # log the data raw - mlflow.log_artifact(file_path) - - # dynamic parameters, including model itself - mlflow.sklearn.log_model(prediction_model, "prediction_model") - mlflow.log_param("retrain", True) - - # clear temp file - if path.exists(file_path): - remove(file_path) - - return "Model retrained successfully", experiment - - def retrain_model(self, data: pd.DataFrame, model_name: str) -> tuple: - """ - Orchestrate the complete model retraining workflow. - - This method coordinates the entire model retraining process by: - 1. Creating the MLFlow experiment environment - 2. Loading existing production models - 3. Executing the retraining process - 4. Returning comprehensive retraining results - - Args: - data (pd.DataFrame): Training data for model retraining - model_name (str): Name of the MLFlow model to retrain - - Returns: - tuple: (status_message, experiment_name) - - status_message (str): Retraining operation status - - experiment_name (str): MLFlow experiment identifier - """ - prediction_model, data_model, experiment = self.create_model_experiment( - model_name, data) - retrain_result = self.perform_model_retrain( - prediction_model, data_model, experiment, model_name, data) - return retrain_result - - def get_experiment(self, experiment_name: str) -> int: - """ - Retrieve MLFlow experiment ID by experiment name. - - This method searches for an MLFlow experiment by name and - returns its unique identifier. It provides error handling - for non-existent experiments. - - Args: - experiment_name (str): Name of the MLFlow experiment - - Returns: - int: MLFlow experiment ID - - Raises: - ValueError: If the experiment name is not found - """ - experiment = mlflow.get_experiment_by_name(experiment_name) - - if experiment is None: - raise ValueError(f'Experiment {experiment_name} not found') - - return int(experiment.experiment_id) - - def get_experiment_last_run(self, experiment_id: int) -> str: - """ - Retrieve the most recent retraining run ID for an experiment. - - This method searches for the latest run in an MLFlow experiment - that has been marked as a retraining run. It filters runs by - the 'retrain' parameter and orders them by completion time. - - Args: - experiment_id (int): MLFlow experiment ID - - Returns: - str: MLFlow run ID of the most recent retraining run - - Raises: - ValueError: If runs data is not in expected DataFrame format - """ - runs = mlflow.search_runs( - experiment_ids=[experiment_id], - filter_string="", # Sem filtro no MLflow ainda - output_format="pandas" - ) - - if not isinstance(runs, pd.DataFrame): - raise ValueError('Runs is not a pandas DataFrame') - - # Filtrar apenas as runs onde params.retrain == True - filtered_runs = runs[runs["params.retrain"] == 'True'] - - # Converter a coluna 'end_time' para datetime - filtered_runs['end_time'] = pd.to_datetime(filtered_runs['end_time']) - - # Ordenar o DataFrame de forma descendente pela coluna 'end_time' - filtered_runs = filtered_runs.sort_values( - by='end_time', ascending=False) - - # Pegar a última run_id do DataFrame filtrado e ordenado - latest_run_id = filtered_runs.iloc[0]['run_id'] - - return latest_run_id - - def update_production_model_by_run_id(self, run_id: str, model_name: str) -> dict: - """ - Update production model with a specific MLFlow run. - - This method promotes a model from a specific MLFlow run to - production stage. It handles model registration, versioning, - and stage transitions with proper error handling. - - Args: - run_id (str): MLFlow run ID containing the model to promote - model_name (str): Name of the MLFlow model - - Returns: - dict: Model update metadata containing: - - model_name (str): Name of the updated model - - version (str): New model version number - - mlflow_run_id (str): Source run ID - - Update Process: - 1. Registers the model from the specified run - 2. Retrieves the latest model version - 3. Transitions the model to 'Production' stage - 4. Archives existing production versions - """ - # Registrar o modelo - # Aqui estamos assumindo que você já tem um modelo salvo, caso contrário você precisará treiná-lo e salvá-lo primeiro. - # Se o modelo já está registrado, você pode usar o método register_model() ou pyfunc.load_model() para isso. - mlflow.register_model( - f"runs:/{run_id}/prediction_model", model_name) - - # Colocar a versão do modelo em produção - # Depois de registrar o modelo, precisamos pegar a versão mais recente do modelo e movê-lo para o estágio 'Production' - client = mlflow.tracking.MlflowClient() - - # Obter a versão mais recente registrada do modelo - model_versions = client.get_registered_model( - model_name).latest_versions - - if not isinstance(model_versions, list): - raise ValueError('Model versions is not a list') - - max_version = max(model_versions, key=lambda x: int(x.version)).version - - # Mover a versão mais recente do modelo para o estágio de 'Production' - client.transition_model_version_stage( - name=model_name, - version=max_version, - stage="Production", - archive_existing_versions=True - ) - - return { - 'model_name': model_name, - 'version': max_version, - 'mlflow_run_id': run_id - } - - def update_production_model(self, experiment: str, model_name: str) -> dict: - """ - Update production model using the latest retraining run. - - This method orchestrates the complete production model update - process by identifying the most recent retraining run and - promoting it to production stage. - - Args: - experiment (str): MLFlow experiment name - model_name (str): Name of the MLFlow model - - Returns: - dict: Complete model update metadata containing: - - model_name (str): Name of the updated model - - version (str): New model version number - - mlflow_run_id (str): Source run ID - - mlflow_experiment_id (int): Experiment ID - """ - experiment_id = self.get_experiment(experiment) - run_id = self.get_experiment_last_run(experiment_id) - metadata = self.update_production_model_by_run_id(run_id, model_name) - - metadata['mlflow_experiment_id'] = experiment_id - - return metadata diff --git a/laborious/utils/repository/opc_repository.py b/laborious/utils/repository/opc_repository.py index 96ecfef..a0856cd 100644 --- a/laborious/utils/repository/opc_repository.py +++ b/laborious/utils/repository/opc_repository.py @@ -1,359 +1,893 @@ -import asyncio -import traceback +""" +Synchronous OPC UA client repository using asyncua ``sync`` API. + +``asyncua.sync.Client`` runs the asyncio stack on a background thread so Temporal +activities and other callers stay blocking while preserving the same session +lifecycle, security policy, reconnect semantics, and write error classification +as the async ``main`` implementation at ``fcc8920a8be4`` (async → sync/thread conversion). +""" + +import json +import threading import time +import traceback from datetime import datetime from pathlib import Path from typing import Any -from asyncua import Client -from asyncua.crypto.security_policies import SecurityPolicyBasic256 -from asyncua.ua import DataValue, Variant, VariantType, DateTime -from regex import F + +from asyncua import ua +from asyncua.crypto import security_policies +from asyncua.sync import Client +from asyncua.ua.uaerrors import UaStatusCodeError from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.notifications.models import NotificationLevel from sientia_do.observability.logger import Logger +from sientia_do.observability.metrics_controller import MetricsController +from sientia_do.observability.sientia_monitoring import SientiaMonitoring + from laborious import metrics +# Requested session and secure channel lifetime (ms) before server revision; 10 minutes. +OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS = 10 * 60 * 1000 + + +class OpcClientAlreadyExistsError(RuntimeError): + """Raised when _create_client is called while self.client is already set.""" + + +class OpcSessionAlreadyConnectedError(RuntimeError): + """Raised when _open_session is called while a UA session is already open.""" + + +class OpcClientNotInitializedError(RuntimeError): + """Raised when _open_session is called before _create_client.""" + + +RECONNECTABLE_OPC_BAD_NAMES: frozenset[str] = frozenset( + { + 'BadSessionIdInvalid', + 'BadSessionClosed', + 'BadSessionNotActivated', + 'BadSecureChannelIdInvalid', + 'BadSecureChannelClosed', + 'BadSecureChannelTokenUnknown', + 'BadTcpSecureChannelUnknown', + 'BadServerNotConnected', + 'BadConnectionClosed', + 'BadDisconnect', + 'BadConnectionRejected', + 'BadCommunicationError', + 'BadRequestInterrupted', + 'BadUnknownResponse', + 'BadTimeout', + 'BadRequestTimeout', + 'BadSequenceNumberInvalid', + 'BadSequenceNumberUnknown', + 'BadSecurityModeInsufficient', + 'BadRequestHeaderInvalid', + 'BadInvalidState', + } +) + + +def _opc_authentication_token_str(client: Client | None) -> str: + """ + Serialize the current OPC UA authentication token (session handle) for logging and metrics. + + Return: + str: Token string, or "unknown" if unavailable. + """ + if client is None: + return 'unknown' + try: + proto = client.aio_obj.uaclient.protocol + if proto is None: + return 'unknown' + tok = getattr(proto, 'authentication_token', None) + if tok is None: + return 'unknown' + return str(tok) + except Exception: + return 'unknown' + + +def _opc_status_from_exception(exc: BaseException) -> str: + """ + Resolve OPC UA status name from an exception, including chained UaStatusCodeError causes. + + Args: + exc (BaseException): Raised error from asyncua. + + Return: + str: Status class name or generic Python exception name. + """ + current: BaseException | None = exc + while current is not None: + if isinstance(current, UaStatusCodeError): + return type(current).__name__ + current = current.__cause__ + return type(exc).__name__ + + +def is_reconnectable_opcua_bad(exc: BaseException) -> bool: + """ + Return whether the exception is a Tier-1 OPC UA Bad* that should trigger reconnect. + + Args: + exc (BaseException): Raised error from get_node or write_value. + + Return: + bool: True if reconnect should be scheduled. + """ + return _opc_status_from_exception(exc) in RECONNECTABLE_OPC_BAD_NAMES + + +def _model_labels_from_write_metadata(metadata: dict[str, Any] | None) -> dict[str, str]: + """ + Extract model_id and model_name from write metadata for Prometheus labels. + + Args: + metadata (dict[str, Any] | None): Context passed into write_data; may omit keys. + + Return: + dict[str, str]: Labels model_id and model_name, defaulting to "unknown". + """ + if not metadata: + return {'model_id': 'unknown', 'model_name': 'unknown'} + return { + 'model_id': str(metadata.get('model_id', 'unknown')), + 'model_name': str(metadata.get('model_name', 'unknown')), + } + + data_type_map = { 'float': { 'converter': float, - 'opc_type': VariantType.Float, + 'opc_type': ua.VariantType.Float, }, 'double': { 'converter': float, - 'opc_type': VariantType.Double, + 'opc_type': ua.VariantType.Double, }, 'int': { 'converter': int, - 'opc_type': VariantType.Int32, + 'opc_type': ua.VariantType.Int32, }, 'bool': { 'converter': bool, - 'opc_type': VariantType.Boolean, + 'opc_type': ua.VariantType.Boolean, }, 'str': { 'converter': str, - 'opc_type': VariantType.String, - } + 'opc_type': ua.VariantType.String, + }, } -class OpcRepository(): - def __init__(self, id: str, url: str, logger: Logger, - notification_handler: NotificationHandler, - reconnection_interval: int = 60, server_uri: str = None, cert_path: str = None, - private_key_path: str = None, server_cert_path: str = None, pod_id: str = None): +class OpcRepository(SientiaMonitoring): + """ + Synchronous OPC UA repository for connect/disconnect and typed writes. + + Uses ``asyncua.sync.Client`` with the same session metrics, Tier-1 Bad* reconnect, + and structured write error payloads as the async repository on ``main``. + """ + + def __init__( + self, + opc_id: str, + url: str, + server_name: str, + logger: Logger, + notification_handler: NotificationHandler, + metrics_controller: MetricsController, + reconnection_interval: int = 60, + server_uri: str | None = None, + cert_path: str | None = None, + private_key_path: str | None = None, + server_cert_path: str | None = None, + ): self.url = url - self.id = id + self.id = opc_id + self.server_name = server_name self.server_uri = server_uri self.cert_path = cert_path self.private_key_path = private_key_path self.server_cert_path = server_cert_path - self.logger = logger - self.error_count = 0 self.reconnection_interval = reconnection_interval - self.last_reconnection_time = None + self.last_reconnection_time: None | datetime = None + self.disconnection_interval = 10.0 self.notification_handler = notification_handler - self.client = None - self.pod_id = pod_id + self.client: None | Client = None + + SientiaMonitoring.__init__(self, logger, notification_handler, metrics_controller) self.metadata = { 'model_name': '-', 'model_id': '-', 'workflow_name': 'opc_repository', - 'schedule_name': '-' + 'schedule_name': '-', + } + self._last_write_mono: float | None = None + self._connection_lock = threading.Lock() + self._session_ready = threading.Event() + self._reconnect_thread: threading.Thread | None = None + self._allow_reconnect = True + + def _opc_debug_tags(self, session_id: str) -> dict[str, str]: + """ + Build Prometheus/log label tags for OPC session-scoped metrics. + + Args: + session_id (str): OPC UA session token string. + + Return: + dict[str, str]: Labels pod_id, server_name, runtime, opc_server_id, session_id. + """ + return { + 'pod_id': str(getattr(self, 'pod_id', 'unknown')), + 'server_name': self.server_name, + 'runtime': str(getattr(self, 'runtime', 'unknown')), + 'opc_server_id': self.id, + 'session_id': session_id, } - async def set_security(self): - """ - Configures the security settings for the OPC UA client. - This method sets up the security policy, certificates, and timeouts - required for establishing a secure connection with the OPC UA server. - Raises: - ValueError: If either the certificate path or private key path is not provided. - Attributes: - - cert_path (str): Path to the client's certificate file. - - private_key_path (str): Path to the client's private key file. - - server_cert_path (str, optional): Path to the server's certificate file. - - server_uri (str): The URI of the server to be used as the application URI. - - client (opcua.Client): The OPC UA client instance. - - logger (logging.Logger): Logger instance for logging information. - Security Settings: - - Security Policy: Basic256 - - Secure Channel Timeout: 10,000,000 ms - - Session Timeout: 10,000,000 ms + def _is_session_open(self) -> bool: """ + Return whether the asyncua client has an open transport session. - if not all([self.cert_path, self.private_key_path]): + Return: + bool: True when protocol exists and is not closed. + """ + if self.client is None: + return False + try: + proto = self.client.aio_obj.uaclient.protocol + return proto is not None and proto.state != 'closed' + except Exception: + return False + + def _reconnection_window_elapsed(self) -> bool: + """ + Return whether enough time has passed since the last reconnect attempt. + + Return: + bool: True if a new reconnect is allowed. + """ + if self.last_reconnection_time is None: + return True + return ( + datetime.now() - self.last_reconnection_time + ).total_seconds() > self.reconnection_interval + + def _not_connected_error(self) -> dict[str, Any]: + """ + Build the standard error payload when validate_connection finds no open protocol. + + Return: + dict[str, Any]: Notification fields for OPC_CONNECTION_NOT_READY. + """ + return { + 'notification_id': f'OPC_CONNECTION_NOT_READY_{self.id}', + 'message': f'OPC server {self.id} is not connected', + 'block': 'opc_repository', + 'level': NotificationLevel.WARNING, + } + + def set_security(self) -> None: + """ + Configure certificates and timeouts on the sync asyncua client. + + Raises: + ValueError: If cert paths or client are missing. + """ + if self.cert_path is None or self.private_key_path is None: raise ValueError( - "Certificate and private key paths must be provided for secure connection.") + 'Certificate and private key paths must be provided for secure connection.' + ) + cert = Path(self.cert_path) private_key = Path(self.private_key_path) - server_cert = Path( - self.server_cert_path) if self.server_cert_path else None + server_cert = Path(self.server_cert_path) if self.server_cert_path else None + + if self.client is None: + raise ValueError('Client must be initialized before setting security') self.client.application_uri = self.server_uri - self.logger.custom_info('Setting security...', self.metadata) - await self.client.set_security( - SecurityPolicyBasic256, - certificate=str(cert), - private_key=str(private_key), - server_certificate=str(server_cert) + self.info('Setting security...', self.metadata) + self.client.set_security( + security_policies.SecurityPolicyBasic256, + str(cert), + str(private_key), + None, + str(server_cert) if server_cert else None, ) - self.client.secure_channel_timeout = 10000000 - self.client.session_timeout = 10000000 + aio = self.client.aio_obj + aio.secure_channel_timeout = OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS + aio.session_timeout = OPC_UA_SESSION_AND_CHANNEL_TIMEOUT_MS - async def connect(self) -> tuple[bool, dict[str, Any]]: + def _create_client(self) -> None: """ - Establishes a connection to the OPC server. - This method initializes the OPC client using the provided URL and - sets up security if a certificate path is specified. It then - attempts to connect to the server and logs the connection status. + Instantiate the sync Client and apply security when configured. + + Caller must hold _connection_lock. Does not open a UA session. + Raises: - Exception: If the connection to the OPC server fails. + OpcClientAlreadyExistsError: If self.client is already set. """ + if self.client is not None: + raise OpcClientAlreadyExistsError( + f'OPC client already exists for server {self.id}; ' + 'call disconnect() before creating a new client' + ) - self.client = Client(self.url) + self.client = Client(self.url, timeout=10) + aio = self.client.aio_obj + if hasattr(aio, 'watchdog_intervall'): + aio.watchdog_intervall = 50 + aio.name = self.pod_id + aio.description = self.pod_id + pod_uri = self.pod_id.replace('-', ':') + self.client.application_uri = pod_uri + aio.product_uri = pod_uri if self.cert_path: - await self.set_security() - self.logger.custom_info( - f'Starting connection to OPC server {self.id}...', self.metadata) - return await self.try_connect() + self.set_security() - async def try_connect(self) -> tuple[bool, dict[str, Any]]: + def _open_session(self) -> tuple[bool, dict[str, Any]]: """ - Attempt to establish connection to the OPC server. + Open the OPC UA session on the existing client. - This method performs the actual connection attempt to the OPC server - and handles connection failures with comprehensive error reporting. - It updates reconnection timing and provides detailed error information - for operational monitoring and debugging. + Caller must hold _connection_lock. - Returns: - tuple[bool, dict[str, Any]]: Connection result - - bool: True if connection successful, False otherwise - - dict: Error information if connection failed + Raises: + OpcClientNotInitializedError: If self.client is None. + OpcSessionAlreadyConnectedError: If a session is already open. + + Return: + tuple[bool, dict[str, Any]]: Success flag and error payload on connect failure. """ + if self.client is None: + raise OpcClientNotInitializedError( + f'OPC client is not initialized for server {self.id}; ' + 'call _create_client() before opening a session' + ) + if self._is_session_open(): + raise OpcSessionAlreadyConnectedError( + f'OPC session already connected for server {self.id}; ' + 'call disconnect() before connecting again' + ) + + tags = { + 'pod_id': self.pod_id, + 'server_name': self.server_name, + } + self.emit_metric_sync(metrics.OPC_CONNECTIONS_TOTAL, tags) try: - self.last_reconnection_time = datetime.now() - await self.client.connect() - return True, {} - except Exception as e: - trace = traceback.format_exc() - self.logger.custom_error(trace, self.metadata) + self.client.connect() + aio = self.client.aio_obj + session_id = _opc_authentication_token_str(self.client) + revised_session_timeout_ms = int(aio.session_timeout) + revised_secure_channel_timeout_ms = int(aio.secure_channel_timeout) + self.info( + f'OPC new session connected opc_server_id={self.id} session_id={session_id} ' + f'revised_session_timeout_ms={revised_session_timeout_ms} ' + f'revised_secure_channel_timeout_ms={revised_secure_channel_timeout_ms}', + self.metadata, + ) + self.emit_metric_sync( + metrics.OPC_SESSION_CREATED_TOTAL, self._opc_debug_tags(session_id) + ) + self.emit_metric_sync( + metric_object=metrics.OPC_SESSION_REVISED_TIMEOUT_MS, + method='set', + tags=self._opc_debug_tags(session_id), + value=revised_session_timeout_ms, + ) + self.emit_metric_sync( + metric_object=metrics.OPC_CONNECTION_STATUS, + method='set', + tags={**tags, 'server_url': self.url}, + value=1, + ) + + self._last_write_mono = None + self._session_ready.set() + return True, {} + + except Exception as e: + self._disconnect_locked() + trace = traceback.format_exc() + self.error(trace, self.metadata) + self.emit_metric_sync(metrics.OPC_CONNECTIONS_FAILED, tags) return False, { - "notification_id": f"OPC_CONNECTION_ERROR_{self.id}", - "message": f"Failed to connect to OPC server: {e}", - "block": "opc_repository", - "level": NotificationLevel.ERROR, - "attachment_content": trace + 'notification_id': f'OPC_CONNECTION_ERROR_{self.id}', + 'message': f'Failed to connect to OPC server: {e}', + 'block': 'opc_repository', + 'level': NotificationLevel.ERROR, + 'attachment_content': trace, } - async def disconnect(self): + def _connect_locked(self) -> tuple[bool, dict[str, Any]]: """ - Gracefully disconnect from the OPC server. + Create the client when absent, then open a UA session. - This method safely terminates the connection to the OPC server - and cleans up client resources. It handles disconnection errors - gracefully and ensures proper resource cleanup. + Caller must hold _connection_lock. + + Raises: + OpcSessionAlreadyConnectedError: If a session is already open. + + Return: + tuple[bool, dict[str, Any]]: Result from _open_session on connect failure. """ + if self._is_session_open(): + raise OpcSessionAlreadyConnectedError( + f'OPC session already connected for server {self.id}; ' + 'call disconnect() before connecting again' + ) + if self.client is None: + self._create_client() + return self._open_session() + + def _disconnection_fallback(self) -> list[dict[str, Any]]: + """ + Try up to five times to disconnect from the OPC UA server. + """ + assert self.client is not None + error_stack: list[dict[str, Any]] = [] + for i in range(5): + try: + self.info( + f'Disconnecting from OPC UA server, attempt {i + 1} of 5', + self.metadata, + ) + self.client.disconnect() + return [] + except Exception as e: + self.error( + f'Failed to disconnect from OPC UA server in attempt {i + 1} of 5: {e}', + self.metadata, + ) + error_stack.append( + { + 'attempt': i + 1, + 'error': str(e), + 'traceback': traceback.format_exc(), + } + ) + time.sleep(self.disconnection_interval * i) + return error_stack + + def _disconnect_locked(self) -> None: + """ + Tear down the current session and client. + + Caller must hold _connection_lock. + """ + self._last_write_mono = None + self._session_ready.clear() + if self.client is None: return - try: - await self.client.disconnect() - self.logger.custom_info( - 'Disconnected from OPC server', self.metadata) - except Exception as e: - self.logger.custom_error( - f"Failed to disconnect from OPC server: {e}", self.metadata) + + session_id = _opc_authentication_token_str(self.client) + self.info( + f'OPC disconnecting opc_server_id={self.id} session_id={session_id}', + self.metadata, + ) + self.emit_metric_sync(metrics.OPC_SESSION_CLOSED_TOTAL, self._opc_debug_tags(session_id)) + + errors = self._disconnection_fallback() + if errors: + self.send_notification( + metadata=self.metadata, + notification_id=f'OPC_DISCONNECTION_ERROR_{self.id}', + message='Failed to disconnect from OPC server in 5 attempts.', + block='opc_repository', + level=NotificationLevel.ERROR, + attachment_content=json.dumps(errors, indent=4), + ) + else: + self.warning(f'Disconnected from OPC server {self.id} successfully', self.metadata) + + self.emit_metric_sync( + metric_object=metrics.OPC_CONNECTION_STATUS, + method='set', + tags={ + 'pod_id': self.pod_id, + 'server_name': self.server_name, + 'server_url': self.url, + }, + value=0, + ) self.client = None - async def validate_connection(self) -> tuple[bool, dict[str, Any]]: + def _reconnect_locked(self) -> tuple[bool, dict[str, Any]]: """ - Validate and maintain OPC server connection health. + Close the current session and open a new one. - This method performs comprehensive connection validation and - implements automatic reconnection logic for production reliability. - It handles various connection states and implements intelligent - reconnection strategies with error counting and timing controls. + Caller must hold _connection_lock. Records last_reconnection_time for interval gating. - Connection Validation: - 1. Checks client existence and connection state - 2. Implements error counting with automatic disconnection - 3. Enforces reconnection timing windows - 4. Provides detailed error reporting and notifications - - Reconnection Strategy: - - Error Count Threshold: Disconnects after 5 consecutive errors - - Reconnection Window: Enforces minimum intervals between attempts - - Automatic Recovery: Attempts reconnection when conditions allow - - State Monitoring: Continuously monitors connection health - - Args: - None - - Returns: - tuple[bool, dict[str, Any]]: Connection validation result - - bool: True if connection is healthy, False otherwise - - dict: Error information if validation fails + Return: + tuple[bool, dict[str, Any]]: Result from _connect_locked after teardown. """ - if self.client is None: - return await self.connect() + self.last_reconnection_time = datetime.now() + self._disconnect_locked() + return self._connect_locked() - if self.error_count > 5: - self.logger.custom_warning( - f"OPC server {self.id} will be disconnected due to multiple errors", self.metadata) - try: - await self.disconnect() - except Exception as e: - trace = traceback.format_exc() - self.logger.custom_error( - f"Failed to disconnect from OPC server: {e}", self.metadata) - self.logger.custom_error(trace, self.metadata) - self.logger.custom_info( - f"Attempting to reconnect to OPC server {self.id}...", self.metadata) - return await self.connect() + def connect(self) -> tuple[bool, dict[str, Any]]: + """ + Open an OPC UA session under the connection lock (worker initialization). + """ + with self._connection_lock: + self.info( + f'Starting connection to OPC server {self.id}:{self.server_name}...', + self.metadata, + ) + return self._connect_locked() - # Check if client is connected using asyncua's connection state - try: - if self.client.uaclient.protocol is None or self.client.uaclient.protocol.state == "closed": - # OPC server is not connected - self.logger.custom_error( - f"OPC server {self.id} is not connected", self.metadata) - if self.last_reconnection_time is None or (datetime.now() - self.last_reconnection_time).total_seconds( - ) > self.reconnection_interval: - await self.disconnect() - self.logger.custom_info( - f"Trying to reconnect to OPC server {self.id}...", self.metadata) - return await self.connect() + def disconnect(self) -> None: + """ + Gracefully disconnect from the OPC server under the connection lock. - return False, { - "notification_id": f"OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}", - "message": f"OPC server {self.id} is not connected, waiting for next reconnection window...", - "block": "opc_repository", - "level": NotificationLevel.WARNING - } + Disables background reconnect so late writes during worker shutdown do not + respawn sessions. + """ + with self._connection_lock: + self._allow_reconnect = False + self._disconnect_locked() + + def validate_connection(self) -> tuple[bool, dict[str, Any]]: + """ + Read-only check that the asyncua protocol is open. + + Caller must ensure _session_ready before writing. Does not connect or reconnect. + + Return: + tuple[bool, dict[str, Any]]: (True, {}) when open, otherwise (False, error). + """ + if self._is_session_open(): return True, {} - except Exception as e: - trace = traceback.format_exc() - message = f"Failed to validate connection to OPC server: {e}" - self.logger.custom_error(message, self.metadata) - return False, { - "notification_id": f"OPC_CONNECTION_CHECK_ERROR_{self.id}", - "message": message, - "block": "opc_repository", - "level": NotificationLevel.ERROR, - "attachment_content": trace - } + self.error(f'OPC server {self.id} is not connected', self.metadata) + return False, self._not_connected_error() - async def write_data(self, node: str, value: Any, data_type: str, - logger: Logger, metadata: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + def _reconnect_thread_in_progress(self) -> bool: """ - Write data to OPC server with comprehensive validation and monitoring. + Return whether a background reconnect thread is currently running. - This method provides secure and reliable data writing to OPC servers - with automatic connection validation, data type conversion, and - comprehensive error handling. It implements performance monitoring - and metrics collection for operational visibility. + Return: + bool: True when a reconnect thread exists and is alive. + """ + return self._reconnect_thread is not None and self._reconnect_thread.is_alive() - Data Writing Process: - 1. Connection validation and automatic reconnection - 2. Node validation and error handling - 3. Data type conversion and validation - 4. OPC data writing with timestamp - 5. Performance metrics collection - 6. Error handling and notification + def _start_reconnect(self, reason: str, session_id: str) -> None: + """ + Schedule a background reconnect when allowed by interval and thread state. + + Clears _session_ready before starting the thread. No-op when _allow_reconnect is + False, the reconnection window has not elapsed, or a reconnect is already running. Args: - node (str): OPC node identifier to write data to - value (Any): Data value to write to the OPC node - data_type (str): Data type for OPC conversion - logger (Logger): Logger instance for operation logging - metadata (dict[str, Any]): Context metadata for logging and metrics - - Returns: - tuple[bool, dict[str, Any]]: Write operation result - - bool: True if write successful, False otherwise - - dict: Error information if write failed + reason (str): Trigger for reconnect (OPC status name or synthetic reason). + session_id (str): Session token before failure. """ + if not self._allow_reconnect: + return + if not self._reconnection_window_elapsed(): + self.warning( + f'OPC reconnect skipped reason=reconnection_window opc_server_id={self.id} ' + f'reconnect_reason={reason}', + self.metadata, + ) + return + if self._reconnect_thread_in_progress(): + self.warning( + f'OPC reconnect skipped reason=in_progress opc_server_id={self.id} ' + f'reconnect_reason={reason}', + self.metadata, + ) + return - is_connected, error = await self.validate_connection() + self._session_ready.clear() + self.info( + f'OPC reconnect scheduled reconnect_reason={reason} opc_server_id={self.id} ' + f'old_session_id={session_id}', + self.metadata, + ) + self._reconnect_thread = threading.Thread( + target=self._run_reconnect, + args=(reason, session_id), + daemon=True, + ) + self._reconnect_thread.start() + def _run_reconnect(self, reason: str, session_id: str) -> None: + """ + Tear down and re-establish the OPC UA session under the connection lock. + + Args: + reason (str): Trigger for reconnect (OPC status or ProtocolClosed). + session_id (str): Previous session token string. + """ + try: + with self._connection_lock: + self.info( + f'OPC reconnect started reconnect_reason={reason} opc_server_id={self.id} ' + f'old_session_id={session_id}', + self.metadata, + ) + success, error = self._reconnect_locked() + if not success: + self.error( + f'OPC reconnect failed reconnect_reason={reason} opc_server_id={self.id}', + self.metadata, + ) + if error: + self.error(error.get('message', ''), self.metadata) + except Exception: + self.error( + f'OPC reconnect task failed opc_server_id={self.id} reconnect_reason={reason}', + self.metadata, + ) + self.error(traceback.format_exc(), self.metadata) + + def _log_write_inter_arrival(self, session_id: str, node: str) -> None: + """ + Log elapsed wall time since the previous successful OPC write on this repository. + + Args: + session_id (str): Current OPC UA session token string. + node (str): Node id written in this operation. + """ + now = time.monotonic() + if self._last_write_mono is not None: + delta_s = now - self._last_write_mono + self.info( + f'OPC write inter-arrival_s={delta_s:.6f} opc_server_id={self.id} ' + f'session_id={session_id} node={node}', + self.metadata, + ) + if self.client is not None: + session_timeout_ms = float(self.client.aio_obj.session_timeout) + if session_timeout_ms > 0 and delta_s > (session_timeout_ms / 1000.0): + self.emit_metric_sync( + metrics.OPC_WRITE_INTER_ARRIVAL_OVER_SESSION_TIMEOUT_TOTAL, + self._opc_debug_tags(session_id), + ) + self._last_write_mono = now + + def _emit_opc_write_metric( + self, session_id: str, result: str, metadata: dict[str, Any] | None + ) -> None: + self.emit_metric_sync( + metrics.OPC_WRITE_ATTEMPTS_TOTAL, + { + **self._opc_debug_tags(session_id), + **_model_labels_from_write_metadata(metadata), + 'result': result, + }, + ) + + def _write_failure_payload( + self, + notification_id: str, + message: str, + level: NotificationLevel = NotificationLevel.ERROR, + attachment_content: str | None = None, + opc_error_kind: str | None = None, + opc_status: str | None = None, + ) -> dict[str, Any]: + payload: dict[str, Any] = { + 'notification_id': notification_id, + 'message': message, + 'block': 'opc_repository', + 'level': level, + } + if attachment_content is not None: + payload['attachment_content'] = attachment_content + if opc_error_kind is not None: + payload['opc_error_kind'] = opc_error_kind + if opc_status is not None: + payload['opc_status'] = opc_status + return payload + + def _handle_tier1_bad( + self, + exc: BaseException, + session_id: str, + node: str, + metadata: dict[str, Any], + phase: str, + ) -> tuple[bool, dict[str, Any]]: + """ + Record metrics/logs and schedule reconnect after a Tier-1 Bad* error. + + Args: + exc (BaseException): Tier-1 OPC UA error. + session_id (str): Session token at failure time. + node (str): Node id being written. + metadata (dict[str, Any]): Write context. + phase (str): get_node or write_value. + + Return: + tuple[bool, dict[str, Any]]: Always (False, error payload). + """ + opc_status = _opc_status_from_exception(exc) + trace = traceback.format_exc() + self.error(trace, metadata) + self._emit_opc_write_metric(session_id, opc_status, metadata) + self.error( + f'OPC write failed opc_status={opc_status} opc_server_id={self.id} ' + f'session_id={session_id} model_id={metadata.get("model_id", "unknown")} ' + f'model_name={metadata.get("model_name", "unknown")} node={node} phase={phase}', + metadata, + ) + self._start_reconnect(opc_status, session_id) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}', + message=f'Failed to {phase} on OPC server: {exc} | metadata: {metadata}', + attachment_content=trace, + opc_error_kind='session_bad', + opc_status=opc_status, + ) + + @staticmethod + def _write_node_value( + node_obj: Any, + ua_data: ua.DataValue, + data: Any, + variant_type: ua.VariantType, + ) -> None: + """ + Write a DataValue to a node, falling back to set_value when write_value is unavailable. + + Args: + node_obj: Sync or async node wrapper from asyncua. + ua_data (ua.DataValue): Encoded value for write_value. + data: Scalar converted value for set_value fallback. + variant_type (ua.VariantType): OPC UA type for set_value fallback. + """ + if hasattr(node_obj, 'write_value'): + try: + node_obj.write_value(ua_data) + return + except (AttributeError, TypeError): + pass + node_obj.set_value(data, variant_type) + + def _write_reconnect_in_progress(self, metadata: dict[str, Any]) -> tuple[bool, dict[str, Any]]: + """ + Fail a write because a background reconnect thread is already running. + + Args: + metadata (dict[str, Any]): Write context passed through to the activity. + + Return: + tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind reconnect_in_progress). + """ + self._emit_opc_write_metric('unknown', 'ReconnectInProgress', metadata) + self.warning( + f'OPC write rejected reconnect_in_progress opc_server_id={self.id} ' + f'model_id={metadata.get("model_id", "unknown")} ' + f'model_name={metadata.get("model_name", "unknown")}', + metadata, + ) + return False, { + 'notification_id': f'OPC_WRITE_RECONNECT_IN_PROGRESS_{self.id}', + 'message': f'OPC write skipped: reconnect in progress | metadata: {metadata}', + 'block': 'opc_repository', + 'level': NotificationLevel.WARNING, + 'opc_error_kind': 'reconnect_in_progress', + } + + def _write_connection_lost( + self, metadata: dict[str, Any], opc_status: str + ) -> tuple[bool, dict[str, Any]]: + """ + Fail a write after scheduling reconnect for a closed or stale session. + + Args: + metadata (dict[str, Any]): Write context passed through to the activity. + opc_status (str): Synthetic reason (ProtocolClosed, SessionNotReady). + + Return: + tuple[bool, dict[str, Any]]: (False, error info with opc_error_kind connection_lost). + """ + self._emit_opc_write_metric('unknown', opc_status, metadata) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_CONNECTION_LOST_{self.id}', + message=f'OPC write skipped: connection lost ({opc_status}) | metadata: {metadata}', + level=NotificationLevel.WARNING, + opc_error_kind='connection_lost', + opc_status=opc_status, + ) + + def write_data( + self, node: str, value: Any, data_type: str, metadata: dict[str, Any] + ) -> tuple[bool, dict[str, Any]]: + """ + Write data to OPC server with a single attempt and background reconnect scheduling. + + Reconnect is scheduled on Tier-1 Bad*, closed protocol, or stale session readiness. + There is no retry within the same call. + + Args: + node (str): OPC UA node id to write. + value (Any): Value to convert and send. + data_type (str): Logical type key (float, int, bool, str, double). + metadata (dict[str, Any]): Activity context (model_id, model_name, etc.). + + Return: + tuple[bool, dict[str, Any]]: (True, {response_time}) on success, or + (False, structured error info) on failure. + """ + if self._reconnect_thread_in_progress(): + return self._write_reconnect_in_progress(metadata) + + if not self._session_ready.is_set(): + session_id = _opc_authentication_token_str(self.client) + self._start_reconnect('SessionNotReady', session_id) + if self._reconnect_thread_in_progress(): + return self._write_reconnect_in_progress(metadata) + return self._write_connection_lost(metadata, 'SessionNotReady') + + is_connected, _error = self.validate_connection() if not is_connected: - return False, error + session_id = _opc_authentication_token_str(self.client) + self._start_reconnect('ProtocolClosed', session_id) + return self._write_connection_lost(metadata, 'ProtocolClosed') start_time = time.time() + session_id = _opc_authentication_token_str(self.client) try: + assert self.client is not None node_obj = self.client.get_node(node) except Exception as e: + if is_reconnectable_opcua_bad(e): + return self._handle_tier1_bad(e, session_id, node, metadata, 'get_node') trace = traceback.format_exc() - logger.custom_error(trace, metadata.get('schedule_name', 'N/A')) - self.error_count += 1 - return False, { - "notification_id": f"OPC_WRITE_GET_NODE_ERROR_{self.id}", - "message": f"Failed to get node from OPC server: {e} | metadata: {metadata}", - "block": "opc_repository", - "level": NotificationLevel.ERROR, - "attachment_content": trace - } + self.error(trace, metadata) + self._emit_opc_write_metric(session_id, f'GetNodeError:{type(e).__name__}', metadata) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_GET_NODE_ERROR_{self.id}', + message=f'Failed to get node from OPC server: {e} | metadata: {metadata}', + attachment_content=trace, + ) if data_type not in data_type_map: - return False, { - "notification_id": f"OPC_WRITE_DATA_TYPE_ERROR_{self.id}", - "message": f"Unsupported data type: {data_type} | metadata: {metadata}", - "block": "opc_repository", - "level": NotificationLevel.ERROR - } + self._emit_opc_write_metric(session_id, 'UnsupportedDataType', metadata) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_DATA_TYPE_ERROR_{self.id}', + message=f'Unsupported data type: {data_type} | metadata: {metadata}', + ) data = data_type_map[data_type]['converter'](value) - logger.custom_info( - f'Writing {data} - {type(data)} to {node}', metadata) - now = datetime.now() - ua_data = DataValue( - Variant(data, data_type_map[data_type]['opc_type']), - SourceTimestamp=DateTime( - now.year, - now.month, - now.day, - now.hour, - now.minute, - now.second, - now.microsecond - ) + self.info(f'Writing {data} - {type(data)} to {node}', metadata) + variant_type = data_type_map[data_type]['opc_type'] + ua_data = ua.DataValue( + ua.Variant(data, variant_type), ) try: - await node_obj.write_value(ua_data) - - metrics.PREDICTION_OPC_WRITING_COUNT.labels( - pod_id=self.pod_id, - model_name=metadata['model_name'], - pipeline_name=metadata['workflow_name'], - opc_server_id=self.id - ).inc() - + self._write_node_value(node_obj, ua_data, data, variant_type) end_time = time.time() response_time = end_time - start_time - metrics.PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR.labels( - pod_id=self.pod_id, - model_name=metadata['model_name'], - pipeline_name=metadata['workflow_name'], - opc_server_id=self.id - ).observe(response_time) - except Exception as e: + if is_reconnectable_opcua_bad(e): + return self._handle_tier1_bad(e, session_id, node, metadata, 'write_value') trace = traceback.format_exc() - logger.custom_error(trace, metadata) - self.error_count += 1 - return False, { - "notification_id": f"OPC_WRITE_DATA_ERROR_{self.id}", - "message": f"Failed to write data to OPC server: {e} | metadata: {metadata}", - "block": "opc_repository", - "level": NotificationLevel.ERROR, - "attachment_content": trace - } - self.error_count = 0 + self.error(trace, metadata) + self._emit_opc_write_metric(session_id, type(e).__name__, metadata) + return False, self._write_failure_payload( + notification_id=f'OPC_WRITE_DATA_ERROR_{self.id}', + message=f'Failed to write data to OPC server: {e} | metadata: {metadata}', + attachment_content=trace, + ) - return True, {} + self._emit_opc_write_metric(session_id, 'OK', metadata) + self._log_write_inter_arrival(session_id, node) + + return True, { + 'response_time': response_time, + } diff --git a/laborious/worker/worker.py b/laborious/worker/worker.py index 567a257..f284a8c 100644 --- a/laborious/worker/worker.py +++ b/laborious/worker/worker.py @@ -1,78 +1,88 @@ """ Laborious Worker Module -This module provides the main worker implementation for the Sientia DataOps Laborious system. -It orchestrates Temporal workers, manages task queues, and handles the lifecycle of -prediction and retraining workflows. +Entry process that connects to Temporal, registers Laborious activities, and runs four workers in +parallel. Each worker shares the same ``Activities`` instance (single Postgres pool, single MLflow +repository, single PluginStore handle) but polls a different task queue. -The worker supports two main task queues: -- predictions_batch-queue: Handles batch prediction workflows -- minimal_retrain-queue: Handles model retraining workflows +Task queues (see ``sientia_do.temporal.worker.prepare_worker``): +- ``predictions_batch-{runtime}-queue`` + sub-workflows on the same queue (ML-heavy path). +- ``minimal_retrain-{runtime}-queue`` (retrain + promote + export). +- ``drift-queue`` and ``simple_metrics-queue`` without a runtime suffix so existing schedulers + keep stable queue names. -Key Features: -- Automatic scaling with PollerBehaviorAutoscaling -- Prometheus metrics integration -- Comprehensive error handling and logging -- Graceful shutdown with cleanup -- Multiple worker instances for different workflow types +Bootstrap order: +1. Prometheus app metrics and Mongo-backed notification handler. +2. ``RUNTIME`` validation and ``PluginStore.install_runtime`` so ``SientiaModel`` code is importable. +3. ``Activities`` construction (builds ``SientiaMLflowRepository`` internally from env). +4. OPC client initialization inside activities. +5. Temporal ``Runtime`` with SDK Prometheus bind, client connect, then ``prepare_worker`` per workflow. + +Shutdown closes workers, notifications, activities (pools + OPC), and clears ``app_up``. Environment Variables: -- TEMPORAL_HOST: Temporal server address (default: localhost:7233) -- TEMPORAL_NAMESPACE: Temporal namespace (default: laborious) -- POD_ID: Kubernetes pod identifier for metrics -- HTTP_METRICS_PORT: Prometheus metrics server port (default: 9090) -- HTTP_SDK_METRICS_PORT: Temporal SDK metrics port (default: 9091) -- PROJECT_NAME: Project name for notifications (default: laborious) +- RUNTIME: Required non-empty string passed to ``install_runtime``. +- STORE_* / PYPI_*: Plugin store and private index (see ``build_plugin_store_config``). +- TEMPORAL_HOST, TEMPORAL_NAMESPACE: Cluster connection. +- POD_ID, HTTP_METRICS_PORT, HTTP_SDK_METRICS_PORT: Observability. +- PROJECT_NAME, MONGODB_*: Notifications (via ``build_mongodb_config`` in handler). +- POSTGRES_*, MINIO_*, OPC_*, PI_WEB_API_*, MLFLOW_*: Passed through ``Activities`` helpers. """ -from temporalio import workflow, client -from temporalio.worker import Worker, PollerBehaviorAutoscaling -from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig +from temporalio import client, workflow +from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig with workflow.unsafe.imports_passed_through(): + import asyncio import os import sys - import asyncio - from laborious.workflows.minimal_retrain import MinimalRetrain - from laborious.workflows.predictions_batch import PredictionsBatch - from laborious.workflows.sub_workflows.prediction_process import PredictionProcess - from laborious.workflows.sub_workflows.format_and_export_prediction import \ - FormatAndExportPrediction - from laborious.activities.activities import Activities - from laborious.utils.connectors_config import ( - build_postgres_config, - build_mlflow_config, - build_opc_config, - build_mongodb_config - ) + + from prometheus_client import start_http_server from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler from sientia_do.observability.logger import get_logger - from laborious import metrics - from prometheus_client import start_http_server + from sientia_do.observability.metrics_controller import MetricsController + from sientia_do.temporal.worker.prepare_worker import prepare_worker + from sientia_do.utils.connectors_config import ( + build_api_config, + build_mongodb_config, + build_postgres_config, + ) + from sientia_model.model_repository.plugin_store import PluginStore -POD_ID = os.getenv('POD_ID') -SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091")) + from laborious import metrics + from laborious.activities.activities import Activities + from laborious.utils.connectors_config import ( + build_minio_config, + build_opc_config, + build_plugin_store_config, + ) + from laborious.workflows.drift import Drift + from laborious.workflows.minimal_retrain import MinimalRetrain + from laborious.workflows.predictions_batch import PredictionsBatch + from laborious.workflows.simple_metrics import SimpleMetrics + from laborious.workflows.sub_workflows.format_and_export_prediction import ( + FormatAndExportPrediction, + ) + from laborious.workflows.sub_workflows.prediction_process import PredictionProcess + +POD_ID = os.getenv('HOSTNAME') +SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091')) async def main(): """ - Main entry point for the Laborious worker application. + Run the full worker lifecycle: metrics, notifications, runtime install, workers, gather. - This function initializes and starts all components of the worker: - 1. Sets up logging and metadata - 2. Starts Prometheus metrics server - 3. Initializes notification handler - 4. Creates and configures activities - 5. Initializes OPC connections - 6. Starts Temporal client and workers - 7. Manages worker lifecycle and graceful shutdown - - The function runs indefinitely until interrupted or an error occurs. - On error, it performs cleanup and exits with a non-zero status code. + Exits the process with code 0 on normal completion of all worker tasks, or 1 after logging + if any worker raises. ``finally`` always shuts down notifications and activities and sets + ``app_up`` to 0 before ``sys.exit``. Raises: - Exception: Any unhandled exception during worker execution - SystemExit: On graceful shutdown or error conditions + Exception: Propagated from ``asyncio.gather`` only before ``finally`` handling; typically + workers run until cancelled. + + Return: + None (process terminates via ``sys.exit`` from the ``finally`` block). """ host = os.getenv('TEMPORAL_HOST', 'localhost:7233') logger = get_logger(__name__) @@ -87,7 +97,7 @@ async def main(): logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata) - logger.custom_info("Starting prometheus client...", metadata) + logger.custom_info('Starting prometheus client...', metadata) start_prometheus_server() logger.custom_info('Starting Notification Handler...', metadata) @@ -97,29 +107,77 @@ async def main(): connection_string=mongo_config['connection_string'], database=mongo_config['database_name'], logger=logger, - project_name=os.getenv('PROJECT_NAME', 'laborious') + project_name=os.getenv('PROJECT_NAME', 'laborious'), ) + metrics_controller = MetricsController(logger=logger) + + runtime = os.getenv('RUNTIME', '').strip() + if not runtime: + logger.custom_critical( + 'RUNTIME environment variable is required and must be non-empty', + metadata, + ) + metrics.APP_UP.labels(pod_id=POD_ID).set(0) + sys.exit(1) + + metadata_runtime = {**metadata, 'runtime': runtime} + logger.custom_info(f'Installing PluginStore runtime: {runtime}', metadata_runtime) + + ps_cfg = build_plugin_store_config() + plugin_store = PluginStore( + base_url=ps_cfg['base_url'], + owner=ps_cfg['owner'], + repo=ps_cfg['repo'], + username=ps_cfg['username'], + password=ps_cfg['password'], + branch=ps_cfg['branch'], + cache_ttl_seconds=ps_cfg['cache_ttl_seconds'], + pypi_index_url=ps_cfg['pypi_index_url'], + pypi_username=ps_cfg['pypi_username'], + pypi_password=ps_cfg['pypi_password'], + logger=logger, + notification_handler=notification_handler, + metrics_controller=metrics_controller, + ) + + if runtime == 'legacy': + to_install_runtime = 'single' + else: + to_install_runtime = runtime + + try: + await plugin_store.install_runtime( + runtime_name=to_install_runtime, metadata=metadata_runtime + ) + except Exception as exc: + logger.custom_critical( + f'Failed to install runtime {to_install_runtime}: {exc}', metadata_runtime + ) + metrics.APP_UP.labels(pod_id=POD_ID).set(0) + sys.exit(1) + logger.custom_info('Starting Activities...', metadata) activities = Activities( postgres_config=build_postgres_config(), - mlflow_config=build_mlflow_config(), + plugin_store=plugin_store, + minio_config=build_minio_config(), opc_config=build_opc_config(), + pi_web_api_config=build_api_config(), logger=logger, - notification_handler=notification_handler + notification_handler=notification_handler, + metrics_controller=metrics_controller, ) logger.custom_info('Initializing OPC...', metadata) - await activities.init_opc() + activities.init_opc() - logger.custom_info( - f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata) + logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata) new_runtime = Runtime( telemetry=TelemetryConfig( - metrics=PrometheusConfig( - bind_address=f"0.0.0.0:{SDK_METRICS_PORT}") + metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}') ) ) @@ -128,34 +186,55 @@ async def main(): temporal_client = await client.Client.connect( target_host=host, namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'), - runtime=new_runtime + runtime=new_runtime, ) logger.custom_info('Starting Workers...', metadata) workers = [ - Worker( - temporal_client, - task_queue='minimal_retrain-queue', - workflows=[MinimalRetrain], + prepare_worker( + temporal_client=temporal_client, + main_workflow=MinimalRetrain, + other_workflows=[], activities=[ - activities.load_custom_query, + activities.load_query_with_minio_offload, activities.retrain_model, activities.update_production_model, - activities.export_data_to_postgres + activities.format_retrain_report, + activities.export_data_to_postgres, ], - max_concurrent_workflow_tasks=50, - max_concurrent_activities=50, - max_concurrent_local_activities=50, - max_cached_workflows=200, - workflow_task_poller_behavior=PollerBehaviorAutoscaling(), - activity_task_poller_behavior=PollerBehaviorAutoscaling() + logger=logger, + runtime=runtime, ), - Worker( - temporal_client, - task_queue='predictions_batch-queue', - workflows=[PredictionsBatch, PredictionProcess, - FormatAndExportPrediction], + prepare_worker( + temporal_client=temporal_client, + main_workflow=SimpleMetrics, + other_workflows=[], + activities=[ + activities.load_custom_query, + activities.calculate_simple_metrics, + activities.export_data_to_postgres, + ], + logger=logger, + runtime='core', + ), + prepare_worker( + temporal_client=temporal_client, + main_workflow=Drift, + other_workflows=[], + activities=[ + activities.load_custom_query, + activities.get_reference_data, + activities.calculate_drift, + activities.export_data_to_postgres, + ], + logger=logger, + runtime='core', + ), + prepare_worker( + temporal_client=temporal_client, + main_workflow=PredictionsBatch, + other_workflows=[PredictionProcess, FormatAndExportPrediction], activities=[ # MLFlow activities.request_predict, @@ -164,24 +243,24 @@ async def main(): activities.input_gate, activities.mlflow_response_gate, activities.mlflow_content_gate, + activities.format_transformed_data, activities.format_prediction, activities.format_default_prediction, - activities.get_last_timestamp, # OPC activities.write_opc_data, - # Postgres - activities.load_custom_query, + # Postgres / MinIO offload + activities.load_query_with_minio_offload, + activities.cleanup_minio_objects_expired, activities.repeat_last_prediction, activities.export_data_to_postgres, - activities.write_metrics + activities.export_payload_to_postgres, + activities.write_metrics, + # Pi Web API + activities.write_pi_web_api_data, ], - max_concurrent_workflow_tasks=50, - max_concurrent_activities=50, - max_concurrent_local_activities=50, - max_cached_workflows=200, - workflow_task_poller_behavior=PollerBehaviorAutoscaling(), - activity_task_poller_behavior=PollerBehaviorAutoscaling() - ) + logger=logger, + runtime=runtime, + ), ] handlers = [] @@ -190,20 +269,17 @@ async def main(): logger.custom_info('Workers started successfully', metadata) + exit_code = 0 try: - # This will run the workers and wait for them to complete. - # If an exception occurs in any of the worker handlers, it will be propagated here. await asyncio.gather(*handlers) except BaseException as e: # NOSONAR - logger.custom_error(f"An unhandled exception occurred: {e}", metadata) + logger.custom_error(f'An unhandled exception occurred: {e}', metadata) + exit_code = 1 finally: - if notification_handler: - notification_handler.shutdown() - if activities: - await activities.shutdown() - # Exit with a non-zero status code to indicate failure to Kubernetes + notification_handler.shutdown() + activities.shutdown() metrics.APP_UP.labels(pod_id=POD_ID).set(0) # Mark app as DOWN - sys.exit(1) + sys.exit(exit_code) def start_prometheus_server(): @@ -224,12 +300,12 @@ def start_prometheus_server(): SystemExit: If the metrics server fails to start """ try: - port = int(os.getenv("HTTP_METRICS_PORT", 9090)) + port = int(os.getenv('HTTP_METRICS_PORT', 9090)) start_http_server(port) - print(f"Prometheus server started on port {port}.") + print(f'Prometheus server started on port {port}.') metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP except Exception as e: - print(f"Failed to start Prometheus server: {e}") + print(f'Failed to start Prometheus server: {e}') os._exit(1) diff --git a/laborious/workflows/drift.py b/laborious/workflows/drift.py new file mode 100644 index 0000000..2faff32 --- /dev/null +++ b/laborious/workflows/drift.py @@ -0,0 +1,107 @@ +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from datetime import timedelta + from typing import Any + + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities + + +@workflow.defn(name='drift') +class Drift: + @workflow.run + async def run(self, input_data: dict[str, Any]): + """ + Execute the drift workflow. + + This method orchestrates the complete drift process by: + 1. Loading data using the provided custom SQL query + 2. Preparing prediction configuration and filters + 3. Delegating to the PredictionProcess workflow for ML operations + """ + + metadata = { + 'metadata': { + 'schedule_name': input_data['schedule_name'], + 'model_name': input_data['model_name'], + 'model_id': input_data['model_id'], + 'workflow_name': 'drift', + } + } + + print(f'Input data: {input_data}', metadata) + + model_config = input_data['model_config'] + target_name = model_config['target'] + + gathering_query = f""" + SELECT * + FROM "{input_data['schema']}"."{input_data['source_table_name']}" + WHERE + model_id = '{input_data['model_id']}' AND + timestamp > NOW() - INTERVAL '{input_data['interval']} minutes' + ORDER BY timestamp ASC + """ # nosec B608 - values come from internal Temporal workflow config, not user input + + target_data_handler = workflow.start_activity_method( + Activities.load_custom_query, + { + **metadata, + 'query': gathering_query, + 'datetime_columns': ['timestamp', 'created_at'], + 'orient': 'records', + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=300), + ) + + reference_data_handler = workflow.start_activity_method( + Activities.get_reference_data, + {**metadata, 'model_name': input_data['model_name']}, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=300), + ) + + target_data = await target_data_handler + reference_data = await reference_data_handler + + if not target_data: + return + + drift_data = await workflow.execute_local_activity_method( + Activities.calculate_drift, + { + **metadata, + 'target_data': target_data, + 'reference_data': reference_data, + 'model_name': input_data['model_name'], + 'model_id': input_data['model_id'], + 'target_name': target_name, + 'drift_metrics': input_data.get( + 'drift_metrics', ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein'] + ), + 'chunk_period': input_data.get('chunk_period', 'min'), + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=300), + ) + + if drift_data: + await workflow.execute_activity_method( + Activities.export_data_to_postgres, + { + **metadata, + 'data': drift_data, + 'schema': input_data['schema'], + 'table_name': input_data['target_table_name'], + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=300), + ) diff --git a/laborious/workflows/minimal_retrain.py b/laborious/workflows/minimal_retrain.py index 1893e25..f8f5151 100644 --- a/laborious/workflows/minimal_retrain.py +++ b/laborious/workflows/minimal_retrain.py @@ -1,14 +1,17 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities - from typing import Any - from sientia_do.temporal.policies import retry_policy from datetime import timedelta + from typing import Any + + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities + from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload -@workflow.defn(name="minimal_retrain") -class MinimalRetrain(): +@workflow.defn(name='minimal_retrain') +class MinimalRetrain: """ Automated model retraining workflow for the Laborious system. @@ -63,44 +66,62 @@ class MinimalRetrain(): 'schedule_name': input_data['schedule_name'], 'model_name': input_data['model_name'], 'model_id': input_data['model_id'], - 'workflow_name': 'minimal_retrain' + 'workflow_name': 'minimal_retrain', } } model_name = input_data['model_name'] + model_config = input_data.get('model_config', {}) - data = await workflow.execute_local_activity_method( - Activities.load_custom_query, + storage_result = await workflow.execute_activity_method( + Activities.load_query_with_minio_offload, { **metadata, 'query': input_data['query'], - 'datetime_columns': input_data.get('datetime_columns', []) + 'datetime_columns': input_data.get('datetime_columns', []), + 'model_name': model_name, }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=600), ) + storage_payload = MinioDataFramePayload.from_dict(storage_result) + if not storage_payload.has_data(): + raise ValueError('No data returned from query') + experiment_response = await workflow.execute_activity_method( Activities.retrain_model, { **metadata, - 'data': data, - 'model_name': model_name + 'data': storage_result, + 'model_name': model_name, + 'model_config': model_config, }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(hours=1), ) - report = await workflow.execute_activity_method( - Activities.update_production_model, + if experiment_response['success']: + update_report = await workflow.execute_activity_method( + Activities.update_production_model, + {**metadata, 'model_name': model_name, **experiment_response}, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) + else: + update_report = {} + + report = await workflow.execute_local_activity_method( + Activities.format_retrain_report, { **metadata, + 'experiment_response': experiment_response, 'model_name': model_name, 'model_id': input_data['model_id'], - **experiment_response + 'update_report': update_report, }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) await workflow.execute_activity_method( @@ -109,8 +130,8 @@ class MinimalRetrain(): **metadata, 'data': report, 'schema': input_data['schema'], - 'table_name': input_data['table_name'] + 'table_name': input_data['table_name'], }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=600), ) diff --git a/laborious/workflows/predictions_batch.py b/laborious/workflows/predictions_batch.py index e522eaa..e68c215 100644 --- a/laborious/workflows/predictions_batch.py +++ b/laborious/workflows/predictions_batch.py @@ -1,14 +1,16 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities - from typing import Any - from sientia_do.temporal.policies import retry_policy from datetime import timedelta + from typing import Any + + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities -@workflow.defn(name="predictions_batch") -class PredictionsBatch(): +@workflow.defn(name='predictions_batch') +class PredictionsBatch: """ Main batch prediction workflow for the Laborious system. @@ -59,7 +61,10 @@ class PredictionsBatch(): - model_retention (int, optional): Model retention period in minutes - path_priority (list[str]): Decision path priority configuration - opc_output_config (dict, optional): OPC server export configuration + - pi_web_api_output_config (dict, optional): PI Web API export configuration - datetime_columns (list[str], optional): Columns to treat as datetime + - save_transform (bool, optional): Whether to save transformed data (default: True) + - prediction_store_policy (str, optional): Data retention policy (default: 'lts:1') Returns: None: The workflow completes successfully when the child workflow finishes @@ -74,20 +79,21 @@ class PredictionsBatch(): 'schedule_name': input_data['schedule_name'], 'model_name': input_data['model_name'], 'model_id': input_data['model_id'], - 'workflow_name': 'predictions_batch' + 'workflow_name': 'predictions_batch', } } - # Load data using custom query - data = await workflow.execute_local_activity_method( - Activities.load_custom_query, + # Load data using custom query with optional MinIO offload for large frames + data = await workflow.execute_activity_method( + Activities.load_query_with_minio_offload, { **metadata, 'query': input_data['query'], - 'datetime_columns': input_data.get('datetime_columns', []) + 'datetime_columns': input_data.get('datetime_columns', []), + 'model_name': input_data['model_name'], }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=300) + start_to_close_timeout=timedelta(seconds=300), ) # Prepare input for prediction_process workflow @@ -96,30 +102,26 @@ class PredictionsBatch(): 'data': data, 'schema': input_data['schema'], 'table_name': input_data['table_name'], + 'transform_table_name': input_data['transform_table_name'], 'model_id': input_data['model_id'], 'model_name': input_data['model_name'], - 'input_filters': input_data.get('input_filters', { - 'EMPTY_DATA': { - 'POLICY': 'STOP' - } - }), - 'mlflow_transform_filters': input_data.get('mlflow_transform_filters', { - 'API_ERROR': { - 'POLICY': 'STOP' - } - }), - 'mlflow_predict_filters': input_data.get('mlflow_predict_filters', { - 'API_ERROR': { - 'POLICY': 'STOP' - } - }), + 'input_filters': input_data.get( + 'input_filters', {'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}}} + ), + 'mlflow_transform_filters': input_data.get( + 'mlflow_transform_filters', {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}} + ), + 'mlflow_predict_filters': input_data.get( + 'mlflow_predict_filters', {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}} + ), 'model_config': input_data.get('model_config', {}), 'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']), 'opc_output_config': input_data.get('opc_output_config', {}), - 'prediction_store_policy': input_data.get( - 'prediction_store_policy', 'lts:1') + 'on_conflict': input_data.get('on_conflict', 'error'), + 'pi_web_api_output_config': input_data.get('pi_web_api_output_config', {}), + 'prediction_store_policy': input_data.get('prediction_store_policy', 'lts:1'), + 'save_transform': input_data.get('save_transform', True), } # Execute prediction process workflow - await workflow.execute_child_workflow( - 'prediction_process', prediction_input) + await workflow.execute_child_workflow('subworkflow.prediction_process', prediction_input) diff --git a/laborious/workflows/simple_metrics.py b/laborious/workflows/simple_metrics.py new file mode 100644 index 0000000..38b47c4 --- /dev/null +++ b/laborious/workflows/simple_metrics.py @@ -0,0 +1,95 @@ +from temporalio import workflow + +with workflow.unsafe.imports_passed_through(): + from datetime import timedelta + from typing import Any + + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities + + +@workflow.defn(name='simple_metrics') +class SimpleMetrics: + @workflow.run + async def run(self, input_data: dict[str, Any]): + """ + Execute the simple metrics workflow. + """ + metadata = { + 'metadata': { + 'model_id': input_data['model_id'], + 'model_name': input_data['model_name'], + 'workflow_name': 'simple_metrics', + 'schedule_name': input_data['schedule_name'], + } + } + + model_id = input_data['model_id'] + interval_minutes = input_data['interval_minutes'] + + model_config = input_data['model_config'] + target_name = model_config['target'] + + query = f""" + select p."timestamp", p.prediction, ld.value as "target" + from "{input_data['schema']}"."{input_data['predictions_table_name']}" p + inner join "{input_data['schema']}"."{input_data['data_table_name']}" ld + on p."timestamp" = ld."timestamp" + where + p.model_id = '{model_id}' and + p.prediction is not null and + ld.variable = '{target_name}' and + ld.value is not null and + p."timestamp" >= NOW() - INTERVAL '{interval_minutes} minutes' + order by + p."timestamp" desc; + """ # nosec B608 - values come from internal Temporal workflow config, not user input + + target_data = await workflow.execute_activity_method( + Activities.load_custom_query, + { + **metadata, + 'query': query, + 'datetime_columns': ['timestamp'], + 'orient': 'records', + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=300), + ) + + if not target_data: + return + + simple_metrics = await workflow.execute_local_activity_method( + Activities.calculate_simple_metrics, + { + **metadata, + 'model_id': model_id, + 'target_data': target_data, + 'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']), + 'interval_minutes': interval_minutes, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=300), + ) + + if not simple_metrics: + return + + await workflow.execute_activity_method( + Activities.export_data_to_postgres, + { + **metadata, + 'data': simple_metrics, + 'schema': input_data['schema'], + 'table_name': input_data['target_table_name'], + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=300), + ) diff --git a/laborious/workflows/sub_workflows/format_and_export_prediction.py b/laborious/workflows/sub_workflows/format_and_export_prediction.py index 8e7df07..cf950ce 100644 --- a/laborious/workflows/sub_workflows/format_and_export_prediction.py +++ b/laborious/workflows/sub_workflows/format_and_export_prediction.py @@ -1,15 +1,17 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities - from typing import Any from datetime import timedelta - from sientia_do.temporal.policies import retry_policy + from typing import Any + from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities -@workflow.defn(name="format_and_export_prediction") -class FormatAndExportPrediction(): +@workflow.defn(name='subworkflow.format_and_export_prediction') +class FormatAndExportPrediction: """ Data formatting and export workflow for prediction results. @@ -24,6 +26,7 @@ class FormatAndExportPrediction(): Export Destinations: - PostgreSQL Database: Persistent storage with timestamp conversion + - PI Web API: Real-time industrial system integration for prediction and confidence values - OPC Servers: Real-time industrial system integration - Prometheus Metrics: Performance monitoring and operational visibility """ @@ -36,9 +39,10 @@ class FormatAndExportPrediction(): This method orchestrates the complete data export process by: 1. Determining the appropriate formatting strategy based on path_flag 2. Formatting prediction data according to quality and requirements - 3. Exporting data to OPC servers for real-time industrial access - 4. Persisting data to PostgreSQL database with comprehensive metadata - 5. Recording performance metrics for operational monitoring + 3. Exporting data to PI Web API for real-time industrial access (if configured) + 4. Exporting data to OPC servers for real-time industrial access (if configured) + 5. Persisting data to PostgreSQL database with comprehensive metadata + 6. Recording performance metrics for operational monitoring The method implements flexible formatting strategies: - Normal predictions: Full data formatting with confidence scores @@ -48,29 +52,50 @@ class FormatAndExportPrediction(): Args: input_data: Complete configuration for the export workflow Required keys: + - metadata (dict): Workflow execution metadata - path_flag (str | None): Decision path flag for formatting strategy + - None: Normal prediction path with full formatting + - Any other value: Default prediction path for error conditions - data (dict[str, Any]): Prediction data to format and export - prediction_confidence (float): Confidence score for the prediction - timestamp (str): ISO-formatted timestamp for the prediction - model_id (int): Unique identifier for the ML model - model_name (str): Name of the ML model - - model_retention (str): Model retention policy configuration - - comment (str): Operational comment or error description - schema (str): Database schema for data storage - table_name (str): Target table for data persistence + Optional keys: - opc_output_config (dict[str, Any]): OPC server export configuration - - prediction_store_policy (str, optional): Data retention policy + - pi_web_api_output_config (dict[str, Any]): PI Web API export configuration + Contains endpoint, prediction_tags, and confidence_tags mappings + - transformed_data (dict[str, Any]): Transformed data to export separately + Only processed when path_flag is None + - transform_table_name (str): Target table for transformed data export + Required if transformed_data is provided + - prediction_store_policy (str): Data retention policy (e.g., 'lts:1', 'erl:2') + Required when path_flag is None + - comment (str): Operational comment or error description + Required when path_flag is not None Returns: - bool: True if the workflow completes successfully, False otherwise + None: The workflow completes successfully when all export operations finish + + Note: + When transformed_data is provided and path_flag is None, the workflow will: + 1. Format the transformed data using format_transformed_data + 2. Export it to a separate table (transform_table_name) asynchronously + 3. Wait for both prediction and transformed data exports to complete """ metadata = input_data['metadata'] path_flag = input_data['path_flag'] data = input_data['data'] + transformed_data = input_data.get('transformed_data', None) prediction_confidence = input_data['prediction_confidence'] + opc_output_config = input_data.get('opc_output_config', None) + pi_web_api_output_config = input_data.get('pi_web_api_output_config', None) + if path_flag is None: - # proceed with formatting and exporting + # Normal prediction path: format prediction data with full metadata prediction = await workflow.execute_local_activity_method( Activities.format_prediction, { @@ -79,14 +104,48 @@ class FormatAndExportPrediction(): 'timestamp': input_data['timestamp'], 'model_id': input_data['model_id'], 'prediction_confidence': prediction_confidence, - 'prediction_store_policy': input_data['prediction_store_policy'] + 'prediction_store_policy': input_data['prediction_store_policy'], + 'model_name': input_data['model_name'], }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) + # Optionally format and export transformed data to separate table + if transformed_data is not None: + transformed = await workflow.execute_local_activity_method( + Activities.format_transformed_data, + { + **metadata, + 'data': transformed_data, + 'model_id': input_data['model_id'], + 'model_name': input_data['model_name'], + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) + + write_transformed_handler = workflow.start_activity_method( + Activities.export_payload_to_postgres, + { + **metadata, + 'schema': input_data['schema'], + 'table_name': input_data['transform_table_name'], + 'data': transformed, + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) + + else: + write_transformed_handler = None + else: - # create default prediction + # Error path: create default prediction with error indicators prediction = await workflow.execute_local_activity_method( Activities.format_default_prediction, { @@ -94,23 +153,41 @@ class FormatAndExportPrediction(): 'timestamp': input_data['timestamp'], 'model_id': input_data['model_id'], 'prediction_confidence': prediction_confidence, - 'comment': input_data['comment'] + 'comment': input_data['comment'], }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), + ) + + write_transformed_handler = None + + opc_metrics: dict[str, dict[str, float | None]] = {} + + # write to pi web api + if pi_web_api_output_config: + prediction = await workflow.execute_activity_method( + Activities.write_pi_web_api_data, + { + 'pi_web_api_output_config': pi_web_api_output_config, + 'data': prediction, + **metadata, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), ) # write to opc - prediction = await workflow.execute_activity_method( - Activities.write_opc_data, - { - **metadata, - 'opc_output_config': input_data['opc_output_config'], - 'data': prediction - }, - retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) - ) + if opc_output_config: + prediction, opc_metrics = await workflow.execute_activity_method( + Activities.write_opc_data, + { + 'opc_output_config': opc_output_config, + 'data': prediction, + **metadata, + }, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(seconds=60), + ) # write to postgres await workflow.execute_activity_method( @@ -120,21 +197,24 @@ class FormatAndExportPrediction(): 'schema': input_data['schema'], 'table_name': input_data['table_name'], 'data': prediction, - 'timestamp_conversion': { - 'column': 'timestamp', - 'format': DATETIME_FORMAT_WITH_TZ - } + 'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ}, + 'on_conflict': input_data.get('on_conflict', 'error'), + 'unique_columns': ['model_id', 'timestamp'], }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=180), ) + if write_transformed_handler is not None: + await write_transformed_handler + await workflow.execute_activity_method( Activities.write_metrics, { **metadata, - 'prediction': prediction + 'prediction': prediction, + 'opc_metrics': opc_metrics, }, retry_policy=retry_policy, - start_to_close_timeout=timedelta(seconds=60) + start_to_close_timeout=timedelta(seconds=60), ) diff --git a/laborious/workflows/sub_workflows/prediction_process.py b/laborious/workflows/sub_workflows/prediction_process.py index 777fa1c..0f86fa1 100644 --- a/laborious/workflows/sub_workflows/prediction_process.py +++ b/laborious/workflows/sub_workflows/prediction_process.py @@ -1,14 +1,16 @@ from temporalio import workflow with workflow.unsafe.imports_passed_through(): - from laborious.activities.activities import Activities - from typing import Any - from sientia_do.temporal.policies import retry_policy from datetime import timedelta + from typing import Any + + from sientia_do.temporal.policies import retry_policy + + from laborious.activities.activities import Activities -@workflow.defn(name="prediction_process") -class PredictionProcess(): +@workflow.defn(name='subworkflow.prediction_process') +class PredictionProcess: """ Core prediction processing workflow for the Laborious system. @@ -64,7 +66,10 @@ class PredictionProcess(): - mlflow_predict_filters (dict): MLFlow prediction filters - model_retention (int): Model retention period in minutes - path_priority (list[str]): Decision path priority configuration - - opc_output_config (dict): OPC server export configuration + - opc_output_config (dict, optional): OPC server export configuration + - pi_web_api_output_config (dict, optional): PI Web API export configuration + - save_transform (bool, optional): Whether to save transformed data (default: True) + - prediction_store_policy (str, optional): Data retention policy (default: 'lts:1') Returns: None: The workflow completes successfully when export workflow finishes @@ -80,24 +85,51 @@ class PredictionProcess(): model_id = input_data['model_id'] model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) + save_transform = input_data.get('save_transform', True) - # Get last timestamp for incremental processing - last_timestamp = await workflow.execute_local_activity_method( - Activities.get_last_timestamp, - { - **metadata, - 'data': data - }, - retry_policy=retry_policy, - start_to_close_timeout=timedelta(minutes=1), - ) + try: + await self._run_prediction_pipeline( + input_data, + metadata, + data, + model_id, + model_name, + model_config, + save_transform, + ) + await workflow.execute_activity_method( + Activities.cleanup_minio_objects_expired, + {**metadata, 'data': data}, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(minutes=5), + ) + except Exception as e: + await workflow.execute_activity_method( + Activities.cleanup_minio_objects_expired, + {**metadata, 'data': data}, + retry_policy=retry_policy, + start_to_close_timeout=timedelta(minutes=5), + ) + raise e + + async def _run_prediction_pipeline( + self, + input_data: dict[str, Any], + metadata: dict[str, Any], + data: dict[str, Any], + model_id: str, + model_name: str, + model_config: dict[str, Any], + save_transform: bool, + ) -> None: + last_timestamp = data['last_timestamp'] # Apply input data quality gates gate_input = { **metadata, 'filters': input_data['input_filters'], 'data': data, - 'path_priority': input_data['path_priority'] + 'path_priority': input_data['path_priority'], } path_flag, confidence, comment = await workflow.execute_local_activity_method( @@ -114,14 +146,9 @@ class PredictionProcess(): return # Request MLFlow model transformation - response_data = await workflow.execute_local_activity_method( + transformed_data = await workflow.execute_activity_method( Activities.request_transform, - { - **metadata, - 'data': data, - 'model_name': model_name, - 'model_config': model_config - }, + {**metadata, 'data': data, 'model_name': model_name, 'model_config': model_config}, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=5), ) @@ -132,9 +159,9 @@ class PredictionProcess(): { **metadata, 'filters': input_data['mlflow_transform_filters'], - 'data': response_data, + 'data': transformed_data, 'type': 'transform', - 'path_priority': input_data['path_priority'] + 'path_priority': input_data['path_priority'], }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -146,8 +173,6 @@ class PredictionProcess(): ): return - transformed_data = response_data['content'] - path_flag, confidence, comment = await workflow.execute_local_activity_method( Activities.mlflow_content_gate, { @@ -155,7 +180,7 @@ class PredictionProcess(): 'filters': input_data['mlflow_transform_filters'], 'data': transformed_data, 'type': 'transform', - 'path_priority': input_data['path_priority'] + 'path_priority': input_data['path_priority'], }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -166,13 +191,13 @@ class PredictionProcess(): ): return - response_data = await workflow.execute_local_activity_method( + predicted_data = await workflow.execute_activity_method( Activities.request_predict, { **metadata, 'data': transformed_data, 'model_name': model_name, - 'model_config': model_config + 'model_config': model_config, }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=5), @@ -184,9 +209,9 @@ class PredictionProcess(): { **metadata, 'filters': input_data['mlflow_predict_filters'], - 'data': response_data, + 'data': predicted_data, 'type': 'predict', - 'path_priority': input_data['path_priority'] + 'path_priority': input_data['path_priority'], }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -200,26 +225,37 @@ class PredictionProcess(): # Delegate to export workflow for data persistence await workflow.execute_child_workflow( - 'format_and_export_prediction', + 'subworkflow.format_and_export_prediction', { 'metadata': metadata, + 'on_conflict': input_data.get('on_conflict', 'error'), 'path_flag': path_flag, - 'data': response_data['content'], + 'data': predicted_data, + 'transformed_data': transformed_data if save_transform else None, 'prediction_confidence': confidence, 'timestamp': last_timestamp, 'model_id': model_id, 'model_name': model_name, 'model_config': model_config, 'opc_output_config': input_data['opc_output_config'], + 'pi_web_api_output_config': input_data['pi_web_api_output_config'], 'schema': input_data['schema'], 'table_name': input_data['table_name'], + 'transform_table_name': input_data['transform_table_name'], 'comment': comment, - 'prediction_store_policy': input_data['prediction_store_policy'] - } + 'prediction_store_policy': input_data['prediction_store_policy'], + }, ) - async def path_flag_handler(self, data: dict, path_flag: str, input_data: dict, - confidence: int, last_timestamp: str, comment: str) -> bool: + async def path_flag_handler( + self, + data: dict[str, Any], + path_flag: str | None, + input_data: dict, + confidence: int, + last_timestamp: str, + comment: str, + ) -> bool: """ Handle path decisions based on filter results and confidence levels. @@ -230,7 +266,17 @@ class PredictionProcess(): Args: data: Input data for processing path_flag: Path decision from filter (STOP, CONTINUE, REPEAT) - input_data: Complete workflow input configuration + input_data: Complete workflow input configuration including: + - metadata (dict): Workflow execution metadata + - schema (str): Database schema + - table_name (str): Target table for predictions + - transform_table_name (str): Target table for transformed data + - model_id (str): ML model identifier + - model_name (str): ML model name + - model_config (dict, optional): Model configuration + - opc_output_config (dict, optional): OPC server export configuration + - pi_web_api_output_config (dict, optional): PI Web API export configuration + - prediction_store_policy (str, optional): Data retention policy confidence: Confidence level from filter validation last_timestamp: Last processed timestamp comment: Additional information about the filter result @@ -240,13 +286,14 @@ class PredictionProcess(): Path Handling: - STOP: Terminates workflow execution - - CONTINUE: Proceeds with normal processing + - CONTINUE: Delegates to FormatAndExportPrediction workflow with current data - REPEAT: Repeats last prediction if available """ metadata = input_data['metadata'] schema = input_data['schema'] table_name = input_data['table_name'] + transform_table_name = input_data['transform_table_name'] model_id = input_data['model_id'] model_name = input_data['model_name'] model_config = input_data.get('model_config', {}) @@ -265,7 +312,7 @@ class PredictionProcess(): 'schema': schema, 'table_name': table_name, 'model': model_id, - 'last_timestamp': last_timestamp + 'last_timestamp': last_timestamp, }, retry_policy=retry_policy, start_to_close_timeout=timedelta(minutes=1), @@ -274,7 +321,7 @@ class PredictionProcess(): elif path_flag == 'CONTINUE': # call write workflow await workflow.execute_child_workflow( - 'format_and_export_prediction', + 'subworkflow.format_and_export_prediction', { 'metadata': metadata, 'path_flag': path_flag, @@ -286,10 +333,13 @@ class PredictionProcess(): 'model_config': model_config, 'schema': schema, 'table_name': table_name, + 'transform_table_name': transform_table_name, 'comment': comment, 'opc_output_config': input_data['opc_output_config'], - 'prediction_store_policy': input_data['prediction_store_policy'] - } + 'pi_web_api_output_config': input_data['pi_web_api_output_config'], + 'prediction_store_policy': input_data['prediction_store_policy'], + 'on_conflict': input_data.get('on_conflict', 'error'), + }, ) return True