gitea_admin/sientia-dataops-laborious-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1231

Browse Source 
Update .gitignore and refactor metrics.py for improved logging and consistency

- Added coverage.xml to .gitignore to prevent tracking of coverage reports.
- Refactored metric labels in metrics.py for consistency in string formatting and improved readability.
- Enhanced logging messages in various activities to ensure uniformity in message formatting.
This commit is contained in:
Vitor Pimentel dos Santos
2025-10-15 16:00:18 -03:00
parent a5d2b0d3fd
commit ac795c7c53
39 changed files with 4122 additions and 2602 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
laborious/activities/activities.py
View File

@@ -1,13 +1,15 @@
from temporalio import activity, workflow
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from laborious.activities.storage import Storage
from typing import Any
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
from laborious.activities.mlflow import MLFlow
from laborious.activities.gates import Gates
from laborious.activities.mlflow import MLFlow
from laborious.activities.opc import OPC
from typing import Any
from laborious.activities.storage import Storage
class Activities(Storage, MLFlow, Gates, OPC):
@@ -32,13 +34,15 @@ class Activities(Storage, MLFlow, Gates, OPC):
notification_handler (NotificationHandler): Notification management instance
"""
def __init__(self,
postgres_config: dict[str, Any],
mlflow_config: dict[str, Any],
minio_config: dict[str, Any],
opc_config: dict[str, Any],
logger: Logger,
notification_handler: NotificationHandler):
def __init__(
self,
postgres_config: dict[str, Any],
mlflow_config: dict[str, Any],
minio_config: dict[str, Any],
opc_config: dict[str, Any],
logger: Logger,
notification_handler: NotificationHandler,
):
"""
Initialize the Activities orchestrator with all required configurations.
@@ -59,32 +63,36 @@ class Activities(Storage, MLFlow, Gates, OPC):
Exception: If any parent class initialization fails
"""
# Initialize parent classes
Storage.__init__(self, host=postgres_config['host'],
port=postgres_config['port'],
user=postgres_config['user'],
password=postgres_config['password'],
dbname=postgres_config['dbname'],
min_connections=postgres_config['min_connections'],
max_connections=postgres_config['max_connections'],
minio_config=minio_config,
logger=logger,
notification_handler=notification_handler)
Storage.__init__(
self,
host=postgres_config['host'],
port=postgres_config['port'],
user=postgres_config['user'],
password=postgres_config['password'],
dbname=postgres_config['dbname'],
min_connections=postgres_config['min_connections'],
max_connections=postgres_config['max_connections'],
minio_config=minio_config,
logger=logger,
notification_handler=notification_handler,
)
MLFlow.__init__(self, mlflow_host=mlflow_config['host'],
mlflow_port=mlflow_config['port'],
mlflow_username=mlflow_config['username'],
mlflow_password=mlflow_config['password'],
minio_config=minio_config,
logger=logger,
notification_handler=notification_handler)
MLFlow.__init__(
self,
mlflow_host=mlflow_config['host'],
mlflow_port=mlflow_config['port'],
mlflow_username=mlflow_config['username'],
mlflow_password=mlflow_config['password'],
minio_config=minio_config,
logger=logger,
notification_handler=notification_handler,
)
Gates.__init__(self, logger=logger,
notification_handler=notification_handler)
Gates.__init__(self, logger=logger, notification_handler=notification_handler)
OPC.__init__(self,
opc_servers=opc_config,
logger=logger,
notification_handler=notification_handler)
OPC.__init__(
self, opc_servers=opc_config, logger=logger, notification_handler=notification_handler
)
async def shutdown(self):
"""

320
laborious/activities/gates.py
View File

@@ -1,55 +1,66 @@
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import traceback
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.observability.logger import Logger
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
from sientia_do.formatters import create_sample_dict
from laborious.utils.filters.mlflow_filters import nan_values_filter, api_error_filter
from typing import Any
from laborious.utils.filters.conditional_filters import (
filter_empty_data,
filter_specific_variables_null_values
)
from pandas import DataFrame
from laborious import metrics
from collections.abc import Callable, Mapping
from os import path
from shutil import rmtree
from typing import Any
from pandas import DataFrame
from sientia_do.formatters import create_sample_dict
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ, now
from laborious import metrics
from laborious.utils.filters.conditional_filters import (
filter_empty_data,
filter_specific_variables_null_values,
)
from laborious.utils.filters.mlflow_filters import api_error_filter, nan_values_filter
# Strongly-typed filter function signatures
InputFilterFunc = Callable[[DataFrame, dict[str, Any]], bool]
ResponseFilterFunc = Callable[[dict[str, Any], dict[str, Any]], bool]
ContentFilterFunc = Callable[[DataFrame, dict[str, Any]], bool]
# Input filter function mappings
input_filter_functions = {
input_filter_functions: dict[str, InputFilterFunc] = {
'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values,
'EMPTY_DATA': filter_empty_data,
'path_confidence': {
'STOP': -1,
'CONTINUE': 2,
'REPEAT': -1
}
}
# Confidence mappings kept separate from function maps to avoid Union types
input_path_confidence: Mapping[str, int] = {
'STOP': -1,
'CONTINUE': 2,
'REPEAT': -1,
}
# MLFlow response filter function mappings
mlflow_response_filter_functions = {
mlflow_response_filter_functions: dict[str, ResponseFilterFunc] = {
'API_ERROR': api_error_filter,
'path_confidence': {
'STOP': -1,
'CONTINUE': 10,
'REPEAT': -1
},
}
mlflow_response_path_confidence: Mapping[str, int] = {
'STOP': -1,
'CONTINUE': 10,
'REPEAT': -1,
}
# MLFlow content filter function mappings
mlflow_content_filter_functions = {
mlflow_content_filter_functions: dict[str, ContentFilterFunc] = {
'NAN_VALUES': nan_values_filter,
'EMPTY_DATA': filter_empty_data,
'path_confidence': {
'STOP': -1,
'CONTINUE': 18,
'REPEAT': -1
}
}
mlflow_content_path_confidence: Mapping[str, int] = {
'STOP': -1,
'CONTINUE': 18,
'REPEAT': -1,
}
@@ -84,10 +95,9 @@ class Gates(BaseActivity):
Raises:
Exception: If BaseActivity initialization fails
"""
BaseActivity.__init__(
self, logger, notification_handler, set_error_counter=True)
BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True)
@activity.defn(name="input_gate")
@activity.defn(name='input_gate')
async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Apply input data quality filters and validation.
@@ -123,7 +133,7 @@ class Gates(BaseActivity):
"""
metadata = input_data['metadata']
self.info("Performing input gate...", metadata)
self.info('Performing input gate...', metadata)
filters = input_data['filters']
data = DataFrame(input_data['data'])
@@ -131,40 +141,38 @@ class Gates(BaseActivity):
filter_output = []
self.debug(f"Input data: {data.head(5).to_string()}", metadata)
self.debug(f"Filters: {filters}", metadata)
self.debug(f'Input data: {data.head(5).to_string()}', metadata)
self.debug(f'Filters: {filters}', metadata)
# Apply each configured filter
for fil, config in filters.items():
if fil not in input_filter_functions:
self.error(f"Filter {fil} not found", metadata)
self.error(f'Filter {fil} not found', metadata)
continue
try:
if input_filter_functions[fil](data, config['config']):
self.debug(
f"Data not passed the input filter {fil}:{config}", metadata)
self.debug(f'Data not passed the input filter {fil}:{config}', metadata)
filter_output.append(config['policy'])
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id=f"INTPUT_GATE_ERROR__{fil}",
message=f"Error in filter {fil}:{config}: \n {e}",
block="input_gate",
notification_id=f'INTPUT_GATE_ERROR__{fil}',
message=f'Error in filter {fil}:{config}: \n {e}',
block='input_gate',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
for path_flag in path_priority:
if path_flag in filter_output:
self.info(f"Input gate result: {path_flag}", metadata)
return path_flag, input_filter_functions['path_confidence'][path_flag], \
"Input data with bad quality"
self.info(f'Input gate result: {path_flag}', metadata)
return path_flag, input_path_confidence[path_flag], 'Input data with bad quality'
self.info("Nothing was filtered by the input gate", metadata)
return None, 0, ""
self.info('Nothing was filtered by the input gate', metadata)
return None, 0, ''
@activity.defn(name="mlflow_response_gate")
@activity.defn(name='mlflow_response_gate')
async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Validate MLFlow API response quality and integrity.
@@ -199,7 +207,7 @@ class Gates(BaseActivity):
Exception: If response validation fails or configuration is invalid
"""
metadata = input_data['metadata']
self.info("Performing mlflow response gate...", metadata)
self.info('Performing mlflow response gate...', metadata)
filters = input_data['filters']
data = input_data['data']
@@ -208,9 +216,8 @@ class Gates(BaseActivity):
filter_output = []
self.debug(
f"Input data: \n {create_sample_dict(data, max_items=5, max_depth=5)}", metadata)
self.debug(f"Filters: {filters}", metadata)
self.debug(f'Input data: \n {create_sample_dict(data, max_items=5, max_depth=5)}', metadata)
self.debug(f'Filters: {filters}', metadata)
comments = []
for fil, config in filters.items():
@@ -222,34 +229,32 @@ class Gates(BaseActivity):
comments.append(data['content']['message'])
self.send_notification(
metadata=metadata,
notification_id=f"{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}",
notification_id=f'{gate_type.upper()}_GATE_RESPONSE_FILTER__{fil}',
message=data['content']['message'],
block="mlflow_gate",
block='mlflow_gate',
level=NotificationLevel.ERROR,
attachment_content=data['content']['traceback']
attachment_content=data['content']['traceback'],
)
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id=f"MLFLOW_GATE_RESPONSE_FILTER__{fil}",
message=f"Error in filter {fil}:{config}: \n {e}",
block="mlflow_gate",
notification_id=f'MLFLOW_GATE_RESPONSE_FILTER__{fil}',
message=f'Error in filter {fil}:{config}: \n {e}',
block='mlflow_gate',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
for path_flag in path_priority:
if path_flag in filter_output:
self.info(
f"Mlflow response gate result: {path_flag}", metadata)
return path_flag, mlflow_response_filter_functions['path_confidence'][path_flag], \
", ".join(comments)
self.info(f'Mlflow response gate result: {path_flag}', metadata)
return path_flag, mlflow_response_path_confidence[path_flag], ', '.join(comments)
self.info("Nothing was filtered by the mlflow response gate", metadata)
return None, 0, ""
self.info('Nothing was filtered by the mlflow response gate', metadata)
return None, 0, ''
@activity.defn(name="mlflow_content_gate")
@activity.defn(name='mlflow_content_gate')
async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
"""
Validate MLFlow prediction content quality and integrity.
@@ -284,7 +289,7 @@ class Gates(BaseActivity):
Exception: If content validation fails or configuration is invalid
"""
metadata = input_data['metadata']
self.info("Performing mlflow content gate...", metadata)
self.info('Performing mlflow content gate...', metadata)
filters = input_data['filters']
data = DataFrame(input_data['data'])
@@ -293,8 +298,8 @@ class Gates(BaseActivity):
filter_output = []
self.debug(f"Input data:\n {data.head(5).to_string()}", metadata)
self.debug(f"Filters: \n {filters}", metadata)
self.debug(f'Input data:\n {data.head(5).to_string()}', metadata)
self.debug(f'Filters: \n {filters}', metadata)
for fil, config in filters.items():
if fil not in mlflow_content_filter_functions:
@@ -304,36 +309,38 @@ class Gates(BaseActivity):
filter_output.append(config['policy'])
self.send_notification(
metadata=metadata,
notification_id=f"{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}",
message=f"Data not passed the content filter {fil}:{config}",
block="mlflow_gate",
notification_id=f'{gate_type.upper()}_GATE_CONTENT_FILTER__{fil}',
message=f'Data not passed the content filter {fil}:{config}',
block='mlflow_gate',
level=NotificationLevel.WARNING,
attachment_content=data.to_string()
attachment_content=data.to_string(),
)
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id=f"MLFLOW_GATE_CONTENT_FILTER__{fil}",
message=f"Error in filter {fil}:{config}: \n {e}",
block="mlflow_gate",
notification_id=f'MLFLOW_GATE_CONTENT_FILTER__{fil}',
message=f'Error in filter {fil}:{config}: \n {e}',
block='mlflow_gate',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
for path_flag in path_priority:
if path_flag in filter_output:
self.info(
f"Mlflow content gate result: {path_flag}", metadata)
return path_flag, mlflow_content_filter_functions['path_confidence'][path_flag], \
"Transformed data not passed the content filter"
self.info(f'Mlflow content gate result: {path_flag}', metadata)
return (
path_flag,
mlflow_content_path_confidence[path_flag],
'Transformed data not passed the content filter',
)
self.info("Nothing was filtered by the mlflow content gate", metadata)
return None, 0, ""
self.info('Nothing was filtered by the mlflow content gate', metadata)
return None, 0, ''
def get_prediction_store_policy(self,
prediction_store_policy: str,
metadata: dict[str, Any]) -> tuple[str, int]:
def get_prediction_store_policy(
self, prediction_store_policy: str, metadata: dict[str, Any]
) -> tuple[str, int]:
"""
Parse and validate prediction store policy configuration.
@@ -359,7 +366,9 @@ class Gates(BaseActivity):
if len(policy_elements) < 2:
self.error(
f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata)
f'Invalid prediction store policy: {prediction_store_policy}, using default policy',
metadata,
)
return 'lts', 1
policy_type = policy_elements[0]
@@ -367,14 +376,20 @@ class Gates(BaseActivity):
# If the policy_type is not lts or erl, we use the default policy
# If the policty_value is not a number or 0, we use the default policy
if policy_type not in ['lts', 'erl'] or not policy_value.isdigit() or int(policy_value) == 0:
if (
policy_type not in ['lts', 'erl']
or not policy_value.isdigit()
or int(policy_value) == 0
):
self.error(
f"Invalid prediction store policy: {prediction_store_policy}, using default policy", metadata)
f'Invalid prediction store policy: {prediction_store_policy}, using default policy',
metadata,
)
return 'lts', 1
return policy_type, int(policy_value)
@activity.defn(name="format_prediction")
@activity.defn(name='format_prediction')
async def format_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
"""
Format prediction data according to configured storage policies.
@@ -401,7 +416,7 @@ class Gates(BaseActivity):
"""
metadata = input_data['metadata']
prediction_store_policy = input_data['prediction_store_policy']
self.info("Formatting prediction...", metadata)
self.info('Formatting prediction...', metadata)
data = DataFrame(input_data['data'])
@@ -409,48 +424,45 @@ class Gates(BaseActivity):
data['timestamp'] = data.index
data = data.reset_index(drop=True)
self.debug(
f"Prediction store policy: {prediction_store_policy}", metadata)
self.debug(f"Prediction data: {data.head(5).to_string()}", metadata)
self.debug(f'Prediction store policy: {prediction_store_policy}', metadata)
self.debug(f'Prediction data: {data.head(5).to_string()}', metadata)
policy_type, policy_value = self.get_prediction_store_policy(
prediction_store_policy, metadata)
prediction_store_policy, metadata
)
# If data has no timestamp, we use the default timestamp and not sort the data
self.info(
f"Sorting data by timestamp and applying policy: {policy_type}:{policy_value}", metadata)
f'Sorting data by timestamp and applying policy: {policy_type}:{policy_value}', metadata
)
# If policy_type is lts, we need to sort the data by timestamp descending and take the first policy_value rows
if policy_type == 'lts':
self.debug(
"Sorting data by timestamp descending", metadata)
self.debug('Sorting data by timestamp descending', metadata)
data = data.sort_values(by='timestamp', ascending=False)
# If policy_type is erl, we need to sort the data by timestamp ascending and take the first policy_value rows
elif policy_type == 'erl':
self.debug(
"Sorting data by timestamp ascending", metadata)
self.debug('Sorting data by timestamp ascending', metadata)
data = data.sort_values(by='timestamp', ascending=True)
else:
self.error(
f"Invalid policy type: {policy_type}, using default policy", metadata)
raise ValueError(
f"Invalid policy type: {policy_type}")
self.error(f'Invalid policy type: {policy_type}, using default policy', metadata)
raise ValueError(f'Invalid policy type: {policy_type}')
data = data.head(int(policy_value))
data['model_id'] = input_data['model_id']
data['prediction_confidence'] = input_data['prediction_confidence']
data['prediction_status'] = 'Good'
data['comments'] = ""
data['comments'] = ''
data = data.sort_values(by='timestamp', ascending=False)
data = data.reset_index(drop=True)
self.info(f"Prediction formatted: {len(data)} rows", metadata)
self.debug(f"Prediction data: {data.head(5).to_string()}", metadata)
self.info(f'Prediction formatted: {len(data)} rows', metadata)
self.debug(f'Prediction data: {data.head(5).to_string()}', metadata)
return data.to_dict()
@activity.defn(name="format_default_prediction")
@activity.defn(name='format_default_prediction')
async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
"""
Create and format default prediction data for error conditions.
@@ -478,40 +490,44 @@ class Gates(BaseActivity):
"""
metadata = input_data['metadata']
self.debug("Formatting default prediction...", metadata)
self.debug('Formatting default prediction...', metadata)
data = DataFrame({
'prediction': [0],
'response_time': [0],
'timestamp': [input_data['timestamp']],
'model_id': [input_data['model_id']],
'prediction_confidence': [input_data['prediction_confidence']],
'prediction_status': ['Bad'],
'comments': [input_data['comment']]
})
data = DataFrame(
{
'prediction': [0],
'response_time': [0],
'timestamp': [input_data['timestamp']],
'model_id': [input_data['model_id']],
'prediction_confidence': [input_data['prediction_confidence']],
'prediction_status': ['Bad'],
'comments': [input_data['comment']],
}
)
self.info(f"Default prediction formatted: {data.size} rows", metadata)
self.info(f'Default prediction formatted: {data.size} rows', metadata)
return data.to_dict()
@activity.defn(name="format_retrain_report")
@activity.defn(name='format_retrain_report')
async def format_retrain_report(self, input_data: dict[str, Any]) -> dict[Any, Any]:
"""
Format retrain report data according to configured storage policies.
"""
metadata = input_data['metadata']
self.info("Formatting retrain report...", metadata)
self.info('Formatting retrain report...', metadata)
experiment_response = input_data['experiment_response']
update_report = input_data['update_report']
model_id = input_data['model_id']
model_name = input_data['model_name']
report = DataFrame({
'model_id': [model_id],
'model_name': [model_name],
'timestamp': [experiment_response['timestamp']],
'status': [experiment_response['message']]
})
report = DataFrame(
{
'model_id': [model_id],
'model_name': [model_name],
'timestamp': [experiment_response['timestamp']],
'status': [experiment_response['message']],
}
)
if experiment_response['success']:
# Retrain was successfull
@@ -519,11 +535,11 @@ class Gates(BaseActivity):
report['mlflow_run_id'] = update_report['mlflow_run_id']
report['mlflow_experiment_id'] = update_report['mlflow_experiment_id']
self.debug(f"Retrain report: {report.to_csv()}", metadata)
self.debug(f'Retrain report: {report.to_csv()}', metadata)
return report.to_dict()
@activity.defn(name="get_last_timestamp")
@activity.defn(name='get_last_timestamp')
async def get_last_timestamp(self, input_data: dict[str, Any]) -> str:
"""
Extract the most recent timestamp from prediction data.
@@ -548,24 +564,22 @@ class Gates(BaseActivity):
"""
metadata = input_data['metadata']
self.info("Getting last timestamp...", metadata)
self.info('Getting last timestamp...', metadata)
data = DataFrame(input_data['data'])
self.debug(f"Input data: {data.head(5).to_string()}", metadata)
self.debug(f'Input data: {data.head(5).to_string()}', metadata)
if data.empty:
return now().strftime(DATETIME_FORMAT_WITH_TZ)
max_timestamp = max(
data['timestamp'].values.tolist())
max_timestamp = max(data['timestamp'].values.tolist())
self.info(
f"Last timestamp: {max_timestamp}", metadata)
self.info(f'Last timestamp: {max_timestamp}', metadata)
return max_timestamp
@activity.defn(name="write_metrics")
@activity.defn(name='write_metrics')
async def write_metrics(self, input_data: dict[str, Any]):
"""
Write prediction performance metrics to Prometheus monitoring system.
@@ -593,42 +607,40 @@ class Gates(BaseActivity):
prediction_confidence = prediction['prediction_confidence'].values[0]
response_time = prediction['response_time'].values[0]
self.info(
f"Writing metrics for model {metadata['model_name']}", metadata)
self.info(f'Writing metrics for model {metadata["model_name"]}', metadata)
metrics.PREDICTIONS_WRITTEN_COUNT.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name']
pipeline_name=metadata['workflow_name'],
).inc()
metrics.PREDICTION_CONFIDENCE_MONITOR.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name']
pipeline_name=metadata['workflow_name'],
).set(prediction_confidence)
metrics.PREDICTION_RESPONSE_TIME_MONITOR.labels(
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name']
pipeline_name=metadata['workflow_name'],
).observe(response_time)
self.info(
f"Metrics written for model {metadata['model_name']}", metadata)
self.info(f'Metrics written for model {metadata["model_name"]}', metadata)
@activity.defn(name="clean_tmp_files")
@activity.defn(name='clean_tmp_files')
async def clean_tmp_files(self, input_data: dict[str, Any]):
"""
Clean temporary files in the tmp directory.
"""
model_name = input_data['model_name']
metadata = input_data['metadata']
self.info(f"Cleaning tmp files for model {model_name}...", metadata)
self.info(f'Cleaning tmp files for model {model_name}...', metadata)
if path.exists(f"tmp/retrain_data/{model_name}"):
rmtree(f"tmp/retrain_data/{model_name}")
if path.exists(f"tmp/artifacts/{model_name}"):
rmtree(f"tmp/artifacts/{model_name}")
if path.exists(f'tmp/retrain_data/{model_name}'):
rmtree(f'tmp/retrain_data/{model_name}')
if path.exists(f'tmp/artifacts/{model_name}'):
rmtree(f'tmp/artifacts/{model_name}')
self.info("Tmp files cleaned", metadata)
self.info('Tmp files cleaned', metadata)

144
laborious/activities/mlflow.py
View File

@@ -1,21 +1,25 @@
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ, now
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
from pandas import to_datetime
from sientia_do.temporal.constants import DATETIME_FORMAT, DATETIME_FORMAT_WITH_TZ
from sientia_do.temporal.activities.base import BaseActivity
import traceback
from typing import Any
import numpy as np
from pandas import DataFrame, to_datetime
from sientia_do.formatters import create_sample_dict
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.formatters import create_sample_dict
from laborious.utils.repository.model_repository import MLFlowRepository
from typing import Any
import numpy as np
from pandas import DataFrame
import traceback
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.temporal.constants import (
DATETIME_FORMAT,
DATETIME_FORMAT_MS_WITH_TZ,
DATETIME_FORMAT_WITH_TZ,
now,
)
from laborious.utils.repository.minio_repository import MinioRepository
from laborious.utils.repository.model_repository import MLFlowRepository
class MLFlow(BaseActivity):
@@ -37,9 +41,16 @@ class MLFlow(BaseActivity):
model_monitoring_repository (MLFlowRepository): Repository for MLFlow operations
"""
def __init__(self, mlflow_host: str, mlflow_port: int, mlflow_username: str,
minio_config: dict[str, Any], mlflow_password: str,
logger: Logger, notification_handler: NotificationHandler):
def __init__(
self,
mlflow_host: str,
mlflow_port: int,
mlflow_username: str,
minio_config: dict[str, Any],
mlflow_password: str,
logger: Logger,
notification_handler: NotificationHandler,
):
"""
Initialize MLFlow activities with server configuration.
@@ -54,26 +65,18 @@ class MLFlow(BaseActivity):
Raises:
Exception: If MLFlowRepository initialization fails
"""
BaseActivity.__init__(
self, logger, notification_handler, set_error_counter=True)
BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True)
self.mlflow_host = mlflow_host
self.mlflow_port = mlflow_port
self.mlflow_username = mlflow_username
self.mlflow_password = mlflow_password
self.model_monitoring_repository = MLFlowRepository(
f"{mlflow_host}:{mlflow_port}", mlflow_username, mlflow_password, logger
f'{mlflow_host}:{mlflow_port}', mlflow_username, mlflow_password, logger
)
if not hasattr(self, 'minio_repository'):
self.minio_repository = MinioRepository(
logger=logger,
notification_handler=notification_handler,
minio_endpoint_url=minio_config['endpoint_url'],
minio_access_key=minio_config['access_key'],
minio_secret_key=minio_config['secret_key'],
minio_region_name=minio_config['region_name'],
minio_default_bucket=minio_config['default_bucket'])
self.minio_repository: MinioRepository | None = None
if self.minio_repository is None:
self.minio_repository = MinioRepository(
@@ -83,9 +86,10 @@ class MLFlow(BaseActivity):
minio_access_key=minio_config['access_key'],
minio_secret_key=minio_config['secret_key'],
minio_region_name=minio_config['region_name'],
minio_default_bucket=minio_config['default_bucket'])
minio_default_bucket=minio_config['default_bucket'],
)
@activity.defn(name="request_transform")
@activity.defn(name='request_transform')
async def request_transform(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Transform input data using MLFlow models.
@@ -121,7 +125,7 @@ class MLFlow(BaseActivity):
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
self.debug("Raw input data:", metadata)
self.debug('Raw input data:', metadata)
self.debug(data.head(5).to_string(), metadata)
# Sort by created_at in descending order and keep first occurrence of each variable/timestamp pair
@@ -130,14 +134,12 @@ class MLFlow(BaseActivity):
)
# Pivot data for model input format
data = data.pivot(
index='timestamp', columns='variable',
values='value')
data = data.pivot(index='timestamp', columns='variable', values='value')
data.fillna(np.nan, inplace=True)
# data.reset_index(inplace=True)
data.columns.name = None
self.debug("Processed input data:", metadata)
self.debug('Processed input data:', metadata)
self.debug(data.head(5).to_string(), metadata)
# Request transformation from MLFlow model
@@ -146,16 +148,20 @@ class MLFlow(BaseActivity):
)
self.debug(
f"Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata)
f'Transform raw response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}',
metadata,
)
self.debug(
f"Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata)
f'Transform response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}',
metadata,
)
self.info("Data transformed successfully", metadata)
self.info('Data transformed successfully', metadata)
return response_data
@activity.defn(name="request_predict")
@activity.defn(name='request_predict')
async def request_predict(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Execute predictions using MLFlow models.
@@ -191,14 +197,15 @@ class MLFlow(BaseActivity):
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
self.debug(f"Input data for: \n {data.head(5).to_string()}", metadata)
self.debug(f'Input data for: \n {data.head(5).to_string()}', metadata)
# Convert numpy.nan to None for model compatibility
data.replace(np.nan, None, inplace=True)
data['timestamp'] = data.index
data['timestamp'] = to_datetime(
data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT)
data['timestamp'], format=DATETIME_FORMAT_WITH_TZ
).dt.strftime(DATETIME_FORMAT)
# Request prediction from MLFlow model
response_data = self.model_monitoring_repository.predict(
@@ -206,13 +213,15 @@ class MLFlow(BaseActivity):
)
self.debug(
f"Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}", metadata)
f'Prediction response data: \n {create_sample_dict(response_data, max_items=5, max_depth=5)}',
metadata,
)
self.info("Data predicted successfully", metadata)
self.info('Data predicted successfully', metadata)
return response_data
@activity.defn(name="retrain_model")
@activity.defn(name='retrain_model')
async def retrain_model(self, input_data: dict[str, Any]) -> dict[str, Any]:
"""
Retrain MLFlow models with updated training data.
@@ -244,15 +253,19 @@ class MLFlow(BaseActivity):
Raises:
Exception: If retraining fails or encounters critical errors
"""
if self.minio_repository is None:
raise ValueError('Minio repository not initialized')
metadata = input_data['metadata']
object_key = input_data['object_key']
self.info(f'Loading retrain data from Key: {object_key}', metadata)
try:
data = self.minio_repository.get_parquet_as_dataframe(
object_key=object_key, metadata=metadata)
object_key=object_key, metadata=metadata
)
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
@@ -261,18 +274,17 @@ class MLFlow(BaseActivity):
message=f'Error loading retrain data: {e}',
block='retrain_model',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
self.error(trace, metadata)
return {
'success': False,
'message': f'Error loading retrain data: {e}',
'traceback': trace,
'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ)
'timestamp': now().strftime(DATETIME_FORMAT_MS_WITH_TZ),
}
self.debug(
f'Retrain data loaded successfully: shape {data.shape}', metadata)
self.debug(f'Retrain data loaded successfully: shape {data.shape}', metadata)
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
@@ -291,47 +303,38 @@ class MLFlow(BaseActivity):
data.drop(columns=['created_at'], inplace=True, errors='ignore')
# Pivot data for model input format
data = data.pivot(
index='timestamp', columns='variable',
values='value')
data = data.pivot(index='timestamp', columns='variable', values='value')
data.fillna(np.nan, inplace=True)
# data.reset_index(inplace=True)
data.columns.name = None
data['timestamp'] = data.index
data['timestamp'] = to_datetime(
data['timestamp'], format=DATETIME_FORMAT_WITH_TZ).dt.strftime(DATETIME_FORMAT)
data['timestamp'] = to_datetime(
data['timestamp'], format=DATETIME_FORMAT)
data['timestamp'], format=DATETIME_FORMAT_WITH_TZ
).dt.strftime(DATETIME_FORMAT)
data['timestamp'] = to_datetime(data['timestamp'], format=DATETIME_FORMAT)
data.columns.name = None
retrain_output = self.model_monitoring_repository.retrain_model(
data=data,
model_name=model_name,
model_config=model_config,
metadata=metadata
data=data, model_name=model_name, model_config=model_config, metadata=metadata
)
if not retrain_output['success']:
trace = retrain_output['traceback']
self.send_notification(
metadata=metadata,
notification_id='RETRAIN_MODEL_ERROR',
message=f"Error retraining model {model_name}: {retrain_output['message']}",
message=f'Error retraining model {model_name}: {retrain_output["message"]}',
block='retrain_model',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
self.error(trace, metadata=metadata)
return {
**retrain_output,
'timestamp': timestamp
}
return {**retrain_output, 'timestamp': timestamp}
@activity.defn(name="update_production_model")
@activity.defn(name='update_production_model')
async def update_production_model(self, input_data: dict[str, Any]) -> dict[Any, Any]:
"""
Update production model with newly trained model version.
@@ -372,16 +375,15 @@ class MLFlow(BaseActivity):
model_name = input_data['model_name']
experiment = input_data['experiment']
self.info(
f'Updating production model {model_name} from experiment {experiment}...', metadata)
f'Updating production model {model_name} from experiment {experiment}...', metadata
)
try:
response = self.model_monitoring_repository.update_production_model(
experiment=experiment,
model_name=model_name
experiment=experiment, model_name=model_name, metadata=metadata
)
self.info(
f'Production model {model_name} updated successfully', metadata)
self.info(f'Production model {model_name} updated successfully', metadata)
return response
except Exception as e:
@@ -392,7 +394,7 @@ class MLFlow(BaseActivity):
message=f'Error updating production model {model_name}: {e}',
block='update_production_model',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
self.error(trace, metadata=metadata)
raise e

119
laborious/activities/opc.py
View File

@@ -1,15 +1,16 @@
from temporalio import activity, workflow
with workflow.unsafe.imports_passed_through():
import traceback
from typing import Any
from pandas import DataFrame
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.activities.base import BaseActivity
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.base import BaseActivity
from laborious.utils.repository.opc_repository import OpcRepository
from typing import Any
import traceback
from pandas import DataFrame
OPC_WRITTING_ERROR_CONFIDENCE = 12
@@ -33,15 +34,17 @@ class OPC(BaseActivity):
notification_handler (NotificationHandler): Notification management instance
"""
def __init__(self, opc_servers: dict[str, dict[str, Any]],
logger: Logger, notification_handler: NotificationHandler):
def __init__(
self,
opc_servers: dict[str, dict[str, Any]],
logger: Logger,
notification_handler: NotificationHandler,
):
self.logger = logger
self.notification_handler = notification_handler
self.opc_servers = opc_servers
BaseActivity.__init__(
self, logger, notification_handler, set_error_counter=True)
BaseActivity.__init__(self, logger, notification_handler, set_error_counter=True)
self.opc_repository: dict[str, OpcRepository] = {}
self.opc_servers = opc_servers
@@ -70,10 +73,10 @@ class OPC(BaseActivity):
the initialization of other OPC servers. Each server is handled
independently to ensure maximum availability.
"""
self.logger.info("Initializing OPC servers...")
for id, server in self.opc_servers.items():
self.opc_repository[id] = OpcRepository(
id=server['id'],
self.logger.info('Initializing OPC servers...')
for opc_id, server in self.opc_servers.items():
self.opc_repository[opc_id] = OpcRepository(
opc_id=server['id'],
url=server['url'],
logger=self.logger,
server_uri=server['server_uri'],
@@ -82,30 +85,35 @@ class OPC(BaseActivity):
server_cert_path=server['server_cert_path'],
notification_handler=self.notification_handler,
reconnection_interval=server['reconnection_interval'],
pod_id=self.pod_id
pod_id=self.pod_id,
)
is_connected, error_data = await self.opc_repository[id].connect()
is_connected, error_data = await self.opc_repository[opc_id].connect()
if not is_connected:
self.send_notification(
metadata={
'model_id': '-',
'model_name': '-',
'workflow_name': '-',
'schedule_name': 'INITIALIZATION'
'schedule_name': 'INITIALIZATION',
},
notification_id=error_data['notification_id'],
message=error_data['message'],
block=error_data['block'],
level=error_data.get('level', NotificationLevel.ERROR),
attachment_content=error_data.get(
'attachment_content', None)
attachment_content=error_data.get('attachment_content', None),
)
else:
self.logger.info(
f"OPC server {id} connected successfully.")
self.logger.info(f'OPC server {opc_id} connected successfully.')
async def write_data(self, server_id: str, tag: str, data: Any,
data_type: str, tag_type: str, metadata: dict[str, Any]) -> bool:
async def write_data(
self,
server_id: str,
tag: str,
data: Any,
data_type: str,
tag_type: str,
metadata: dict[str, Any],
) -> bool:
"""
Write data to a specific OPC server tag with comprehensive error handling.
@@ -127,7 +135,8 @@ class OPC(BaseActivity):
try:
is_success, error_data = await self.opc_repository[server_id].write_data(
tag, data, data_type, self.logger, metadata)
tag, data, data_type, self.logger, metadata
)
if not is_success:
self.send_notification(
metadata=metadata,
@@ -135,8 +144,7 @@ class OPC(BaseActivity):
message=error_data['message'],
block=error_data['block'],
level=error_data.get('level', NotificationLevel.ERROR),
attachment_content=error_data.get(
'attachment_content', None)
attachment_content=error_data.get('attachment_content', None),
)
return False
return True
@@ -144,11 +152,11 @@ class OPC(BaseActivity):
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id=f"WRITE_OPC_{tag_type.upper()}_ERROR",
message=f"Error writing data to OPC server: {e}",
block="write_opc_data",
notification_id=f'WRITE_OPC_{tag_type.upper()}_ERROR',
message=f'Error writing data to OPC server: {e}',
block='write_opc_data',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
raise e
@@ -174,21 +182,26 @@ class OPC(BaseActivity):
This helps operators quickly identify configuration issues.
"""
if self.opc_repository.get(server_id) is None:
message = f"OPC server {server_id} not found to perform write operation."
message = f'OPC server {server_id} not found to perform write operation.'
self.send_notification(
metadata=metadata,
notification_id="OPC_SERVER_NOT_FOUND",
notification_id='OPC_SERVER_NOT_FOUND',
message=message,
block="write_opc_data",
block='write_opc_data',
level=NotificationLevel.ERROR,
attachment_content=f"OPC servers: {list(self.opc_repository.keys())}"
attachment_content=f'OPC servers: {list(self.opc_repository.keys())}',
)
return False
return True
async def manage_output_tags(
self, server_id: str, config: dict[str, Any], data: DataFrame,
metadata: dict[str, Any], success: bool) -> tuple[bool, int]:
self,
server_id: str,
config: dict[str, Any],
data: DataFrame,
metadata: dict[str, Any],
success: bool,
) -> tuple[bool, int]:
"""
Manage the writing of prediction and confidence data to OPC server tags.
@@ -225,11 +238,13 @@ class OPC(BaseActivity):
data=data.head(1)['prediction'].values[0],
data_type=tag_config['data_type'],
tag_type='prediction',
metadata=metadata
metadata=metadata,
)
if local_success:
self.info(
f"Prediction data written to OPC server {server_id} for tag {tag}.", metadata)
f'Prediction data written to OPC server {server_id} for tag {tag}.',
metadata,
)
count += 1
success = success and local_success
@@ -241,11 +256,13 @@ class OPC(BaseActivity):
data=data.head(1)['prediction_confidence'].values[0],
data_type=tag_config['data_type'],
tag_type='confidence',
metadata=metadata
metadata=metadata,
)
if local_success:
self.info(
f"Confidence data written to OPC server {server_id} for tag {tag}.", metadata)
f'Confidence data written to OPC server {server_id} for tag {tag}.',
metadata,
)
count += 1
success = success and local_success
@@ -271,29 +288,33 @@ class OPC(BaseActivity):
"""
metadata = input_data['metadata']
self.info("Writing data to OPC servers...", metadata)
self.info('Writing data to OPC servers...', metadata)
data = DataFrame(input_data['data'])
opc_output_config = input_data['opc_output_config']
self.info(f"Data to write: {data.size} rows", metadata)
self.info(f'Data to write: {data.size} rows', metadata)
success = True
for server_id, config in opc_output_config.items():
if not self.validate_server(server_id, metadata):
success = False
continue
local_success, local_count = await self.manage_output_tags(
server_id, config, data, metadata, success)
server_id, config, data, metadata, success
)
success = success and local_success
self.info(
f"Process completed for OPC server {server_id}: {local_count} of {len(config.get('prediction_tags', []))} prediction tags and {len(config.get('confidence_tags', []))} confidence tags", metadata)
f'Process completed for OPC server {server_id}: {local_count} of {len(config.get("prediction_tags", []))} prediction tags and {len(config.get("confidence_tags", []))} confidence tags',
metadata,
)
return self.process_confidence(data, success, metadata)
def process_confidence(self, data: DataFrame, success: bool, metadata: dict[str, Any]) -> dict[Any, Any]:
def process_confidence(
self, data: DataFrame, success: bool, metadata: dict[str, Any]
) -> dict[Any, Any]:
"""
Process prediction confidence based on OPC write operation success.
@@ -323,12 +344,12 @@ class OPC(BaseActivity):
if not success:
data['prediction_confidence'] = OPC_WRITTING_ERROR_CONFIDENCE
self.debug(
f"Some data could not be written to OPC servers, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.",
metadata
f'Some data could not be written to OPC servers, setting confidence to {OPC_WRITTING_ERROR_CONFIDENCE}.',
metadata,
)
else:
self.debug("Data written to OPC servers successfully.", metadata)
self.debug('Data written to OPC servers successfully.', metadata)
return data.to_dict()

121
laborious/activities/storage.py
View File

@@ -1,20 +1,21 @@
from temporalio import activity, workflow
from laborious.utils.repository.minio_repository import MinioRepository
with workflow.unsafe.imports_passed_through():
# Extend the Temporal Postgres activities for convenient query -> MinIO export
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
from sientia_do.temporal.constants import now
from sientia_do.notifications.models import NotificationLevel
from typing import Any
import traceback
import pandas as pd
from typing import Any
DATETIME_FILENAME_FORMAT = "%Y-%m-%d_%H-%M-%S"
import pandas as pd
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from sientia_do.temporal.activities.postgres import Postgres
from sientia_do.temporal.constants import now
from laborious.utils.repository.minio_repository import MinioRepository
DATETIME_FILENAME_FORMAT = '%Y-%m-%d_%H-%M-%S'
class Storage(Postgres):
@@ -23,36 +24,33 @@ class Storage(Postgres):
directly to MinIO as Parquet and return the object name.
"""
def __init__(self,
host: str,
port: int,
user: str,
password: str,
dbname: str,
min_connections: int,
max_connections: int,
minio_config: dict[str, Any],
logger: Logger,
notification_handler: NotificationHandler):
super().__init__(host=host,
port=port,
user=user,
password=password,
dbname=dbname,
min_connections=min_connections,
max_connections=max_connections,
logger=logger,
notification_handler=notification_handler)
def __init__(
self,
host: str,
port: int,
user: str,
password: str,
dbname: str,
min_connections: int,
max_connections: int,
minio_config: dict[str, Any],
logger: Logger,
notification_handler: NotificationHandler,
):
super().__init__(
host=host,
port=port,
user=user,
password=password,
dbname=dbname,
min_connections=min_connections,
max_connections=max_connections,
logger=logger,
notification_handler=notification_handler,
)
if not hasattr(self, 'minio_repository'):
self.minio_repository = MinioRepository(
logger=logger,
notification_handler=notification_handler,
minio_endpoint_url=minio_config['endpoint_url'],
minio_access_key=minio_config['access_key'],
minio_secret_key=minio_config['secret_key'],
minio_region_name=minio_config['region_name'],
minio_default_bucket=minio_config['default_bucket'])
self.minio_repository: MinioRepository | None = None
if self.minio_repository is None:
self.minio_repository = MinioRepository(
@@ -62,7 +60,8 @@ class Storage(Postgres):
minio_access_key=minio_config['access_key'],
minio_secret_key=minio_config['secret_key'],
minio_region_name=minio_config['region_name'],
minio_default_bucket=minio_config['default_bucket'])
minio_default_bucket=minio_config['default_bucket'],
)
@activity.defn(name='query_to_minio')
async def query_to_minio(self, input_data: dict[str, Any]) -> dict[str, Any]:
@@ -79,64 +78,60 @@ class Storage(Postgres):
dict: { success: bool, object_name: str, uri: str }
"""
if self.minio_repository is None:
raise ValueError('Minio repository not initialized')
metadata = input_data.get('metadata', {})
object_prefix = input_data.get('object_prefix', 'datasets/retrain')
timestamp = now().strftime(DATETIME_FILENAME_FORMAT)
object_name = f"{object_prefix}_{timestamp}.parquet"
uri = f"s3://{self.minio_repository.minio_bucket}/{object_name}"
object_name = f'{object_prefix}_{timestamp}.parquet'
uri = f's3://{self.minio_repository.minio_bucket}/{object_name}'
try:
data = await self.load_custom_query(input_data)
if not data:
self.error(
f"query_to_minio failed: No data returned from query", metadata)
return {"success": False, "message": "No data returned from query"}
self.error('query_to_minio failed: No data returned from query', metadata)
return {'success': False, 'message': 'No data returned from query'}
# Ensure we have a DataFrame
data = pd.DataFrame(data)
# Write parquet to memory and upload via persistent client
self.minio_repository.store_dataframe_as_parquet(
dataframe=data,
uri=uri,
object_name=object_name,
metadata=metadata
dataframe=data, uri=uri, object_name=object_name, metadata=metadata
)
return {"success": True, "object_key": object_name, "uri": uri}
return {'success': True, 'object_key': object_name, 'uri': uri}
except Exception as e:
trace = traceback.format_exc()
self.send_notification(
metadata=metadata,
notification_id="ERROR_LOADING_CUSTOM_QUERY",
message=f"Error fetching data from query: {e}",
block="load_custom_query",
notification_id='ERROR_STORING_QUERY_TO_MINIO',
message=f'Error storing query to MinIO: {e}',
block='query_to_minio',
level=NotificationLevel.ERROR,
attachment_content=trace
attachment_content=trace,
)
self.error(trace, metadata)
return {"success": False, "message": str(e)}
return {'success': False, 'message': str(e)}
def close(self) -> None:
"""Close Storage resources (MinIO client and Postgres engine)."""
try:
if hasattr(self, 's3_client') and self.s3_client is not None:
if hasattr(self, 'minio_repository') and self.minio_repository is not None:
try:
self.s3_client.close()
self.minio_repository.close()
finally:
self.s3_client = None
self.minio_repository = None
finally:
# Ensure Postgres resources are disposed as well
try:
super().close()
except Exception:
pass
self.logger.error('Error closing Postgres resources')
def __del__(self):
try:
self.close()
except Exception:
pass
self.close()

38
laborious/metrics.py
View File

@@ -23,50 +23,50 @@ Metric Labels:
- opc_server_id: Identifier for OPC server operations
"""
from prometheus_client import Gauge, Counter, Histogram
from prometheus_client import Counter, Gauge, Histogram
# Application health metric
APP_UP = Gauge(
"app_up",
"Indicates if the application is running (1) or shutting down (0)",
["pod_id"],
'app_up',
'Indicates if the application is running (1) or shutting down (0)',
['pod_id'],
)
# Core labels used across multiple metrics
CORE_LABELS = ["pod_id", "model_name", "pipeline_name"]
CORE_LABELS = ['pod_id', 'model_name', 'pipeline_name']
# Prediction operation metrics
PREDICTIONS_WRITTEN_COUNT = Counter(
"laborious_predictions_written_count",
"Number of predictions written to the database table predictions",
'laborious_predictions_written_count',
'Number of predictions written to the database table predictions',
CORE_LABELS,
)
# Prediction quality metrics
PREDICTION_CONFIDENCE_MONITOR = Gauge(
"laborious_prediction_confidence_monitor",
"Current confidence of each prediction",
'laborious_prediction_confidence_monitor',
'Current confidence of each prediction',
CORE_LABELS,
)
# Performance monitoring metrics
PREDICTION_RESPONSE_TIME_MONITOR = Histogram(
"laborious_prediction_response_time_monitor",
"Current response time of each prediction",
'laborious_prediction_response_time_monitor',
'Current response time of each prediction',
CORE_LABELS,
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0]
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
)
# OPC export metrics
PREDICTION_OPC_WRITING_COUNT = Counter(
"laborious_prediction_opc_writing_count",
"Number of predictions written to the OPC server",
[*CORE_LABELS, "opc_server_id"],
'laborious_prediction_opc_writing_count',
'Number of predictions written to the OPC server',
[*CORE_LABELS, 'opc_server_id'],
)
PREDICTION_OPC_WRITING_RESPONSE_TIME_MONITOR = Histogram(
"laborious_prediction_opc_writing_response_time_monitor",
"Current response time of each prediction written to the OPC server",
[*CORE_LABELS, "opc_server_id"],
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0]
'laborious_prediction_opc_writing_response_time_monitor',
'Current response time of each prediction written to the OPC server',
[*CORE_LABELS, 'opc_server_id'],
buckets=[0.01, 0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0],
)

24
laborious/utils/connectors_config.py
View File

@@ -1,9 +1,9 @@
from os import getenv
import json
from typing import Dict, Any
from os import getenv
from typing import Any
def build_postgres_config() -> Dict[str, Any]:
def build_postgres_config() -> dict[str, Any]:
"""
Build PostgreSQL database configuration from environment variables.
@@ -30,11 +30,11 @@ def build_postgres_config() -> Dict[str, Any]:
'password': getenv('POSTGRES_PASSWORD', 'sientia'),
'dbname': getenv('POSTGRES_DBNAME', 'sientia'),
'min_connections': int(getenv('POSTGRES_MIN_CONNECTIONS', '5')),
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20'))
'max_connections': int(getenv('POSTGRES_MAX_CONNECTIONS', '20')),
}
def build_mlflow_config() -> Dict[str, Any]:
def build_mlflow_config() -> dict[str, Any]:
"""
Build MLFlow server configuration from environment variables.
@@ -55,11 +55,11 @@ def build_mlflow_config() -> Dict[str, Any]:
'host': getenv('MLFLOW_HOST', 'http://localhost'),
'port': int(getenv('MLFLOW_PORT', '5080')),
'username': getenv('MLFLOW_USERNAME', 'aignosi'),
'password': getenv('MLFLOW_PASSWORD', 'aignosi')
'password': getenv('MLFLOW_PASSWORD', 'aignosi'),
}
def build_opc_config() -> Dict[str, Any]:
def build_opc_config() -> dict[str, Any]:
"""
Build OPC server configuration from environment variables.
@@ -93,12 +93,12 @@ def build_opc_config() -> Dict[str, Any]:
'cert_path': getenv('OPC_CERT_PATH', None),
'private_key_path': getenv('OPC_PRIVATE_KEY_PATH', None),
'server_cert_path': getenv('OPC_SERVER_CERT_PATH', None),
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120'))
'reconnection_interval': int(getenv('OPC_RECONNECTION_INTERVAL', '120')),
}
}
def build_mongodb_config() -> Dict[str, Any]:
def build_mongodb_config() -> dict[str, Any]:
"""
Build MongoDB configuration from environment variables.
@@ -125,11 +125,11 @@ def build_mongodb_config() -> Dict[str, Any]:
return {
'connection_string': connection_string,
'database_name': getenv('MONGODB_DATABASE_NAME', 'sientia'),
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600
'ttl_index_seconds': int(getenv('MONGODB_TTL_INDEX_HOURS', '1')) * 3600,
}
def build_minio_config() -> Dict[str, Any]:
def build_minio_config() -> dict[str, Any]:
"""
Build MinIO (S3-compatible) configuration from environment variables.
@@ -148,5 +148,5 @@ def build_minio_config() -> Dict[str, Any]:
'access_key': getenv('MINIO_ACCESS_KEY', 'minioadmin'),
'secret_key': getenv('MINIO_SECRET_KEY', 'minioadmin'),
'region_name': getenv('MINIO_REGION_NAME', 'us-east-1'),
'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious')
'default_bucket': getenv('MINIO_DEFAULT_BUCKET', 'laborious'),
}

3
laborious/utils/filters/conditional_filters.py
View File

@@ -24,8 +24,7 @@ def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool
if data.empty:
return False
return not data[
data['variable'].isin(config['variables']) & data['value'].isna()].empty
return not data[data['variable'].isin(config['variables']) & data['value'].isna()].empty
def filter_empty_data(data: DataFrame, _config: dict) -> bool:

7
laborious/utils/filters/mlflow_filters.py
View File

@@ -52,8 +52,11 @@ def nan_values_filter(predictions: DataFrame, _config: dict) -> bool:
bool: True if data should be filtered (too many NaN values), False otherwise
"""
data = predictions.replace({None: np.nan}).drop(
columns=['timestamp'], errors='ignore').infer_objects()
data = (
predictions.replace({None: np.nan})
.drop(columns=['timestamp'], errors='ignore')
.infer_objects()
)
if data.isna().all().all():
return True

90
laborious/utils/repository/minio_repository.py
View File

@@ -1,40 +1,41 @@
from io import BytesIO
import traceback
from typing import Any
import boto3
from botocore.config import Config
from pandas import DataFrame, read_parquet
from sientia_do.observability.logger import Logger
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from typing import Any
from botocore.exceptions import ClientError
from pandas import DataFrame, read_parquet
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import Logger
class MinioRepository():
def __init__(self,
minio_endpoint_url: str,
minio_access_key: str,
minio_secret_key: str,
minio_region_name: str,
minio_default_bucket: str,
logger: Logger,
notification_handler: NotificationHandler):
class MinioRepository:
def __init__(
self,
minio_endpoint_url: str,
minio_access_key: str,
minio_secret_key: str,
minio_region_name: str,
minio_default_bucket: str,
logger: Logger,
notification_handler: NotificationHandler,
):
# MinIO settings shared with pandas s3fs
self.storage_options = {
'key': minio_access_key,
'secret': minio_secret_key,
'client_kwargs': {'endpoint_url': minio_endpoint_url}
'client_kwargs': {'endpoint_url': minio_endpoint_url},
}
self.minio_bucket = minio_default_bucket
self.minio_endpoint_url = minio_endpoint_url
self.minio_region_name = minio_region_name
logger.info(
f"Connecting to MinIO at {self.minio_endpoint_url}, default bucket: {self.minio_bucket}")
f'Connecting to MinIO at {self.minio_endpoint_url}, default bucket: {self.minio_bucket}'
)
# Reusable MinIO client
self.s3_client = boto3.client(
self.s3_client: Any = boto3.client(
's3',
endpoint_url=self.minio_endpoint_url,
aws_access_key_id=self.storage_options['key'],
@@ -48,67 +49,46 @@ class MinioRepository():
read_timeout=120,
),
)
self._bucket_checked = False
self.logger = logger
self.notification_handler = notification_handler
def close(self):
self.s3_client.close()
def ensure_bucket_exists(self, metadata: dict[str, Any]) -> bool:
"""
Ensure the MinIO bucket exists; create it if necessary.
"""
if self._bucket_checked:
return True
try:
self.logger.custom_info(
f"Checking if bucket '{self.minio_bucket}' exists", metadata)
self.logger.custom_info(f"Checking if bucket '{self.minio_bucket}' exists", metadata)
self.s3_client.head_bucket(Bucket=self.minio_bucket)
self._bucket_checked = True
return True
except ClientError:
try:
self.logger.custom_info(
f"Creating bucket '{self.minio_bucket}'", metadata)
self.s3_client.create_bucket(Bucket=self.minio_bucket)
self._bucket_checked = True
return True
except ClientError as ce:
trace = traceback.format_exc()
self.notification_handler.send_notification(
metadata=metadata,
notification_id="ERROR_CREATING_MINIO_BUCKET",
message=f"Failed to ensure bucket '{self.minio_bucket}': {ce}",
block="ensure_bucket_exists",
level=NotificationLevel.ERROR,
attachment_content=str(ce)
)
self.logger.custom_error(trace, metadata)
return False
self.logger.custom_info(f"Creating bucket '{self.minio_bucket}'", metadata)
self.s3_client.create_bucket(Bucket=self.minio_bucket)
def store_dataframe_as_parquet(self, dataframe: DataFrame, uri: str,
object_name: str, metadata: dict[str, Any]):
return True
def store_dataframe_as_parquet(
self, dataframe: DataFrame, uri: str, object_name: str, metadata: dict[str, Any]
):
self.ensure_bucket_exists(metadata)
self.logger.custom_info(
f"Storing dataframe as parquet in {uri}", metadata)
self.logger.custom_info(f'Storing dataframe as parquet in {uri}', metadata)
buffer = BytesIO()
dataframe.to_parquet(buffer, engine='pyarrow', index=True)
buffer.seek(0)
self.s3_client.put_object(
Bucket=self.minio_bucket, Key=object_name, Body=buffer.getvalue())
self.s3_client.put_object(Bucket=self.minio_bucket, Key=object_name, Body=buffer.getvalue())
self.logger.custom_info(
f"Dataframe stored as parquet in {uri}", metadata)
self.logger.custom_info(f'Dataframe stored as parquet in {uri}', metadata)
def get_parquet_as_dataframe(self, object_key: str, metadata: dict[str, Any]) -> DataFrame:
self.logger.custom_info(
f"Getting parquet as dataframe from {object_key}", metadata)
self.logger.custom_info(f'Getting parquet as dataframe from {object_key}', metadata)
response = self.s3_client.get_object(
Bucket=self.minio_bucket, Key=object_key)
response = self.s3_client.get_object(Bucket=self.minio_bucket, Key=object_key)
# Read the content into a BytesIO buffer to support seek operations
buffer = BytesIO(response['Body'].read())

765
laborious/utils/repository/model_repository.py
View File

File diff suppressed because it is too large Load Diff

182
laborious/utils/repository/opc_repository.py
View File

@@ -1,16 +1,16 @@
import asyncio
import traceback
import time
import traceback
from datetime import datetime
from pathlib import Path
from typing import Any
from asyncua import Client
from asyncua.crypto.security_policies import SecurityPolicyBasic256
from asyncua.ua import DataValue, Variant, VariantType, DateTime
from regex import F
from asyncua.ua import DataValue, DateTime, Variant, VariantType
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.notifications.models import NotificationLevel
from sientia_do.observability.logger import Logger
from laborious import metrics
data_type_map = {
@@ -33,17 +33,26 @@ data_type_map = {
'str': {
'converter': str,
'opc_type': VariantType.String,
}
},
}
class OpcRepository():
def __init__(self, id: str, url: str, logger: Logger,
notification_handler: NotificationHandler,
reconnection_interval: int = 60, server_uri: str = None, cert_path: str = None,
private_key_path: str = None, server_cert_path: str = None, pod_id: str = None):
class OpcRepository:
def __init__(
self,
opc_id: str,
url: str,
logger: Logger,
notification_handler: NotificationHandler,
reconnection_interval: int = 60,
server_uri: str | None = None,
cert_path: str | None = None,
private_key_path: str | None = None,
server_cert_path: str | None = None,
pod_id: str | None = None,
):
self.url = url
self.id = id
self.id = opc_id
self.server_uri = server_uri
self.cert_path = cert_path
self.private_key_path = private_key_path
@@ -51,16 +60,16 @@ class OpcRepository():
self.logger = logger
self.error_count = 0
self.reconnection_interval = reconnection_interval
self.last_reconnection_time = None
self.last_reconnection_time: None | datetime = None
self.notification_handler = notification_handler
self.client = None
self.client: None | Client = None
self.pod_id = pod_id
self.metadata = {
'model_name': '-',
'model_id': '-',
'workflow_name': 'opc_repository',
'schedule_name': '-'
'schedule_name': '-',
}
async def set_security(self):
@@ -85,11 +94,18 @@ class OpcRepository():
if not all([self.cert_path, self.private_key_path]):
raise ValueError(
"Certificate and private key paths must be provided for secure connection.")
'Certificate and private key paths must be provided for secure connection.'
)
if self.cert_path is None or self.private_key_path is None:
raise ValueError('Certificate and private key paths cannot be None')
cert = Path(self.cert_path)
private_key = Path(self.private_key_path)
server_cert = Path(
self.server_cert_path) if self.server_cert_path else None
server_cert = Path(self.server_cert_path) if self.server_cert_path else None
if self.client is None:
raise ValueError('Client must be initialized before setting security')
self.client.application_uri = self.server_uri
self.logger.custom_info('Setting security...', self.metadata)
@@ -97,7 +113,7 @@ class OpcRepository():
SecurityPolicyBasic256,
certificate=str(cert),
private_key=str(private_key),
server_certificate=str(server_cert)
server_certificate=str(server_cert) if server_cert else None,
)
self.client.secure_channel_timeout = 10000000
self.client.session_timeout = 10000000
@@ -115,8 +131,7 @@ class OpcRepository():
self.client = Client(self.url)
if self.cert_path:
await self.set_security()
self.logger.custom_info(
f'Starting connection to OPC server {self.id}...', self.metadata)
self.logger.custom_info(f'Starting connection to OPC server {self.id}...', self.metadata)
return await self.try_connect()
async def try_connect(self) -> tuple[bool, dict[str, Any]]:
@@ -136,6 +151,13 @@ class OpcRepository():
try:
self.last_reconnection_time = datetime.now()
if self.client is None:
return False, {
'notification_id': f'OPC_CONNECTION_ERROR_{self.id}',
'message': 'Client is not initialized',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
}
await self.client.connect()
return True, {}
except Exception as e:
@@ -143,11 +165,11 @@ class OpcRepository():
self.logger.custom_error(trace, self.metadata)
return False, {
"notification_id": f"OPC_CONNECTION_ERROR_{self.id}",
"message": f"Failed to connect to OPC server: {e}",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
'notification_id': f'OPC_CONNECTION_ERROR_{self.id}',
'message': f'Failed to connect to OPC server: {e}',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': trace,
}
async def disconnect(self):
@@ -162,11 +184,9 @@ class OpcRepository():
return
try:
await self.client.disconnect()
self.logger.custom_info(
'Disconnected from OPC server', self.metadata)
self.logger.custom_info('Disconnected from OPC server', self.metadata)
except Exception as e:
self.logger.custom_error(
f"Failed to disconnect from OPC server: {e}", self.metadata)
self.logger.custom_error(f'Failed to disconnect from OPC server: {e}', self.metadata)
self.client = None
async def validate_connection(self) -> tuple[bool, dict[str, Any]]:
@@ -203,52 +223,62 @@ class OpcRepository():
if self.error_count > 5:
self.logger.custom_warning(
f"OPC server {self.id} will be disconnected due to multiple errors", self.metadata)
f'OPC server {self.id} will be disconnected due to multiple errors', self.metadata
)
try:
await self.disconnect()
except Exception as e:
trace = traceback.format_exc()
self.logger.custom_error(
f"Failed to disconnect from OPC server: {e}", self.metadata)
f'Failed to disconnect from OPC server: {e}', self.metadata
)
self.logger.custom_error(trace, self.metadata)
self.logger.custom_info(
f"Attempting to reconnect to OPC server {self.id}...", self.metadata)
f'Attempting to reconnect to OPC server {self.id}...', self.metadata
)
return await self.connect()
# Check if client is connected using asyncua's connection state
try:
if self.client.uaclient.protocol is None or self.client.uaclient.protocol.state == "closed":
if (
self.client.uaclient.protocol is None
or self.client.uaclient.protocol.state == 'closed'
):
# OPC server is not connected
self.logger.custom_error(
f"OPC server {self.id} is not connected", self.metadata)
if self.last_reconnection_time is None or (datetime.now() - self.last_reconnection_time).total_seconds(
) > self.reconnection_interval:
self.logger.custom_error(f'OPC server {self.id} is not connected', self.metadata)
if (
self.last_reconnection_time is None
or (datetime.now() - self.last_reconnection_time).total_seconds()
> self.reconnection_interval
):
await self.disconnect()
self.logger.custom_info(
f"Trying to reconnect to OPC server {self.id}...", self.metadata)
f'Trying to reconnect to OPC server {self.id}...', self.metadata
)
return await self.connect()
return False, {
"notification_id": f"OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}",
"message": f"OPC server {self.id} is not connected, waiting for next reconnection window...",
"block": "opc_repository",
"level": NotificationLevel.WARNING
'notification_id': f'OPC_CONNECTION_AWAITING_RECONNECTION_WINDOW_{self.id}',
'message': f'OPC server {self.id} is not connected, waiting for next reconnection window...',
'block': 'opc_repository',
'level': NotificationLevel.WARNING,
}
return True, {}
except Exception as e:
trace = traceback.format_exc()
message = f"Failed to validate connection to OPC server: {e}"
message = f'Failed to validate connection to OPC server: {e}'
self.logger.custom_error(message, self.metadata)
return False, {
"notification_id": f"OPC_CONNECTION_CHECK_ERROR_{self.id}",
"message": message,
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
'notification_id': f'OPC_CONNECTION_CHECK_ERROR_{self.id}',
'message': message,
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': trace,
}
async def write_data(self, node: str, value: Any, data_type: str,
logger: Logger, metadata: dict[str, Any]) -> tuple[bool, dict[str, Any]]:
async def write_data(
self, node: str, value: Any, data_type: str, logger: Logger, metadata: dict[str, Any]
) -> tuple[bool, dict[str, Any]]:
"""
Write data to OPC server with comprehensive validation and monitoring.
@@ -286,42 +316,42 @@ class OpcRepository():
start_time = time.time()
try:
if self.client is None:
return False, {
'notification_id': f'OPC_WRITE_GET_NODE_ERROR_{self.id}',
'message': 'Client is not initialized',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
}
node_obj = self.client.get_node(node)
except Exception as e:
trace = traceback.format_exc()
logger.custom_error(trace, metadata.get('schedule_name', 'N/A'))
self.error_count += 1
return False, {
"notification_id": f"OPC_WRITE_GET_NODE_ERROR_{self.id}",
"message": f"Failed to get node from OPC server: {e} | metadata: {metadata}",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
'notification_id': f'OPC_WRITE_GET_NODE_ERROR_{self.id}',
'message': f'Failed to get node from OPC server: {e} | metadata: {metadata}',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': trace,
}
if data_type not in data_type_map:
return False, {
"notification_id": f"OPC_WRITE_DATA_TYPE_ERROR_{self.id}",
"message": f"Unsupported data type: {data_type} | metadata: {metadata}",
"block": "opc_repository",
"level": NotificationLevel.ERROR
'notification_id': f'OPC_WRITE_DATA_TYPE_ERROR_{self.id}',
'message': f'Unsupported data type: {data_type} | metadata: {metadata}',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
}
data = data_type_map[data_type]['converter'](value)
logger.custom_info(
f'Writing {data} - {type(data)} to {node}', metadata)
logger.custom_info(f'Writing {data} - {type(data)} to {node}', metadata)
now = datetime.now()
ua_data = DataValue(
Variant(data, data_type_map[data_type]['opc_type']),
SourceTimestamp=DateTime(
now.year,
now.month,
now.day,
now.hour,
now.minute,
now.second,
now.microsecond
)
now.year, now.month, now.day, now.hour, now.minute, now.second, now.microsecond
),
)
try:
@@ -331,7 +361,7 @@ class OpcRepository():
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=self.id
opc_server_id=self.id,
).inc()
end_time = time.time()
@@ -340,7 +370,7 @@ class OpcRepository():
pod_id=self.pod_id,
model_name=metadata['model_name'],
pipeline_name=metadata['workflow_name'],
opc_server_id=self.id
opc_server_id=self.id,
).observe(response_time)
except Exception as e:
@@ -348,11 +378,11 @@ class OpcRepository():
logger.custom_error(trace, metadata)
self.error_count += 1
return False, {
"notification_id": f"OPC_WRITE_DATA_ERROR_{self.id}",
"message": f"Failed to write data to OPC server: {e} | metadata: {metadata}",
"block": "opc_repository",
"level": NotificationLevel.ERROR,
"attachment_content": trace
'notification_id': f'OPC_WRITE_DATA_ERROR_{self.id}',
'message': f'Failed to write data to OPC server: {e} | metadata: {metadata}',
'block': 'opc_repository',
'level': NotificationLevel.ERROR,
'attachment_content': trace,
}
self.error_count = 0

77
laborious/worker/worker.py
View File

@@ -25,37 +25,37 @@ Environment Variables:
- PROJECT_NAME: Project name for notifications (default: laborious)
"""
from temporalio import workflow, client
from temporalio.worker import Worker, PollerBehaviorAutoscaling
from temporalio.runtime import Runtime, TelemetryConfig, PrometheusConfig
from temporalio import client, workflow
from temporalio.runtime import PrometheusConfig, Runtime, TelemetryConfig
from temporalio.worker import PollerBehaviorAutoscaling, Worker
with workflow.unsafe.imports_passed_through():
import asyncio
import os
import sys
import asyncio
from laborious.workflows.minimal_retrain import MinimalRetrain
from laborious.workflows.predictions_batch import PredictionsBatch
from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
from laborious.workflows.sub_workflows.format_and_export_prediction import \
FormatAndExportPrediction
from laborious.activities.activities import Activities
from laborious.utils.connectors_config import (
build_postgres_config,
build_mlflow_config,
build_minio_config,
build_opc_config,
build_mongodb_config
)
from prometheus_client import start_http_server
from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
from sientia_do.observability.logger import get_logger
from laborious import metrics
from prometheus_client import start_http_server
import lzma
import dataclasses
from laborious import metrics
from laborious.activities.activities import Activities
from laborious.utils.connectors_config import (
build_minio_config,
build_mlflow_config,
build_mongodb_config,
build_opc_config,
build_postgres_config,
)
from laborious.workflows.minimal_retrain import MinimalRetrain
from laborious.workflows.predictions_batch import PredictionsBatch
from laborious.workflows.sub_workflows.format_and_export_prediction import (
FormatAndExportPrediction,
)
from laborious.workflows.sub_workflows.prediction_process import PredictionProcess
POD_ID = os.getenv('POD_ID')
SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', "9091"))
SDK_METRICS_PORT = int(os.getenv('HTTP_SDK_METRICS_PORT', '9091'))
async def main():
@@ -91,7 +91,7 @@ async def main():
logger.custom_info(f'Starting Worker with POD_ID: {POD_ID}', metadata)
logger.custom_info("Starting prometheus client...", metadata)
logger.custom_info('Starting prometheus client...', metadata)
start_prometheus_server()
logger.custom_info('Starting Notification Handler...', metadata)
@@ -101,7 +101,7 @@ async def main():
connection_string=mongo_config['connection_string'],
database=mongo_config['database_name'],
logger=logger,
project_name=os.getenv('PROJECT_NAME', 'laborious')
project_name=os.getenv('PROJECT_NAME', 'laborious'),
)
logger.custom_info('Starting Activities...', metadata)
@@ -112,19 +112,17 @@ async def main():
minio_config=build_minio_config(),
opc_config=build_opc_config(),
logger=logger,
notification_handler=notification_handler
notification_handler=notification_handler,
)
logger.custom_info('Initializing OPC...', metadata)
await activities.init_opc()
logger.custom_info(
f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
logger.custom_info(f'Starting SDK Metrics Server on port {SDK_METRICS_PORT}...', metadata)
new_runtime = Runtime(
telemetry=TelemetryConfig(
metrics=PrometheusConfig(
bind_address=f"0.0.0.0:{SDK_METRICS_PORT}")
metrics=PrometheusConfig(bind_address=f'0.0.0.0:{SDK_METRICS_PORT}')
)
)
@@ -133,7 +131,7 @@ async def main():
temporal_client = await client.Client.connect(
target_host=host,
namespace=os.getenv('TEMPORAL_NAMESPACE', 'laborious'),
runtime=new_runtime
runtime=new_runtime,
)
logger.custom_info('Starting Workers...', metadata)
@@ -156,13 +154,12 @@ async def main():
max_concurrent_local_activities=50,
max_cached_workflows=200,
workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
activity_task_poller_behavior=PollerBehaviorAutoscaling()
activity_task_poller_behavior=PollerBehaviorAutoscaling(),
),
Worker(
temporal_client,
task_queue='predictions_batch-queue',
workflows=[PredictionsBatch, PredictionProcess,
FormatAndExportPrediction],
workflows=[PredictionsBatch, PredictionProcess, FormatAndExportPrediction],
activities=[
# MLFlow
activities.request_predict,
@@ -181,15 +178,15 @@ async def main():
activities.load_custom_query,
activities.repeat_last_prediction,
activities.export_data_to_postgres,
activities.write_metrics
activities.write_metrics,
],
max_concurrent_workflow_tasks=50,
max_concurrent_activities=50,
max_concurrent_local_activities=50,
max_cached_workflows=200,
workflow_task_poller_behavior=PollerBehaviorAutoscaling(),
activity_task_poller_behavior=PollerBehaviorAutoscaling()
)
activity_task_poller_behavior=PollerBehaviorAutoscaling(),
),
]
handlers = []
@@ -203,7 +200,7 @@ async def main():
# If an exception occurs in any of the worker handlers, it will be propagated here.
await asyncio.gather(*handlers)
except BaseException as e: # NOSONAR
logger.custom_error(f"An unhandled exception occurred: {e}", metadata)
logger.custom_error(f'An unhandled exception occurred: {e}', metadata)
finally:
if notification_handler:
notification_handler.shutdown()
@@ -232,12 +229,12 @@ def start_prometheus_server():
SystemExit: If the metrics server fails to start
"""
try:
port = int(os.getenv("HTTP_METRICS_PORT", 9090))
port = int(os.getenv('HTTP_METRICS_PORT', 9090))
start_http_server(port)
print(f"Prometheus server started on port {port}.")
print(f'Prometheus server started on port {port}.')
metrics.APP_UP.labels(pod_id=POD_ID).set(1) # Mark app as UP
except Exception as e:
print(f"Failed to start Prometheus server: {e}")
print(f'Failed to start Prometheus server: {e}')
os._exit(1)

44
laborious/workflows/minimal_retrain.py
View File

@@ -1,14 +1,16 @@
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from laborious.activities.activities import Activities
from typing import Any
from sientia_do.temporal.policies import retry_policy
from datetime import timedelta
from typing import Any
from sientia_do.temporal.policies import retry_policy
from laborious.activities.activities import Activities
@workflow.defn(name="minimal_retrain")
class MinimalRetrain():
@workflow.defn(name='minimal_retrain')
class MinimalRetrain:
"""
Automated model retraining workflow for the Laborious system.
@@ -63,24 +65,24 @@ class MinimalRetrain():
'schedule_name': input_data['schedule_name'],
'model_name': input_data['model_name'],
'model_id': input_data['model_id'],
'workflow_name': 'minimal_retrain'
'workflow_name': 'minimal_retrain',
}
}
model_name = input_data['model_name']
model_config = input_data.get('model_config', {})
storage_result = await workflow.execute_local_activity_method(
storage_result = await workflow.execute_activity_method(
Activities.query_to_minio,
{
**metadata,
'query': input_data['query'],
'datetime_columns': input_data.get('datetime_columns', []),
'model_name': model_name,
'object_prefix': f'retrain_datasets/{model_name}/data'
'object_prefix': f'retrain_datasets/{model_name}/data',
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=600)
start_to_close_timeout=timedelta(seconds=600),
)
if not storage_result['success']:
@@ -92,38 +94,32 @@ class MinimalRetrain():
**metadata,
'object_key': storage_result['object_key'],
'model_name': model_name,
'model_config': model_config
'model_config': model_config,
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(hours=1)
start_to_close_timeout=timedelta(hours=1),
)
if experiment_response['success']:
update_report = await workflow.execute_activity_method(
Activities.update_production_model,
{
**metadata,
'model_name': model_name,
**experiment_response
},
{**metadata, 'model_name': model_name, **experiment_response},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
else:
update_report = {}
report = await workflow.execute_activity_method(
report = await workflow.execute_local_activity_method(
Activities.format_retrain_report,
{
**metadata,
'experiment_response': experiment_response,
'model_id': input_data['model_id'],
'model_name': model_name,
'update_report': update_report
'update_report': update_report,
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
await workflow.execute_activity_method(
@@ -132,8 +128,8 @@ class MinimalRetrain():
**metadata,
'data': report,
'schema': input_data['schema'],
'table_name': input_data['table_name']
'table_name': input_data['table_name'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=600)
start_to_close_timeout=timedelta(seconds=600),
)

46
laborious/workflows/predictions_batch.py
View File

@@ -1,14 +1,16 @@
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from laborious.activities.activities import Activities
from typing import Any
from sientia_do.temporal.policies import retry_policy
from datetime import timedelta
from typing import Any
from sientia_do.temporal.policies import retry_policy
from laborious.activities.activities import Activities
@workflow.defn(name="predictions_batch")
class PredictionsBatch():
@workflow.defn(name='predictions_batch')
class PredictionsBatch:
"""
Main batch prediction workflow for the Laborious system.
@@ -74,7 +76,7 @@ class PredictionsBatch():
'schedule_name': input_data['schedule_name'],
'model_name': input_data['model_name'],
'model_id': input_data['model_id'],
'workflow_name': 'predictions_batch'
'workflow_name': 'predictions_batch',
}
}
@@ -84,10 +86,10 @@ class PredictionsBatch():
{
**metadata,
'query': input_data['query'],
'datetime_columns': input_data.get('datetime_columns', [])
'datetime_columns': input_data.get('datetime_columns', []),
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=300)
start_to_close_timeout=timedelta(seconds=300),
)
# Prepare input for prediction_process workflow
@@ -98,28 +100,18 @@ class PredictionsBatch():
'table_name': input_data['table_name'],
'model_id': input_data['model_id'],
'model_name': input_data['model_name'],
'input_filters': input_data.get('input_filters', {
'EMPTY_DATA': {
'POLICY': 'STOP'
}
}),
'mlflow_transform_filters': input_data.get('mlflow_transform_filters', {
'API_ERROR': {
'POLICY': 'STOP'
}
}),
'mlflow_predict_filters': input_data.get('mlflow_predict_filters', {
'API_ERROR': {
'POLICY': 'STOP'
}
}),
'input_filters': input_data.get('input_filters', {'EMPTY_DATA': {'POLICY': 'STOP'}}),
'mlflow_transform_filters': input_data.get(
'mlflow_transform_filters', {'API_ERROR': {'POLICY': 'STOP'}}
),
'mlflow_predict_filters': input_data.get(
'mlflow_predict_filters', {'API_ERROR': {'POLICY': 'STOP'}}
),
'model_config': input_data.get('model_config', {}),
'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']),
'opc_output_config': input_data.get('opc_output_config', {}),
'prediction_store_policy': input_data.get(
'prediction_store_policy', 'lts:1')
'prediction_store_policy': input_data.get('prediction_store_policy', 'lts:1'),
}
# Execute prediction process workflow
await workflow.execute_child_workflow(
'prediction_process', prediction_input)
await workflow.execute_child_workflow('prediction_process', prediction_input)

42
laborious/workflows/sub_workflows/format_and_export_prediction.py
View File

@@ -1,15 +1,17 @@
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from laborious.activities.activities import Activities
from typing import Any
from datetime import timedelta
from sientia_do.temporal.policies import retry_policy
from typing import Any
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from sientia_do.temporal.policies import retry_policy
from laborious.activities.activities import Activities
@workflow.defn(name="format_and_export_prediction")
class FormatAndExportPrediction():
@workflow.defn(name='format_and_export_prediction')
class FormatAndExportPrediction:
"""
Data formatting and export workflow for prediction results.
@@ -79,10 +81,10 @@ class FormatAndExportPrediction():
'timestamp': input_data['timestamp'],
'model_id': input_data['model_id'],
'prediction_confidence': prediction_confidence,
'prediction_store_policy': input_data['prediction_store_policy']
'prediction_store_policy': input_data['prediction_store_policy'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
else:
@@ -94,22 +96,18 @@ class FormatAndExportPrediction():
'timestamp': input_data['timestamp'],
'model_id': input_data['model_id'],
'prediction_confidence': prediction_confidence,
'comment': input_data['comment']
'comment': input_data['comment'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
# write to opc
prediction = await workflow.execute_activity_method(
Activities.write_opc_data,
{
**metadata,
'opc_output_config': input_data['opc_output_config'],
'data': prediction
},
{**metadata, 'opc_output_config': input_data['opc_output_config'], 'data': prediction},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
# write to postgres
@@ -120,21 +118,15 @@ class FormatAndExportPrediction():
'schema': input_data['schema'],
'table_name': input_data['table_name'],
'data': prediction,
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ
}
'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ},
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)
await workflow.execute_activity_method(
Activities.write_metrics,
{
**metadata,
'prediction': prediction
},
{**metadata, 'prediction': prediction},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(seconds=60)
start_to_close_timeout=timedelta(seconds=60),
)

55
laborious/workflows/sub_workflows/prediction_process.py
View File

@@ -1,14 +1,16 @@
from temporalio import workflow
with workflow.unsafe.imports_passed_through():
from laborious.activities.activities import Activities
from typing import Any
from sientia_do.temporal.policies import retry_policy
from datetime import timedelta
from typing import Any
from sientia_do.temporal.policies import retry_policy
from laborious.activities.activities import Activities
@workflow.defn(name="prediction_process")
class PredictionProcess():
@workflow.defn(name='prediction_process')
class PredictionProcess:
"""
Core prediction processing workflow for the Laborious system.
@@ -84,10 +86,7 @@ class PredictionProcess():
# Get last timestamp for incremental processing
last_timestamp = await workflow.execute_local_activity_method(
Activities.get_last_timestamp,
{
**metadata,
'data': data
},
{**metadata, 'data': data},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(minutes=1),
)
@@ -97,7 +96,7 @@ class PredictionProcess():
**metadata,
'filters': input_data['input_filters'],
'data': data,
'path_priority': input_data['path_priority']
'path_priority': input_data['path_priority'],
}
path_flag, confidence, comment = await workflow.execute_local_activity_method(
@@ -116,12 +115,7 @@ class PredictionProcess():
# Request MLFlow model transformation
response_data = await workflow.execute_local_activity_method(
Activities.request_transform,
{
**metadata,
'data': data,
'model_name': model_name,
'model_config': model_config
},
{**metadata, 'data': data, 'model_name': model_name, 'model_config': model_config},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(minutes=5),
)
@@ -134,7 +128,7 @@ class PredictionProcess():
'filters': input_data['mlflow_transform_filters'],
'data': response_data,
'type': 'transform',
'path_priority': input_data['path_priority']
'path_priority': input_data['path_priority'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(minutes=1),
@@ -155,7 +149,7 @@ class PredictionProcess():
'filters': input_data['mlflow_transform_filters'],
'data': transformed_data,
'type': 'transform',
'path_priority': input_data['path_priority']
'path_priority': input_data['path_priority'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(minutes=1),
@@ -172,7 +166,7 @@ class PredictionProcess():
**metadata,
'data': transformed_data,
'model_name': model_name,
'model_config': model_config
'model_config': model_config,
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(minutes=5),
@@ -186,7 +180,7 @@ class PredictionProcess():
'filters': input_data['mlflow_predict_filters'],
'data': response_data,
'type': 'predict',
'path_priority': input_data['path_priority']
'path_priority': input_data['path_priority'],
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(minutes=1),
@@ -214,12 +208,19 @@ class PredictionProcess():
'schema': input_data['schema'],
'table_name': input_data['table_name'],
'comment': comment,
'prediction_store_policy': input_data['prediction_store_policy']
}
'prediction_store_policy': input_data['prediction_store_policy'],
},
)
async def path_flag_handler(self, data: dict, path_flag: str, input_data: dict,
confidence: int, last_timestamp: str, comment: str) -> bool:
async def path_flag_handler(
self,
data: dict,
path_flag: str,
input_data: dict,
confidence: int,
last_timestamp: str,
comment: str,
) -> bool:
"""
Handle path decisions based on filter results and confidence levels.
@@ -265,7 +266,7 @@ class PredictionProcess():
'schema': schema,
'table_name': table_name,
'model': model_id,
'last_timestamp': last_timestamp
'last_timestamp': last_timestamp,
},
retry_policy=retry_policy,
start_to_close_timeout=timedelta(minutes=1),
@@ -288,8 +289,8 @@ class PredictionProcess():
'table_name': table_name,
'comment': comment,
'opc_output_config': input_data['opc_output_config'],
'prediction_store_policy': input_data['prediction_store_policy']
}
'prediction_store_policy': input_data['prediction_store_policy'],
},
)
return True