Code import - branch release/SIENTIAPDE-1646
This commit is contained in:
0
laborious/workflows/__init__.py
Normal file
0
laborious/workflows/__init__.py
Normal file
107
laborious/workflows/drift.py
Normal file
107
laborious/workflows/drift.py
Normal file
@@ -0,0 +1,107 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name='drift')
|
||||
class Drift:
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the drift workflow.
|
||||
|
||||
This method orchestrates the complete drift process by:
|
||||
1. Loading data using the provided custom SQL query
|
||||
2. Preparing prediction configuration and filters
|
||||
3. Delegating to the PredictionProcess workflow for ML operations
|
||||
"""
|
||||
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'workflow_name': 'drift',
|
||||
}
|
||||
}
|
||||
|
||||
print(f'Input data: {input_data}', metadata)
|
||||
|
||||
model_config = input_data['model_config']
|
||||
target_name = model_config['target']
|
||||
|
||||
gathering_query = f"""
|
||||
SELECT *
|
||||
FROM "{input_data['schema']}"."{input_data['source_table_name']}"
|
||||
WHERE
|
||||
model_id = '{input_data['model_id']}' AND
|
||||
timestamp > NOW() - INTERVAL '{input_data['interval']} minutes'
|
||||
ORDER BY timestamp ASC
|
||||
""" # nosec B608 - values come from internal Temporal workflow config, not user input
|
||||
|
||||
target_data_handler = workflow.start_activity_method(
|
||||
Activities.load_custom_query,
|
||||
{
|
||||
**metadata,
|
||||
'query': gathering_query,
|
||||
'datetime_columns': ['timestamp', 'created_at'],
|
||||
'orient': 'records',
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
reference_data_handler = workflow.start_activity_method(
|
||||
Activities.get_reference_data,
|
||||
{**metadata, 'model_name': input_data['model_name']},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
target_data = await target_data_handler
|
||||
reference_data = await reference_data_handler
|
||||
|
||||
if not target_data:
|
||||
return
|
||||
|
||||
drift_data = await workflow.execute_local_activity_method(
|
||||
Activities.calculate_drift,
|
||||
{
|
||||
**metadata,
|
||||
'target_data': target_data,
|
||||
'reference_data': reference_data,
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'target_name': target_name,
|
||||
'drift_metrics': input_data.get(
|
||||
'drift_metrics', ['kolmogorov_smirnov', 'jensen_shannon', 'wasserstein']
|
||||
),
|
||||
'chunk_period': input_data.get('chunk_period', 'min'),
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
if drift_data:
|
||||
await workflow.execute_activity_method(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'data': drift_data,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['target_table_name'],
|
||||
'timestamp_conversion': {
|
||||
'column': 'timestamp',
|
||||
'format': DATETIME_FORMAT_WITH_TZ,
|
||||
},
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
137
laborious/workflows/minimal_retrain.py
Normal file
137
laborious/workflows/minimal_retrain.py
Normal file
@@ -0,0 +1,137 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
from laborious.utils.models.minio_dataframe_payload import MinioDataFramePayload
|
||||
|
||||
|
||||
@workflow.defn(name='minimal_retrain')
|
||||
class MinimalRetrain:
|
||||
"""
|
||||
Automated model retraining workflow for the Laborious system.
|
||||
|
||||
This workflow implements a complete model retraining pipeline that loads
|
||||
training data, executes model retraining, updates production models,
|
||||
and maintains comprehensive audit trails. It's designed for automated
|
||||
model lifecycle management with minimal manual intervention.
|
||||
|
||||
The workflow provides a robust retraining process with:
|
||||
- Automated data loading from configured data sources
|
||||
- MLFlow model retraining with quality validation
|
||||
- Production model updates with version control
|
||||
- Comprehensive reporting and audit trail maintenance
|
||||
- Error handling and notification integration
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the automated model retraining workflow.
|
||||
|
||||
This method orchestrates the complete model retraining process by:
|
||||
1. Loading training data using the provided custom SQL query
|
||||
2. Executing MLFlow model retraining with the loaded data
|
||||
3. Updating production models with newly trained versions
|
||||
4. Persisting comprehensive retraining reports to database
|
||||
|
||||
The method implements comprehensive error handling and ensures all
|
||||
required parameters are properly configured before proceeding.
|
||||
|
||||
Args:
|
||||
input_data: Complete configuration for the retraining workflow
|
||||
Required keys:
|
||||
- schedule_name (str): Schedule identifier for the retraining
|
||||
- model_name (str): Name of the ML model to retrain
|
||||
- model_id (int): Unique identifier for the model version
|
||||
- query (str): SQL query for training data loading
|
||||
- schema (str, optional): Database schema for report storage
|
||||
- table_name (str, optional): Target table for retraining reports
|
||||
- datetime_columns (list[str], optional): Columns to treat as datetime
|
||||
|
||||
Returns:
|
||||
None: The workflow completes successfully when all steps finish
|
||||
|
||||
Raises:
|
||||
Exception: If any required parameters are missing or if the workflow fails
|
||||
during data loading, retraining, or model update operations
|
||||
"""
|
||||
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'workflow_name': 'minimal_retrain',
|
||||
}
|
||||
}
|
||||
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
|
||||
storage_result = await workflow.execute_activity_method(
|
||||
Activities.load_query_with_minio_offload,
|
||||
{
|
||||
**metadata,
|
||||
'query': input_data['query'],
|
||||
'datetime_columns': input_data.get('datetime_columns', []),
|
||||
'model_name': model_name,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=600),
|
||||
)
|
||||
|
||||
storage_payload = MinioDataFramePayload.from_dict(storage_result)
|
||||
if not storage_payload.has_data():
|
||||
raise ValueError('No data returned from query')
|
||||
|
||||
experiment_response = await workflow.execute_activity_method(
|
||||
Activities.retrain_model,
|
||||
{
|
||||
**metadata,
|
||||
'data': storage_result,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(hours=1),
|
||||
)
|
||||
|
||||
if experiment_response['success']:
|
||||
update_report = await workflow.execute_activity_method(
|
||||
Activities.update_production_model,
|
||||
{**metadata, 'model_name': model_name, **experiment_response},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
else:
|
||||
update_report = {}
|
||||
|
||||
report = await workflow.execute_local_activity_method(
|
||||
Activities.format_retrain_report,
|
||||
{
|
||||
**metadata,
|
||||
'experiment_response': experiment_response,
|
||||
'model_name': model_name,
|
||||
'model_id': input_data['model_id'],
|
||||
'update_report': update_report,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
await workflow.execute_activity_method(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'data': report,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=600),
|
||||
)
|
||||
127
laborious/workflows/predictions_batch.py
Normal file
127
laborious/workflows/predictions_batch.py
Normal file
@@ -0,0 +1,127 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name='predictions_batch')
|
||||
class PredictionsBatch:
|
||||
"""
|
||||
Main batch prediction workflow for the Laborious system.
|
||||
|
||||
This workflow orchestrates the complete batch prediction process, handling
|
||||
data loading, configuration management, and workflow delegation. It serves
|
||||
as the primary entry point for batch prediction operations and ensures
|
||||
proper data preparation before ML model inference.
|
||||
|
||||
The workflow implements a robust data processing pipeline with:
|
||||
- Custom SQL query execution for data loading
|
||||
- Comprehensive configuration management
|
||||
- Data quality filter application
|
||||
- MLFlow model integration
|
||||
- Workflow delegation to specialized sub-workflows
|
||||
|
||||
Workflow Execution:
|
||||
1. Data Loading: Executes custom SQL query to load prediction data
|
||||
2. Configuration Preparation: Sets up prediction parameters and filters
|
||||
3. Workflow Delegation: Spawns PredictionProcess child workflow
|
||||
4. Error Handling: Implements comprehensive error handling and retry policies
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the batch prediction workflow.
|
||||
|
||||
This method orchestrates the complete batch prediction process by:
|
||||
1. Loading data using the provided custom SQL query
|
||||
2. Preparing prediction configuration and filters
|
||||
3. Delegating to the PredictionProcess workflow for ML operations
|
||||
|
||||
The method implements comprehensive error handling and ensures all
|
||||
required parameters are properly configured before proceeding.
|
||||
|
||||
Args:
|
||||
input_data: Complete configuration for the batch prediction
|
||||
Required keys:
|
||||
- schedule_name (str): Schedule identifier for the prediction
|
||||
- model_name (str): Name of the ML model to use
|
||||
- model_id (int): Unique identifier for the model
|
||||
- query (str): SQL query for data loading
|
||||
- schema (dict, optional): Data schema definition
|
||||
- table_name (str, optional): Target table for predictions
|
||||
- input_filters (dict, optional): Data quality filters
|
||||
- mlflow_transform_filters (dict, optional): MLFlow transform filters
|
||||
- mlflow_predict_filters (dict, optional): MLFlow prediction filters
|
||||
- model_retention (int, optional): Model retention period in minutes
|
||||
- path_priority (list[str]): Decision path priority configuration
|
||||
- opc_output_config (dict, optional): OPC server export configuration
|
||||
- pi_web_api_output_config (dict, optional): PI Web API export configuration
|
||||
- datetime_columns (list[str], optional): Columns to treat as datetime
|
||||
- save_transform (bool, optional): Whether to save transformed data (default: True)
|
||||
- prediction_store_policy (str, optional): Data retention policy (default: 'lts:1')
|
||||
|
||||
Returns:
|
||||
None: The workflow completes successfully when the child workflow finishes
|
||||
|
||||
Raises:
|
||||
Exception: If any required parameters are missing or if the workflow fails
|
||||
during data loading or workflow delegation
|
||||
"""
|
||||
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'workflow_name': 'predictions_batch',
|
||||
}
|
||||
}
|
||||
|
||||
# Load data using custom query with optional MinIO offload for large frames
|
||||
data = await workflow.execute_activity_method(
|
||||
Activities.load_query_with_minio_offload,
|
||||
{
|
||||
**metadata,
|
||||
'query': input_data['query'],
|
||||
'datetime_columns': input_data.get('datetime_columns', []),
|
||||
'model_name': input_data['model_name'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
# Prepare input for prediction_process workflow
|
||||
prediction_input = {
|
||||
'metadata': metadata,
|
||||
'data': data,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'transform_table_name': input_data['transform_table_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'model_name': input_data['model_name'],
|
||||
'input_filters': input_data.get(
|
||||
'input_filters', {'EMPTY_DATA': {'POLICY': 'STOP', 'CONFIG': {}}}
|
||||
),
|
||||
'mlflow_transform_filters': input_data.get(
|
||||
'mlflow_transform_filters', {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}}
|
||||
),
|
||||
'mlflow_predict_filters': input_data.get(
|
||||
'mlflow_predict_filters', {'API_ERROR': {'POLICY': 'STOP', 'CONFIG': {}}}
|
||||
),
|
||||
'model_config': input_data.get('model_config', {}),
|
||||
'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']),
|
||||
'opc_output_config': input_data.get('opc_output_config', {}),
|
||||
'on_conflict': input_data.get('on_conflict', 'error'),
|
||||
'pi_web_api_output_config': input_data.get('pi_web_api_output_config', {}),
|
||||
'prediction_store_policy': input_data.get('prediction_store_policy', 'lts:1'),
|
||||
'save_transform': input_data.get('save_transform', True),
|
||||
}
|
||||
|
||||
# Execute prediction process workflow
|
||||
await workflow.execute_child_workflow('subworkflow.prediction_process', prediction_input)
|
||||
95
laborious/workflows/simple_metrics.py
Normal file
95
laborious/workflows/simple_metrics.py
Normal file
@@ -0,0 +1,95 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name='simple_metrics')
|
||||
class SimpleMetrics:
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the simple metrics workflow.
|
||||
"""
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'model_id': input_data['model_id'],
|
||||
'model_name': input_data['model_name'],
|
||||
'workflow_name': 'simple_metrics',
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
}
|
||||
}
|
||||
|
||||
model_id = input_data['model_id']
|
||||
interval_minutes = input_data['interval_minutes']
|
||||
|
||||
model_config = input_data['model_config']
|
||||
target_name = model_config['target']
|
||||
|
||||
query = f"""
|
||||
select p."timestamp", p.prediction, ld.value as "target"
|
||||
from "{input_data['schema']}"."{input_data['predictions_table_name']}" p
|
||||
inner join "{input_data['schema']}"."{input_data['data_table_name']}" ld
|
||||
on p."timestamp" = ld."timestamp"
|
||||
where
|
||||
p.model_id = '{model_id}' and
|
||||
p.prediction is not null and
|
||||
ld.variable = '{target_name}' and
|
||||
ld.value is not null and
|
||||
p."timestamp" >= NOW() - INTERVAL '{interval_minutes} minutes'
|
||||
order by
|
||||
p."timestamp" desc;
|
||||
""" # nosec B608 - values come from internal Temporal workflow config, not user input
|
||||
|
||||
target_data = await workflow.execute_activity_method(
|
||||
Activities.load_custom_query,
|
||||
{
|
||||
**metadata,
|
||||
'query': query,
|
||||
'datetime_columns': ['timestamp'],
|
||||
'orient': 'records',
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
if not target_data:
|
||||
return
|
||||
|
||||
simple_metrics = await workflow.execute_local_activity_method(
|
||||
Activities.calculate_simple_metrics,
|
||||
{
|
||||
**metadata,
|
||||
'model_id': model_id,
|
||||
'target_data': target_data,
|
||||
'metrics': input_data.get('metrics', ['rmse', 'mse', 'mae', 'r2']),
|
||||
'interval_minutes': interval_minutes,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
|
||||
if not simple_metrics:
|
||||
return
|
||||
|
||||
await workflow.execute_activity_method(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'data': simple_metrics,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['target_table_name'],
|
||||
'timestamp_conversion': {
|
||||
'column': 'timestamp',
|
||||
'format': DATETIME_FORMAT_WITH_TZ,
|
||||
},
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300),
|
||||
)
|
||||
0
laborious/workflows/sub_workflows/__init__.py
Normal file
0
laborious/workflows/sub_workflows/__init__.py
Normal file
@@ -0,0 +1,220 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name='subworkflow.format_and_export_prediction')
|
||||
class FormatAndExportPrediction:
|
||||
"""
|
||||
Data formatting and export workflow for prediction results.
|
||||
|
||||
This workflow handles the final stages of the prediction pipeline, including
|
||||
data formatting, database persistence, OPC server export, and metrics recording.
|
||||
It implements flexible formatting based on prediction quality and provides
|
||||
comprehensive export capabilities to multiple destinations.
|
||||
|
||||
The workflow supports two main prediction paths:
|
||||
1. Normal Prediction: Formats and exports successful prediction results
|
||||
2. Default Prediction: Creates fallback predictions for error conditions
|
||||
|
||||
Export Destinations:
|
||||
- PostgreSQL Database: Persistent storage with timestamp conversion
|
||||
- PI Web API: Real-time industrial system integration for prediction and confidence values
|
||||
- OPC Servers: Real-time industrial system integration
|
||||
- Prometheus Metrics: Performance monitoring and operational visibility
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the prediction formatting and export workflow.
|
||||
|
||||
This method orchestrates the complete data export process by:
|
||||
1. Determining the appropriate formatting strategy based on path_flag
|
||||
2. Formatting prediction data according to quality and requirements
|
||||
3. Exporting data to PI Web API for real-time industrial access (if configured)
|
||||
4. Exporting data to OPC servers for real-time industrial access (if configured)
|
||||
5. Persisting data to PostgreSQL database with comprehensive metadata
|
||||
6. Recording performance metrics for operational monitoring
|
||||
|
||||
The method implements flexible formatting strategies:
|
||||
- Normal predictions: Full data formatting with confidence scores
|
||||
- Error predictions: Default formatting with error indicators
|
||||
- Comprehensive export: Multi-destination data distribution
|
||||
|
||||
Args:
|
||||
input_data: Complete configuration for the export workflow
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- path_flag (str | None): Decision path flag for formatting strategy
|
||||
- None: Normal prediction path with full formatting
|
||||
- Any other value: Default prediction path for error conditions
|
||||
- data (dict[str, Any]): Prediction data to format and export
|
||||
- prediction_confidence (float): Confidence score for the prediction
|
||||
- timestamp (str): ISO-formatted timestamp for the prediction
|
||||
- model_id (int): Unique identifier for the ML model
|
||||
- model_name (str): Name of the ML model
|
||||
- schema (str): Database schema for data storage
|
||||
- table_name (str): Target table for data persistence
|
||||
Optional keys:
|
||||
- opc_output_config (dict[str, Any]): OPC server export configuration
|
||||
- pi_web_api_output_config (dict[str, Any]): PI Web API export configuration
|
||||
Contains endpoint, prediction_tags, and confidence_tags mappings
|
||||
- transformed_data (dict[str, Any]): Transformed data to export separately
|
||||
Only processed when path_flag is None
|
||||
- transform_table_name (str): Target table for transformed data export
|
||||
Required if transformed_data is provided
|
||||
- prediction_store_policy (str): Data retention policy (e.g., 'lts:1', 'erl:2')
|
||||
Required when path_flag is None
|
||||
- comment (str): Operational comment or error description
|
||||
Required when path_flag is not None
|
||||
|
||||
Returns:
|
||||
None: The workflow completes successfully when all export operations finish
|
||||
|
||||
Note:
|
||||
When transformed_data is provided and path_flag is None, the workflow will:
|
||||
1. Format the transformed data using format_transformed_data
|
||||
2. Export it to a separate table (transform_table_name) asynchronously
|
||||
3. Wait for both prediction and transformed data exports to complete
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
path_flag = input_data['path_flag']
|
||||
data = input_data['data']
|
||||
transformed_data = input_data.get('transformed_data', None)
|
||||
prediction_confidence = input_data['prediction_confidence']
|
||||
|
||||
opc_output_config = input_data.get('opc_output_config', None)
|
||||
pi_web_api_output_config = input_data.get('pi_web_api_output_config', None)
|
||||
|
||||
if path_flag is None:
|
||||
# Normal prediction path: format prediction data with full metadata
|
||||
prediction = await workflow.execute_local_activity_method(
|
||||
Activities.format_prediction,
|
||||
{
|
||||
**metadata,
|
||||
'data': data,
|
||||
'timestamp': input_data['timestamp'],
|
||||
'model_id': input_data['model_id'],
|
||||
'prediction_confidence': prediction_confidence,
|
||||
'prediction_store_policy': input_data['prediction_store_policy'],
|
||||
'model_name': input_data['model_name'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
# Optionally format and export transformed data to separate table
|
||||
if transformed_data is not None:
|
||||
transformed = await workflow.execute_local_activity_method(
|
||||
Activities.format_transformed_data,
|
||||
{
|
||||
**metadata,
|
||||
'data': transformed_data,
|
||||
'model_id': input_data['model_id'],
|
||||
'model_name': input_data['model_name'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
write_transformed_handler = workflow.start_activity_method(
|
||||
Activities.export_payload_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['transform_table_name'],
|
||||
'data': transformed,
|
||||
'timestamp_conversion': {
|
||||
'column': 'timestamp',
|
||||
'format': DATETIME_FORMAT_WITH_TZ,
|
||||
},
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
else:
|
||||
write_transformed_handler = None
|
||||
|
||||
else:
|
||||
# Error path: create default prediction with error indicators
|
||||
prediction = await workflow.execute_local_activity_method(
|
||||
Activities.format_default_prediction,
|
||||
{
|
||||
**metadata,
|
||||
'timestamp': input_data['timestamp'],
|
||||
'model_id': input_data['model_id'],
|
||||
'prediction_confidence': prediction_confidence,
|
||||
'comment': input_data['comment'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
write_transformed_handler = None
|
||||
|
||||
opc_metrics: dict[str, dict[str, float | None]] = {}
|
||||
|
||||
# write to pi web api
|
||||
if pi_web_api_output_config:
|
||||
prediction = await workflow.execute_activity_method(
|
||||
Activities.write_pi_web_api_data,
|
||||
{
|
||||
'pi_web_api_output_config': pi_web_api_output_config,
|
||||
'data': prediction,
|
||||
**metadata,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
# write to opc
|
||||
if opc_output_config:
|
||||
prediction, opc_metrics = await workflow.execute_activity_method(
|
||||
Activities.write_opc_data,
|
||||
{
|
||||
'opc_output_config': opc_output_config,
|
||||
'data': prediction,
|
||||
**metadata,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
# write to postgres
|
||||
await workflow.execute_activity_method(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'data': prediction,
|
||||
'timestamp_conversion': {'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ},
|
||||
'on_conflict': input_data.get('on_conflict', 'error'),
|
||||
'unique_columns': ['model_id', 'timestamp'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=180),
|
||||
)
|
||||
|
||||
if write_transformed_handler is not None:
|
||||
await write_transformed_handler
|
||||
|
||||
await workflow.execute_activity_method(
|
||||
Activities.write_metrics,
|
||||
{
|
||||
**metadata,
|
||||
'prediction': prediction,
|
||||
'opc_metrics': opc_metrics,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
346
laborious/workflows/sub_workflows/prediction_process.py
Normal file
346
laborious/workflows/sub_workflows/prediction_process.py
Normal file
@@ -0,0 +1,346 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
|
||||
from laborious.activities.activities import Activities
|
||||
|
||||
|
||||
@workflow.defn(name='subworkflow.prediction_process')
|
||||
class PredictionProcess:
|
||||
"""
|
||||
Core prediction processing workflow for the Laborious system.
|
||||
|
||||
This workflow implements the complete ML model inference pipeline, handling
|
||||
data quality validation, MLFlow model interactions, and prediction processing.
|
||||
It serves as the central orchestrator for all prediction operations and ensures
|
||||
data quality throughout the entire process.
|
||||
|
||||
The workflow implements a robust data processing pipeline with:
|
||||
- Data quality validation using configurable filters
|
||||
- MLFlow model transformation and prediction
|
||||
- Response validation and quality assurance
|
||||
- Flexible decision path handling
|
||||
- Comprehensive error handling and retry policies
|
||||
|
||||
Workflow Execution:
|
||||
1. Timestamp Retrieval: Gets last processed timestamp for incremental processing
|
||||
2. Input Data Gate: Applies data quality filters
|
||||
3. Path Decision: Determines processing path based on filter results
|
||||
4. MLFlow Transform: Requests data transformation using MLFlow models
|
||||
5. Response Validation: Filters transform responses for quality assurance
|
||||
6. MLFlow Prediction: Executes prediction using transformed data
|
||||
7. Content Validation: Filters prediction responses for final quality check
|
||||
8. Export Delegation: Delegates to FormatAndExportPrediction workflow
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the prediction process workflow.
|
||||
|
||||
This method orchestrates the complete prediction processing pipeline by:
|
||||
1. Retrieving the last processed timestamp for incremental processing
|
||||
2. Applying data quality filters to validate input data
|
||||
3. Executing MLFlow model transformation and prediction
|
||||
4. Validating all responses for quality assurance
|
||||
5. Delegating to export workflow for data persistence
|
||||
|
||||
The method implements comprehensive error handling and ensures all
|
||||
data quality requirements are met before proceeding with ML operations.
|
||||
|
||||
Args:
|
||||
input_data: Complete configuration for the prediction process
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict): Input data for prediction processing
|
||||
- schema (dict): Data schema definition
|
||||
- table_name (str): Target table for predictions
|
||||
- model_id (str): ML model identifier
|
||||
- model_name (str): ML model name
|
||||
- input_filters (dict): Data quality filters
|
||||
- mlflow_transform_filters (dict): MLFlow transform filters
|
||||
- mlflow_predict_filters (dict): MLFlow prediction filters
|
||||
- model_retention (int): Model retention period in minutes
|
||||
- path_priority (list[str]): Decision path priority configuration
|
||||
- opc_output_config (dict, optional): OPC server export configuration
|
||||
- pi_web_api_output_config (dict, optional): PI Web API export configuration
|
||||
- save_transform (bool, optional): Whether to save transformed data (default: True)
|
||||
- prediction_store_policy (str, optional): Data retention policy (default: 'lts:1')
|
||||
|
||||
Returns:
|
||||
None: The workflow completes successfully when export workflow finishes
|
||||
|
||||
Raises:
|
||||
Exception: If any required parameters are missing or if the workflow fails
|
||||
during data processing, MLFlow operations, or workflow delegation
|
||||
|
||||
"""
|
||||
|
||||
metadata = input_data['metadata']
|
||||
data = input_data['data']
|
||||
model_id = input_data['model_id']
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
save_transform = input_data.get('save_transform', True)
|
||||
|
||||
try:
|
||||
await self._run_prediction_pipeline(
|
||||
input_data,
|
||||
metadata,
|
||||
data,
|
||||
model_id,
|
||||
model_name,
|
||||
model_config,
|
||||
save_transform,
|
||||
)
|
||||
await workflow.execute_activity_method(
|
||||
Activities.cleanup_minio_objects_expired,
|
||||
{**metadata, 'data': data},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
except Exception as e:
|
||||
await workflow.execute_activity_method(
|
||||
Activities.cleanup_minio_objects_expired,
|
||||
{**metadata, 'data': data},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
raise e
|
||||
|
||||
async def _run_prediction_pipeline(
|
||||
self,
|
||||
input_data: dict[str, Any],
|
||||
metadata: dict[str, Any],
|
||||
data: dict[str, Any],
|
||||
model_id: str,
|
||||
model_name: str,
|
||||
model_config: dict[str, Any],
|
||||
save_transform: bool,
|
||||
) -> None:
|
||||
last_timestamp = data['last_timestamp']
|
||||
|
||||
# Apply input data quality gates
|
||||
gate_input = {
|
||||
**metadata,
|
||||
'filters': input_data['input_filters'],
|
||||
'data': data,
|
||||
'path_priority': input_data['path_priority'],
|
||||
}
|
||||
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.input_gate,
|
||||
gate_input,
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Handle path decision based on filter results
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
# Request MLFlow model transformation
|
||||
transformed_data = await workflow.execute_activity_method(
|
||||
Activities.request_transform,
|
||||
{**metadata, 'data': data, 'model_name': model_name, 'model_config': model_config},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
|
||||
# Validate MLFlow transform response
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_response_gate,
|
||||
{
|
||||
**metadata,
|
||||
'filters': input_data['mlflow_transform_filters'],
|
||||
'data': transformed_data,
|
||||
'type': 'transform',
|
||||
'path_priority': input_data['path_priority'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Handle path decision based on transform validation
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_content_gate,
|
||||
{
|
||||
**metadata,
|
||||
'filters': input_data['mlflow_transform_filters'],
|
||||
'data': transformed_data,
|
||||
'type': 'transform',
|
||||
'path_priority': input_data['path_priority'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
predicted_data = await workflow.execute_activity_method(
|
||||
Activities.request_predict,
|
||||
{
|
||||
**metadata,
|
||||
'data': transformed_data,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
|
||||
# Validate MLFlow prediction response
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_response_gate,
|
||||
{
|
||||
**metadata,
|
||||
'filters': input_data['mlflow_predict_filters'],
|
||||
'data': predicted_data,
|
||||
'type': 'predict',
|
||||
'path_priority': input_data['path_priority'],
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Handle path decision based on prediction validation
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
# Delegate to export workflow for data persistence
|
||||
await workflow.execute_child_workflow(
|
||||
'subworkflow.format_and_export_prediction',
|
||||
{
|
||||
'metadata': metadata,
|
||||
'on_conflict': input_data.get('on_conflict', 'error'),
|
||||
'path_flag': path_flag,
|
||||
'data': predicted_data,
|
||||
'transformed_data': transformed_data if save_transform else None,
|
||||
'prediction_confidence': confidence,
|
||||
'timestamp': last_timestamp,
|
||||
'model_id': model_id,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config,
|
||||
'opc_output_config': input_data['opc_output_config'],
|
||||
'pi_web_api_output_config': input_data['pi_web_api_output_config'],
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'transform_table_name': input_data['transform_table_name'],
|
||||
'comment': comment,
|
||||
'prediction_store_policy': input_data['prediction_store_policy'],
|
||||
},
|
||||
)
|
||||
|
||||
async def path_flag_handler(
|
||||
self,
|
||||
data: dict[str, Any],
|
||||
path_flag: str | None,
|
||||
input_data: dict,
|
||||
confidence: int,
|
||||
last_timestamp: str,
|
||||
comment: str,
|
||||
) -> bool:
|
||||
"""
|
||||
Handle path decisions based on filter results and confidence levels.
|
||||
|
||||
This method determines the appropriate action based on the path flag
|
||||
returned by data quality filters. It can stop processing, continue,
|
||||
or repeat operations based on the configured path priority.
|
||||
|
||||
Args:
|
||||
data: Input data for processing
|
||||
path_flag: Path decision from filter (STOP, CONTINUE, REPEAT)
|
||||
input_data: Complete workflow input configuration including:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- schema (str): Database schema
|
||||
- table_name (str): Target table for predictions
|
||||
- transform_table_name (str): Target table for transformed data
|
||||
- model_id (str): ML model identifier
|
||||
- model_name (str): ML model name
|
||||
- model_config (dict, optional): Model configuration
|
||||
- opc_output_config (dict, optional): OPC server export configuration
|
||||
- pi_web_api_output_config (dict, optional): PI Web API export configuration
|
||||
- prediction_store_policy (str, optional): Data retention policy
|
||||
confidence: Confidence level from filter validation
|
||||
last_timestamp: Last processed timestamp
|
||||
comment: Additional information about the filter result
|
||||
|
||||
Returns:
|
||||
bool: True if processing should stop, False to continue
|
||||
|
||||
Path Handling:
|
||||
- STOP: Terminates workflow execution
|
||||
- CONTINUE: Delegates to FormatAndExportPrediction workflow with current data
|
||||
- REPEAT: Repeats last prediction if available
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
schema = input_data['schema']
|
||||
table_name = input_data['table_name']
|
||||
transform_table_name = input_data['transform_table_name']
|
||||
model_id = input_data['model_id']
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
|
||||
path_flag = path_flag.upper() if path_flag else ''
|
||||
|
||||
if path_flag == 'STOP':
|
||||
# Stop processing and exit workflow
|
||||
return True
|
||||
elif path_flag == 'REPEAT':
|
||||
# Repeat last prediction if available
|
||||
await workflow.execute_activity_method(
|
||||
Activities.repeat_last_prediction,
|
||||
{
|
||||
**metadata,
|
||||
'schema': schema,
|
||||
'table_name': table_name,
|
||||
'model': model_id,
|
||||
'last_timestamp': last_timestamp,
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
return True
|
||||
elif path_flag == 'CONTINUE':
|
||||
# call write workflow
|
||||
await workflow.execute_child_workflow(
|
||||
'subworkflow.format_and_export_prediction',
|
||||
{
|
||||
'metadata': metadata,
|
||||
'path_flag': path_flag,
|
||||
'data': data,
|
||||
'prediction_confidence': confidence,
|
||||
'timestamp': last_timestamp,
|
||||
'model_id': model_id,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config,
|
||||
'schema': schema,
|
||||
'table_name': table_name,
|
||||
'transform_table_name': transform_table_name,
|
||||
'comment': comment,
|
||||
'opc_output_config': input_data['opc_output_config'],
|
||||
'pi_web_api_output_config': input_data['pi_web_api_output_config'],
|
||||
'prediction_store_policy': input_data['prediction_store_policy'],
|
||||
'on_conflict': input_data.get('on_conflict', 'error'),
|
||||
},
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
Reference in New Issue
Block a user