SIENTIAPDE-1243: Refactor: Rename 'laborious' package to 'model_manager'
This commit renames the 'laborious' package to 'model_manager' across the entire project. This includes renaming directories, modules, references in code, configuration files, and documentation to reflect the new package name. This change improves clarity and consistency within the project.
This commit is contained in:
Bruno Domingues
0
model_manager/workflows/__init__.py
Normal file
0
model_manager/workflows/__init__.py
Normal file
116
model_manager/workflows/minimal_retrain.py
Normal file
116
model_manager/workflows/minimal_retrain.py
Normal file
@@ -0,0 +1,116 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from model_manager.activities.activities import Activities
|
||||
from typing import Any
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from datetime import timedelta
|
||||
|
||||
|
||||
@workflow.defn(name="minimal_retrain")
|
||||
class MinimalRetrain():
|
||||
"""
|
||||
Automated model retraining workflow for the Laborious system.
|
||||
|
||||
This workflow implements a complete model retraining pipeline that loads
|
||||
training data, executes model retraining, updates production models,
|
||||
and maintains comprehensive audit trails. It's designed for automated
|
||||
model lifecycle management with minimal manual intervention.
|
||||
|
||||
The workflow provides a robust retraining process with:
|
||||
- Automated data loading from configured data sources
|
||||
- MLFlow model retraining with quality validation
|
||||
- Production model updates with version control
|
||||
- Comprehensive reporting and audit trail maintenance
|
||||
- Error handling and notification integration
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the automated model retraining workflow.
|
||||
|
||||
This method orchestrates the complete model retraining process by:
|
||||
1. Loading training data using the provided custom SQL query
|
||||
2. Executing MLFlow model retraining with the loaded data
|
||||
3. Updating production models with newly trained versions
|
||||
4. Persisting comprehensive retraining reports to database
|
||||
|
||||
The method implements comprehensive error handling and ensures all
|
||||
required parameters are properly configured before proceeding.
|
||||
|
||||
Args:
|
||||
input_data: Complete configuration for the retraining workflow
|
||||
Required keys:
|
||||
- schedule_name (str): Schedule identifier for the retraining
|
||||
- model_name (str): Name of the ML model to retrain
|
||||
- model_id (int): Unique identifier for the model version
|
||||
- query (str): SQL query for training data loading
|
||||
- schema (str, optional): Database schema for report storage
|
||||
- table_name (str, optional): Target table for retraining reports
|
||||
- datetime_columns (list[str], optional): Columns to treat as datetime
|
||||
|
||||
Returns:
|
||||
None: The workflow completes successfully when all steps finish
|
||||
|
||||
Raises:
|
||||
Exception: If any required parameters are missing or if the workflow fails
|
||||
during data loading, retraining, or model update operations
|
||||
"""
|
||||
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'workflow_name': 'minimal_retrain'
|
||||
}
|
||||
}
|
||||
|
||||
model_name = input_data['model_name']
|
||||
|
||||
data = await workflow.execute_local_activity_method(
|
||||
Activities.load_custom_query,
|
||||
{
|
||||
**metadata,
|
||||
'query': input_data['query'],
|
||||
'datetime_columns': input_data.get('datetime_columns', [])
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
|
||||
experiment_response = await workflow.execute_activity_method(
|
||||
Activities.retrain_model,
|
||||
{
|
||||
**metadata,
|
||||
'data': data,
|
||||
'model_name': model_name
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
|
||||
report = await workflow.execute_activity_method(
|
||||
Activities.update_production_model,
|
||||
{
|
||||
**metadata,
|
||||
'model_name': model_name,
|
||||
'model_id': input_data['model_id'],
|
||||
**experiment_response
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
|
||||
await workflow.execute_activity_method(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'data': report,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name']
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
125
model_manager/workflows/predictions_batch.py
Normal file
125
model_manager/workflows/predictions_batch.py
Normal file
@@ -0,0 +1,125 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from model_manager.activities.activities import Activities
|
||||
from typing import Any
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from datetime import timedelta
|
||||
|
||||
|
||||
@workflow.defn(name="predictions_batch")
|
||||
class PredictionsBatch():
|
||||
"""
|
||||
Main batch prediction workflow for the Laborious system.
|
||||
|
||||
This workflow orchestrates the complete batch prediction process, handling
|
||||
data loading, configuration management, and workflow delegation. It serves
|
||||
as the primary entry point for batch prediction operations and ensures
|
||||
proper data preparation before ML model inference.
|
||||
|
||||
The workflow implements a robust data processing pipeline with:
|
||||
- Custom SQL query execution for data loading
|
||||
- Comprehensive configuration management
|
||||
- Data quality filter application
|
||||
- MLFlow model integration
|
||||
- Workflow delegation to specialized sub-workflows
|
||||
|
||||
Workflow Execution:
|
||||
1. Data Loading: Executes custom SQL query to load prediction data
|
||||
2. Configuration Preparation: Sets up prediction parameters and filters
|
||||
3. Workflow Delegation: Spawns PredictionProcess child workflow
|
||||
4. Error Handling: Implements comprehensive error handling and retry policies
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the batch prediction workflow.
|
||||
|
||||
This method orchestrates the complete batch prediction process by:
|
||||
1. Loading data using the provided custom SQL query
|
||||
2. Preparing prediction configuration and filters
|
||||
3. Delegating to the PredictionProcess workflow for ML operations
|
||||
|
||||
The method implements comprehensive error handling and ensures all
|
||||
required parameters are properly configured before proceeding.
|
||||
|
||||
Args:
|
||||
input_data: Complete configuration for the batch prediction
|
||||
Required keys:
|
||||
- schedule_name (str): Schedule identifier for the prediction
|
||||
- model_name (str): Name of the ML model to use
|
||||
- model_id (int): Unique identifier for the model
|
||||
- query (str): SQL query for data loading
|
||||
- schema (dict, optional): Data schema definition
|
||||
- table_name (str, optional): Target table for predictions
|
||||
- input_filters (dict, optional): Data quality filters
|
||||
- mlflow_transform_filters (dict, optional): MLFlow transform filters
|
||||
- mlflow_predict_filters (dict, optional): MLFlow prediction filters
|
||||
- model_retention (int, optional): Model retention period in minutes
|
||||
- path_priority (list[str]): Decision path priority configuration
|
||||
- opc_output_config (dict, optional): OPC server export configuration
|
||||
- datetime_columns (list[str], optional): Columns to treat as datetime
|
||||
|
||||
Returns:
|
||||
None: The workflow completes successfully when the child workflow finishes
|
||||
|
||||
Raises:
|
||||
Exception: If any required parameters are missing or if the workflow fails
|
||||
during data loading or workflow delegation
|
||||
"""
|
||||
|
||||
metadata = {
|
||||
'metadata': {
|
||||
'schedule_name': input_data['schedule_name'],
|
||||
'model_name': input_data['model_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'workflow_name': 'predictions_batch'
|
||||
}
|
||||
}
|
||||
|
||||
# Load data using custom query
|
||||
data = await workflow.execute_local_activity_method(
|
||||
Activities.load_custom_query,
|
||||
{
|
||||
**metadata,
|
||||
'query': input_data['query'],
|
||||
'datetime_columns': input_data.get('datetime_columns', [])
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=300)
|
||||
)
|
||||
|
||||
# Prepare input for prediction_process workflow
|
||||
prediction_input = {
|
||||
'metadata': metadata,
|
||||
'data': data,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'model_id': input_data['model_id'],
|
||||
'model_name': input_data['model_name'],
|
||||
'input_filters': input_data.get('input_filters', {
|
||||
'EMPTY_DATA': {
|
||||
'POLICY': 'STOP'
|
||||
}
|
||||
}),
|
||||
'mlflow_transform_filters': input_data.get('mlflow_transform_filters', {
|
||||
'API_ERROR': {
|
||||
'POLICY': 'STOP'
|
||||
}
|
||||
}),
|
||||
'mlflow_predict_filters': input_data.get('mlflow_predict_filters', {
|
||||
'API_ERROR': {
|
||||
'POLICY': 'STOP'
|
||||
}
|
||||
}),
|
||||
'model_config': input_data.get('model_config', {}),
|
||||
'path_priority': input_data.get('path_priority', ['STOP', 'CONTINUE', 'REPEAT']),
|
||||
'opc_output_config': input_data.get('opc_output_config', {}),
|
||||
'prediction_store_policy': input_data.get(
|
||||
'prediction_store_policy', 'lts:1')
|
||||
}
|
||||
|
||||
# Execute prediction process workflow
|
||||
await workflow.execute_child_workflow(
|
||||
'prediction_process', prediction_input)
|
||||
0
model_manager/workflows/sub_workflows/__init__.py
Normal file
0
model_manager/workflows/sub_workflows/__init__.py
Normal file
@@ -0,0 +1,140 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from model_manager.activities.activities import Activities
|
||||
from typing import Any
|
||||
from datetime import timedelta
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
|
||||
|
||||
|
||||
@workflow.defn(name="format_and_export_prediction")
|
||||
class FormatAndExportPrediction():
|
||||
"""
|
||||
Data formatting and export workflow for prediction results.
|
||||
|
||||
This workflow handles the final stages of the prediction pipeline, including
|
||||
data formatting, database persistence, OPC server export, and metrics recording.
|
||||
It implements flexible formatting based on prediction quality and provides
|
||||
comprehensive export capabilities to multiple destinations.
|
||||
|
||||
The workflow supports two main prediction paths:
|
||||
1. Normal Prediction: Formats and exports successful prediction results
|
||||
2. Default Prediction: Creates fallback predictions for error conditions
|
||||
|
||||
Export Destinations:
|
||||
- PostgreSQL Database: Persistent storage with timestamp conversion
|
||||
- OPC Servers: Real-time industrial system integration
|
||||
- Prometheus Metrics: Performance monitoring and operational visibility
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the prediction formatting and export workflow.
|
||||
|
||||
This method orchestrates the complete data export process by:
|
||||
1. Determining the appropriate formatting strategy based on path_flag
|
||||
2. Formatting prediction data according to quality and requirements
|
||||
3. Exporting data to OPC servers for real-time industrial access
|
||||
4. Persisting data to PostgreSQL database with comprehensive metadata
|
||||
5. Recording performance metrics for operational monitoring
|
||||
|
||||
The method implements flexible formatting strategies:
|
||||
- Normal predictions: Full data formatting with confidence scores
|
||||
- Error predictions: Default formatting with error indicators
|
||||
- Comprehensive export: Multi-destination data distribution
|
||||
|
||||
Args:
|
||||
input_data: Complete configuration for the export workflow
|
||||
Required keys:
|
||||
- path_flag (str | None): Decision path flag for formatting strategy
|
||||
- data (dict[str, Any]): Prediction data to format and export
|
||||
- prediction_confidence (float): Confidence score for the prediction
|
||||
- timestamp (str): ISO-formatted timestamp for the prediction
|
||||
- model_id (int): Unique identifier for the ML model
|
||||
- model_name (str): Name of the ML model
|
||||
- model_retention (str): Model retention policy configuration
|
||||
- comment (str): Operational comment or error description
|
||||
- schema (str): Database schema for data storage
|
||||
- table_name (str): Target table for data persistence
|
||||
- opc_output_config (dict[str, Any]): OPC server export configuration
|
||||
- prediction_store_policy (str, optional): Data retention policy
|
||||
|
||||
Returns:
|
||||
bool: True if the workflow completes successfully, False otherwise
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
path_flag = input_data['path_flag']
|
||||
data = input_data['data']
|
||||
prediction_confidence = input_data['prediction_confidence']
|
||||
|
||||
if path_flag is None:
|
||||
# proceed with formatting and exporting
|
||||
prediction = await workflow.execute_local_activity_method(
|
||||
Activities.format_prediction,
|
||||
{
|
||||
**metadata,
|
||||
'data': data,
|
||||
'timestamp': input_data['timestamp'],
|
||||
'model_id': input_data['model_id'],
|
||||
'prediction_confidence': prediction_confidence,
|
||||
'prediction_store_policy': input_data['prediction_store_policy']
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
|
||||
else:
|
||||
# create default prediction
|
||||
prediction = await workflow.execute_local_activity_method(
|
||||
Activities.format_default_prediction,
|
||||
{
|
||||
**metadata,
|
||||
'timestamp': input_data['timestamp'],
|
||||
'model_id': input_data['model_id'],
|
||||
'prediction_confidence': prediction_confidence,
|
||||
'comment': input_data['comment']
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
|
||||
# write to opc
|
||||
prediction = await workflow.execute_activity_method(
|
||||
Activities.write_opc_data,
|
||||
{
|
||||
**metadata,
|
||||
'opc_output_config': input_data['opc_output_config'],
|
||||
'data': prediction
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
|
||||
# write to postgres
|
||||
await workflow.execute_activity_method(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
**metadata,
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'data': prediction,
|
||||
'timestamp_conversion': {
|
||||
'column': 'timestamp',
|
||||
'format': DATETIME_FORMAT_WITH_TZ
|
||||
}
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
|
||||
await workflow.execute_activity_method(
|
||||
Activities.write_metrics,
|
||||
{
|
||||
**metadata,
|
||||
'prediction': prediction
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(seconds=60)
|
||||
)
|
||||
296
model_manager/workflows/sub_workflows/prediction_process.py
Normal file
296
model_manager/workflows/sub_workflows/prediction_process.py
Normal file
@@ -0,0 +1,296 @@
|
||||
from temporalio import workflow
|
||||
|
||||
with workflow.unsafe.imports_passed_through():
|
||||
from model_manager.activities.activities import Activities
|
||||
from typing import Any
|
||||
from sientia_do.temporal.policies import retry_policy
|
||||
from datetime import timedelta
|
||||
|
||||
|
||||
@workflow.defn(name="prediction_process")
|
||||
class PredictionProcess():
|
||||
"""
|
||||
Core prediction processing workflow for the Laborious system.
|
||||
|
||||
This workflow implements the complete ML model inference pipeline, handling
|
||||
data quality validation, MLFlow model interactions, and prediction processing.
|
||||
It serves as the central orchestrator for all prediction operations and ensures
|
||||
data quality throughout the entire process.
|
||||
|
||||
The workflow implements a robust data processing pipeline with:
|
||||
- Data quality validation using configurable filters
|
||||
- MLFlow model transformation and prediction
|
||||
- Response validation and quality assurance
|
||||
- Flexible decision path handling
|
||||
- Comprehensive error handling and retry policies
|
||||
|
||||
Workflow Execution:
|
||||
1. Timestamp Retrieval: Gets last processed timestamp for incremental processing
|
||||
2. Input Data Gate: Applies data quality filters
|
||||
3. Path Decision: Determines processing path based on filter results
|
||||
4. MLFlow Transform: Requests data transformation using MLFlow models
|
||||
5. Response Validation: Filters transform responses for quality assurance
|
||||
6. MLFlow Prediction: Executes prediction using transformed data
|
||||
7. Content Validation: Filters prediction responses for final quality check
|
||||
8. Export Delegation: Delegates to FormatAndExportPrediction workflow
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Execute the prediction process workflow.
|
||||
|
||||
This method orchestrates the complete prediction processing pipeline by:
|
||||
1. Retrieving the last processed timestamp for incremental processing
|
||||
2. Applying data quality filters to validate input data
|
||||
3. Executing MLFlow model transformation and prediction
|
||||
4. Validating all responses for quality assurance
|
||||
5. Delegating to export workflow for data persistence
|
||||
|
||||
The method implements comprehensive error handling and ensures all
|
||||
data quality requirements are met before proceeding with ML operations.
|
||||
|
||||
Args:
|
||||
input_data: Complete configuration for the prediction process
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict): Input data for prediction processing
|
||||
- schema (dict): Data schema definition
|
||||
- table_name (str): Target table for predictions
|
||||
- model_id (str): ML model identifier
|
||||
- model_name (str): ML model name
|
||||
- input_filters (dict): Data quality filters
|
||||
- mlflow_transform_filters (dict): MLFlow transform filters
|
||||
- mlflow_predict_filters (dict): MLFlow prediction filters
|
||||
- model_retention (int): Model retention period in minutes
|
||||
- path_priority (list[str]): Decision path priority configuration
|
||||
- opc_output_config (dict): OPC server export configuration
|
||||
|
||||
Returns:
|
||||
None: The workflow completes successfully when export workflow finishes
|
||||
|
||||
Raises:
|
||||
Exception: If any required parameters are missing or if the workflow fails
|
||||
during data processing, MLFlow operations, or workflow delegation
|
||||
|
||||
"""
|
||||
|
||||
metadata = input_data['metadata']
|
||||
data = input_data['data']
|
||||
model_id = input_data['model_id']
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
|
||||
# Get last timestamp for incremental processing
|
||||
last_timestamp = await workflow.execute_local_activity_method(
|
||||
Activities.get_last_timestamp,
|
||||
{
|
||||
**metadata,
|
||||
'data': data
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Apply input data quality gates
|
||||
gate_input = {
|
||||
**metadata,
|
||||
'filters': input_data['input_filters'],
|
||||
'data': data,
|
||||
'path_priority': input_data['path_priority']
|
||||
}
|
||||
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.input_gate,
|
||||
gate_input,
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Handle path decision based on filter results
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
# Request MLFlow model transformation
|
||||
response_data = await workflow.execute_local_activity_method(
|
||||
Activities.request_transform,
|
||||
{
|
||||
**metadata,
|
||||
'data': data,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
|
||||
# Validate MLFlow transform response
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_response_gate,
|
||||
{
|
||||
**metadata,
|
||||
'filters': input_data['mlflow_transform_filters'],
|
||||
'data': response_data,
|
||||
'type': 'transform',
|
||||
'path_priority': input_data['path_priority']
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Handle path decision based on transform validation
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
transformed_data = response_data['content']
|
||||
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_content_gate,
|
||||
{
|
||||
**metadata,
|
||||
'filters': input_data['mlflow_transform_filters'],
|
||||
'data': transformed_data,
|
||||
'type': 'transform',
|
||||
'path_priority': input_data['path_priority']
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
response_data = await workflow.execute_local_activity_method(
|
||||
Activities.request_predict,
|
||||
{
|
||||
**metadata,
|
||||
'data': transformed_data,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=5),
|
||||
)
|
||||
|
||||
# Validate MLFlow prediction response
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_response_gate,
|
||||
{
|
||||
**metadata,
|
||||
'filters': input_data['mlflow_predict_filters'],
|
||||
'data': response_data,
|
||||
'type': 'predict',
|
||||
'path_priority': input_data['path_priority']
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Handle path decision based on prediction validation
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
# Delegate to export workflow for data persistence
|
||||
await workflow.execute_child_workflow(
|
||||
'format_and_export_prediction',
|
||||
{
|
||||
'metadata': metadata,
|
||||
'path_flag': path_flag,
|
||||
'data': response_data['content'],
|
||||
'prediction_confidence': confidence,
|
||||
'timestamp': last_timestamp,
|
||||
'model_id': model_id,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config,
|
||||
'opc_output_config': input_data['opc_output_config'],
|
||||
'schema': input_data['schema'],
|
||||
'table_name': input_data['table_name'],
|
||||
'comment': comment,
|
||||
'prediction_store_policy': input_data['prediction_store_policy']
|
||||
}
|
||||
)
|
||||
|
||||
async def path_flag_handler(self, data: dict, path_flag: str, input_data: dict,
|
||||
confidence: int, last_timestamp: str, comment: str) -> bool:
|
||||
"""
|
||||
Handle path decisions based on filter results and confidence levels.
|
||||
|
||||
This method determines the appropriate action based on the path flag
|
||||
returned by data quality filters. It can stop processing, continue,
|
||||
or repeat operations based on the configured path priority.
|
||||
|
||||
Args:
|
||||
data: Input data for processing
|
||||
path_flag: Path decision from filter (STOP, CONTINUE, REPEAT)
|
||||
input_data: Complete workflow input configuration
|
||||
confidence: Confidence level from filter validation
|
||||
last_timestamp: Last processed timestamp
|
||||
comment: Additional information about the filter result
|
||||
|
||||
Returns:
|
||||
bool: True if processing should stop, False to continue
|
||||
|
||||
Path Handling:
|
||||
- STOP: Terminates workflow execution
|
||||
- CONTINUE: Proceeds with normal processing
|
||||
- REPEAT: Repeats last prediction if available
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
schema = input_data['schema']
|
||||
table_name = input_data['table_name']
|
||||
model_id = input_data['model_id']
|
||||
model_name = input_data['model_name']
|
||||
model_config = input_data.get('model_config', {})
|
||||
|
||||
path_flag = path_flag.upper() if path_flag else ''
|
||||
|
||||
if path_flag == 'STOP':
|
||||
# Stop processing and exit workflow
|
||||
return True
|
||||
elif path_flag == 'REPEAT':
|
||||
# Repeat last prediction if available
|
||||
await workflow.execute_activity_method(
|
||||
Activities.repeat_last_prediction,
|
||||
{
|
||||
**metadata,
|
||||
'schema': schema,
|
||||
'table_name': table_name,
|
||||
'model': model_id,
|
||||
'last_timestamp': last_timestamp
|
||||
},
|
||||
retry_policy=retry_policy,
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
return True
|
||||
elif path_flag == 'CONTINUE':
|
||||
# call write workflow
|
||||
await workflow.execute_child_workflow(
|
||||
'format_and_export_prediction',
|
||||
{
|
||||
'metadata': metadata,
|
||||
'path_flag': path_flag,
|
||||
'data': data,
|
||||
'prediction_confidence': confidence,
|
||||
'timestamp': last_timestamp,
|
||||
'model_id': model_id,
|
||||
'model_name': model_name,
|
||||
'model_config': model_config,
|
||||
'schema': schema,
|
||||
'table_name': table_name,
|
||||
'comment': comment,
|
||||
'opc_output_config': input_data['opc_output_config'],
|
||||
'prediction_store_policy': input_data['prediction_store_policy']
|
||||
}
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
Reference in New Issue
Block a user