SIENTIAPDE-1182
Remove Docker configuration files and refactor project structure - Deleted docker-compose.yml and Dockerfile as part of the project restructuring. - Updated README.md to reflect changes in project setup and configuration. - Introduced a new __init__.py file in the laborious package to provide an overview of the system. - Enhanced documentation across various modules, including metrics, activities, and workflows, to improve clarity and usability. - Added comprehensive docstrings and comments to key classes and methods for better maintainability.
This commit is contained in:
vitor-aignosi
@@ -0,0 +1,10 @@
|
||||
"""
|
||||
Laborious Workflows Package
|
||||
|
||||
This package contains all Temporal workflow definitions for the Laborious system,
|
||||
including batch prediction workflows, model retraining workflows, and specialized
|
||||
sub-workflows for data processing and export operations.
|
||||
|
||||
Workflows orchestrate the execution of activities and implement the business
|
||||
process logic for ML model inference and data processing pipelines.
|
||||
"""
|
||||
|
||||
@@ -9,31 +9,53 @@ with workflow.unsafe.imports_passed_through():
|
||||
|
||||
@workflow.defn(name="minimal_retrain")
|
||||
class MinimalRetrain():
|
||||
"""
|
||||
Automated model retraining workflow for the Laborious system.
|
||||
|
||||
This workflow implements a complete model retraining pipeline that loads
|
||||
training data, executes model retraining, updates production models,
|
||||
and maintains comprehensive audit trails. It's designed for automated
|
||||
model lifecycle management with minimal manual intervention.
|
||||
|
||||
The workflow provides a robust retraining process with:
|
||||
- Automated data loading from configured data sources
|
||||
- MLFlow model retraining with quality validation
|
||||
- Production model updates with version control
|
||||
- Comprehensive reporting and audit trail maintenance
|
||||
- Error handling and notification integration
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
This workflow runs a minimal retrain of a model.
|
||||
Execute the automated model retraining workflow.
|
||||
|
||||
The workflow executes in four steps:
|
||||
1. Loads the data from the database
|
||||
2. Formats the data and perform the retrain
|
||||
3. Updates the production model
|
||||
4. Saves a model
|
||||
This method orchestrates the complete model retraining process by:
|
||||
1. Loading training data using the provided custom SQL query
|
||||
2. Executing MLFlow model retraining with the loaded data
|
||||
3. Updating production models with newly trained versions
|
||||
4. Persisting comprehensive retraining reports to database
|
||||
|
||||
The method implements comprehensive error handling and ensures all
|
||||
required parameters are properly configured before proceeding.
|
||||
|
||||
Args:
|
||||
- input_data (dict[str, Any]): The input data for the workflow.
|
||||
- schedule_name (str): The name of the schedule.
|
||||
- model_name (str): The name of the model.
|
||||
- model_id (int): The id of the model.
|
||||
- query (str): The SQL query to be executed to load data.
|
||||
- schema (dict, optional): The schema to store the report.
|
||||
- table_name (str, optional): The name of the table to store report.
|
||||
input_data: Complete configuration for the retraining workflow
|
||||
Required keys:
|
||||
- schedule_name (str): Schedule identifier for the retraining
|
||||
- model_name (str): Name of the ML model to retrain
|
||||
- model_id (int): Unique identifier for the model version
|
||||
- query (str): SQL query for training data loading
|
||||
- schema (str, optional): Database schema for report storage
|
||||
- table_name (str, optional): Target table for retraining reports
|
||||
- datetime_columns (list[str], optional): Columns to treat as datetime
|
||||
|
||||
Returns:
|
||||
None
|
||||
None: The workflow completes successfully when all steps finish
|
||||
|
||||
Raises:
|
||||
Exception: If any of the required parameters are missing or if the workflow fails.
|
||||
Exception: If any required parameters are missing or if the workflow fails
|
||||
during data loading, retraining, or model update operations
|
||||
"""
|
||||
|
||||
metadata = {
|
||||
|
||||
@@ -9,35 +9,80 @@ with workflow.unsafe.imports_passed_through():
|
||||
|
||||
@workflow.defn(name="predictions_batch")
|
||||
class PredictionsBatch():
|
||||
"""
|
||||
Main batch prediction workflow for the Laborious system.
|
||||
|
||||
This workflow orchestrates the complete batch prediction process, handling
|
||||
data loading, configuration management, and workflow delegation. It serves
|
||||
as the primary entry point for batch prediction operations and ensures
|
||||
proper data preparation before ML model inference.
|
||||
|
||||
The workflow implements a robust data processing pipeline with:
|
||||
- Custom SQL query execution for data loading
|
||||
- Comprehensive configuration management
|
||||
- Data quality filter application
|
||||
- MLFlow model integration
|
||||
- Workflow delegation to specialized sub-workflows
|
||||
|
||||
Workflow Execution:
|
||||
1. Data Loading: Executes custom SQL query to load prediction data
|
||||
2. Configuration Preparation: Sets up prediction parameters and filters
|
||||
3. Workflow Delegation: Spawns PredictionProcess child workflow
|
||||
4. Error Handling: Implements comprehensive error handling and retry policies
|
||||
|
||||
Example:
|
||||
>>> # Start the workflow
|
||||
>>> await client.start_workflow(
|
||||
... PredictionsBatch.run,
|
||||
... id="batch_pred_001",
|
||||
... task_queue="predictions_batch-queue",
|
||||
... input_data={
|
||||
... "schedule_name": "hourly_predictions",
|
||||
... "model_name": "temperature_model",
|
||||
... "model_id": "temp_001",
|
||||
... "query": "SELECT * FROM sensor_data WHERE timestamp > NOW() - INTERVAL '1 hour'",
|
||||
... "schema": {"timestamp": "datetime", "temperature": "float"},
|
||||
... "table_name": "predictions"
|
||||
... }
|
||||
... )
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
This workflow runs a batch of predictions based on the input data.
|
||||
Execute the batch prediction workflow.
|
||||
|
||||
The workflow executes in two main steps:
|
||||
1. Prepares the activity with schedule and model information
|
||||
2. Loads data using a custom query and executes the prediction process
|
||||
This method orchestrates the complete batch prediction process by:
|
||||
1. Loading data using the provided custom SQL query
|
||||
2. Preparing prediction configuration and filters
|
||||
3. Delegating to the PredictionProcess workflow for ML operations
|
||||
|
||||
The method implements comprehensive error handling and ensures all
|
||||
required parameters are properly configured before proceeding.
|
||||
|
||||
Args:
|
||||
- input_data (dict[str, Any]): The input data for the workflow.
|
||||
Contains the following keys:
|
||||
- schedule_name (str): The name of the schedule.
|
||||
- model_name (str): The name of the model.
|
||||
- model_id (int): The id of the model.
|
||||
- query (str): The SQL query to be executed to load data.
|
||||
- schema (dict, optional): The schema definition for the data.
|
||||
- table_name (str, optional): The name of the table to process.
|
||||
- input_filters (dict, optional): Filters to be applied during prediction.
|
||||
- mlflow_transform_filters (dict, optional): Filters to be applied
|
||||
during prediction.
|
||||
- mlflow_predict_filters (dict, optional): Filters to be applied during prediction.
|
||||
- model_retention (int, optional): The model retention period in minutes.
|
||||
- path_priority (list[str]): The path priority.
|
||||
input_data: Complete configuration for the batch prediction
|
||||
Required keys:
|
||||
- schedule_name (str): Schedule identifier for the prediction
|
||||
- model_name (str): Name of the ML model to use
|
||||
- model_id (int): Unique identifier for the model
|
||||
- query (str): SQL query for data loading
|
||||
- schema (dict, optional): Data schema definition
|
||||
- table_name (str, optional): Target table for predictions
|
||||
- input_filters (dict, optional): Data quality filters
|
||||
- mlflow_transform_filters (dict, optional): MLFlow transform filters
|
||||
- mlflow_predict_filters (dict, optional): MLFlow prediction filters
|
||||
- model_retention (int, optional): Model retention period in minutes
|
||||
- path_priority (list[str]): Decision path priority configuration
|
||||
- opc_output_config (dict, optional): OPC server export configuration
|
||||
- datetime_columns (list[str], optional): Columns to treat as datetime
|
||||
|
||||
Returns:
|
||||
None
|
||||
None: The workflow completes successfully when the child workflow finishes
|
||||
|
||||
Raises:
|
||||
Exception: If any of the required parameters are missing or if the workflow fails.
|
||||
Exception: If any required parameters are missing or if the workflow fails
|
||||
during data loading or workflow delegation
|
||||
"""
|
||||
|
||||
metadata = {
|
||||
@@ -49,6 +94,7 @@ class PredictionsBatch():
|
||||
}
|
||||
}
|
||||
|
||||
# Load data using custom query
|
||||
data = await workflow.execute_local_activity_method(
|
||||
Activities.load_custom_query,
|
||||
{
|
||||
@@ -88,5 +134,6 @@ class PredictionsBatch():
|
||||
'opc_output_config': input_data.get('opc_output_config', {})
|
||||
}
|
||||
|
||||
# Execute prediction process workflow
|
||||
await workflow.execute_child_workflow(
|
||||
'prediction_process', prediction_input)
|
||||
|
||||
@@ -10,32 +10,59 @@ with workflow.unsafe.imports_passed_through():
|
||||
|
||||
@workflow.defn(name="format_and_export_prediction")
|
||||
class FormatAndExportPrediction():
|
||||
"""
|
||||
Data formatting and export workflow for prediction results.
|
||||
|
||||
This workflow handles the final stages of the prediction pipeline, including
|
||||
data formatting, database persistence, OPC server export, and metrics recording.
|
||||
It implements flexible formatting based on prediction quality and provides
|
||||
comprehensive export capabilities to multiple destinations.
|
||||
|
||||
The workflow supports two main prediction paths:
|
||||
1. Normal Prediction: Formats and exports successful prediction results
|
||||
2. Default Prediction: Creates fallback predictions for error conditions
|
||||
|
||||
Export Destinations:
|
||||
- PostgreSQL Database: Persistent storage with timestamp conversion
|
||||
- OPC Servers: Real-time industrial system integration
|
||||
- Prometheus Metrics: Performance monitoring and operational visibility
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
This workflow formats and exports predictions based on path_flag:
|
||||
- If path_flag is None: formats prediction
|
||||
using input data, timestamp, model_id and confidence
|
||||
- If path_flag exists: creates default prediction
|
||||
with timestamp, model_id, confidence and comment
|
||||
Finally exports formatted prediction to postgres table
|
||||
Execute the prediction formatting and export workflow.
|
||||
|
||||
This method orchestrates the complete data export process by:
|
||||
1. Determining the appropriate formatting strategy based on path_flag
|
||||
2. Formatting prediction data according to quality and requirements
|
||||
3. Exporting data to OPC servers for real-time industrial access
|
||||
4. Persisting data to PostgreSQL database with comprehensive metadata
|
||||
5. Recording performance metrics for operational monitoring
|
||||
|
||||
The method implements flexible formatting strategies:
|
||||
- Normal predictions: Full data formatting with confidence scores
|
||||
- Error predictions: Default formatting with error indicators
|
||||
- Comprehensive export: Multi-destination data distribution
|
||||
|
||||
Args:
|
||||
input_data(dict[str, Any]): The input data for the workflow.
|
||||
Contains the following keys:
|
||||
- path_flag(str): The path flag to determine the type of prediction to format
|
||||
- data(dict[str, Any]): The data to format
|
||||
- prediction_confidence(float): The prediction confidence to be registered
|
||||
- timestamp(str): The timestamp of the prediction, synchronized with the data
|
||||
- model_id(int): The model id of the prediction
|
||||
- model_name(str): The model name of the prediction
|
||||
- model_retention(str): The model retention of the prediction
|
||||
- comment(str): The comment to be registered
|
||||
- schema(str): The schema of the prediction
|
||||
- table_name(str): The table name of the prediction
|
||||
- opc_output_config(dict[str, Any]): The opc output config of the prediction
|
||||
input_data: Complete configuration for the export workflow
|
||||
Required keys:
|
||||
- path_flag (str | None): Decision path flag for formatting strategy
|
||||
- data (dict[str, Any]): Prediction data to format and export
|
||||
- prediction_confidence (float): Confidence score for the prediction
|
||||
- timestamp (str): ISO-formatted timestamp for the prediction
|
||||
- model_id (int): Unique identifier for the ML model
|
||||
- model_name (str): Name of the ML model
|
||||
- model_retention (str): Model retention policy configuration
|
||||
- comment (str): Operational comment or error description
|
||||
- schema (str): Database schema for data storage
|
||||
- table_name (str): Target table for data persistence
|
||||
- opc_output_config (dict[str, Any]): OPC server export configuration
|
||||
- prediction_store_policy (str, optional): Data retention policy
|
||||
|
||||
Returns:
|
||||
bool: True if the workflow was successful, False otherwise.
|
||||
bool: True if the workflow completes successfully, False otherwise
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
path_flag = input_data['path_flag']
|
||||
|
||||
@@ -9,36 +9,70 @@ with workflow.unsafe.imports_passed_through():
|
||||
|
||||
@workflow.defn(name="prediction_process")
|
||||
class PredictionProcess():
|
||||
"""
|
||||
Core prediction processing workflow for the Laborious system.
|
||||
|
||||
This workflow implements the complete ML model inference pipeline, handling
|
||||
data quality validation, MLFlow model interactions, and prediction processing.
|
||||
It serves as the central orchestrator for all prediction operations and ensures
|
||||
data quality throughout the entire process.
|
||||
|
||||
The workflow implements a robust data processing pipeline with:
|
||||
- Data quality validation using configurable filters
|
||||
- MLFlow model transformation and prediction
|
||||
- Response validation and quality assurance
|
||||
- Flexible decision path handling
|
||||
- Comprehensive error handling and retry policies
|
||||
|
||||
Workflow Execution:
|
||||
1. Timestamp Retrieval: Gets last processed timestamp for incremental processing
|
||||
2. Input Data Gate: Applies data quality filters
|
||||
3. Path Decision: Determines processing path based on filter results
|
||||
4. MLFlow Transform: Requests data transformation using MLFlow models
|
||||
5. Response Validation: Filters transform responses for quality assurance
|
||||
6. MLFlow Prediction: Executes prediction using transformed data
|
||||
7. Content Validation: Filters prediction responses for final quality check
|
||||
8. Export Delegation: Delegates to FormatAndExportPrediction workflow
|
||||
"""
|
||||
|
||||
@workflow.run
|
||||
async def run(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
This workflow runs a prediction process based on the input data.
|
||||
Execute the prediction process workflow.
|
||||
|
||||
The workflow executes in two main steps:
|
||||
1. Prepares the activity with schedule and model information
|
||||
2. Loads data using a custom query and executes the prediction process
|
||||
This method orchestrates the complete prediction processing pipeline by:
|
||||
1. Retrieving the last processed timestamp for incremental processing
|
||||
2. Applying data quality filters to validate input data
|
||||
3. Executing MLFlow model transformation and prediction
|
||||
4. Validating all responses for quality assurance
|
||||
5. Delegating to export workflow for data persistence
|
||||
|
||||
The method implements comprehensive error handling and ensures all
|
||||
data quality requirements are met before proceeding with ML operations.
|
||||
|
||||
Args:
|
||||
- input_data (dict[str, Any]): The input data for the workflow.
|
||||
Contains the following keys:
|
||||
- data (dict[str, Any]): The data to be used for the prediction.
|
||||
- schema (str): The schema of the table.
|
||||
- table_name (str): The name of the table.
|
||||
- model_id (int): The id of the model.
|
||||
- input_filters (dict, optional): Filters to be applied during prediction.
|
||||
- mlflow_transform_filters (dict, optional): Filters to be
|
||||
applied during prediction.
|
||||
- mlflow_predict_filters (dict, optional): Filters to be
|
||||
applied during prediction.
|
||||
- model_name (str): The name of the model.
|
||||
- model_retention (int, optional): The model retention period in minutes.
|
||||
- path_priority (list[str]): The path priority.
|
||||
- opc_output_config (dict[str, Any]): The opc output config of the prediction.
|
||||
input_data: Complete configuration for the prediction process
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict): Input data for prediction processing
|
||||
- schema (dict): Data schema definition
|
||||
- table_name (str): Target table for predictions
|
||||
- model_id (str): ML model identifier
|
||||
- model_name (str): ML model name
|
||||
- input_filters (dict): Data quality filters
|
||||
- mlflow_transform_filters (dict): MLFlow transform filters
|
||||
- mlflow_predict_filters (dict): MLFlow prediction filters
|
||||
- model_retention (int): Model retention period in minutes
|
||||
- path_priority (list[str]): Decision path priority configuration
|
||||
- opc_output_config (dict): OPC server export configuration
|
||||
|
||||
Returns:
|
||||
None
|
||||
None: The workflow completes successfully when export workflow finishes
|
||||
|
||||
Raises:
|
||||
Exception: If any of the required parameters are missing or if the workflow fails.
|
||||
Exception: If any required parameters are missing or if the workflow fails
|
||||
during data processing, MLFlow operations, or workflow delegation
|
||||
|
||||
"""
|
||||
|
||||
metadata = input_data['metadata']
|
||||
@@ -47,6 +81,7 @@ class PredictionProcess():
|
||||
model_name = input_data['model_name']
|
||||
model_retention = input_data['model_retention']
|
||||
|
||||
# Get last timestamp for incremental processing
|
||||
last_timestamp = await workflow.execute_local_activity_method(
|
||||
Activities.get_last_timestamp,
|
||||
{
|
||||
@@ -57,6 +92,7 @@ class PredictionProcess():
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Apply input data quality gates
|
||||
gate_input = {
|
||||
**metadata,
|
||||
'filters': input_data['input_filters'],
|
||||
@@ -71,11 +107,13 @@ class PredictionProcess():
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Handle path decision based on filter results
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
# Request MLFlow model transformation
|
||||
response_data = await workflow.execute_local_activity_method(
|
||||
Activities.request_transform,
|
||||
{
|
||||
@@ -88,6 +126,7 @@ class PredictionProcess():
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Validate MLFlow transform response
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_response_gate,
|
||||
{
|
||||
@@ -101,6 +140,7 @@ class PredictionProcess():
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Handle path decision based on transform validation
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
@@ -138,6 +178,7 @@ class PredictionProcess():
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Validate MLFlow prediction response
|
||||
path_flag, confidence, comment = await workflow.execute_local_activity_method(
|
||||
Activities.mlflow_response_gate,
|
||||
{
|
||||
@@ -151,11 +192,13 @@ class PredictionProcess():
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
|
||||
# Handle path decision based on prediction validation
|
||||
if await self.path_flag_handler(
|
||||
data, path_flag, input_data, confidence, last_timestamp, comment
|
||||
):
|
||||
return
|
||||
|
||||
# Delegate to export workflow for data persistence
|
||||
await workflow.execute_child_workflow(
|
||||
'format_and_export_prediction',
|
||||
{
|
||||
@@ -174,30 +217,31 @@ class PredictionProcess():
|
||||
}
|
||||
)
|
||||
|
||||
async def path_flag_handler(self, data: dict[str, Any], path_flag: str,
|
||||
input_data: dict[str, Any], confidence: int,
|
||||
last_timestamp: str, comment: str):
|
||||
"""
|
||||
This function handles the path flag and the confidence of the prediction.
|
||||
It returns True if the prediction should be stopped. If path_flag is 'repeat',
|
||||
it repeats the last prediction.
|
||||
If path_flag is 'continue', it calls the write workflow. If path_flag is 'stop',
|
||||
it stops the prediction process.
|
||||
Args:
|
||||
data (dict[str, Any]): The data to be used for the prediction.
|
||||
path_flag (str): The path flag to determine the type of prediction to format
|
||||
confidence (int): The confidence of the prediction
|
||||
schema (str): The schema of the prediction
|
||||
table_name (str): The table name of the prediction
|
||||
model_id (int): The model id of the prediction
|
||||
last_timestamp (str): The timestamp of the last prediction
|
||||
model_name (str): The model name of the prediction
|
||||
model_retention (int): The model retention of the prediction
|
||||
comment (str): The comment of the prediction
|
||||
Returns:
|
||||
bool: True if the prediction should be stopped, False otherwise.
|
||||
async def path_flag_handler(self, data: dict, path_flag: str, input_data: dict,
|
||||
confidence: int, last_timestamp: str, comment: str) -> bool:
|
||||
"""
|
||||
Handle path decisions based on filter results and confidence levels.
|
||||
|
||||
This method determines the appropriate action based on the path flag
|
||||
returned by data quality filters. It can stop processing, continue,
|
||||
or repeat operations based on the configured path priority.
|
||||
|
||||
Args:
|
||||
data: Input data for processing
|
||||
path_flag: Path decision from filter (STOP, CONTINUE, REPEAT)
|
||||
input_data: Complete workflow input configuration
|
||||
confidence: Confidence level from filter validation
|
||||
last_timestamp: Last processed timestamp
|
||||
comment: Additional information about the filter result
|
||||
|
||||
Returns:
|
||||
bool: True if processing should stop, False to continue
|
||||
|
||||
Path Handling:
|
||||
- STOP: Terminates workflow execution
|
||||
- CONTINUE: Proceeds with normal processing
|
||||
- REPEAT: Repeats last prediction if available
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
schema = input_data['schema']
|
||||
@@ -209,10 +253,10 @@ class PredictionProcess():
|
||||
path_flag = path_flag.upper() if path_flag else ''
|
||||
|
||||
if path_flag == 'STOP':
|
||||
# Stop processing and exit workflow
|
||||
return True
|
||||
|
||||
elif path_flag == 'REPEAT':
|
||||
# repeat last prediction
|
||||
# Repeat last prediction if available
|
||||
await workflow.execute_activity_method(
|
||||
Activities.repeat_last_prediction,
|
||||
{
|
||||
@@ -226,7 +270,6 @@ class PredictionProcess():
|
||||
start_to_close_timeout=timedelta(minutes=1),
|
||||
)
|
||||
return True
|
||||
|
||||
elif path_flag == 'CONTINUE':
|
||||
# call write workflow
|
||||
await workflow.execute_child_workflow(
|
||||
|
||||
Reference in New Issue
Block a user