SIENTIAPDE-1273
Enhance data handling and export processes in Laborious workflows - Updated `gates.py` to improve data quality validation, filtering, and formatting operations, including enhanced metrics recording. - Refined `mlflow.py` to better manage model transformations and reference data retrieval from MLflow Model Registry. - Enhanced `format_and_export_prediction.py` to support separate export of transformed data, improving flexibility in data handling. - Added comprehensive test coverage for new functionalities, including transformed data formatting and retrain report generation. - Improved documentation in `README.md` to reflect changes in activities and workflows, ensuring clarity on data processing and export paths.
This commit is contained in:
vitor-aignosi
@@ -405,7 +405,32 @@ class Gates(SientiaMonitoring):
|
||||
@activity.defn(name='format_transformed_data')
|
||||
async def format_transformed_data(self, input_data: dict[str, Any]) -> dict:
|
||||
"""
|
||||
Format transformed data according to configured storage policies.
|
||||
Format transformed data for storage and export operations.
|
||||
|
||||
This method formats transformed data from MLFlow model transformations
|
||||
into a standardized format suitable for database storage. It converts
|
||||
wide-format data (columns as variables) into long-format (melted)
|
||||
with proper timestamp handling and model identification.
|
||||
|
||||
The formatting process includes:
|
||||
1. Converting input data dictionary to DataFrame
|
||||
2. Extracting timestamps from DataFrame index
|
||||
3. Resetting index to create sequential row numbers
|
||||
4. Melting data from wide format to long format (variable-value pairs)
|
||||
5. Adding model_id for data lineage tracking
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- data (dict[str, Any]): Transformed data to format (DataFrame-compatible dict)
|
||||
- model_id (str): Unique identifier for the ML model
|
||||
|
||||
Returns:
|
||||
dict: Formatted data dictionary with keys:
|
||||
- timestamp (dict): Timestamp values indexed by row number
|
||||
- variable (dict): Variable names indexed by row number
|
||||
- value (dict): Variable values indexed by row number
|
||||
- model_id (dict): Model identifiers indexed by row number
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
@@ -544,7 +569,45 @@ class Gates(SientiaMonitoring):
|
||||
@activity.defn(name='format_retrain_report')
|
||||
async def format_retrain_report(self, input_data: dict[str, Any]) -> dict:
|
||||
"""
|
||||
Format retrain report data according to configured storage policies.
|
||||
Format retrain report data for storage and audit trail maintenance.
|
||||
|
||||
This method formats model retraining operation results into a standardized
|
||||
report format suitable for database storage and operational monitoring.
|
||||
It captures retraining status, timestamps, and model version information
|
||||
for comprehensive audit trails and operational visibility.
|
||||
|
||||
The formatting process includes:
|
||||
1. Extracting retraining experiment response data
|
||||
2. Capturing model update report information (version, MLflow IDs)
|
||||
3. Formatting timestamps and status information
|
||||
4. Conditionally including version information for successful retrains
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- experiment_response (dict): Retraining experiment response containing:
|
||||
- success (bool): Retraining operation success status
|
||||
- timestamp (str): Timestamp of the retraining operation
|
||||
- message (str): Status message or error description
|
||||
- update_report (dict): Model update report containing:
|
||||
- version (str): New model version identifier
|
||||
- mlflow_run_id (str): MLflow run identifier
|
||||
- mlflow_experiment_id (str): MLflow experiment identifier
|
||||
- model_id (str): Unique identifier for the ML model
|
||||
- model_name (str): Name of the ML model
|
||||
|
||||
Returns:
|
||||
dict: Formatted retrain report dictionary with keys:
|
||||
- model_id (dict): Model identifiers indexed by row number
|
||||
- model_name (dict): Model names indexed by row number
|
||||
- timestamp (dict): Retraining timestamps indexed by row number
|
||||
- status (dict): Retraining status messages indexed by row number
|
||||
- version (dict, optional): Model versions indexed by row number
|
||||
Only included if experiment_response['success'] is True
|
||||
- mlflow_run_id (dict, optional): MLflow run IDs indexed by row number
|
||||
Only included if experiment_response['success'] is True
|
||||
- mlflow_experiment_id (dict, optional): MLflow experiment IDs indexed by row number
|
||||
Only included if experiment_response['success'] is True
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info('Formatting retrain report...', metadata)
|
||||
|
||||
@@ -424,14 +424,30 @@ class MLFlow(SientiaMonitoring):
|
||||
"""
|
||||
Get reference data from the MLflow Model Registry.
|
||||
|
||||
This method retrieves evaluation reference data stored as artifacts in the
|
||||
MLflow Model Registry. The reference data is typically used for model
|
||||
drift detection, performance comparison, and quality validation. The method
|
||||
loads the data from a CSV artifact file and formats timestamps for
|
||||
consistent processing.
|
||||
|
||||
The method handles:
|
||||
1. Loading evaluation data artifact from MLflow Model Registry
|
||||
2. Timestamp parsing and formatting for consistency
|
||||
3. Data conversion to dictionary format for workflow consumption
|
||||
4. Graceful handling of missing reference data
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- model_name (str): Name of the MLFlow model to get reference data from
|
||||
|
||||
Returns:
|
||||
list[dict[Hashable, Any]] | None: Reference data from the MLflow Model Registry.
|
||||
list[dict[Hashable, Any]] | None: Reference data from the MLflow Model Registry
|
||||
as a list of dictionaries. Returns None if reference data is not found
|
||||
or if the artifact does not exist.
|
||||
|
||||
Raises:
|
||||
Exception: If artifact loading fails or encounters errors during processing
|
||||
"""
|
||||
|
||||
metadata = input_data['metadata']
|
||||
|
||||
@@ -50,21 +50,36 @@ class FormatAndExportPrediction:
|
||||
Args:
|
||||
input_data: Complete configuration for the export workflow
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- path_flag (str | None): Decision path flag for formatting strategy
|
||||
- None: Normal prediction path with full formatting
|
||||
- Any other value: Default prediction path for error conditions
|
||||
- data (dict[str, Any]): Prediction data to format and export
|
||||
- prediction_confidence (float): Confidence score for the prediction
|
||||
- timestamp (str): ISO-formatted timestamp for the prediction
|
||||
- model_id (int): Unique identifier for the ML model
|
||||
- model_name (str): Name of the ML model
|
||||
- model_retention (str): Model retention policy configuration
|
||||
- comment (str): Operational comment or error description
|
||||
- schema (str): Database schema for data storage
|
||||
- table_name (str): Target table for data persistence
|
||||
- opc_output_config (dict[str, Any]): OPC server export configuration
|
||||
- prediction_store_policy (str, optional): Data retention policy
|
||||
Optional keys:
|
||||
- transformed_data (dict[str, Any]): Transformed data to export separately
|
||||
Only processed when path_flag is None
|
||||
- transform_table_name (str): Target table for transformed data export
|
||||
Required if transformed_data is provided
|
||||
- prediction_store_policy (str): Data retention policy (e.g., 'lts:1', 'erl:2')
|
||||
Required when path_flag is None
|
||||
- comment (str): Operational comment or error description
|
||||
Required when path_flag is not None
|
||||
|
||||
Returns:
|
||||
bool: True if the workflow completes successfully, False otherwise
|
||||
None: The workflow completes successfully when all export operations finish
|
||||
|
||||
Note:
|
||||
When transformed_data is provided and path_flag is None, the workflow will:
|
||||
1. Format the transformed data using format_transformed_data
|
||||
2. Export it to a separate table (transform_table_name) asynchronously
|
||||
3. Wait for both prediction and transformed data exports to complete
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
path_flag = input_data['path_flag']
|
||||
@@ -73,7 +88,7 @@ class FormatAndExportPrediction:
|
||||
prediction_confidence = input_data['prediction_confidence']
|
||||
|
||||
if path_flag is None:
|
||||
# proceed with formatting and exporting
|
||||
# Normal prediction path: format prediction data with full metadata
|
||||
prediction = await workflow.execute_local_activity_method(
|
||||
Activities.format_prediction,
|
||||
{
|
||||
@@ -88,6 +103,7 @@ class FormatAndExportPrediction:
|
||||
start_to_close_timeout=timedelta(seconds=60),
|
||||
)
|
||||
|
||||
# Optionally format and export transformed data to separate table
|
||||
if transformed_data is not None:
|
||||
transformed = await workflow.execute_local_activity_method(
|
||||
Activities.format_transformed_data,
|
||||
@@ -120,7 +136,7 @@ class FormatAndExportPrediction:
|
||||
write_transformed_handler = None
|
||||
|
||||
else:
|
||||
# create default prediction
|
||||
# Error path: create default prediction with error indicators
|
||||
prediction = await workflow.execute_local_activity_method(
|
||||
Activities.format_default_prediction,
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user