SIENTIAPDE-1273

Enhance data handling and export processes in Laborious workflows

- Updated `gates.py` to improve data quality validation, filtering, and formatting operations, including enhanced metrics recording.
- Refined `mlflow.py` to better manage model transformations and reference data retrieval from MLflow Model Registry.
- Enhanced `format_and_export_prediction.py` to support separate export of transformed data, improving flexibility in data handling.
- Added comprehensive test coverage for new functionalities, including transformed data formatting and retrain report generation.
- Improved documentation in `README.md` to reflect changes in activities and workflows, ensuring clarity on data processing and export paths.

laborious/workflows/sub_workflows/format_and_export_prediction.py

@@ -50,21 +50,36 @@ class FormatAndExportPrediction:
         Args:
             input_data: Complete configuration for the export workflow
                 Required keys:
+                    - metadata (dict): Workflow execution metadata
                     - path_flag (str | None): Decision path flag for formatting strategy
+                        - None: Normal prediction path with full formatting
+                        - Any other value: Default prediction path for error conditions
                     - data (dict[str, Any]): Prediction data to format and export
                     - prediction_confidence (float): Confidence score for the prediction
                     - timestamp (str): ISO-formatted timestamp for the prediction
                     - model_id (int): Unique identifier for the ML model
                     - model_name (str): Name of the ML model
-                    - model_retention (str): Model retention policy configuration
-                    - comment (str): Operational comment or error description
                     - schema (str): Database schema for data storage
                     - table_name (str): Target table for data persistence
                     - opc_output_config (dict[str, Any]): OPC server export configuration
-                    - prediction_store_policy (str, optional): Data retention policy
+                Optional keys:
+                    - transformed_data (dict[str, Any]): Transformed data to export separately
+                        Only processed when path_flag is None
+                    - transform_table_name (str): Target table for transformed data export
+                        Required if transformed_data is provided
+                    - prediction_store_policy (str): Data retention policy (e.g., 'lts:1', 'erl:2')
+                        Required when path_flag is None
+                    - comment (str): Operational comment or error description
+                        Required when path_flag is not None
 
         Returns:
-            bool: True if the workflow completes successfully, False otherwise
+            None: The workflow completes successfully when all export operations finish
+
+        Note:
+            When transformed_data is provided and path_flag is None, the workflow will:
+            1. Format the transformed data using format_transformed_data
+            2. Export it to a separate table (transform_table_name) asynchronously
+            3. Wait for both prediction and transformed data exports to complete
         """
         metadata = input_data['metadata']
         path_flag = input_data['path_flag']
@@ -73,7 +88,7 @@ class FormatAndExportPrediction:
         prediction_confidence = input_data['prediction_confidence']
 
         if path_flag is None:
-            # proceed with formatting and exporting
+            # Normal prediction path: format prediction data with full metadata
             prediction = await workflow.execute_local_activity_method(
                 Activities.format_prediction,
                 {
@@ -88,6 +103,7 @@ class FormatAndExportPrediction:
                 start_to_close_timeout=timedelta(seconds=60),
             )
 
+            # Optionally format and export transformed data to separate table
             if transformed_data is not None:
                 transformed = await workflow.execute_local_activity_method(
                     Activities.format_transformed_data,
@@ -120,7 +136,7 @@ class FormatAndExportPrediction:
                 write_transformed_handler = None
 
         else:
-            # create default prediction
+            # Error path: create default prediction with error indicators
             prediction = await workflow.execute_local_activity_method(
                 Activities.format_default_prediction,
                 {