SIENTIAPDE-1182
Remove Docker configuration files and refactor project structure - Deleted docker-compose.yml and Dockerfile as part of the project restructuring. - Updated README.md to reflect changes in project setup and configuration. - Introduced a new __init__.py file in the laborious package to provide an overview of the system. - Enhanced documentation across various modules, including metrics, activities, and workflows, to improve clarity and usability. - Added comprehensive docstrings and comments to key classes and methods for better maintainability.
This commit is contained in:
vitor-aignosi
@@ -17,6 +17,7 @@ with workflow.unsafe.imports_passed_through():
|
||||
from pandas import DataFrame
|
||||
from laborious import metrics
|
||||
|
||||
# Input filter function mappings
|
||||
input_filter_functions = {
|
||||
'SPECIFIC_VARIABLES_NULL_VALUES': filter_specific_variables_null_values,
|
||||
'EMPTY_DATA': filter_empty_data,
|
||||
@@ -27,6 +28,7 @@ input_filter_functions = {
|
||||
}
|
||||
}
|
||||
|
||||
# MLFlow response filter function mappings
|
||||
mlflow_response_filter_functions = {
|
||||
'API_ERROR': api_error_filter,
|
||||
'path_confidence': {
|
||||
@@ -36,6 +38,7 @@ mlflow_response_filter_functions = {
|
||||
},
|
||||
}
|
||||
|
||||
# MLFlow content filter function mappings
|
||||
mlflow_content_filter_functions = {
|
||||
'NAN_VALUES': nan_values_filter,
|
||||
'path_confidence': {
|
||||
@@ -47,24 +50,72 @@ mlflow_content_filter_functions = {
|
||||
|
||||
|
||||
class Gates(BaseActivity):
|
||||
"""
|
||||
Data quality gates and filtering activities for the Laborious system.
|
||||
|
||||
This class implements comprehensive data quality validation and filtering
|
||||
mechanisms that can be applied at different stages of the prediction pipeline.
|
||||
It provides configurable filters with policy-based decision making to ensure
|
||||
data integrity and quality throughout the ML workflow.
|
||||
|
||||
The class supports multiple filter types and implements a flexible policy
|
||||
system that can be configured for different validation requirements. Each
|
||||
filter returns a path decision (STOP, CONTINUE, REPEAT) along with confidence
|
||||
scores and detailed comments for monitoring and debugging.
|
||||
|
||||
Attributes:
|
||||
input_filter_functions (dict): Mapping of input filter names to functions
|
||||
mlflow_response_filter_functions (dict): Mapping of MLFlow response filter names to functions
|
||||
mlflow_content_filter_functions (dict): Mapping of MLFlow content filter names to functions
|
||||
"""
|
||||
|
||||
def __init__(self, logger: Logger, notification_handler: NotificationHandler):
|
||||
"""
|
||||
Initialize data quality gates with logging and notification capabilities.
|
||||
|
||||
Args:
|
||||
logger: Logger instance for observability and debugging
|
||||
notification_handler: Notification handler for alerts and monitoring
|
||||
|
||||
Raises:
|
||||
Exception: If BaseActivity initialization fails
|
||||
"""
|
||||
BaseActivity.__init__(
|
||||
self, logger, notification_handler, set_error_counter=True)
|
||||
|
||||
@activity.defn(name="input_gate")
|
||||
async def input_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
"""
|
||||
Filters the data based on the filters. The return value is a tuple with the first element
|
||||
being the policy and the second element being the confidence status.
|
||||
Apply input data quality filters and validation.
|
||||
|
||||
This activity validates input data quality using configurable filters
|
||||
before proceeding with ML operations. It applies multiple filter types
|
||||
and returns a path decision based on the filter results and configured
|
||||
policies.
|
||||
|
||||
The method implements a comprehensive filtering system that:
|
||||
1. Applies configured filters to input data
|
||||
2. Evaluates filter results against policy configurations
|
||||
3. Determines appropriate path decisions (STOP, CONTINUE, REPEAT)
|
||||
4. Provides confidence scores and detailed comments
|
||||
5. Handles errors gracefully with notification integration
|
||||
|
||||
Args:
|
||||
- input_data (dict): The input data. Contains:
|
||||
- filters (dict): The filters to apply.
|
||||
The key is the filter name and the value is the filter configuration.
|
||||
- data (dict[str, Any]): The data to filter.
|
||||
- path_priority (list[str]): The path priority.
|
||||
input_data: Configuration and data for input validation
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- filters (dict): Filter configuration and policies
|
||||
- data (dict): Input data to validate
|
||||
- path_priority (list[str]): Priority order for path decisions
|
||||
|
||||
Returns:
|
||||
tuple[str | None, int, str]: (policy, confidence, comments) based in priority
|
||||
list and filter configuration and functions.
|
||||
tuple: (path_flag, confidence, comment)
|
||||
- path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
|
||||
- confidence (int): Confidence score for the decision
|
||||
- comment (str): Detailed explanation of the decision
|
||||
|
||||
Raises:
|
||||
Exception: If filter execution fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
@@ -81,6 +132,7 @@ class Gates(BaseActivity):
|
||||
self.debug(f"Input data:\n {data}", metadata)
|
||||
self.debug(f"Filters: {filters}", metadata)
|
||||
|
||||
# Apply each configured filter
|
||||
for fil, config in filters.items():
|
||||
if fil not in input_filter_functions:
|
||||
self.error(f"Filter {fil} not found", metadata)
|
||||
@@ -113,20 +165,37 @@ class Gates(BaseActivity):
|
||||
@activity.defn(name="mlflow_response_gate")
|
||||
async def mlflow_response_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
"""
|
||||
Filters the data based on the mlflow response filters.
|
||||
The return value is a tuple with the first element
|
||||
being the policy and the second element being the confidence status.
|
||||
Args:
|
||||
- input_data (dict): The input data. Contains:
|
||||
- filters (dict): The filter configuration to apply.
|
||||
- data (dict[str, Any]): The data to filter.
|
||||
- path_priority (list[str]): The path priority list.
|
||||
- type (str): The type of the gate.
|
||||
Returns:
|
||||
tuple[str | None, int, str]: (policy, confidence, comments) based in priority list
|
||||
and filter configuration and functions.
|
||||
"""
|
||||
Validate MLFlow API response quality and integrity.
|
||||
|
||||
This activity validates MLFlow API responses to ensure they meet quality
|
||||
standards before proceeding with further processing. It applies response-specific
|
||||
filters and determines appropriate path decisions based on response quality.
|
||||
|
||||
The method implements response validation that:
|
||||
1. Applies MLFlow response-specific filters
|
||||
2. Evaluates API response quality and integrity
|
||||
3. Determines path decisions based on response validation results
|
||||
4. Provides confidence scores and detailed validation comments
|
||||
5. Handles API errors and response validation failures
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for response validation
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- filters (dict): Response filter configuration and policies
|
||||
- data (dict): MLFlow API response data to validate
|
||||
- type (str): Type of MLFlow operation (transform, predict)
|
||||
- path_priority (list[str]): Priority order for path decisions
|
||||
|
||||
Returns:
|
||||
tuple: (path_flag, confidence, comment)
|
||||
- path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
|
||||
- confidence (int): Confidence score for the decision
|
||||
- comment (str): Detailed explanation of the decision
|
||||
|
||||
Raises:
|
||||
Exception: If response validation fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info("Performing mlflow response gate...", metadata)
|
||||
|
||||
@@ -143,6 +212,7 @@ class Gates(BaseActivity):
|
||||
comments = []
|
||||
for fil, config in filters.items():
|
||||
if fil not in mlflow_response_filter_functions:
|
||||
self.error(f"Filter {fil} not found", metadata)
|
||||
continue
|
||||
try:
|
||||
if mlflow_response_filter_functions[fil](data, config):
|
||||
@@ -180,20 +250,37 @@ class Gates(BaseActivity):
|
||||
@activity.defn(name="mlflow_content_gate")
|
||||
async def mlflow_content_gate(self, input_data: dict[str, Any]) -> tuple[str | None, int, str]:
|
||||
"""
|
||||
Filters the data based on the mlflow content filters.
|
||||
The return value is a tuple with the first element
|
||||
being the policy and the second element being the confidence status.
|
||||
Args:
|
||||
- input_data (dict): The input data. Contains:
|
||||
- filters (dict): The filter configuration to apply.
|
||||
- data (dict[str, Any]): The data to filter.
|
||||
- path_priority (list[str]): The path priority list.
|
||||
- type (str): The type of the gate.
|
||||
Returns:
|
||||
tuple[str | None, int, str]: (policy, confidence, comments) based in priority
|
||||
list and filter configuration and functions.
|
||||
"""
|
||||
Validate MLFlow prediction content quality and integrity.
|
||||
|
||||
This activity validates the content of MLFlow predictions to ensure they
|
||||
meet quality standards before export and persistence. It applies content-specific
|
||||
filters and determines appropriate path decisions based on content quality.
|
||||
|
||||
The method implements content validation that:
|
||||
1. Applies MLFlow content-specific filters
|
||||
2. Evaluates prediction content quality and integrity
|
||||
3. Determines path decisions based on content validation results
|
||||
4. Provides confidence scores and detailed validation comments
|
||||
5. Handles content validation failures and quality issues
|
||||
|
||||
Args:
|
||||
input_data: Configuration and data for content validation
|
||||
Required keys:
|
||||
- metadata (dict): Workflow execution metadata
|
||||
- filters (dict): Content filter configuration and policies
|
||||
- data (dict): MLFlow prediction content to validate
|
||||
- type (str): Type of MLFlow operation (transform, predict)
|
||||
- path_priority (list[str]): Priority order for path decisions
|
||||
|
||||
Returns:
|
||||
tuple: (path_flag, confidence, comment)
|
||||
- path_flag (str | None): Decision path (STOP, CONTINUE, REPEAT, or None)
|
||||
- confidence (int): Confidence score for the decision
|
||||
- comment (str): Detailed explanation of the decision
|
||||
|
||||
Raises:
|
||||
Exception: If content validation fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
self.info("Performing mlflow content gate...", metadata)
|
||||
|
||||
@@ -245,6 +332,27 @@ class Gates(BaseActivity):
|
||||
def get_prediction_store_policy(self,
|
||||
prediction_store_policy: str,
|
||||
metadata: dict[str, Any]) -> tuple[str, int]:
|
||||
"""
|
||||
Parse and validate prediction store policy configuration.
|
||||
|
||||
This method parses prediction store policy strings in the format 'type:value'
|
||||
and validates them against allowed policy types and values. It provides
|
||||
sensible defaults for invalid configurations and logs policy validation
|
||||
failures for operational monitoring.
|
||||
|
||||
Supported Policy Types:
|
||||
- 'lts': Latest timestamp - sorts data by timestamp descending
|
||||
- 'erl': Earliest timestamp - sorts data by timestamp ascending
|
||||
|
||||
Args:
|
||||
prediction_store_policy (str): Policy string in format 'type:value'
|
||||
metadata (dict[str, Any]): Context metadata for logging and notifications
|
||||
|
||||
Returns:
|
||||
tuple[str, int]: (policy_type, policy_value)
|
||||
- policy_type (str): Validated policy type ('lts' or 'erl')
|
||||
- policy_value (int): Number of rows to retain
|
||||
"""
|
||||
policy_elements = prediction_store_policy.split(':')
|
||||
|
||||
if len(policy_elements) < 2:
|
||||
@@ -267,16 +375,27 @@ class Gates(BaseActivity):
|
||||
@activity.defn(name="format_prediction")
|
||||
async def format_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
"""
|
||||
Formats the prediction data.
|
||||
Format prediction data according to configured storage policies.
|
||||
|
||||
This method formats prediction data for storage and export operations.
|
||||
It applies timestamp-based sorting policies, adds metadata fields,
|
||||
and ensures data consistency before persistence. The method supports
|
||||
multiple storage policies for flexible data retention strategies.
|
||||
|
||||
Storage Policies:
|
||||
- 'lts:N': Latest timestamp - retains N most recent predictions
|
||||
- 'erl:N': Earliest timestamp - retains N oldest predictions
|
||||
|
||||
Args:
|
||||
- input_data (dict): The input data. Contains:
|
||||
- data (dict[str, Any]): The data to format.
|
||||
- timestamp (str): The timestamp of the data.
|
||||
- model_id (str): The id of the model.
|
||||
- prediction_confidence (float): The confidence of the prediction.
|
||||
- prediction_store_policy (str): The policy to store the prediction.
|
||||
input_data (dict): Input data containing:
|
||||
- data (dict[str, Any]): Raw prediction data to format
|
||||
- timestamp (str): Default timestamp if data lacks timestamp column
|
||||
- model_id (str): Unique identifier for the ML model
|
||||
- prediction_confidence (float): Confidence score for the prediction
|
||||
- prediction_store_policy (str): Storage policy in format 'type:value'
|
||||
|
||||
Returns:
|
||||
dict: The formatted data.
|
||||
dict: Formatted prediction data ready for storage and export
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
prediction_store_policy = input_data['prediction_store_policy']
|
||||
@@ -332,17 +451,28 @@ class Gates(BaseActivity):
|
||||
@activity.defn(name="format_default_prediction")
|
||||
async def format_default_prediction(self, input_data: dict[str, Any]) -> dict[Any, Any]:
|
||||
"""
|
||||
Creates and formats the default prediction data, with zero value in prediction,
|
||||
and usefull information in the other fields.
|
||||
Create and format default prediction data for error conditions.
|
||||
|
||||
This method generates default prediction data when the main prediction
|
||||
pipeline encounters errors or quality issues. It creates a standardized
|
||||
data structure with zero values for predictions and useful metadata
|
||||
for operational monitoring and debugging.
|
||||
|
||||
The default prediction serves as a fallback mechanism to:
|
||||
1. Maintain data pipeline continuity during failures
|
||||
2. Provide operational visibility into prediction quality issues
|
||||
3. Enable downstream systems to handle error conditions gracefully
|
||||
4. Support debugging and troubleshooting efforts
|
||||
|
||||
Args:
|
||||
- input_data (dict): The input data. Contains:
|
||||
- timestamp (str): The timestamp of the data.
|
||||
- model_id (str): The id of the model.
|
||||
- prediction_confidence (float): The confidence of the prediction.
|
||||
- comment (str): The comment of the prediction.
|
||||
input_data (dict): Input data containing:
|
||||
- timestamp (str): Timestamp for the default prediction
|
||||
- model_id (str): Unique identifier for the ML model
|
||||
- prediction_confidence (float): Confidence score (typically low for errors)
|
||||
- comment (str): Error description or operational comment
|
||||
|
||||
Returns:
|
||||
dict: The formatted data.
|
||||
dict: Formatted default prediction data with error indicators
|
||||
"""
|
||||
|
||||
metadata = input_data['metadata']
|
||||
@@ -364,12 +494,25 @@ class Gates(BaseActivity):
|
||||
@activity.defn(name="get_last_timestamp")
|
||||
async def get_last_timestamp(self, input_data: dict[str, Any]) -> str:
|
||||
"""
|
||||
Gets the last timestamp of the data.
|
||||
Extract the most recent timestamp from prediction data.
|
||||
|
||||
This method analyzes prediction data to find the latest timestamp,
|
||||
enabling incremental processing and data continuity tracking.
|
||||
It handles empty datasets gracefully by returning the current time
|
||||
as a fallback timestamp.
|
||||
|
||||
The method is essential for:
|
||||
1. Incremental data processing workflows
|
||||
2. Data continuity validation
|
||||
3. Timestamp-based data loading optimization
|
||||
4. Workflow execution tracking
|
||||
|
||||
Args:
|
||||
- input_data (dict): The input data. Contains:
|
||||
- data (dict[str, Any]): The data to get the last timestamp from.
|
||||
input_data (dict): Input data containing:
|
||||
- data (dict[str, Any]): Prediction data to analyze
|
||||
|
||||
Returns:
|
||||
str: The last timestamp of the data.
|
||||
str: Formatted timestamp string in UTC with timezone
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
|
||||
@@ -393,10 +536,25 @@ class Gates(BaseActivity):
|
||||
@activity.defn(name="write_metrics")
|
||||
async def write_metrics(self, input_data: dict[str, Any]):
|
||||
"""
|
||||
Write metrics to the database.
|
||||
input_data:
|
||||
metadata: dict[str, Any]
|
||||
prediction: dict[str, Any]
|
||||
Write prediction performance metrics to Prometheus monitoring system.
|
||||
|
||||
This method records comprehensive metrics for prediction operations,
|
||||
enabling operational monitoring, performance analysis, and alerting.
|
||||
It tracks prediction counts, confidence levels, and response times
|
||||
for each model and pipeline combination.
|
||||
|
||||
Metrics Recorded:
|
||||
1. Prediction Count: Incremental counter for successful predictions
|
||||
2. Confidence Monitor: Current confidence level for predictions
|
||||
3. Response Time Monitor: Histogram of prediction response times
|
||||
|
||||
Args:
|
||||
input_data (dict): Input data containing:
|
||||
- metadata (dict[str, Any]): Workflow execution metadata
|
||||
- prediction (dict[str, Any]): Prediction data with metrics
|
||||
|
||||
Raises:
|
||||
Exception: If metrics writing fails or configuration is invalid
|
||||
"""
|
||||
metadata = input_data['metadata']
|
||||
prediction = DataFrame(input_data['prediction'])
|
||||
|
||||
Reference in New Issue
Block a user