SIENTIAPDE-1255: Refactor data quality gates to training focused metrics and repositories. This commit removes the data quality gates and filters, focusing on training-specific metrics and data repositories. It also updates the README to reflect these changes, including new training metrics and a streamlined data services section.

model_manager/utils/filters/conditional_filters.py

@@ -1,44 +0,0 @@
-from pandas import DataFrame
-
-
-def filter_specific_variables_null_values(data: DataFrame, config: dict) -> bool:
-    """
-    Filter to check if specific variables contain null values.
-
-    This function examines a DataFrame to determine if any of the specified variables
-    contain null (NaN) values. It returns True if null values are found for any of
-    the specified variables, False otherwise.
-
-    Args:
-        data (DataFrame): The pandas DataFrame to be examined. Must contain columns
-            named 'variable' and 'value'.
-        config (dict): Configuration dictionary containing the following key:
-            - variables (list): List of variable names to check for null values
-
-    Returns:
-        bool: True if any of the specified variables contain null values,
-              False if none of the specified variables contain null values.
-
-    """
-    return not data[data['variable'].isin(config['variables']) & data['value'].isna()].empty
-
-
-def filter_empty_data(data: DataFrame, _config: dict) -> bool:
-    """
-    Filter to check if the DataFrame is empty.
-
-    This function determines whether the provided DataFrame contains any data.
-    It's a simple utility function that can be used in conditional logic to
-    handle cases where no data is available.
-
-    Args:
-        data (DataFrame): The pandas DataFrame to be checked for emptiness.
-        _config (dict): Configuration dictionary (unused in this function).
-            The underscore prefix indicates this parameter is required for
-            interface consistency but not used in the implementation.
-
-    Returns:
-        bool: True if the DataFrame is empty (has no rows), False if it contains data.
-
-    """
-    return data.empty

model_manager/utils/filters/mlflow_filters.py

@@ -1,64 +0,0 @@
-import numpy as np
-from pandas import DataFrame
-
-
-def api_error_filter(response: dict, _config: dict) -> bool:
-    """
-    Filter MLFlow API responses for error conditions.
-
-    This function analyzes MLFlow API responses to detect error conditions
-    and determine if the response should be filtered out due to quality
-    or reliability issues.
-
-
-    Args:
-        response: MLFlow API response data (dict)
-        _config: Filter configuration dictionary
-            Required keys:
-                - error_codes (list, optional): List of error codes to detect
-                - error_keywords (list, optional): List of error keywords to detect
-                - check_structure (bool, optional): Whether to validate response structure
-
-    Returns:
-        bool: True if data should be filtered (contains errors), False otherwise
-
-    """
-    if not response:
-        return True
-
-    if not response['success']:
-        return True
-
-    return False
-
-
-def nan_values_filter(predictions: DataFrame, _config: dict) -> bool:
-    """
-    Filter data for NaN (Not a Number) values.
-
-    This function detects NaN values in MLFlow prediction results and
-    determines if the data quality is sufficient for further processing
-    or export operations.
-
-    Args:
-        predictions: DataFrame containing prediction data to check for NaN values
-        _config: Filter configuration dictionary
-            Required keys:
-                - max_nan_ratio (float, optional): Maximum allowed NaN value ratio (0.0 to 1.0)
-                - max_nan_count (int, optional): Maximum allowed NaN value count
-                - check_nested (bool, optional): Whether to check nested data structures
-
-    Returns:
-        bool: True if data should be filtered (too many NaN values), False otherwise
-
-    """
-    data = (
-        predictions.replace({None: np.nan})
-        .infer_objects(copy=False)
-        .drop(columns=['timestamp'], errors='ignore')
-    )
-
-    if data.isna().all().all():
-        return True
-
-    return False