Code import - branch feature/SIENTIAPDE-1646

scouter/utils/quality/filters.py

@@ -0,0 +1,81 @@
+from typing import Any
+
+import numpy as np
+from pandas import DataFrame
+
+
+def check_data_range(value: float | int | None, val_range: list) -> bool:
+    """
+    Check if a value falls outside the specified range.
+
+    This function validates if a numeric value is within the acceptable range
+    defined by the minimum and maximum bounds. It handles edge cases including
+    None values and NaN values.
+
+    Args:
+        value (float | int | None): The numeric value to validate
+        val_range (list): List containing [min_value, max_value] bounds
+
+    Returns:
+        bool: True if value is outside the range, False if within range
+
+    Note:
+        None and NaN values are considered out of range (return True)
+    """
+    if value is None or np.isnan(value):
+        return True
+
+    bottom = val_range[0]
+    up = val_range[-1]
+
+    return value < bottom or value > up
+
+
+def out_of_bounds_filter(df: DataFrame, model_tags: dict[str, Any]) -> DataFrame:
+    """
+    Filter DataFrame rows where values are outside configured ranges.
+
+    This function applies range validation to each row in the DataFrame based
+    on tag-specific configuration. Rows with values outside the configured
+    ranges are filtered out.
+
+    Args:
+        df (DataFrame): DataFrame containing sensor data with 'name' and 'value' columns
+        model_tags (dict[str, Any]): Tag configuration containing data_range for each tag.
+            If a tag doesn't have data_range, it's considered to have infinite bounds.
+
+    Returns:
+        DataFrame: Filtered DataFrame with out-of-bounds values removed
+
+    Note:
+        Tags without data_range configuration are treated as having infinite bounds
+    """
+    return df[
+        df.apply(
+            lambda x: check_data_range(
+                x['value'], model_tags[x['name']].get('data_range', (-np.inf, np.inf))
+            ),
+            axis=1,
+        )
+    ]
+
+
+def null_values_filter(df: DataFrame, _model_tags: dict[str, Any]) -> DataFrame:
+    """
+    Filter DataFrame rows containing null values.
+
+    This function removes rows where the 'value' column contains null values.
+    It's used for data quality filtering to ensure only complete data records
+    are processed.
+
+    Args:
+        df (DataFrame): DataFrame containing sensor data with 'value' column
+        _model_tags (dict[str, Any]): Tag configuration (unused in this filter)
+
+    Returns:
+        DataFrame: Filtered DataFrame with null values removed
+
+    Note:
+        The _model_tags parameter is included for interface consistency but not used
+    """
+    return df[df['value'].isnull()]