Code import - branch feature/SIENTIAPDE-1646

tests/utils/quality/test_filters.py

@@ -0,0 +1,139 @@
+import numpy as np
+import pandas as pd
+import pytest
+from pandas.testing import assert_frame_equal
+
+from scouter.utils.quality.filters import check_data_range, null_values_filter, out_of_bounds_filter
+
+# Fixtures
+
+
+@pytest.fixture
+def sample_dataframe():
+    """Fixture providing a sample DataFrame for testing."""
+    return pd.DataFrame(
+        {
+            'tag': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
+            'name': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
+            'value': [25, 35, 95, 105, 60, None],
+            'timestamp': pd.date_range(start='2023-01-01', periods=6),
+        }
+    )
+
+
+@pytest.fixture
+def nodes_data_range():
+    """Fixture providing data ranges for different tags."""
+    return {
+        'temp': {'data_range': [10, 30]},
+        'pressure': {'data_range': [90, 100]},
+        'humidity': {'data_range': [40, 80]},
+        'wind_speed': {'data_range': [0, 50]},
+    }
+
+
+# Parameterized test data
+CHECK_DATA_RANGE_CASES = [
+    # (value, val_range, expected)
+    # Values within range
+    (5, [0, 10], False),
+    (0, [0, 10], False),  # Edge case: value equals lower bound
+    (10, [0, 10], False),  # Edge case: value equals upper bound
+    # Values outside range
+    (-1, [0, 10], True),
+    (11, [0, 10], True),
+    # Single value range
+    (5, [5, 5], False),
+    (4, [5, 5], True),
+    # Empty or None value
+    (None, [0, 10], True),
+    (np.nan, [0, 10], True),
+]
+
+# Tests for check_data_range
+
+
+@pytest.mark.parametrize('value,val_range,expected', CHECK_DATA_RANGE_CASES)
+def test_check_data_range(value, val_range, expected):
+    """Test the check_data_range function with various input scenarios."""
+    result = check_data_range(value, val_range)
+    if isinstance(value, float) and np.isnan(value):
+        assert result is True
+    else:
+        assert result == expected
+
+
+# Tests for out_of_bounds_filter
+
+
+def test_out_of_bounds_filter(sample_dataframe, nodes_data_range):
+    """Test filtering out-of-bounds values from a DataFrame."""
+    # Expected result: rows where value is outside the defined range
+    expected_data = {
+        'tag': ['temp', 'pressure', 'wind_speed'],
+        'name': ['temp', 'pressure', 'wind_speed'],
+        'value': [35, 105, None],
+        'timestamp': [
+            pd.Timestamp('2023-01-02'),
+            pd.Timestamp('2023-01-04'),
+            pd.Timestamp('2023-01-06'),
+        ],
+    }
+    expected_df = pd.DataFrame(expected_data)
+
+    result = out_of_bounds_filter(sample_dataframe, nodes_data_range)
+    result = result.reset_index(drop=True)
+    expected_df = expected_df.reset_index(drop=True)
+
+    assert_frame_equal(result, expected_df)
+
+
+def test_out_of_bounds_filter_empty_df(nodes_data_range):
+    """Test with an empty DataFrame."""
+    df = pd.DataFrame(columns=['tag', 'name', 'value', 'timestamp'])
+    result = out_of_bounds_filter(df, nodes_data_range)
+    assert result.empty
+    assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']
+
+
+# Tests for null_values_filter
+
+
+def test_null_values_filter(sample_dataframe, nodes_data_range):
+    """Test filtering null values from a DataFrame."""
+    expected_data = {
+        'tag': ['wind_speed'],
+        'name': ['wind_speed'],
+        'value': [np.nan],
+        'timestamp': [pd.Timestamp('2023-01-06')],
+    }
+    expected_df = pd.DataFrame(expected_data)
+
+    result = null_values_filter(sample_dataframe, nodes_data_range)
+    result = result.reset_index(drop=True)
+    expected_df = expected_df.reset_index(drop=True)
+
+    assert_frame_equal(result, expected_df, check_dtype=False)
+
+
+def test_null_values_filter_no_nulls(nodes_data_range):
+    """Test with a DataFrame containing no null values."""
+    df = pd.DataFrame(
+        {
+            'tag': ['temp', 'pressure'],
+            'name': ['temp', 'pressure'],
+            'value': [25, 100],
+            'timestamp': pd.date_range(start='2023-01-01', periods=2),
+        }
+    )
+    result = null_values_filter(df, nodes_data_range)
+    assert result.empty
+    assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']
+
+
+def test_null_values_filter_empty_df(nodes_data_range):
+    """Test with an empty DataFrame."""
+    df = pd.DataFrame(columns=['tag', 'name', 'value', 'timestamp'])
+    result = null_values_filter(df, nodes_data_range)
+    assert result.empty
+    assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']

tests/utils/test_connectors_config.py

@@ -0,0 +1,42 @@
+import os
+from unittest.mock import patch
+
+import pytest
+
+from scouter.utils.connectors_config import (
+    build_kafka_config,
+)
+
+
+@pytest.fixture
+def mock_env_vars():
+    with patch.dict(os.environ, {}, clear=True):
+        yield
+
+
+@pytest.mark.usefixtures('mock_env_vars')
+def test_build_kafka_config_defaults():
+    """Test that build_kafka_config returns default values when no env vars are set"""
+    config = build_kafka_config()
+
+    assert config == {
+        'bootstrap_servers': 'localhost:9092',
+        'polling_time': 1000,
+        'group_id': 'scouter-group',
+    }
+
+
+@pytest.mark.usefixtures('mock_env_vars')
+def test_build_kafka_config_with_env_vars():
+    """Test that build_kafka_config uses env vars when set"""
+    with patch.dict(
+        os.environ,
+        {'KAFKA_BOOTSTRAP_SERVERS': 'kafka.example.com:9092', 'KAFKA_POLLING_TIME': '5000'},
+    ):
+        config = build_kafka_config()
+
+        assert config == {
+            'bootstrap_servers': 'kafka.example.com:9092',
+            'polling_time': 5000,
+            'group_id': 'scouter-group',
+        }