SIENTIAPDE-1005

Implement workflows for fake data generation, scouter processing, and core scouter operations

- Added `FakeData` workflow to generate random data and send it to a Kafka topic.
- Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow.
- Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL.
- Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling.
- Enhanced Redis and Postgres activities with robust testing for data handling and error notifications.
- Introduced quality filters for data validation and implemented tests to verify their functionality.

tests/utils/quality/test_filters.py

@@ -0,0 +1,132 @@
+import pytest
+import pandas as pd
+import numpy as np
+from pandas.testing import assert_frame_equal
+from scouter.utils.quality.filters import check_data_range, out_of_bounds_filter, null_values_filter
+
+# Fixtures
+
+
+@pytest.fixture
+def sample_dataframe():
+    """Fixture providing a sample DataFrame for testing."""
+    return pd.DataFrame({
+        'tag': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
+        'name': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
+        'value': [25, 35, 95, 105, 60, None],
+        'timestamp': pd.date_range(start='2023-01-01', periods=6)
+    })
+
+
+@pytest.fixture
+def nodes_data_range():
+    """Fixture providing data ranges for different tags."""
+    return {
+        'temp': {'data_range': [10, 30]},
+        'pressure': {'data_range': [90, 100]},
+        'humidity': {'data_range': [40, 80]},
+        'wind_speed': {'data_range': [0, 50]}
+    }
+
+
+# Parameterized test data
+CHECK_DATA_RANGE_CASES = [
+    # (value, val_range, expected)
+    # Values within range
+    (5, [0, 10], False),
+    (0, [0, 10], False),  # Edge case: value equals lower bound
+    (10, [0, 10], False),  # Edge case: value equals upper bound
+    # Values outside range
+    (-1, [0, 10], True),
+    (11, [0, 10], True),
+    # Single value range
+    (5, [5, 5], False),
+    (4, [5, 5], True),
+    # Empty or None value
+    (None, [0, 10], True),
+    (np.nan, [0, 10], True),
+]
+
+# Tests for check_data_range
+
+
+@pytest.mark.parametrize('value,val_range,expected', CHECK_DATA_RANGE_CASES)
+def test_check_data_range(value, val_range, expected):
+    """Test the check_data_range function with various input scenarios."""
+    result = check_data_range(value, val_range)
+    if isinstance(value, float) and np.isnan(value):
+        assert result is True
+    else:
+        assert result == expected
+
+# Tests for out_of_bounds_filter
+
+
+def test_out_of_bounds_filter(sample_dataframe, nodes_data_range):
+    """Test filtering out-of-bounds values from a DataFrame."""
+    # Expected result: rows where value is outside the defined range
+    expected_data = {
+        'tag': ['temp', 'pressure', 'wind_speed'],
+        'name': ['temp', 'pressure', 'wind_speed'],
+        'value': [35, 105, None],
+        'timestamp': [
+            pd.Timestamp('2023-01-02'),
+            pd.Timestamp('2023-01-04'),
+            pd.Timestamp('2023-01-06')
+        ]
+    }
+    expected_df = pd.DataFrame(expected_data)
+
+    result = out_of_bounds_filter(sample_dataframe, nodes_data_range)
+    result = result.reset_index(drop=True)
+    expected_df = expected_df.reset_index(drop=True)
+
+    assert_frame_equal(result, expected_df)
+
+
+def test_out_of_bounds_filter_empty_df(nodes_data_range):
+    """Test with an empty DataFrame."""
+    df = pd.DataFrame(columns=['tag', 'name', 'value', 'timestamp'])
+    result = out_of_bounds_filter(df, nodes_data_range)
+    assert result.empty
+    assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']
+
+# Tests for null_values_filter
+
+
+def test_null_values_filter(sample_dataframe, nodes_data_range):
+    """Test filtering null values from a DataFrame."""
+    expected_data = {
+        'tag': ['wind_speed'],
+        'name': ['wind_speed'],
+        'value': [None],
+        'timestamp': [pd.Timestamp('2023-01-06')]
+    }
+    expected_df = pd.DataFrame(expected_data)
+
+    result = null_values_filter(sample_dataframe, nodes_data_range)
+    result = result.reset_index(drop=True)
+    expected_df = expected_df.reset_index(drop=True)
+
+    assert_frame_equal(result, expected_df, check_dtype=False)
+
+
+def test_null_values_filter_no_nulls(nodes_data_range):
+    """Test with a DataFrame containing no null values."""
+    df = pd.DataFrame({
+        'tag': ['temp', 'pressure'],
+        'name': ['temp', 'pressure'],
+        'value': [25, 100],
+        'timestamp': pd.date_range(start='2023-01-01', periods=2)
+    })
+    result = null_values_filter(df, nodes_data_range)
+    assert result.empty
+    assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']
+
+
+def test_null_values_filter_empty_df(nodes_data_range):
+    """Test with an empty DataFrame."""
+    df = pd.DataFrame(columns=['tag', 'name', 'value', 'timestamp'])
+    result = null_values_filter(df, nodes_data_range)
+    assert result.empty
+    assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']