SIENTIAPDE-1005

Implement workflows for fake data generation, scouter processing, and core scouter operations

- Added `FakeData` workflow to generate random data and send it to a Kafka topic.
- Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow.
- Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL.
- Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling.
- Enhanced Redis and Postgres activities with robust testing for data handling and error notifications.
- Introduced quality filters for data validation and implemented tests to verify their functionality.

tests/activities/test_base.py

@@ -0,0 +1,36 @@
+from unittest.mock import MagicMock
+from pytest import fixture
+from sientia_do.notifications.models import Notification
+from scouter.activities.base import BaseActivity
+
+
+@fixture
+def base_activity():
+    return BaseActivity(
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+    )
+
+
+def test_prepare_activity(base_activity):
+    base_activity.notification_handler.base_notification = Notification(
+        project="project",
+        pipeline="pipeline",
+        trigger="-",
+        model_name="-",
+        model_id="-",
+    )
+
+    base_activity.prepare_activity(
+        {
+            'workflow_name': 'test_workflow',
+            'schedule_name': 'test_schedule',
+            'model_name': 'test_model',
+            'model_id': 'test_model_id',
+        }
+    )
+
+    assert base_activity.notification_handler.base_notification.schedule_name == "test_schedule"
+    assert base_activity.notification_handler.base_notification.model_name == "test_model"
+    assert base_activity.notification_handler.base_notification.model_id == "test_model_id"
+    assert base_activity.notification_handler.base_notification.pipeline_name == "test_workflow"

tests/activities/test_faker.py

@@ -0,0 +1,108 @@
+import pytest
+from unittest.mock import MagicMock, patch, call
+from scouter.activities.faker import Faker
+from logging import Logger
+from sientia_do.notifications.handlers import NotificationHandler
+
+
+@pytest.fixture
+def mock_kafka_producer():
+    with patch('scouter.activities.faker.KafkaProducer') as mock:
+        producer = MagicMock()
+        mock.return_value = producer
+        yield producer
+
+
+@pytest.fixture
+def mock_datetime():
+    with patch('scouter.activities.faker.datetime') as mock_dt:
+        mock_dt.now.return_value.strftime.return_value = '2025-05-14 14:54:24'
+        yield mock_dt
+
+
+@pytest.fixture
+def faker_instance(mock_kafka_producer):
+    logger = MagicMock(spec=Logger)
+    notification_handler = MagicMock(spec=NotificationHandler)
+    return Faker(
+        bootstrap_servers='localhost:9092',
+        logger=logger,
+        notification_handler=notification_handler
+    )
+
+
+@pytest.mark.asyncio
+async def test_faker_init(faker_instance, mock_kafka_producer):
+    """Test Faker initialization with correct parameters"""
+    assert faker_instance.producer is not None
+    assert len(faker_instance.tags) == 6
+
+
+@pytest.mark.asyncio
+async def test_generate_and_send_data_default_count(faker_instance, mock_kafka_producer, mock_datetime):
+    """Test generating data with default message count"""
+    # Mock random.choice to control the output
+    with patch('random.choice') as mock_choice, \
+            patch('random.uniform', return_value=42.5), \
+            patch('random.randint', return_value=3):
+
+        # Setup mock for tag and name selection
+        mock_choice.side_effect = [
+            'ns=1;i=1001',
+            'ns=1;i=1002',
+            'ns=1;i=1003'
+        ]
+
+        # Call the method
+        await faker_instance.generate_and_send_data({'topic': 'test_topic'})
+
+        # Verify the producer was called 3 times (default count)
+        assert mock_kafka_producer.send.call_count == 3
+        mock_kafka_producer.flush.assert_called_once()
+
+        # Verify the message format
+        expected_data = {
+            'tag': 'ns=1;i=1001',
+            'name': 'Temperature Sensor',
+            'timestamp': '2025-05-14 14:54:24',
+            'value': 42.5
+        }
+        mock_kafka_producer.send.assert_any_call(
+            'test_topic', value=expected_data)
+
+
+@pytest.mark.asyncio
+async def test_generate_and_send_data_custom_count(faker_instance, mock_kafka_producer):
+    """Test generating data with custom message count"""
+    # Call the method with custom count
+    await faker_instance.generate_and_send_data({
+        'topic': 'test_topic',
+        'num_messages': 2
+    })
+
+    # Verify the producer was called 2 times
+    assert mock_kafka_producer.send.call_count == 2
+    mock_kafka_producer.flush.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_generate_and_send_data_no_topic(faker_instance):
+    """Test that ValueError is raised when no topic is provided"""
+    with pytest.raises(ValueError, match="Topic must be specified in input_data"):
+        await faker_instance.generate_and_send_data({})
+
+
+@pytest.mark.asyncio
+async def test_generate_and_send_data_random_values(faker_instance, mock_kafka_producer):
+    """Test that random values are within expected ranges"""
+    # Call the method
+    await faker_instance.generate_and_send_data({'topic': 'test_topic'})
+
+    # Get the call arguments
+    call_args = mock_kafka_producer.send.call_args[1]['value']
+
+    # Verify the data structure
+    assert 'tag' in call_args
+    assert call_args['tag'] in faker_instance.tags
+    assert 'value' in call_args
+    assert 0 <= call_args['value'] <= 100

tests/activities/test_gates.py

@@ -0,0 +1,290 @@
+from unittest.mock import Mock, patch, MagicMock
+import numpy as np
+import pandas as pd
+import pytest
+from sientia_do.notifications.models import NotificationLevel
+from scouter.activities.gates import Gates
+
+
+@pytest.fixture
+def gates_fixture():
+    """Fixture to create a Gates instance with mocked dependencies."""
+    logger = Mock()
+    notification_handler = MagicMock()
+    return Gates(logger=logger, notification_handler=notification_handler)
+
+
+@pytest.mark.asyncio
+async def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
+    """Test data_quality_gate with NULL_VALUES_FILTER and DISCARD policy."""
+    # Setup test data
+    input_data = {
+        'filters': {
+            'NULL_VALUES_FILTER': 'DISCARD'
+        },
+        'data': {
+            'tag': ['tag1', 'tag2', 'tag3'],
+            'value': [1.0, None, 3.0],
+            'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'],
+        },
+        'model_tags': {
+            'tag1': {'data_range': [0, 100]},
+            'tag2': {'data_range': [0, 100]},
+            'tag3': {'data_range': [0, 100]}
+        }
+    }
+
+    # Execute
+    result = await gates_fixture.data_quality_gate(input_data)
+
+    # Verify
+    assert len(result['tag']) == 2
+    assert 'tag2' not in result['tag']
+    gates_fixture.notification_handler.build_and_send_notification.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture):
+    """Test data_quality_gate with OUT_OF_BOUNDS_FILTER and KEEP policy."""
+    # Setup test data with out of bounds values
+    input_data = {
+        'filters': {
+            'OUT_OF_BOUNDS_FILTER': 'KEEP'
+        },
+        'data': {
+            'tag': ['tag1', 'tag2', 'tag3'],
+            'value': [1.0, 200.0, 3.0],
+            'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03']
+        },
+        'model_tags': {
+            'tag1': {'data_range': [0, 100]},
+            'tag2': {'data_range': [0, 100]},
+            'tag3': {'data_range': [0, 100]}
+        }
+    }
+
+    # Mock the out_of_bounds_filter to return rows with out of bounds values
+    with patch('scouter.activities.gates.quality_gate_filters', {
+        'OUT_OF_BOUNDS_FILTER': lambda df: df[df['tag'] == 'tag2']
+    }):
+        # Execute
+        result = await gates_fixture.data_quality_gate(input_data)
+
+    # Verify data is kept but notification is sent
+    assert len(result['tag']) == 3  # All rows kept
+    gates_fixture.notification_handler.build_and_send_notification.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_data_quality_gate_with_multiple_filters(gates_fixture):
+    """Test data_quality_gate with multiple filters."""
+    # Setup test data
+    input_data = {
+        'filters': {
+            'NULL_VALUES_FILTER': 'DISCARD',
+            'OUT_OF_BOUNDS_FILTER': 'DISCARD'
+        },
+        'data': {
+            'tag': ['tag1', 'tag2', 'tag3', 'tag4'],
+            'name': ['tag1', 'tag2', 'tag3', 'tag4'],
+            'value': [1.0, None, 300.0, 4.0],
+            'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03', '2023-01-04']
+        },
+        'model_tags': {
+            'tag1': {'data_range': [0, 100]},
+            'tag2': {'data_range': [0, 100]},
+            'tag3': {'data_range': [0, 100]},
+            'tag4': {'data_range': [0, 100]}
+        }
+    }
+
+    result = await gates_fixture.data_quality_gate(input_data)
+
+    # Verify only tag1 and tag4 remain (tag2 has null, tag3 is out of bounds)
+    assert result == {'tag': {0: 'tag1', 3: 'tag4'}, 'name': {0: 'tag1', 3: 'tag4'}, 'value': {
+        0: 1.0, 3: 4.0}, 'timestamp': {0: '2023-01-01', 3: '2023-01-04'}}
+    # Should be called twice (once for each filter)
+    assert gates_fixture.notification_handler.build_and_send_notification.call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_data_quality_gate_with_unknown_filter(gates_fixture):
+    """Test data_quality_gate with an unknown filter."""
+    # Setup test data with unknown filter
+    input_data = {
+        'filters': {
+            'UNKNOWN_FILTER': 'DISCARD'
+        },
+        'data': {
+            'tag': ['tag1'],
+            'name': ['tag1'],
+            'value': [1.0],
+            'timestamp': ['2023-01-01']
+        },
+        'model_tags': {
+            'tag1': {'data_range': [0, 100]}
+        }
+    }
+
+    # Execute
+    result = await gates_fixture.data_quality_gate(input_data)
+
+    # Verify data is unchanged and warning is logged
+    assert len(result['tag']) == 1
+    gates_fixture.logger.warning.assert_called_once_with(
+        "Filter UNKNOWN_FILTER not found")
+
+
+@pytest.mark.asyncio
+async def test_data_quality_gate_with_filter_error(gates_fixture):
+    """Test data_quality_gate when a filter raises an exception."""
+    # Setup test data
+    input_data = {
+        'filters': {
+            'NULL_VALUES_FILTER': 'DISCARD'
+        },
+        'data': {
+            'tag': ['tag1'],
+            'name': ['tag1'],
+            'value': [1.0],
+            'timestamp': ['2023-01-01']
+        },
+        'model_tags': {
+            'tag1': {'data_range': [0, 100]}
+        }
+    }
+
+    # Mock the filter to raise an exception
+    def failing_filter(_, _model_tags):
+        raise ValueError("Filter error")
+
+    with patch('scouter.activities.gates.quality_gate_filters', {
+        'NULL_VALUES_FILTER': failing_filter
+    }):
+        # Execute
+        result = await gates_fixture.data_quality_gate(input_data)
+
+    # Verify error notification is sent and data is unchanged
+    assert len(result['tag']) == 1
+    gates_fixture.notification_handler.build_and_send_notification.assert_called_once()
+    call_args = gates_fixture.notification_handler.build_and_send_notification.call_args[1]
+    assert call_args['notification_id'] == "DATA_QUALITY_GATE_ISSUES"
+    assert call_args['level'] == NotificationLevel.ERROR
+    assert "Filter error" in call_args['message']
+
+
+@pytest.mark.asyncio
+async def test_data_quality_gate_with_empty_data(gates_fixture):
+    """Test data_quality_gate with empty input data."""
+    # Setup empty input data
+    input_data = {
+        'filters': {
+            'NULL_VALUES_FILTER': 'DISCARD'
+        },
+        'data': {
+            'tag': [],
+            'name': [],
+            'value': [],
+            'timestamp': []
+        },
+        'model_tags': {}
+    }
+
+    # Execute
+    result = await gates_fixture.data_quality_gate(input_data)
+
+    # Verify empty result and no notifications
+    assert len(result['tag']) == 0
+    gates_fixture.notification_handler.build_and_send_notification.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_data_quality_gate_with_no_filters(gates_fixture):
+    """Test data_quality_gate with no filters specified."""
+    # Setup test data with no filters
+    input_data = {
+        'filters': {},
+        'data': {
+            'tag': ['tag1'],
+            'name': ['tag1'],
+            'value': [1.0],
+            'timestamp': ['2023-01-01']
+        },
+        'model_tags': {
+            'tag1': {'data_range': [0, 100]}
+        }
+    }
+
+    # Execute
+    result = await gates_fixture.data_quality_gate(input_data)
+
+    # Verify data is unchanged and no notifications
+    assert len(result['tag']) == 1
+    gates_fixture.notification_handler.build_and_send_notification.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    "group_data, aggr_function, expected_result",
+    [
+        # Single value case
+        (pd.DataFrame({'value': [10.0]}), 'avg', 10.0),
+        # Multiple values with different aggregation functions
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'avg', 2.5),
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'mdn', 2.5),
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'max', 4.0),
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'min', 1.0),
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'lts', 4.0),
+        # With NaN values
+        (pd.DataFrame({'value': [1.0, np.nan, 3.0, 4.0]}),
+         'avg', 2.6666666666666665),
+        # Empty group after dropping NaN
+        (pd.DataFrame({'value': [np.nan, np.nan]}), 'avg', None),
+        # Invalid aggregation function
+        (pd.DataFrame({'value': [1.0, 2.0]}), 'invalid', 'continue'),
+    ]
+)
+def test_apply_aggregation(gates_fixture, group_data, aggr_function, expected_result):
+    """Test apply_aggregation method with various scenarios."""
+    result = gates_fixture.apply_aggregation(group_data, aggr_function)
+    assert result == expected_result
+
+    # Check notification was sent for invalid function
+    if aggr_function == 'invalid':
+        gates_fixture.notification_handler.build_and_send_notification.assert_called_once()
+    else:
+        gates_fixture.notification_handler.build_and_send_notification.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_aggregate_data(gates_fixture):
+    """Test aggregate_data method with multiple groups and aggregation functions."""
+    input_data = {
+        'data': [
+            {'tag': 'tag1', 'name': 'name1', 'value': 1.0, 'timestamp': '2023-01-01'},
+            {'tag': 'tag1', 'name': 'name1', 'value': 2.0, 'timestamp': '2023-01-02'},
+            {'tag': 'tag1', 'name': 'name1', 'value': 3.0, 'timestamp': '2023-01-03'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'},
+            {'tag': 'tag1', 'name': 'name1',
+                'value': None, 'timestamp': '2023-01-04'},
+        ],
+        'model_tags': {
+            'name1': {'aggr_function': 'avg'},
+            'name2': {'aggr_function': 'max'},
+        }
+    }
+
+    # Expected result
+    expected_result = {'tag': {0: 'tag1', 1: 'tag2'},
+                       'name': {0: 'name1', 1: 'name2'},
+                       'value': {0: 2.0, 1: 6.0},
+                       'timestamp': {0: '2023-01-04', 1: '2023-01-03'},
+                       'aggregation_function': {0: 'avg', 1: 'max'}}
+
+    # Execute
+    result = await gates_fixture.aggregate_data(input_data)
+
+    # Verify
+    assert result == expected_result
+    gates_fixture.notification_handler.build_and_send_notification.assert_not_called()

tests/activities/test_kafka.py

@@ -1,5 +1,5 @@
 from unittest.mock import MagicMock, patch, ANY
-from pytest import fixture
+from pytest import fixture, mark
 from pandas import DataFrame
 from scouter.activities.kafka import Kafka
 
@@ -38,7 +38,8 @@ def test___init__(kafka_consumer):
     )
 
 
-def test_load_from_kafka(kafka):
+@mark.asyncio
+async def test_load_from_kafka(kafka):
     input_data = {"topic": "test-topic"}
 
     data = [
@@ -55,7 +56,7 @@ def test_load_from_kafka(kafka):
 
     expected = DataFrame([d.value for d in data[0][1]]).to_dict()
 
-    result = kafka.load_from_kafka(input_data)
+    result = await kafka.load_from_kafka(input_data)
 
     assert result == expected
 

tests/activities/test_postgres.py

@@ -0,0 +1,90 @@
+from unittest.mock import ANY, MagicMock, patch
+from pytest import fixture
+from pytest import mark
+from sientia_do.notifications.models import NotificationLevel
+
+from scouter.activities.postgres import Postgres
+
+
+@fixture
+@patch("scouter.activities.postgres.create_engine")
+@patch("scouter.activities.postgres.sessionmaker")
+def postgres_client(mock_sessionmaker, mock_engine):
+    # Create a mock session
+    mock_session = MagicMock()
+    mock_session.commit = MagicMock()
+    mock_session.close = MagicMock()
+
+    # Configure the session to work with context management
+    mock_session.__enter__ = MagicMock(return_value=mock_session)
+    mock_session.__exit__ = MagicMock(return_value=None)
+
+    # Configure the sessionmaker to return our mock session
+    mock_sessionmaker.return_value = mock_session
+
+    # Configure the engine to return our mock sessionmaker
+    mock_engine.return_value = MagicMock()
+    mock_engine.return_value.dispose = MagicMock()
+
+    # Create the Postgres client
+    client = Postgres(
+        host="localhost",
+        port=5432,
+        user="postgres",
+        password="postgres",
+        dbname="postgres",
+        min_connections=1,
+        max_connections=10,
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+    )
+
+    # Set up the session factory
+    client.session_factory = mock_sessionmaker
+
+    return client
+
+
+@mark.asyncio
+@patch("scouter.activities.postgres.DataFrame")
+async def test_export_data_to_postgres_success(mock_dataframe, postgres_client):
+    data = {"schema": "test", "table_name": "test",
+            "data": {"a": [1, 2, 3], "b": [4, 5, 6]}}
+    await postgres_client.export_data_to_postgres(data)
+
+    # Verify notification handler wasn't called
+    postgres_client.notification_handler.build_and_send_notification.assert_not_called()
+
+    # Verify session handling
+    mock_dataframe.assert_called_once_with(data["data"])
+    mock_dataframe.return_value.to_sql.assert_called_once_with(
+        data["table_name"],
+        postgres_client.engine,
+        schema=data["schema"],
+        if_exists="append",
+        index=False
+    )
+    postgres_client.session_factory.return_value.commit.assert_called_once()
+    postgres_client.session_factory.return_value.close.assert_called_once()
+
+
+@mark.asyncio
+@patch("scouter.activities.postgres.DataFrame", return_value=MagicMock(
+    to_sql=MagicMock(side_effect=Exception("Error exporting data to postgres"))
+))
+async def test_export_data_to_postgres_error(_mock_dataframe, postgres_client):
+    data = {"schema": "test", "table_name": "test",
+            "data": {"a": [1, 2, 3], "b": [4, 5, 6]}}
+    await postgres_client.export_data_to_postgres(data)
+
+    # Verify error notification was sent
+    postgres_client.notification_handler.build_and_send_notification.assert_called_once_with(
+        notification_id="ERROR_EXPORTING_DATA_TO_POSTGRES",
+        message="Error exporting data to postgres: Error exporting data to postgres",
+        block="export_data_to_postgres",
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY
+    )
+
+    # Verify session handling
+    postgres_client.session_factory.return_value.close.assert_called_once()

tests/activities/test_redis.py

@@ -0,0 +1,208 @@
+import json
+from unittest.mock import MagicMock, patch
+from datetime import datetime
+import pytest
+import numpy as np
+from pandas import DataFrame
+from sientia_do.notifications.handlers import NotificationHandler
+from scouter.activities.redis import Redis
+
+
+@pytest.fixture
+@patch('scouter.activities.redis.redis.Redis')
+def redis_activity(_mock_redis_client):
+    logger = MagicMock()
+    notification_handler = MagicMock(spec=NotificationHandler)
+    return Redis(host='localhost', port=6379,
+                 logger=logger, notification_handler=notification_handler)
+
+
+@patch('scouter.activities.redis.redis.Redis')
+def test_redis_initialization(mock_redis_client):
+    """Test Redis activity initialization"""
+    redis_activity = Redis(host='localhost', port=6379,
+                           logger=MagicMock(), notification_handler=MagicMock())
+    assert redis_activity.host == 'localhost'
+    assert redis_activity.port == 6379
+    mock_redis_client.assert_called_once_with(
+        host='localhost',
+        port=6379,
+        decode_responses=True
+    )
+
+
+def test_get_existing_key(redis_activity):
+    """Test getting an existing key from Redis"""
+    test_data = {'key': 'value'}
+    redis_activity.redis_client.get.return_value = json.dumps(test_data)
+
+    result = redis_activity.get('test_key')
+
+    assert result == test_data
+    redis_activity.redis_client.get.assert_called_once_with('test_key')
+
+
+def test_get_nonexistent_key(redis_activity):
+    """Test getting a non-existent key from Redis"""
+    redis_activity.redis_client.get.return_value = None
+
+    result = redis_activity.get('nonexistent_key')
+
+    assert result is None
+    redis_activity.redis_client.get.assert_called_once_with('nonexistent_key')
+
+
+def test_set_key(redis_activity):
+    """Test setting a key in Redis"""
+    test_data = {'key': 'value'}
+
+    redis_activity.set('test_key', test_data, ttl=300)
+
+    redis_activity.redis_client.set.assert_called_once_with(
+        'test_key',
+        json.dumps(test_data),
+        ex=300
+    )
+
+
+@pytest.mark.asyncio
+async def test_group_and_hold_data_new_key(redis_activity):
+    """Test group_and_hold_data with a new key"""
+    # Setup
+    test_data = {
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame({
+            'name': ['sensor1', 'sensor2'],
+            'value': [25.5, 30.0],
+            'timestamp': ['2023-01-01 12:00:00'] * 2
+        }).to_dict('records')
+    }
+
+    # Mock get to return None for new key
+    redis_activity.get = MagicMock(return_value=None)
+    redis_activity.set = MagicMock()
+
+    # Call the method
+    result = await redis_activity.group_and_hold_data(test_data)
+
+    # Verify the result
+    expected_result = {
+        'timestamp': {0: '2023-01-01 12:00:00', 1: '2023-01-01 12:00:00'},
+        'variable': {0: 'sensor1', 1: 'sensor2'},
+        'value': {0: 25.5, 1: 30.0},
+        'model_id': {0: 1, 1: 1}
+    }
+    assert result == expected_result
+
+    # Verify set was called with correct arguments
+    redis_activity.set.assert_called_once()
+    args, kwargs = redis_activity.set.call_args
+    assert args[0] == 'test_pipeline_test_schedule'
+    assert args[1] == {
+        'sensor1': 25.5,
+        'sensor2': 30.0,
+        'timestamp': '2023-01-01 12:00:00'
+    }
+    assert kwargs['ttl'] == 3600
+
+
+@pytest.mark.asyncio
+async def test_group_and_hold_data_update_existing(redis_activity):
+    """Test updating existing data with group_and_hold_data"""
+    # Setup initial data in Redis
+    existing_data = {
+        'sensor1': 20.0,
+        'sensor2': 28.0,
+        'timestamp': '2023-01-01 11:00:00'
+    }
+
+    # New data to update with
+    test_data = {
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame({
+            'name': ['sensor1', 'sensor3'],
+            'value': [25.5, 42.0],
+            'timestamp': ['2023-01-01 12:00:00'] * 2
+        }).to_dict('records')
+    }
+
+    # Mock get to return existing data
+    redis_activity.get = MagicMock(return_value=existing_data)
+    redis_activity.set = MagicMock()
+
+    # Call the method
+    result = await redis_activity.group_and_hold_data(test_data)
+
+    # Verify the result
+    expected_result = {
+        'timestamp': {0: '2023-01-01 12:00:00', 1: '2023-01-01 12:00:00', 2: '2023-01-01 12:00:00'},
+        'variable': {0: 'sensor1', 1: 'sensor2', 2: 'sensor3'},
+        'value': {0: 25.5, 1: 28.0, 2: 42.0},
+        'model_id': {0: 1, 1: 1, 2: 1}
+    }
+    assert result == expected_result
+
+    # Verify set was called with correct arguments
+    redis_activity.set.assert_called_once()
+    args, kwargs = redis_activity.set.call_args
+    assert args[0] == 'test_workflow_test_schedule'
+    assert args[1] == {
+        'sensor1': 25.5,
+        'sensor2': 28.0,
+        'sensor3': 42.0,
+        'timestamp': '2023-01-01 12:00:00'
+    }
+    assert kwargs['ttl'] == 3600
+
+
+@pytest.mark.asyncio
+async def test_group_and_hold_data_with_none_values(redis_activity):
+    """Test handling of None values in group_and_hold_data"""
+    # Setup test data with None values
+    test_data = {
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame({
+            'name': ['sensor1', 'sensor2'],
+            'value': [None, 30.0],
+            'timestamp': [datetime(2023, 1, 1, 12, 0, 0)] * 2
+        }).to_dict('records')
+    }
+
+    # Mock get to return None for new key
+    redis_activity.get = MagicMock(return_value=None)
+    redis_activity.set = MagicMock()
+
+    # Call the method
+    result = await redis_activity.group_and_hold_data(test_data)
+
+    # Verify None was converted to np.nan and values are as expected
+    assert np.isnan(result['value'][0])
+    assert result['value'][1] == pytest.approx(30.0)
+
+
+@pytest.mark.asyncio
+async def test_group_and_hold_data_empty_dataframe(redis_activity):
+    """Test group_and_hold_data with empty DataFrame"""
+    # Setup test with empty data
+    test_data = {
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records')
+    }
+
+    redis_activity.get = MagicMock(return_value=None)
+
+    # Call the method
+    result = await redis_activity.group_and_hold_data(test_data)
+
+    assert result == {}

tests/utils/quality/test_filters.py

@@ -0,0 +1,132 @@
+import pytest
+import pandas as pd
+import numpy as np
+from pandas.testing import assert_frame_equal
+from scouter.utils.quality.filters import check_data_range, out_of_bounds_filter, null_values_filter
+
+# Fixtures
+
+
+@pytest.fixture
+def sample_dataframe():
+    """Fixture providing a sample DataFrame for testing."""
+    return pd.DataFrame({
+        'tag': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
+        'name': ['temp', 'temp', 'pressure', 'pressure', 'humidity', 'wind_speed'],
+        'value': [25, 35, 95, 105, 60, None],
+        'timestamp': pd.date_range(start='2023-01-01', periods=6)
+    })
+
+
+@pytest.fixture
+def nodes_data_range():
+    """Fixture providing data ranges for different tags."""
+    return {
+        'temp': {'data_range': [10, 30]},
+        'pressure': {'data_range': [90, 100]},
+        'humidity': {'data_range': [40, 80]},
+        'wind_speed': {'data_range': [0, 50]}
+    }
+
+
+# Parameterized test data
+CHECK_DATA_RANGE_CASES = [
+    # (value, val_range, expected)
+    # Values within range
+    (5, [0, 10], False),
+    (0, [0, 10], False),  # Edge case: value equals lower bound
+    (10, [0, 10], False),  # Edge case: value equals upper bound
+    # Values outside range
+    (-1, [0, 10], True),
+    (11, [0, 10], True),
+    # Single value range
+    (5, [5, 5], False),
+    (4, [5, 5], True),
+    # Empty or None value
+    (None, [0, 10], True),
+    (np.nan, [0, 10], True),
+]
+
+# Tests for check_data_range
+
+
+@pytest.mark.parametrize('value,val_range,expected', CHECK_DATA_RANGE_CASES)
+def test_check_data_range(value, val_range, expected):
+    """Test the check_data_range function with various input scenarios."""
+    result = check_data_range(value, val_range)
+    if isinstance(value, float) and np.isnan(value):
+        assert result is True
+    else:
+        assert result == expected
+
+# Tests for out_of_bounds_filter
+
+
+def test_out_of_bounds_filter(sample_dataframe, nodes_data_range):
+    """Test filtering out-of-bounds values from a DataFrame."""
+    # Expected result: rows where value is outside the defined range
+    expected_data = {
+        'tag': ['temp', 'pressure', 'wind_speed'],
+        'name': ['temp', 'pressure', 'wind_speed'],
+        'value': [35, 105, None],
+        'timestamp': [
+            pd.Timestamp('2023-01-02'),
+            pd.Timestamp('2023-01-04'),
+            pd.Timestamp('2023-01-06')
+        ]
+    }
+    expected_df = pd.DataFrame(expected_data)
+
+    result = out_of_bounds_filter(sample_dataframe, nodes_data_range)
+    result = result.reset_index(drop=True)
+    expected_df = expected_df.reset_index(drop=True)
+
+    assert_frame_equal(result, expected_df)
+
+
+def test_out_of_bounds_filter_empty_df(nodes_data_range):
+    """Test with an empty DataFrame."""
+    df = pd.DataFrame(columns=['tag', 'name', 'value', 'timestamp'])
+    result = out_of_bounds_filter(df, nodes_data_range)
+    assert result.empty
+    assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']
+
+# Tests for null_values_filter
+
+
+def test_null_values_filter(sample_dataframe, nodes_data_range):
+    """Test filtering null values from a DataFrame."""
+    expected_data = {
+        'tag': ['wind_speed'],
+        'name': ['wind_speed'],
+        'value': [None],
+        'timestamp': [pd.Timestamp('2023-01-06')]
+    }
+    expected_df = pd.DataFrame(expected_data)
+
+    result = null_values_filter(sample_dataframe, nodes_data_range)
+    result = result.reset_index(drop=True)
+    expected_df = expected_df.reset_index(drop=True)
+
+    assert_frame_equal(result, expected_df, check_dtype=False)
+
+
+def test_null_values_filter_no_nulls(nodes_data_range):
+    """Test with a DataFrame containing no null values."""
+    df = pd.DataFrame({
+        'tag': ['temp', 'pressure'],
+        'name': ['temp', 'pressure'],
+        'value': [25, 100],
+        'timestamp': pd.date_range(start='2023-01-01', periods=2)
+    })
+    result = null_values_filter(df, nodes_data_range)
+    assert result.empty
+    assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']
+
+
+def test_null_values_filter_empty_df(nodes_data_range):
+    """Test with an empty DataFrame."""
+    df = pd.DataFrame(columns=['tag', 'name', 'value', 'timestamp'])
+    result = null_values_filter(df, nodes_data_range)
+    assert result.empty
+    assert list(result.columns) == ['tag', 'name', 'value', 'timestamp']

tests/workflow/sub_workflows/test_core_scouter.py

@@ -0,0 +1,136 @@
+from unittest.mock import AsyncMock, patch, call, ANY
+import pytest
+from scouter.workflow.sub_workflows.core_scouter import CoreScouter
+from scouter.activities.activities import Activities
+
+
+@pytest.fixture
+def core_scouter():
+    return CoreScouter()
+
+
+@pytest.mark.asyncio
+@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
+async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
+    mock_workflow.execute_local_activity_method.side_effect = [
+        'filtered_data', 'grouped_data', 'held_data']
+    await core_scouter.run(
+        input_data={
+            'workflow_name': 'test_workflow',
+            'schedule_name': 'test_schedule',
+            'model_name': 'test_model',
+            'model_id': 'test_model_id',
+            'data': 'test_data',
+            'trigger_laborious': False,
+            'filters': {'test_filter': 'test_value'},
+            'schema': 'test_schema',
+            'table_name': 'test_table',
+            'retention_time': 3600,
+            'model_tags': {}
+        }
+    )
+
+    mock_workflow.execute_local_activity_method.assert_has_calls([
+        call(
+            Activities.data_quality_gate,
+            {
+                'filters': {'test_filter': 'test_value'},
+                'data': 'test_data',
+                'model_tags': {}
+            },
+            retry_policy=ANY,
+            start_to_close_timeout=ANY
+        )])
+    mock_workflow.execute_local_activity_method.assert_has_calls([
+        call(
+            Activities.aggregate_data,
+            {
+                'data': 'filtered_data',
+                'model_tags': {}
+            },
+            retry_policy=ANY,
+            start_to_close_timeout=ANY
+        )])
+    mock_workflow.execute_local_activity_method.assert_has_calls([
+        call(
+            Activities.group_and_hold_data,
+            {
+                'workflow_name': 'test_workflow',
+                'schedule_name': 'test_schedule',
+                'data': 'grouped_data',
+                'model_id': 'test_model_id',
+                'retention_time': 3600
+            },
+            retry_policy=ANY,
+            start_to_close_timeout=ANY
+        )])
+
+    mock_workflow.execute_activity_method.assert_has_calls([
+        call(
+            Activities.export_data_to_postgres,
+            {
+                'schema': 'test_schema',
+                'table_name': 'test_table',
+                'data': 'held_data'},
+            retry_policy=ANY,
+            start_to_close_timeout=ANY
+        )
+    ])
+
+
+@pytest.mark.asyncio
+@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
+async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter):
+    mock_workflow.execute_local_activity_method.return_value = {}
+    await core_scouter.run(
+        input_data={
+            'workflow_name': 'test_workflow',
+            'schedule_name': 'test_schedule',
+            'model_name': 'test_model',
+            'model_id': 'test_model_id',
+            'data': 'test_data',
+            'trigger_laborious': False,
+            'filters': {'test_filter': 'test_value'},
+            'schema': 'test_schema',
+            'table_name': 'test_table',
+            'retention_time': 3600,
+            'model_tags': {}
+        }
+    )
+
+    mock_workflow.execute_local_activity_method.assert_has_calls([
+        call(
+            Activities.data_quality_gate,
+            {
+                'filters': {'test_filter': 'test_value'},
+                'data': 'test_data',
+                'model_tags': {}
+            },
+            retry_policy=ANY,
+            start_to_close_timeout=ANY
+        )])
+    mock_workflow.execute_local_activity_method.assert_has_calls([
+        call(
+            Activities.aggregate_data,
+            {
+                'data': {},
+                'model_tags': {}
+            },
+            retry_policy=ANY,
+            start_to_close_timeout=ANY
+        )])
+    mock_workflow.execute_local_activity_method.assert_has_calls([
+        call(
+            Activities.group_and_hold_data,
+            {
+                'workflow_name': 'test_workflow',
+                'schedule_name': 'test_schedule',
+                'data': {},
+                'model_id': 'test_model_id',
+                'retention_time': 3600
+            },
+            retry_policy=ANY,
+            start_to_close_timeout=ANY
+        )])
+
+    assert mock_workflow.execute_local_activity_method.call_count == 3

tests/workflow/test_fake_data.py

@@ -0,0 +1,29 @@
+from unittest.mock import AsyncMock, patch, ANY
+from pytest import fixture, mark
+from scouter.workflow.fake_data import FakeData
+from scouter.activities.faker import Faker
+
+
+@fixture
+def fake_data():
+    return FakeData()
+
+
+@mark.asyncio
+@patch('scouter.workflow.fake_data.workflow', new_callable=AsyncMock)
+async def test_fake_data_workflow(mock_workflow, fake_data):
+    mock_workflow.execute_activity_method.return_value = None
+    await fake_data.run(
+        {
+            'topic': 'test_topic'
+        }
+    )
+
+    mock_workflow.execute_activity_method.assert_called_once_with(
+        Faker.generate_and_send_data,
+        {
+            'topic': 'test_topic'
+        },
+        retry_policy=ANY,
+        start_to_close_timeout=ANY
+    )

tests/workflow/test_scouter.py

@@ -0,0 +1,38 @@
+from unittest.mock import AsyncMock, patch, ANY
+from pytest import fixture, mark
+from scouter.workflow.scouter import Scouter
+from scouter.activities.activities import Activities
+
+
+@fixture
+def scouter():
+    return Scouter()
+
+
+@mark.asyncio
+@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
+async def test_scouter_workflow(mock_workflow, scouter):
+
+    mock_workflow.execute_activity_method.return_value = 'test_data'
+    await scouter.run(
+        input_data={
+            'topic': 'test_topic'
+        }
+    )
+
+    mock_workflow.execute_activity_method.assert_called_once_with(
+        Activities.load_from_kafka,
+        {
+            'topic': 'test_topic'
+        },
+        retry_policy=ANY,
+        start_to_close_timeout=ANY
+    )
+
+    mock_workflow.execute_child_workflow.assert_called_once_with(
+        'core_scouter',
+        {
+            'topic': 'test_topic',
+            'data': 'test_data'
+        }
+    )