Code import - branch feature/SIENTIAPDE-1646

tests/activities/test_activities.py

@@ -0,0 +1,194 @@
+import inspect
+from unittest.mock import ANY, MagicMock, patch
+
+from sientia_do.temporal.activities.postgres_sync import Postgres
+
+from scouter.activities.activities import Activities
+from scouter.activities.api import API
+from scouter.activities.gates import Gates
+from scouter.activities.mongodb import MongoDB
+from scouter.activities.redis import Redis
+
+
+@patch('scouter.activities.activities.MongoDB.__init__')
+@patch('scouter.activities.activities.Postgres.__init__')
+@patch('scouter.activities.activities.Redis.__init__')
+@patch('scouter.activities.activities.Gates.__init__')
+@patch('scouter.activities.activities.API.__init__')
+@patch('scouter.activities.activities.MetricsController')
+def test___init__(
+    mock_metrics_controller,
+    mock_api_init,
+    mock_gates_init,
+    mock_redis_init,
+    mock_postgres_init,
+    mock_mongodb_init,
+):
+    postgres_config = {
+        'host': 'localhost',
+        'port': 5432,
+        'user': 'postgres',
+        'password': 'postgres',
+        'dbname': 'postgres',
+        'min_connections': 1,
+        'max_connections': 10,
+    }
+
+    redis_config = {'host': 'localhost', 'port': 6379, 'username': 'redis', 'password': 'redis'}
+
+    mongodb_config = {
+        'connection_string': 'mongodb://localhost:27017',
+        'database_name': 'test_database',
+    }
+
+    api_config = {
+        'base_url': 'https://api.example.com',
+        'auth_type': 'bearer',
+        'auth_token': 'test_token',
+    }
+
+    logger = MagicMock()
+    notification_handler = MagicMock()
+
+    activities = Activities(
+        postgres_config=postgres_config,
+        redis_config=redis_config,
+        mongodb_config=mongodb_config,
+        api_config=api_config,
+        logger=logger,
+        notification_handler=notification_handler,
+    )
+
+    assert isinstance(activities, Activities)
+    assert isinstance(activities, Postgres)
+    assert isinstance(activities, Redis)
+    assert isinstance(activities, MongoDB)
+    assert isinstance(activities, Gates)
+    assert isinstance(activities, API)
+
+    mock_postgres_init.assert_called_once_with(
+        ANY,
+        host=postgres_config['host'],
+        port=postgres_config['port'],
+        user=postgres_config['user'],
+        password=postgres_config['password'],
+        dbname=postgres_config['dbname'],
+        min_connections=postgres_config['min_connections'],
+        max_connections=postgres_config['max_connections'],
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_redis_init.assert_called_once_with(
+        ANY,
+        host=redis_config['host'],
+        port=redis_config['port'],
+        username=redis_config['username'],
+        password=redis_config['password'],
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_mongodb_init.assert_called_once_with(
+        ANY,
+        connection_string=mongodb_config['connection_string'],
+        database_name=mongodb_config['database_name'],
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_gates_init.assert_called_once_with(
+        ANY,
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+    mock_api_init.assert_called_once_with(
+        ANY,
+        base_url=api_config['base_url'],
+        auth_type=api_config['auth_type'],
+        auth_token=api_config['auth_token'],
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=mock_metrics_controller.return_value,
+    )
+
+
+@patch('scouter.activities.activities.Postgres.__init__')
+@patch('scouter.activities.activities.Redis.__init__')
+@patch('scouter.activities.activities.Gates.__init__')
+@patch('scouter.activities.activities.MongoDB.__init__')
+@patch('scouter.activities.activities.API.__init__')
+@patch('scouter.activities.activities.Postgres.close')
+@patch('scouter.activities.activities.MongoDB.close')
+@patch('scouter.activities.activities.Redis.close')
+@patch('scouter.activities.activities.Gates.close')
+@patch('scouter.activities.activities.API.close')
+def test_shutdown(
+    mock_api_close,
+    mock_gates_close,
+    mock_redis_close,
+    mock_mongodb_close,
+    mock_postgres_close,
+    _mock_api_init,
+    _mock_mongodb_init,
+    _mock_gates_init,
+    _mock_redis_init,
+    _mock_postgres_init,
+):
+    postgres_config = {
+        'host': 'localhost',
+        'port': 5432,
+        'user': 'postgres',
+        'password': 'postgres',
+        'dbname': 'postgres',
+        'min_connections': 1,
+        'max_connections': 10,
+    }
+
+    redis_config = {'host': 'localhost', 'port': 6379, 'username': 'redis', 'password': 'redis'}
+
+    mongodb_config = {
+        'connection_string': 'mongodb://localhost:27017',
+        'database_name': 'test_database',
+    }
+
+    api_config = {
+        'base_url': 'https://api.example.com',
+        'auth_type': 'bearer',
+        'auth_token': 'test_token',
+    }
+
+    logger = MagicMock()
+    notification_handler = MagicMock()
+
+    activities = Activities(
+        postgres_config=postgres_config,
+        redis_config=redis_config,
+        mongodb_config=mongodb_config,
+        api_config=api_config,
+        logger=logger,
+        notification_handler=notification_handler,
+    )
+
+    activities.shutdown()
+
+    mock_postgres_close.assert_called()
+    mock_mongodb_close.assert_called()
+    mock_redis_close.assert_called()
+    mock_gates_close.assert_called()
+    mock_api_close.assert_called()
+
+
+def test_activity_methods_are_sync():
+    """Every @activity.defn method on Activities must be a synchronous def."""
+    for cls in Activities.__mro__:
+        for name, member in vars(cls).items():
+            if getattr(member, '__temporal_activity_definition', None) is not None:
+                assert not inspect.iscoroutinefunction(member), (
+                    f'{cls.__name__}.{name} must not be async'
+                )

tests/activities/test_api.py

@@ -0,0 +1,340 @@
+from unittest.mock import ANY, MagicMock, Mock, patch
+
+import pandas as pd
+import pytest
+from sientia_do.notifications.models import NotificationLevel
+
+from scouter.activities.api import API
+
+
+@pytest.fixture
+@patch('scouter.activities.api.PIWebAPIClient')
+def api_activity(mock_pi_web_api_client):
+    """Fixture to create an API activity instance with mocked dependencies."""
+    logger = MagicMock()
+    notification_handler = MagicMock()
+    metrics_controller = MagicMock()
+
+    activity = API(
+        base_url='https://pi.example.com',
+        auth_type='basic',
+        auth_token='test_token',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+
+    activity.logger = logger
+    activity.notification_handler = notification_handler
+    activity.metrics_controller = metrics_controller
+    activity.pod_id = 'test_pod_id'
+
+    return activity
+
+
+metadata = {
+    'metadata': {
+        'model_id': 'test_model_id',
+        'model_name': 'test_model',
+        'schedule_name': 'test_schedule',
+        'workflow_name': 'pi_web_api_scouter',
+    }
+}
+
+
+@patch('scouter.activities.api.PIWebAPIClient')
+def test_api_initialization(mock_pi_web_api_client):
+    """Test API activity initialization."""
+    logger = MagicMock()
+    notification_handler = MagicMock()
+    metrics_controller = MagicMock()
+
+    activity = API(
+        base_url='https://pi.example.com',
+        auth_type='basic',
+        auth_token='test_token',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+
+    mock_pi_web_api_client.assert_called_once_with(
+        base_url='https://pi.example.com',
+        auth_config={
+            'type': 'basic',
+            'token': 'test_token',
+        },
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+        headers_config={
+            'Content-Type': 'application/json',
+            'Accept': 'application/json',
+            'x-requested-with': 'piwebapistreams',
+            'User-Agent': 'Aig-Scouter-Agent/1.0',
+        },
+    )
+
+    assert activity.pi_web_api_client is not None
+
+
+@patch('scouter.activities.api.SientiaMonitoring')
+def test_close(mock_sientia_monitoring, api_activity):
+    """Test close method."""
+    api_activity.close()
+    api_activity.pi_web_api_client.close.assert_called_once()
+    mock_sientia_monitoring.shutdown.assert_called_once()
+
+
+def test_get_tag_values_success(api_activity):
+    """Test get_tag_values with successful data retrieval."""
+    # Setup test data
+    test_data = {
+        **metadata,
+        'endpoint': '/streamsets/recorded',
+        'web_ids': {
+            'tag1': {
+                'webid': 'webid1',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+            },
+            'tag2': {
+                'webid': 'webid2',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+            },
+            'tag3': {
+                'webid': 'webid3',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+            },
+        },
+        'period': '*-1d',
+        'max_count': 10,
+        'api_timeout': 30,
+    }
+
+    # Mock DataFrame response
+    mock_df = pd.DataFrame(
+        {
+            'timestamp': [
+                '2023-01-01 12:00:00+0000',
+                '2023-01-01 12:01:00+0000',
+                '2023-01-01 12:02:00+0000',
+            ],
+            'name': ['tag1', 'tag2', 'tag3'],
+            'value': [10.5, 20.3, 30.7],
+            'tag': ['webid1', 'webid2', 'webid3'],
+        }
+    )
+
+    mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'])
+
+    api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df)
+
+    # Execute
+    result = api_activity.get_tag_values(test_data)
+
+    # Verify
+    api_activity.pi_web_api_client.get_latest_values_df.assert_called_once_with(
+        endpoint='/streamsets/recorded',
+        web_ids={
+            'tag1': {'webid': 'webid1', 'aggr_function': 'avg', 'data_range': [0, 100]},
+            'tag2': {'webid': 'webid2', 'aggr_function': 'avg', 'data_range': [0, 100]},
+            'tag3': {'webid': 'webid3', 'aggr_function': 'avg', 'data_range': [0, 100]},
+        },
+        start_time='*-1d',
+        end_time='*',
+        max_count=10,
+        metadata=metadata['metadata'],
+        request_timeout=30,
+    )
+
+    assert len(result) == 3
+    assert result[0]['name'] == 'tag1'
+    assert result[0]['value'] == pytest.approx(10.5)
+    assert result[1]['name'] == 'tag2'
+    assert result[2]['name'] == 'tag3'
+    assert result[0]['timestamp'] == '2023-01-01 12:02:00+0000'
+    assert result[1]['timestamp'] == '2023-01-01 12:02:00+0000'
+    assert result[2]['timestamp'] == '2023-01-01 12:02:00+0000'
+
+
+def test_get_tag_values_with_default_max_count(api_activity):
+    """Test get_tag_values with default max_count value."""
+    # Setup test data without max_count
+    test_data = {
+        **metadata,
+        'endpoint': '/streamsets/recorded',
+        'web_ids': {
+            'tag1': {
+                'webid': 'webid1',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+            }
+        },
+        'period': '*-1h',
+        'api_timeout': 15,
+    }
+
+    mock_df = pd.DataFrame(
+        {
+            'timestamp': ['2023-01-01 12:00:00+0000'],
+            'name': ['tag1'],
+            'value': [42.0],
+            'tag': ['webid1'],
+        }
+    )
+
+    mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'])
+
+    api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df)
+
+    # Execute
+    result = api_activity.get_tag_values(test_data)
+
+    # Verify default max_count is 1
+    api_activity.pi_web_api_client.get_latest_values_df.assert_called_once_with(
+        endpoint='/streamsets/recorded',
+        web_ids={'tag1': {'webid': 'webid1', 'aggr_function': 'avg', 'data_range': [0, 100]}},
+        start_time='*-1h',
+        end_time='*',
+        max_count=1,
+        metadata=metadata['metadata'],
+        request_timeout=15,
+    )
+
+    assert len(result) == 1
+
+
+def test_get_tag_values_with_none_webids(api_activity):
+    """Test get_tag_values with some None WebIds."""
+    # Setup test data with None values
+    test_data = {
+        **metadata,
+        'endpoint': '/streamsets/recorded',
+        'web_ids': {
+            'tag1': {
+                'webid': 'webid1',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+            },
+            'tag2': None,
+            'tag3': {
+                'webid': 'webid3',
+                'aggr_function': 'max',
+                'data_range': [0, 200],
+            },
+        },
+        'period': '*-1h',
+        'max_count': 5,
+        'api_timeout': 20,
+    }
+
+    mock_df = pd.DataFrame(
+        {
+            'timestamp': ['2023-01-01 12:00:00+0000', '2023-01-01 12:01:00+0000'],
+            'name': ['tag1', 'tag3'],
+            'value': [10.5, 30.7],
+            'tag': ['webid1', 'webid3'],
+        }
+    )
+
+    mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'])
+
+    api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df)
+
+    # Execute
+    result = api_activity.get_tag_values(test_data)
+
+    # Verify - should only query non-None WebIds
+    assert len(result) == 2
+    assert all(r['name'] in ['tag1', 'tag3'] for r in result)
+
+
+def test_get_tag_values_api_error(api_activity):
+    """Test get_tag_values when PI Web API client raises an error and sends notification."""
+    # Setup test data
+    test_data = {
+        **metadata,
+        'endpoint': '/streamsets/recorded',
+        'web_ids': {
+            'tag1': {
+                'webid': 'webid1',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+            }
+        },
+        'period': '*-1d',
+        'max_count': 1,
+        'api_timeout': 30,
+    }
+
+    # Mock API error
+    api_activity.pi_web_api_client.get_latest_values_df = Mock(
+        side_effect=Exception('PI Web API connection error')
+    )
+    api_activity.send_notification = MagicMock()
+
+    # Execute and verify exception is raised
+    with pytest.raises(Exception) as exc_info:
+        api_activity.get_tag_values(test_data)
+
+    assert str(exc_info.value) == 'PI Web API connection error'
+
+    # Verify notification was sent
+    api_activity.send_notification.assert_called_once_with(
+        metadata=metadata['metadata'],
+        notification_id='PI_WEB_API_REQUEST_ERROR',
+        message='Error getting tag values from PI Web API: PI Web API connection error',
+        block='get_tag_values',
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY,
+    )
+
+
+def test_get_tag_values_with_nan_values(api_activity):
+    """Test get_tag_values handling NaN values in the DataFrame."""
+    # Setup test data
+    test_data = {
+        **metadata,
+        'endpoint': '/streamsets/recorded',
+        'web_ids': {
+            'tag1': {
+                'webid': 'webid1',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+            },
+            'tag2': {
+                'webid': 'webid2',
+                'aggr_function': 'avg',
+                'data_range': [0, 100],
+            },
+        },
+        'period': '*-1d',
+        'max_count': 1,
+        'api_timeout': 30,
+    }
+
+    # Mock DataFrame with NaN values
+    mock_df = pd.DataFrame(
+        {
+            'timestamp': ['2023-01-01 12:00:00+0000', '2023-01-01 12:00:00+0000'],
+            'name': ['tag1', 'tag2'],
+            'value': [10.0, float('nan')],
+            'tag': ['webid1', 'webid2'],
+        }
+    )
+
+    mock_df['timestamp'] = pd.to_datetime(mock_df['timestamp'])
+
+    api_activity.pi_web_api_client.get_latest_values_df = Mock(return_value=mock_df)
+
+    # Execute
+    result = api_activity.get_tag_values(test_data)
+
+    # Verify
+    assert len(result) == 2
+    assert result[0]['value'] == pytest.approx(10.0)
+    # NaN should be preserved in the result
+    assert pd.isna(result[1]['value'])

tests/activities/test_gates.py

@@ -0,0 +1,409 @@
+from typing import Any
+from unittest.mock import ANY, MagicMock, Mock, call, patch
+
+import numpy as np
+import pandas as pd
+import pytest
+from sientia_do.notifications.models import NotificationLevel
+
+from scouter.activities.gates import Gates
+
+
+@pytest.fixture
+def gates_fixture():
+    """Fixture to create a Gates instance with mocked dependencies."""
+    logger = Mock()
+    notification_handler = MagicMock()
+    metrics_controller = MagicMock()
+    gates = Gates(
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+    gates.send_notification = MagicMock()
+
+    gates.logger = logger
+    gates.notification_handler = notification_handler
+    gates.metrics_controller = metrics_controller
+    gates.pod_id = 'localhost'
+    return gates
+
+
+metadata = {
+    'metadata': {
+        'model_id': 'test_model_id',
+        'model_name': 'test_model',
+        'schedule_name': 'test_schedule',
+        'workflow_name': 'scouter',
+    }
+}
+
+
+@patch('scouter.activities.gates.SientiaMonitoring')
+def test_close(mock_sientia_monitoring, gates_fixture):
+    """Test close method."""
+    gates_fixture.close()
+    mock_sientia_monitoring.shutdown.assert_called_once()
+
+
+def test_data_quality_gate_with_null_values_filter_discard(gates_fixture):
+    """Test data_quality_gate with NULL_VALUES_FILTER and DISCARD policy."""
+    # Setup test data
+    input_data = {
+        'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}},
+        'data': {
+            'name': ['tag1', 'tag2', 'tag3'],
+            'tag': ['tag1', 'tag2', 'tag3'],
+            'value': [1.0, None, 3.0],
+            'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'],
+        },
+        'model_tags': {
+            'tag1': {'data_range': [0, 100]},
+            'tag2': {'data_range': [0, 100]},
+            'tag3': {'data_range': [0, 100]},
+        },
+        **metadata,
+    }
+
+    # Execute
+    result = gates_fixture.data_quality_gate(input_data)
+
+    # Verify
+    assert len(result['tag']) == 2
+    assert 'tag2' not in result['tag']
+    gates_fixture.send_notification.assert_called_once()
+
+
+def test_data_quality_gate_with_out_of_bounds_filter_keep(gates_fixture):
+    """Test data_quality_gate with OUT_OF_BOUNDS_FILTER and KEEP policy."""
+    # Setup test data with out of bounds values
+    input_data = {
+        'filters': {'OUT_OF_BOUNDS_FILTER': {'policy': 'KEEP'}},
+        'data': {
+            'name': ['tag1', 'tag2', 'tag3'],
+            'tag': ['tag1', 'tag2', 'tag3'],
+            'value': [1.0, 200.0, 3.0],
+            'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03'],
+        },
+        'model_tags': {
+            'tag1': {'data_range': [0, 100]},
+            'tag2': {'data_range': [0, 100]},
+            'tag3': {'data_range': [0, 100]},
+        },
+        **metadata,
+    }
+
+    # Mock the out_of_bounds_filter to return rows with out of bounds values
+    with patch(
+        'scouter.activities.gates.quality_gate_filters',
+        {'OUT_OF_BOUNDS_FILTER': lambda df: df[df['tag'] == 'tag2']},
+    ):
+        # Execute
+        result = gates_fixture.data_quality_gate(input_data)
+
+    # Verify data is kept but notification is sent
+    assert len(result['tag']) == 3  # All rows kept
+    gates_fixture.send_notification.assert_called_once()
+
+
+def test_data_quality_gate_with_multiple_filters(gates_fixture):
+    """Test data_quality_gate with multiple filters."""
+    # Setup test data
+    input_data = {
+        'filters': {
+            'NULL_VALUES_FILTER': {'policy': 'DISCARD'},
+            'OUT_OF_BOUNDS_FILTER': {'policy': 'DISCARD'},
+        },
+        'data': {
+            'tag': ['tag1', 'tag2', 'tag3', 'tag4'],
+            'name': ['tag1', 'tag2', 'tag3', 'tag4'],
+            'value': [1.0, None, 300.0, 4.0],
+            'timestamp': ['2023-01-01', '2023-01-02', '2023-01-03', '2023-01-04'],
+        },
+        'model_tags': {
+            'tag1': {'data_range': [0, 100]},
+            'tag2': {'data_range': [0, 100]},
+            'tag3': {'data_range': [0, 100]},
+            'tag4': {'data_range': [0, 100]},
+        },
+        **metadata,
+    }
+
+    result = gates_fixture.data_quality_gate(input_data)
+
+    # Verify only tag1 and tag4 remain (tag2 has null, tag3 is out of bounds)
+    assert result == {
+        'tag': {0: 'tag1', 3: 'tag4'},
+        'name': {0: 'tag1', 3: 'tag4'},
+        'value': {0: 1.0, 3: 4.0},
+        'timestamp': {0: '2023-01-01', 3: '2023-01-04'},
+    }
+    # Should be called twice (once for each filter)
+    assert gates_fixture.send_notification.call_count == 2
+
+
+def test_data_quality_gate_with_unknown_filter(gates_fixture):
+    """Test data_quality_gate with an unknown filter."""
+    # Setup test data with unknown filter
+    gates_fixture.warning = MagicMock()
+    input_data = {
+        'filters': {'UNKNOWN_FILTER': {'policy': 'DISCARD'}},
+        'data': {'tag': ['tag1'], 'name': ['tag1'], 'value': [1.0], 'timestamp': ['2023-01-01']},
+        'model_tags': {'tag1': {'data_range': [0, 100]}},
+        **metadata,
+    }
+
+    # Execute
+    result = gates_fixture.data_quality_gate(input_data)
+
+    # Verify data is unchanged and warning is logged
+    assert len(result['tag']) == 1
+    gates_fixture.warning.assert_called_once_with(
+        'Filter UNKNOWN_FILTER not found', metadata=metadata['metadata']
+    )
+
+
+def test_data_quality_gate_with_filter_error(gates_fixture):
+    """Test data_quality_gate when a filter raises an exception."""
+    # Setup test data
+    input_data = {
+        'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}},
+        'data': {'tag': ['tag1'], 'name': ['tag1'], 'value': [1.0], 'timestamp': ['2023-01-01']},
+        'model_tags': {'tag1': {'data_range': [0, 100]}},
+        **metadata,
+    }
+
+    # Mock the filter to raise an exception
+    def failing_filter(_, _model_tags):
+        raise ValueError('Filter error')
+
+    with patch(
+        'scouter.activities.gates.quality_gate_filters', {'NULL_VALUES_FILTER': failing_filter}
+    ):
+        # Execute
+        result = gates_fixture.data_quality_gate(input_data)
+
+    # Verify error notification is sent and data is unchanged
+    assert len(result['tag']) == 1
+    gates_fixture.send_notification.assert_called_once()
+    call_args = gates_fixture.send_notification.call_args[1]
+    assert call_args['notification_id'] == 'DATA_QUALITY_GATE_ISSUES'
+    assert call_args['level'] == NotificationLevel.ERROR
+    assert 'Filter error' in call_args['message']
+
+
+def test_data_quality_gate_with_empty_data(gates_fixture):
+    """Test data_quality_gate with empty input data."""
+    # Setup empty input data
+    input_data = {
+        'filters': {'NULL_VALUES_FILTER': {'policy': 'DISCARD'}},
+        'data': {'tag': [], 'name': [], 'value': [], 'timestamp': []},
+        'model_tags': {},
+        **metadata,
+    }
+
+    # Execute
+    result = gates_fixture.data_quality_gate(input_data)
+
+    # Verify empty result and no notifications
+    assert len(result['tag']) == 0
+    gates_fixture.send_notification.assert_not_called()
+
+
+def test_data_quality_gate_with_no_filters(gates_fixture):
+    """Test data_quality_gate with no filters specified."""
+    # Setup test data with no filters
+    input_data = {
+        'filters': {},
+        'data': {'tag': ['tag1'], 'name': ['tag1'], 'value': [1.0], 'timestamp': ['2023-01-01']},
+        'model_tags': {'tag1': {'data_range': [0, 100]}},
+        **metadata,
+    }
+
+    # Execute
+    result = gates_fixture.data_quality_gate(input_data)
+
+    # Verify data is unchanged and no notifications
+    assert len(result['tag']) == 1
+    gates_fixture.send_notification.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    'group_data, aggr_function, expected_result',
+    [
+        # Single value case
+        (pd.DataFrame({'value': [10.0]}), 'avg', 10.0),
+        # Multiple values with different aggregation functions
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'avg', 2.5),
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'mdn', 2.5),
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'max', 4.0),
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'min', 1.0),
+        (pd.DataFrame({'value': [1.0, 2.0, 3.0, 4.0]}), 'lts', 4.0),
+        # With NaN values
+        (pd.DataFrame({'value': [1.0, np.nan, 3.0, 4.0]}), 'avg', 2.6666666666666665),
+        # Empty group after dropping NaN
+        (pd.DataFrame({'value': [np.nan, np.nan]}), 'avg', None),
+        # Invalid aggregation function
+        (pd.DataFrame({'value': [1.0, 2.0]}), 'invalid', 'continue'),
+        (pd.DataFrame({'value': [10.0]}), 'invalid', 'continue'),
+    ],
+)
+def test_apply_aggregation(gates_fixture, group_data, aggr_function, expected_result):
+    """Test apply_aggregation method with various scenarios."""
+    result = gates_fixture.apply_aggregation(group_data, aggr_function, metadata)
+    assert result == expected_result
+
+    # Check notification was sent for invalid function
+    if aggr_function == 'invalid':
+        gates_fixture.send_notification.assert_called_once()
+    else:
+        gates_fixture.send_notification.assert_not_called()
+
+
+def test_aggregate_data(gates_fixture):
+    """Test aggregate_data method with multiple groups and aggregation functions."""
+    input_data = {
+        'data': [
+            {'tag': 'tag1', 'name': 'name1', 'value': 1.0, 'timestamp': '2023-01-01'},
+            {'tag': 'tag1', 'name': 'name1', 'value': 2.0, 'timestamp': '2023-01-02'},
+            {'tag': 'tag1', 'name': 'name1', 'value': 3.0, 'timestamp': '2023-01-03'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'},
+            {'tag': 'tag1', 'name': 'name1', 'value': None, 'timestamp': '2023-01-04'},
+        ],
+        'model_tags': {
+            'name1': {'aggr_func': 'avg'},
+            'name2': {'aggr_func': 'max'},
+        },
+        **metadata,
+    }
+
+    # Expected result
+    expected_result = {
+        'tag': {0: 'tag1', 1: 'tag2'},
+        'name': {0: 'name1', 1: 'name2'},
+        'value': {0: 2.0, 1: 6.0},
+        'timestamp': {0: '2023-01-04', 1: '2023-01-03'},
+        'aggregation_function': {0: 'avg', 1: 'max'},
+    }
+
+    # Execute
+    result = gates_fixture.aggregate_data(input_data)
+
+    # Verify
+    assert result == expected_result
+    gates_fixture.send_notification.assert_not_called()
+
+
+def test_aggregate_data_with_continue(gates_fixture):
+    gates_fixture.apply_aggregation = MagicMock(return_value='continue')
+
+    input_data = {
+        'data': [
+            {'tag': 'tag1', 'name': 'name1', 'value': 1.0, 'timestamp': '2023-01-01'},
+            {'tag': 'tag1', 'name': 'name1', 'value': 2.0, 'timestamp': '2023-01-02'},
+            {'tag': 'tag1', 'name': 'name1', 'value': 3.0, 'timestamp': '2023-01-03'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'},
+            {'tag': 'tag1', 'name': 'name1', 'value': None, 'timestamp': '2023-01-04'},
+        ],
+        'model_tags': {
+            'name1': {'aggr_function': 'avg'},
+            'name2': {'aggr_function': 'max'},
+        },
+        **metadata,
+    }
+
+    # Expected result
+    expected_result: dict[str, Any] = {}
+
+    # Execute
+    result = gates_fixture.aggregate_data(input_data)
+
+    # Verify
+    assert result == expected_result
+    gates_fixture.send_notification.assert_not_called()
+
+
+def test_aggregate_data_raise_exception(gates_fixture):
+    gates_fixture.apply_aggregation = MagicMock(side_effect=Exception('Test exception'))
+
+    input_data = {
+        'data': [
+            {'tag': 'tag1', 'name': 'name1', 'value': 1.0, 'timestamp': '2023-01-01'},
+            {'tag': 'tag1', 'name': 'name1', 'value': 2.0, 'timestamp': '2023-01-02'},
+            {'tag': 'tag1', 'name': 'name1', 'value': 3.0, 'timestamp': '2023-01-03'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 4.0, 'timestamp': '2023-01-01'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 5.0, 'timestamp': '2023-01-02'},
+            {'tag': 'tag2', 'name': 'name2', 'value': 6.0, 'timestamp': '2023-01-03'},
+            {'tag': 'tag1', 'name': 'name1', 'value': None, 'timestamp': '2023-01-04'},
+        ],
+        'model_tags': {
+            'name1': {'aggr_function': 'avg'},
+            'name2': {'aggr_function': 'max'},
+        },
+        **metadata,
+    }
+
+    try:
+        gates_fixture.aggregate_data(input_data)
+    except Exception as e:
+        assert str(e) == 'Test exception'
+        gates_fixture.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='AGGREGATION_ISSUES',
+            message='Error aggregating data: Test exception',
+            block='aggregate_data',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+    else:
+        raise AssertionError('Exception not raised')
+
+
+@patch('scouter.activities.gates.metrics')
+def test_write_metrics(mock_metrics, gates_fixture):
+    """Test write_metrics method."""
+    input_data = {
+        'metadata': metadata['metadata'],
+        'tag_values': {
+            'variable': ['tag1', 'tag2', 'tag3'],
+            'value': [1.0, 2.0, None],
+        },
+    }
+    gates_fixture.write_metrics(input_data)
+    mock_metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels.assert_called_once_with(
+        pod_id=gates_fixture.pod_id,
+        model_name=metadata['metadata']['model_name'],
+        workflow_name=metadata['metadata']['workflow_name'],
+    )
+    mock_metrics.LABORIOUS_DATA_WRITTEN_COUNT.labels.return_value.inc.assert_called_once()
+
+    # Only tags with None values should be registered
+    mock_metrics.TAG_CHANGES_MONITOR.labels.return_value.set.assert_has_calls(
+        [
+            call(1.0),
+            call(2.0),
+        ],
+        any_order=True,
+    )
+
+    mock_metrics.TAG_CHANGES_MONITOR.labels.assert_has_calls(
+        [
+            call(
+                pod_id=gates_fixture.pod_id,
+                model_name=metadata['metadata']['model_name'],
+                workflow_name=metadata['metadata']['workflow_name'],
+                tag_name='tag1',
+            ),
+            call(
+                pod_id=gates_fixture.pod_id,
+                model_name=metadata['metadata']['model_name'],
+                workflow_name=metadata['metadata']['workflow_name'],
+                tag_name='tag2',
+            ),
+        ],
+        any_order=True,
+    )

tests/activities/test_mongo.py

@@ -0,0 +1,174 @@
+from datetime import datetime
+from unittest.mock import ANY, MagicMock, Mock, patch
+
+from pytest import fixture
+from sientia_do.notifications.models import NotificationLevel
+from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
+
+from scouter.activities.mongodb import MongoDB
+
+
+@patch('scouter.activities.mongodb.MongoDBRepository')
+def test_mongodb___init__(mock_mongodb_repository):
+    """Test MongoDB __init__"""
+
+    logger = MagicMock()
+    notification_handler = MagicMock()
+    metrics_controller = MagicMock()
+    mongo = MongoDB(
+        connection_string='mongodb://localhost:27017',
+        database_name='test_db',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+
+    mock_mongodb_repository.assert_called_once_with(
+        connection_string='mongodb://localhost:27017',
+        database_name='test_db',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+
+    assert mongo.mongodb_repository is not None
+
+
+@fixture
+@patch('scouter.activities.mongodb.MongoDBRepository')
+def mongodb_activity(mock_mongodb_repository):
+    """Test MongoDB activity"""
+
+    mongo = MongoDB(
+        connection_string='mongodb://localhost:27017',
+        database_name='test_db',
+        logger=MagicMock(),
+        notification_handler=MagicMock(),
+        metrics_controller=MagicMock(),
+    )
+    return mongo
+
+
+@patch('scouter.activities.mongodb.SientiaMonitoring')
+def test_close(mock_sientia_monitoring, mongodb_activity):
+    """Test close"""
+    mongodb_activity.close()
+
+    mongodb_activity.mongodb_repository.close.assert_called_once()
+    mock_sientia_monitoring.shutdown.assert_called_once()
+
+
+def test_del(mongodb_activity):
+    mongodb_activity.close = MagicMock()
+
+    mongodb_activity.__del__()
+
+    mongodb_activity.close.assert_called_once()
+
+
+def test_load_latest_data_none_last_data_timestamp(mongodb_activity):
+    """Test load_latest_data"""
+
+    mongodb_activity.mongodb_repository.find = Mock(
+        return_value=[
+            {
+                'name': 'test1',
+                'value': 1,
+                'inserted_at': datetime.strptime(
+                    '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
+                ),
+            }
+        ]
+    )
+
+    result = mongodb_activity.load_latest_data(
+        {
+            'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+            'collection_name': 'test_collection',
+            'last_data_timestamp': None,
+        }
+    )
+
+    mongodb_activity.mongodb_repository.find.assert_called_once_with(
+        collection_name='test_collection',
+        filters={},
+        metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+    )
+
+    assert result == [
+        {
+            'name': 'test1',
+            'value': 1,
+            'inserted_at': '2023-01-01 12:00:00.000000+0000',
+        }
+    ]
+
+
+def test_load_latest_data_not_none_last_data_timestamp(mongodb_activity):
+    """Test load_latest_data"""
+
+    mongodb_activity.mongodb_repository.find = Mock(
+        return_value=[
+            {
+                'name': 'test1',
+                'value': 1,
+                'inserted_at': datetime.strptime(
+                    '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
+                ),
+            }
+        ]
+    )
+
+    result = mongodb_activity.load_latest_data(
+        {
+            'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+            'collection_name': 'test_collection',
+            'last_data_timestamp': '2023-01-01 12:00:00.000000+0000',
+        }
+    )
+
+    mongodb_activity.mongodb_repository.find.assert_called_once_with(
+        collection_name='test_collection',
+        filters={
+            'inserted_at': {
+                '$gt': datetime.strptime(
+                    '2023-01-01 12:00:00.000000+0000', DATETIME_FORMAT_MS_WITH_TZ
+                )
+            }
+        },
+        metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+    )
+
+    assert result == [
+        {
+            'name': 'test1',
+            'value': 1,
+            'inserted_at': '2023-01-01 12:00:00.000000+0000',
+        }
+    ]
+
+
+def test_load_latest_data_error(mongodb_activity):
+    """Test load_latest_data"""
+    mongodb_activity.mongodb_repository.find.side_effect = Exception('test')
+    mongodb_activity.send_notification = MagicMock()
+
+    try:
+        mongodb_activity.load_latest_data(
+            {
+                'metadata': {'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+                'collection_name': 'test_collection',
+                'last_data_timestamp': '2023-01-01 12:00:00.000000+0000',
+            }
+        )
+    except Exception as e:
+        assert str(e) == 'test'
+
+        mongodb_activity.send_notification.assert_called_once_with(
+            metadata={'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'},
+            notification_id='MONGO_LOAD_ERROR',
+            message='Error loading data from MongoDB: test',
+            block='load_latest_data',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )

tests/activities/test_redis.py

@@ -0,0 +1,525 @@
+from datetime import datetime
+from unittest.mock import ANY, MagicMock, Mock, patch
+
+import numpy as np
+import pytest
+from pandas import DataFrame
+from sientia_do.notifications.handlers import CoreNotificationHandler as NotificationHandler
+from sientia_do.notifications.models import NotificationLevel
+
+from scouter.activities.redis import Redis
+
+
+@pytest.fixture
+@patch('scouter.activities.redis.RedisRepository')
+def redis_activity(mock_redis_repository):
+    logger = MagicMock()
+    notification_handler = MagicMock(spec=NotificationHandler)
+    metrics_controller = MagicMock()
+    activity = Redis(
+        host='localhost',
+        port=6379,
+        logger=logger,
+        notification_handler=notification_handler,
+        username='test',
+        password='test',
+        metrics_controller=metrics_controller,
+    )
+
+    activity.redis_client = MagicMock()
+    activity.logger = logger
+    activity.notification_handler = notification_handler
+    activity.pod_id = 'test_pod_id'
+    return activity
+
+
+metadata = {
+    'metadata': {
+        'model_id': 'test_model_id',
+        'model_name': 'test_model',
+        'schedule_name': 'test_schedule',
+        'workflow_name': 'scouter',
+    }
+}
+
+
+@patch('scouter.activities.redis.SientiaMonitoring')
+def test_close(mock_sientia_monitoring, redis_activity):
+    """Test close method."""
+    redis_activity.close()
+    redis_activity.redis_repository.close.assert_called_once()
+    mock_sientia_monitoring.shutdown.assert_called_once()
+
+
+@patch('scouter.activities.redis.RedisRepository')
+def test_redis_initialization(mock_redis_repository):
+    """Test Redis activity initialization"""
+    logger = MagicMock()
+    notification_handler = MagicMock(spec=NotificationHandler)
+    metrics_controller = MagicMock()
+    activity = Redis(
+        host='localhost',
+        port=6379,
+        logger=logger,
+        notification_handler=notification_handler,
+        username='test',
+        password='test',
+        metrics_controller=metrics_controller,
+    )
+    mock_redis_repository.assert_called_once_with(
+        host='localhost',
+        port=6379,
+        username='test',
+        password='test',
+        logger=logger,
+        notification_handler=notification_handler,
+        metrics_controller=metrics_controller,
+    )
+
+    assert activity.redis_repository is not None
+
+
+def test_get_last_data_timestamp_none(redis_activity):
+    """Test get_last_data_timestamp"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+    }
+
+    redis_activity.redis_repository.get = Mock(return_value=None)
+
+    result = redis_activity.get_last_data_timestamp(test_data)
+
+    redis_activity.redis_repository.get.assert_called_once_with(
+        'last_data_timestamp:test_pipeline:test_schedule',
+        metadata=metadata['metadata'],
+    )
+
+    assert result is None
+
+
+def test_get_last_data_timestamp_not_none(redis_activity):
+    """Test get_last_data_timestamp"""
+    test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
+
+    redis_activity.redis_repository.get = Mock(return_value='2023-01-01 12:00:00')
+
+    result = redis_activity.get_last_data_timestamp(test_data)
+
+    redis_activity.redis_repository.get.assert_called_once_with(
+        'last_data_timestamp:test_pipeline:test_schedule',
+        metadata=metadata['metadata'],
+    )
+
+    assert result == '2023-01-01 12:00:00'
+
+
+def test_get_last_data_timestamp_error(redis_activity):
+    """Test get_last_data_timestamp error"""
+    test_data = {**metadata, 'workflow_name': 'test_pipeline', 'schedule_name': 'test_schedule'}
+
+    redis_activity.send_notification = Mock()
+    redis_activity.redis_repository.get.side_effect = Exception('test')
+
+    try:
+        redis_activity.get_last_data_timestamp(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        redis_activity.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='REDIS_GET_ERROR',
+            message='Error getting last data timestamp: test',
+            block='get_last_data_timestamp',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected exception')
+
+
+def test_put_last_data_timestamp_empty_dataframe(redis_activity):
+    """Test put_last_data_timestamp with empty dataframe"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
+    }
+
+    redis_activity.redis_repository.set = MagicMock()
+
+    result = redis_activity.put_last_data_timestamp(test_data)
+
+    assert result is None
+
+    redis_activity.redis_repository.set.assert_not_called()
+
+
+def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
+    """Test put_last_data_timestamp with not empty dataframe"""
+
+    data = DataFrame(
+        {
+            'name': ['sensor1', 'sensor2'],
+            'value': [25.5, 30.0],
+            'inserted_at': ['2023-01-01 12:00:00', '2023-01-01 12:00:01'],
+        }
+    )
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': data.to_dict('records'),
+    }
+
+    redis_activity.redis_repository.set = Mock()
+
+    result = redis_activity.put_last_data_timestamp(test_data)
+
+    assert result == '2023-01-01 12:00:01'
+
+    redis_activity.redis_repository.set.assert_called_once_with(
+        'last_data_timestamp:test_pipeline:test_schedule',
+        '2023-01-01 12:00:01',
+        ttl=18000,
+        metadata=metadata['metadata'],
+    )
+
+
+def test_put_last_data_timestamp_error(redis_activity):
+    """Test put_last_data_timestamp error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': DataFrame(
+            {
+                'name': ['sensor1', 'sensor2'],
+                'value': [25.5, 30.0],
+                'inserted_at': ['2023-01-01 12:00:00'] * 2,
+            }
+        ).to_dict('records'),
+    }
+
+    redis_activity.send_notification = Mock()
+    redis_activity.redis_repository.set = Mock(side_effect=Exception('test'))
+
+    try:
+        redis_activity.put_last_data_timestamp(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        redis_activity.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='REDIS_SET_ERROR',
+            message='Error setting last data timestamp: test',
+            block='put_last_data_timestamp',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected exception')
+
+
+def test_group_and_hold_data_new_key(redis_activity):
+    """Test group_and_hold_data with a new key"""
+    # Setup
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame(
+            {
+                'name': ['sensor1', 'sensor2'],
+                'value': [25.5, 30.0],
+                'timestamp': ['2023-01-01 12:00:00'] * 2,
+            }
+        ).to_dict('records'),
+        'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
+        'fill_missing_tags': False,
+    }
+
+    # Mock get to return None for new key
+    redis_activity.redis_repository.get = Mock(return_value=None)
+    redis_activity.redis_repository.set = Mock()
+
+    # Call the method
+    result = redis_activity.group_and_hold_data(test_data)
+
+    # Verify the result
+    expected_result = {
+        'timestamp': {0: '2023-01-01 12:00:00', 1: '2023-01-01 12:00:00'},
+        'variable': {0: 'sensor1', 1: 'sensor2'},
+        'value': {0: 25.5, 1: 30.0},
+        'model_id': {0: 1, 1: 1},
+    }
+    assert result == expected_result
+
+    # Verify set was called with correct arguments
+    redis_activity.redis_repository.set.assert_called_once_with(
+        'held_data_test_pipeline_test_schedule',
+        {'sensor1': 25.5, 'sensor2': 30.0, 'timestamp': '2023-01-01 12:00:00'},
+        ttl=3600,
+        metadata=metadata['metadata'],
+    )
+
+
+def test_group_and_hold_data_update_existing_fill_missing(redis_activity):
+    """Test updating existing data with group_and_hold_data"""
+    # Setup initial data in Redis
+    existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
+
+    # New data to update with
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame(
+            {
+                'name': ['sensor1', 'sensor3'],
+                'value': [25.5, 42.0],
+                'timestamp': ['2023-01-01 12:00:00'] * 2,
+            }
+        ).to_dict('records'),
+        'model_tags': {
+            'sensor1': 'sensor1',
+            'sensor2': 'sensor2',
+            'sensor3': 'sensor3',
+            'sensor4': 'sensor4',
+        },
+        'fill_missing_tags': True,
+    }
+
+    # Mock get to return existing data
+    redis_activity.redis_repository.get = Mock(return_value=existing_data)
+    redis_activity.redis_repository.set = Mock()
+
+    # Call the method
+    result = redis_activity.group_and_hold_data(test_data)
+
+    # Verify the result
+    expected_result = {
+        'timestamp': {
+            0: '2023-01-01 12:00:00',
+            1: '2023-01-01 12:00:00',
+            2: '2023-01-01 12:00:00',
+            3: '2023-01-01 12:00:00',
+        },
+        'variable': {0: 'sensor1', 1: 'sensor2', 2: 'sensor3', 3: 'sensor4'},
+        'value': {0: 25.5, 1: 28.0, 2: 42.0, 3: None},
+        'model_id': {0: 1, 1: 1, 2: 1, 3: 1},
+    }
+    assert result == expected_result
+
+    # Verify set was called with correct arguments
+    redis_activity.redis_repository.set.assert_called_once_with(
+        'held_data_test_workflow_test_schedule',
+        {
+            'sensor1': 25.5,
+            'sensor2': 28.0,
+            'sensor3': 42.0,
+            'sensor4': None,
+            'timestamp': '2023-01-01 12:00:00',
+        },
+        ttl=3600,
+        metadata=metadata['metadata'],
+    )
+
+
+def test_group_and_hold_data_with_none_values(redis_activity):
+    """Test handling of None values in group_and_hold_data"""
+    # Setup test data with None values
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame(
+            {
+                'name': ['sensor1', 'sensor2'],
+                'value': [None, 30.0],
+                'timestamp': [datetime(2023, 1, 1, 12, 0, 0)] * 2,
+            }
+        ).to_dict('records'),
+        'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
+        'fill_missing_tags': False,
+    }
+
+    # Mock get to return None for new key
+    redis_activity.redis_repository.get = Mock(return_value=None)
+    redis_activity.redis_repository.set = Mock()
+
+    # Call the method
+    result = redis_activity.group_and_hold_data(test_data)
+
+    # Verify None was converted to np.nan and values are as expected
+    assert np.isnan(result['value'][0])
+    assert result['value'][1] == pytest.approx(30.0)
+
+
+def test_group_and_hold_data_empty_dataframe(redis_activity):
+    """Test group_and_hold_data with empty DataFrame"""
+    # Setup test with empty data
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
+        'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
+        'fill_missing_tags': False,
+    }
+
+    redis_activity.redis_repository.get = Mock(return_value=None)
+
+    # Call the method
+    result = redis_activity.group_and_hold_data(test_data)
+
+    assert result == {}
+
+
+def test_group_and_hold_data_error_get(redis_activity):
+    """Test group_and_hold_data error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
+        'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
+        'fill_missing_tags': False,
+    }
+
+    redis_activity.redis_repository.get = Mock(side_effect=Exception('test'))
+    redis_activity.send_notification = Mock()
+
+    try:
+        redis_activity.group_and_hold_data(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        redis_activity.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id='REDIS_GET_ERROR',
+            message='Error getting held data: test',
+            block='group_and_hold_data',
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY,
+        )
+
+    else:
+        raise AssertionError('Expected exception')
+
+
+def test_group_and_hold_data_error_set(redis_activity):
+    """Test group_and_hold_data error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records'),
+        'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
+        'fill_missing_tags': False,
+    }
+
+    existing_data = {'sensor1': 20.0, 'sensor2': 28.0, 'timestamp': '2023-01-01 11:00:00'}
+
+    # Mock get to return existing data
+    redis_activity.redis_repository.get = Mock(return_value=existing_data)
+    redis_activity.redis_repository.set = Mock(side_effect=Exception('test'))
+    redis_activity.send_notification = Mock()
+
+    try:
+        redis_activity.group_and_hold_data(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+
+def test_store_data_package(redis_activity):
+    """Test store_data_package"""
+    redis_activity.redis_repository.set = Mock()
+
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'held_data': DataFrame(
+            {
+                'name': ['sensor1', 'sensor2'],
+                'value': [25.5, 30.0],
+                'timestamp': ['2023-01-01 12:00:00'] * 2,
+            }
+        ).to_dict(),
+        'data': DataFrame(
+            {
+                'name': ['sensor1', 'sensor2'],
+                'value': [25.5, 30.0],
+                'timestamp': ['2023-01-01 12:00:00'] * 2,
+            }
+        ).to_dict(),
+        'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
+    }
+
+    redis_activity.store_data_package(test_data)
+
+    redis_activity.redis_repository.set.assert_called_once_with(
+        ANY,
+        {'data': test_data['data'], 'held_data': test_data['held_data']},
+        ttl=120,
+        metadata=metadata['metadata'],
+    )
+
+
+def test_store_data_package_error(redis_activity):
+    """Test store_data_package error"""
+    redis_activity.redis_repository.set = Mock(side_effect=ValueError('test'))
+    redis_activity.send_notification = Mock()
+
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'held_data': DataFrame(
+            {
+                'name': ['sensor1', 'sensor2'],
+                'value': [25.5, 30.0],
+                'timestamp': ['2023-01-01 12:00:00'] * 2,
+            }
+        ).to_dict(),
+        'data': DataFrame(
+            {
+                'name': ['sensor1', 'sensor2'],
+                'value': [25.5, 30.0],
+                'timestamp': ['2023-01-01 12:00:00'] * 2,
+            }
+        ).to_dict(),
+        'model_tags': {'sensor1': 'sensor1', 'sensor2': 'sensor2'},
+    }
+
+    with pytest.raises(ValueError):
+        redis_activity.store_data_package(test_data)
+
+    redis_activity.send_notification.assert_called_once_with(
+        metadata=metadata['metadata'],
+        notification_id='REDIS_SET_ERROR',
+        message='Error setting data package: test',
+        block='store_data_package',
+        level=NotificationLevel.ERROR,
+        attachment_content=ANY,
+    )