SIENTIAPDE-1148

Update sonar-project.properties to exclude worker.py from coverage, enhance error handling in worker.py, and add new tests for error scenarios in test_redis.py and test_mongo.py.

scouter/worker/worker.py

@@ -102,7 +102,7 @@ async def main():
     try:
         await asyncio.gather(*handlers)
 
-    except BaseException as e:
+    except BaseException as e:  # NOSONAR
         logger.error("An unhandled exception occurred: %s", e, exc_info=True)
     finally:
         if notification_handler:

sonar-project.properties

@@ -5,6 +5,7 @@ sonar.tests=tests
 sonar.qualitygate.wait=true
 sonar.qualitygate.timeout=300
 sonar.python.coverage.reportPaths=coverage.xml
+sonar.coverage.exclusions=scouter/worker/worker.py
 sonar.python.xunit.reportPath=pytest.xml
 sonar.python.version=3.11
 sonar.projectVersion=1.0.0

tests/activities/test_mongo.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from unittest.mock import ANY, MagicMock, patch
+from unittest.mock import ANY, MagicMock, call, patch
 from pytest import fixture, mark
 from sientia_do.notifications.models import NotificationLevel
 from scouter.activities.mongodb import MongoDB, clear_mongo_id
@@ -12,12 +12,22 @@ def test_clear_mongo_id():
         {'_id': '2', 'name': [{
             '_id': '3',
             'name': 'test3'
-        }]}
+        }]},
+        {'_id': '4', 'name': {
+            '_id': '5',
+            'name': 'test2'
+        }},
+        [{'_id': '6', 'name': 'test2'}]
     ]
 
     result = clear_mongo_id(data)
 
-    assert result == [{'name': 'test1'}, {'name': [{'name': 'test3'}]}]
+    assert result == [
+        {'name': 'test1'},
+        {'name': [{'name': 'test3'}]},
+        {'name': {'name': 'test2'}},
+        [{'name': 'test2'}]
+    ]
 
 
 @patch('scouter.activities.mongodb.MongoClient')

tests/activities/test_redis.py

@@ -88,6 +88,38 @@ async def test_get_last_data_timestamp_not_none(redis_activity):
     assert result == '2023-01-01 12:00:00'
 
 
+@pytest.mark.asyncio
+async def test_get_last_data_timestamp_error(redis_activity):
+    """Test get_last_data_timestamp error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule'
+    }
+
+    redis_activity.send_notification = MagicMock()
+    redis_activity.get = MagicMock(side_effect=Exception('test'))
+
+    try:
+
+        await redis_activity.get_last_data_timestamp(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        redis_activity.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id="REDIS_GET_ERROR",
+            message="Error getting last data timestamp: test",
+            block="get_last_data_timestamp",
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY
+        )
+
+    else:
+        assert False, "Expected exception"
+
+
 @pytest.mark.asyncio
 async def test_put_last_data_timestamp_empty_dataframe(redis_activity):
     """Test put_last_data_timestamp with empty dataframe"""
@@ -136,6 +168,42 @@ async def test_put_last_data_timestamp_not_empty_dataframe(redis_activity):
     )
 
 
+@pytest.mark.asyncio
+async def test_put_last_data_timestamp_error(redis_activity):
+    """Test put_last_data_timestamp error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_pipeline',
+        'schedule_name': 'test_schedule',
+        'data': DataFrame({
+            'name': ['sensor1', 'sensor2'],
+            'value': [25.5, 30.0],
+            'inserted_at': ['2023-01-01 12:00:00'] * 2
+        }).to_dict('records')
+    }
+
+    redis_activity.send_notification = MagicMock()
+    redis_activity.set = MagicMock(side_effect=Exception('test'))
+
+    try:
+        await redis_activity.put_last_data_timestamp(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        redis_activity.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id="REDIS_SET_ERROR",
+            message="Error setting last data timestamp: test",
+            block="put_last_data_timestamp",
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY
+        )
+
+    else:
+        assert False, "Expected exception"
+
+
 @pytest.mark.asyncio
 async def test_group_and_hold_data_new_key(redis_activity):
     """Test group_and_hold_data with a new key"""
@@ -283,6 +351,63 @@ async def test_group_and_hold_data_empty_dataframe(redis_activity):
     assert result == {}
 
 
+@pytest.mark.asyncio
+async def test_group_and_hold_data_error_get(redis_activity):
+    """Test group_and_hold_data error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records')
+    }
+
+    redis_activity.get = MagicMock(side_effect=Exception('test'))
+    redis_activity.send_notification = MagicMock()
+
+    try:
+        await redis_activity.group_and_hold_data(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+        redis_activity.send_notification.assert_called_once_with(
+            metadata=metadata['metadata'],
+            notification_id="REDIS_GET_ERROR",
+            message="Error getting held data: test",
+            block="group_and_hold_data",
+            level=NotificationLevel.ERROR,
+            attachment_content=ANY
+        )
+
+    else:
+        assert False, "Expected exception"
+
+
+@pytest.mark.asyncio
+async def test_group_and_hold_data_error_set(redis_activity):
+    """Test group_and_hold_data error"""
+    test_data = {
+        **metadata,
+        'workflow_name': 'test_workflow',
+        'schedule_name': 'test_schedule',
+        'retention_time': 3600,
+        'model_id': 1,
+        'data': DataFrame(columns=['name', 'value', 'timestamp']).to_dict('records')
+    }
+
+    redis_activity.get = MagicMock(return_value=None)
+    redis_activity.set = MagicMock(side_effect=Exception('test'))
+    redis_activity.send_notification = MagicMock()
+
+    try:
+        await redis_activity.group_and_hold_data(test_data)
+
+    except Exception as e:
+        assert str(e) == 'test'
+
+
 @pytest.mark.asyncio
 async def test_store_data_package(redis_activity):
     """Test store_data_package"""