gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1110

Browse Source 
Refactor Activities class to remove Kafka and Druid dependencies, simplifying initialization. Update values.yaml to set replica count to 1 for reduced resource usage. Adjust Redis activity to set TTL to None for better data retention. Remove unused Kafka and Druid activity files and their associated tests, streamlining the codebase.
This commit is contained in:
vitor-aignosi
2025-07-03 16:25:55 -03:00
parent 1dcba6b48e
commit d08d1b1337
13 changed files with 105 additions and 369 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
tests/workflow/test_scouter.py
View File

@@ -1,4 +1,4 @@
from unittest.mock import AsyncMock, patch, ANY
from unittest.mock import AsyncMock, patch, ANY, call
from pytest import fixture, mark
from scouter.workflow.scouter import Scouter
from scouter.activities.activities import Activities
@@ -13,7 +13,10 @@ def scouter():
@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
async def test_scouter_workflow(mock_workflow, scouter):
mock_workflow.execute_activity_method.return_value = 'test_data'
mock_workflow.execute_local_activity_method.side_effect = [
'test_last_data_timestamp',
'test_data'
]
await scouter.run(
input_data={
'topic': 'test_topic',
@@ -32,11 +35,41 @@ async def test_scouter_workflow(mock_workflow, scouter):
}
}
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.get_last_data_timestamp,
{
**expected_metadata,
'workflow_name': 'scouter',
'schedule_name': 'test_schedule'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.load_latest_data,
{
**expected_metadata,
'collection_name': "raw_test_schedule",
'last_data_timestamp': 'test_last_data_timestamp'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.load_from_kafka,
Activities.put_last_data_timestamp,
{
**expected_metadata,
'topic': 'test_topic'
'data': 'test_data',
'workflow_name': 'scouter',
'schedule_name': 'test_schedule'
},
retry_policy=ANY,
start_to_close_timeout=ANY
@@ -45,6 +78,7 @@ async def test_scouter_workflow(mock_workflow, scouter):
mock_workflow.execute_child_workflow.assert_called_once_with(
'core_scouter',
{
'metadata': expected_metadata,
'topic': 'test_topic',
'data': 'test_data',
'workflow_name': 'scouter',
@@ -58,7 +92,10 @@ async def test_scouter_workflow(mock_workflow, scouter):
@mark.asyncio
@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
async def test_scouter_workflow_empty(mock_workflow, scouter):
mock_workflow.execute_activity_method.return_value = {}
mock_workflow.execute_local_activity_method.side_effect = [
'test_last_data_timestamp',
{}
]
await scouter.run(
input_data={
'topic': 'test_topic',
@@ -77,14 +114,19 @@ async def test_scouter_workflow_empty(mock_workflow, scouter):
}
}
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.load_from_kafka,
{
**expected_metadata,
'topic': 'test_topic'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.load_latest_data,
{
**expected_metadata,
'collection_name': "raw_test_schedule",
'last_data_timestamp': 'test_last_data_timestamp'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_activity_method.assert_not_called()
mock_workflow.execute_child_workflow.assert_not_called()