SIENTIAPDE-1005
Implement workflows for fake data generation, scouter processing, and core scouter operations - Added `FakeData` workflow to generate random data and send it to a Kafka topic. - Implemented `Scouter` workflow to load data from Kafka and trigger the core scouter workflow. - Created `CoreScouter` workflow to process data through quality gates, aggregation, and export to PostgreSQL. - Developed comprehensive unit tests for activities and workflows, ensuring proper functionality and error handling. - Enhanced Redis and Postgres activities with robust testing for data handling and error notifications. - Introduced quality filters for data validation and implemented tests to verify their functionality.
This commit is contained in:
vitor-aignosi
136
tests/workflow/sub_workflows/test_core_scouter.py
Normal file
136
tests/workflow/sub_workflows/test_core_scouter.py
Normal file
@@ -0,0 +1,136 @@
|
||||
from unittest.mock import AsyncMock, patch, call, ANY
|
||||
import pytest
|
||||
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
|
||||
from scouter.activities.activities import Activities
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def core_scouter():
|
||||
return CoreScouter()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
|
||||
async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
|
||||
mock_workflow.execute_local_activity_method.side_effect = [
|
||||
'filtered_data', 'grouped_data', 'held_data']
|
||||
await core_scouter.run(
|
||||
input_data={
|
||||
'workflow_name': 'test_workflow',
|
||||
'schedule_name': 'test_schedule',
|
||||
'model_name': 'test_model',
|
||||
'model_id': 'test_model_id',
|
||||
'data': 'test_data',
|
||||
'trigger_laborious': False,
|
||||
'filters': {'test_filter': 'test_value'},
|
||||
'schema': 'test_schema',
|
||||
'table_name': 'test_table',
|
||||
'retention_time': 3600,
|
||||
'model_tags': {}
|
||||
}
|
||||
)
|
||||
|
||||
mock_workflow.execute_local_activity_method.assert_has_calls([
|
||||
call(
|
||||
Activities.data_quality_gate,
|
||||
{
|
||||
'filters': {'test_filter': 'test_value'},
|
||||
'data': 'test_data',
|
||||
'model_tags': {}
|
||||
},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY
|
||||
)])
|
||||
mock_workflow.execute_local_activity_method.assert_has_calls([
|
||||
call(
|
||||
Activities.aggregate_data,
|
||||
{
|
||||
'data': 'filtered_data',
|
||||
'model_tags': {}
|
||||
},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY
|
||||
)])
|
||||
mock_workflow.execute_local_activity_method.assert_has_calls([
|
||||
call(
|
||||
Activities.group_and_hold_data,
|
||||
{
|
||||
'workflow_name': 'test_workflow',
|
||||
'schedule_name': 'test_schedule',
|
||||
'data': 'grouped_data',
|
||||
'model_id': 'test_model_id',
|
||||
'retention_time': 3600
|
||||
},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY
|
||||
)])
|
||||
|
||||
mock_workflow.execute_activity_method.assert_has_calls([
|
||||
call(
|
||||
Activities.export_data_to_postgres,
|
||||
{
|
||||
'schema': 'test_schema',
|
||||
'table_name': 'test_table',
|
||||
'data': 'held_data'},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY
|
||||
)
|
||||
])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
|
||||
async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter):
|
||||
mock_workflow.execute_local_activity_method.return_value = {}
|
||||
await core_scouter.run(
|
||||
input_data={
|
||||
'workflow_name': 'test_workflow',
|
||||
'schedule_name': 'test_schedule',
|
||||
'model_name': 'test_model',
|
||||
'model_id': 'test_model_id',
|
||||
'data': 'test_data',
|
||||
'trigger_laborious': False,
|
||||
'filters': {'test_filter': 'test_value'},
|
||||
'schema': 'test_schema',
|
||||
'table_name': 'test_table',
|
||||
'retention_time': 3600,
|
||||
'model_tags': {}
|
||||
}
|
||||
)
|
||||
|
||||
mock_workflow.execute_local_activity_method.assert_has_calls([
|
||||
call(
|
||||
Activities.data_quality_gate,
|
||||
{
|
||||
'filters': {'test_filter': 'test_value'},
|
||||
'data': 'test_data',
|
||||
'model_tags': {}
|
||||
},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY
|
||||
)])
|
||||
mock_workflow.execute_local_activity_method.assert_has_calls([
|
||||
call(
|
||||
Activities.aggregate_data,
|
||||
{
|
||||
'data': {},
|
||||
'model_tags': {}
|
||||
},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY
|
||||
)])
|
||||
mock_workflow.execute_local_activity_method.assert_has_calls([
|
||||
call(
|
||||
Activities.group_and_hold_data,
|
||||
{
|
||||
'workflow_name': 'test_workflow',
|
||||
'schedule_name': 'test_schedule',
|
||||
'data': {},
|
||||
'model_id': 'test_model_id',
|
||||
'retention_time': 3600
|
||||
},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY
|
||||
)])
|
||||
|
||||
assert mock_workflow.execute_local_activity_method.call_count == 3
|
||||
29
tests/workflow/test_fake_data.py
Normal file
29
tests/workflow/test_fake_data.py
Normal file
@@ -0,0 +1,29 @@
|
||||
from unittest.mock import AsyncMock, patch, ANY
|
||||
from pytest import fixture, mark
|
||||
from scouter.workflow.fake_data import FakeData
|
||||
from scouter.activities.faker import Faker
|
||||
|
||||
|
||||
@fixture
|
||||
def fake_data():
|
||||
return FakeData()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('scouter.workflow.fake_data.workflow', new_callable=AsyncMock)
|
||||
async def test_fake_data_workflow(mock_workflow, fake_data):
|
||||
mock_workflow.execute_activity_method.return_value = None
|
||||
await fake_data.run(
|
||||
{
|
||||
'topic': 'test_topic'
|
||||
}
|
||||
)
|
||||
|
||||
mock_workflow.execute_activity_method.assert_called_once_with(
|
||||
Faker.generate_and_send_data,
|
||||
{
|
||||
'topic': 'test_topic'
|
||||
},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY
|
||||
)
|
||||
38
tests/workflow/test_scouter.py
Normal file
38
tests/workflow/test_scouter.py
Normal file
@@ -0,0 +1,38 @@
|
||||
from unittest.mock import AsyncMock, patch, ANY
|
||||
from pytest import fixture, mark
|
||||
from scouter.workflow.scouter import Scouter
|
||||
from scouter.activities.activities import Activities
|
||||
|
||||
|
||||
@fixture
|
||||
def scouter():
|
||||
return Scouter()
|
||||
|
||||
|
||||
@mark.asyncio
|
||||
@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
|
||||
async def test_scouter_workflow(mock_workflow, scouter):
|
||||
|
||||
mock_workflow.execute_activity_method.return_value = 'test_data'
|
||||
await scouter.run(
|
||||
input_data={
|
||||
'topic': 'test_topic'
|
||||
}
|
||||
)
|
||||
|
||||
mock_workflow.execute_activity_method.assert_called_once_with(
|
||||
Activities.load_from_kafka,
|
||||
{
|
||||
'topic': 'test_topic'
|
||||
},
|
||||
retry_policy=ANY,
|
||||
start_to_close_timeout=ANY
|
||||
)
|
||||
|
||||
mock_workflow.execute_child_workflow.assert_called_once_with(
|
||||
'core_scouter',
|
||||
{
|
||||
'topic': 'test_topic',
|
||||
'data': 'test_data'
|
||||
}
|
||||
)
|
||||
Reference in New Issue
Block a user