gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1316

Browse Source 
Update .gitignore and refactor metrics.py, activities.py, and gates.py for improved clarity and consistency. Added coverage.xml and cache directories to .gitignore. Standardized string formatting and parameter handling in metrics and activities classes, enhancing code readability. Removed the deprecated faker.py file and adjusted related tests accordingly.
This commit is contained in:
vitor-aignosi
2025-10-16 13:31:12 -03:00
parent 8bdbf049b8
commit 97eb5bc904
27 changed files with 1101 additions and 1336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

253
tests/workflow/sub_workflows/test_core_scouter.py
View File

@@ -1,9 +1,11 @@
from unittest.mock import AsyncMock, patch, call, ANY
from unittest.mock import ANY, AsyncMock, call, patch
import pytest
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
from scouter.activities.activities import Activities
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from scouter.activities.activities import Activities
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
@pytest.fixture
def core_scouter():
@@ -14,7 +16,10 @@ def core_scouter():
@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
mock_workflow.execute_local_activity_method.side_effect = [
'filtered_data', 'grouped_data', 'held_data']
'filtered_data',
'grouped_data',
'held_data',
]
await core_scouter.run(
input_data={
'metadata': {
@@ -22,7 +27,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
'workflow_name': 'test_workflow',
}
},
'workflow_name': 'test_workflow',
@@ -36,7 +41,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
'table_name': 'test_table',
'retention_time': 3600,
'model_tags': {},
'debug_data_package': True
'debug_data_package': True,
}
)
@@ -45,81 +50,90 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
'workflow_name': 'test_workflow',
}
}
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.aggregate_data,
{
**expected_metadata,
'data': 'filtered_data',
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': 'grouped_data',
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.aggregate_data,
{**expected_metadata, 'data': 'filtered_data', 'model_tags': {}},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': 'grouped_data',
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_has_calls([
call(
Activities.export_data_to_postgres,
{
**expected_metadata,
'schema': 'test_schema',
'table_name': 'test_table',
'data': 'held_data',
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ
}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_activity_method.assert_has_calls(
[
call(
Activities.export_data_to_postgres,
{
**expected_metadata,
'schema': 'test_schema',
'table_name': 'test_table',
'data': 'held_data',
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ,
},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_has_calls([
call(
Activities.store_data_package,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'held_data': 'held_data',
'data': 'test_data'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
])
mock_workflow.execute_activity_method.assert_has_calls(
[
call(
Activities.store_data_package,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'held_data': 'held_data',
'data': 'test_data',
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
@pytest.mark.asyncio
@@ -133,7 +147,7 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
'workflow_name': 'test_workflow',
}
},
'workflow_name': 'test_workflow',
@@ -146,7 +160,7 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'model_tags': {}
'model_tags': {},
}
)
@@ -155,47 +169,52 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
'workflow_name': 'test_workflow',
}
}
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.aggregate_data,
{
**expected_metadata,
'data': {},
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': {},
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {}
},
retry_policy=ANY,
start_to_close_timeout=ANY
)])
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.aggregate_data,
{**expected_metadata, 'data': {}, 'model_tags': {}},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': {},
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
assert mock_workflow.execute_local_activity_method.call_count == 3