gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1110

Browse Source 
Update sientia-dataops-library version to 1.2.0 in requirements.txt; refactor logging in activities to use a unified Logger instance and include metadata in log messages across various activities.
This commit is contained in:
vitor-aignosi
2025-06-26 15:15:31 -03:00
parent 9355af1709
commit 4a90b77786
15 changed files with 238 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
tests/workflow/sub_workflows/test_core_scouter.py
View File

@@ -30,10 +30,20 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
}
}
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {}
@@ -45,6 +55,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
call(
Activities.aggregate_data,
{
**expected_metadata,
'data': 'filtered_data',
'model_tags': {}
},
@@ -55,6 +66,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': 'grouped_data',
@@ -69,6 +81,7 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
call(
Activities.export_data_to_postgres,
{
**expected_metadata,
'schema': 'test_schema',
'table_name': 'test_table',
'data': 'held_data'},
@@ -98,10 +111,20 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow'
}
}
mock_workflow.execute_local_activity_method.assert_has_calls([
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {}
@@ -113,6 +136,7 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
call(
Activities.aggregate_data,
{
**expected_metadata,
'data': {},
'model_tags': {}
},
@@ -123,6 +147,7 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': {},

36
tests/workflow/test_scouter.py
View File

@@ -23,21 +23,19 @@ async def test_scouter_workflow(mock_workflow, scouter):
}
)
mock_workflow.execute_local_activity_method.assert_called_once_with(
Activities.prepare_activity,
{
'workflow_name': 'scouter',
'schedule_name': 'test_schedule',
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'model_id': 'test_model_id'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.load_from_kafka,
{
**expected_metadata,
'topic': 'test_topic'
},
retry_policy=ANY,
@@ -70,21 +68,19 @@ async def test_scouter_workflow_empty(mock_workflow, scouter):
}
)
mock_workflow.execute_local_activity_method.assert_called_once_with(
Activities.prepare_activity,
{
'workflow_name': 'scouter',
'schedule_name': 'test_schedule',
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'model_id': 'test_model_id'
},
retry_policy=ANY,
start_to_close_timeout=ANY
)
'schedule_name': 'test_schedule',
'workflow_name': 'scouter'
}
}
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.load_from_kafka,
{
**expected_metadata,
'topic': 'test_topic'
},
retry_policy=ANY,