gitea_admin/sientia-dataops-scouter_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Code import - branch feature/SIENTIAPDE-1646

Browse Source
This commit is contained in:
gitea_admin
2026-06-28 03:03:00 +00:00
commit 1be8c97e5a
87 changed files with 10783 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
tests/workflow/__init__.py Normal file
View File

397
tests/workflow/sub_workflows/test_core_scouter.py Normal file
View File

@@ -0,0 +1,397 @@
from unittest.mock import ANY, AsyncMock, call, patch
import pytest
from sientia_do.temporal.constants import DATETIME_FORMAT_WITH_TZ
from scouter.activities.activities import Activities
from scouter.workflow.sub_workflows.core_scouter import CoreScouter
@pytest.fixture
def core_scouter():
return CoreScouter()
@pytest.mark.asyncio
@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
async def test_core_scouter_workflow_success(mock_workflow, core_scouter):
mock_workflow.execute_local_activity_method.side_effect = [
'filtered_data',
'grouped_data',
'held_data',
]
mock_workflow.execute_activity_method.side_effect = [
{'affected_rows': 10}, # export_data_to_postgres
None, # write_metrics
None, # store_data_package
]
await core_scouter.run(
input_data={
'metadata': {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow',
}
},
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id',
'data': 'test_data',
'trigger_laborious': False,
'filters': {'test_filter': 'test_value'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'model_tags': {},
'debug_data_package': True,
'fill_missing_tags': False,
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow',
}
}
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.aggregate_data,
{**expected_metadata, 'data': 'filtered_data', 'model_tags': {}},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': 'grouped_data',
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {},
'fill_missing_tags': False,
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_has_calls(
[
call(
Activities.export_data_to_postgres,
{
**expected_metadata,
'schema': 'test_schema',
'table_name': 'test_table',
'data': 'held_data',
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ,
},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_has_calls(
[
call(
Activities.write_metrics,
{
**expected_metadata,
'tag_values': 'held_data',
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_has_calls(
[
call(
Activities.store_data_package,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'held_data': 'held_data',
'data': 'test_data',
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
@pytest.mark.asyncio
@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter):
mock_workflow.execute_local_activity_method.return_value = {}
await core_scouter.run(
input_data={
'metadata': {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow',
}
},
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id',
'data': 'test_data',
'trigger_laborious': False,
'filters': {'test_filter': 'test_value'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'model_tags': {},
'fill_missing_tags': False,
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow',
}
}
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.data_quality_gate,
{
**expected_metadata,
'filters': {'test_filter': 'test_value'},
'data': 'test_data',
'model_tags': {},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.aggregate_data,
{**expected_metadata, 'data': {}, 'model_tags': {}},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.group_and_hold_data,
{
**expected_metadata,
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'data': {},
'model_id': 'test_model_id',
'retention_time': 3600,
'model_tags': {},
'fill_missing_tags': False,
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
assert mock_workflow.execute_local_activity_method.call_count == 3
mock_workflow.execute_activity_method.assert_not_called()
@pytest.mark.asyncio
@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
async def test_core_scouter_workflow_with_zero_affected_rows(mock_workflow, core_scouter):
"""
Test that workflow stops after export when no rows are affected
"""
mock_workflow.execute_local_activity_method.side_effect = [
'filtered_data',
'grouped_data',
'held_data',
]
mock_workflow.execute_activity_method.return_value = {'affected_rows': 0}
await core_scouter.run(
input_data={
'metadata': {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow',
}
},
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id',
'data': 'test_data',
'trigger_laborious': False,
'filters': {'test_filter': 'test_value'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'model_tags': {},
'debug_data_package': True,
'fill_missing_tags': False,
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow',
}
}
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.export_data_to_postgres,
{
**expected_metadata,
'schema': 'test_schema',
'table_name': 'test_table',
'data': 'held_data',
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ,
},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
@pytest.mark.asyncio
@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock)
async def test_core_scouter_workflow_without_debug_data_package(mock_workflow, core_scouter):
"""
Test that store_data_package is not called when debug_data_package is False
"""
mock_workflow.execute_local_activity_method.side_effect = [
'filtered_data',
'grouped_data',
'held_data',
]
mock_workflow.execute_activity_method.side_effect = [
{'affected_rows': 5},
None,
]
await core_scouter.run(
input_data={
'metadata': {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow',
}
},
'workflow_name': 'test_workflow',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id',
'data': 'test_data',
'trigger_laborious': False,
'filters': {'test_filter': 'test_value'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'model_tags': {},
'debug_data_package': False,
'fill_missing_tags': False,
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'test_workflow',
}
}
mock_workflow.execute_activity_method.assert_has_calls(
[
call(
Activities.export_data_to_postgres,
{
**expected_metadata,
'schema': 'test_schema',
'table_name': 'test_table',
'data': 'held_data',
'timestamp_conversion': {
'column': 'timestamp',
'format': DATETIME_FORMAT_WITH_TZ,
},
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_has_calls(
[
call(
Activities.write_metrics,
{
**expected_metadata,
'tag_values': 'held_data',
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
assert mock_workflow.execute_activity_method.call_count == 2

133
tests/workflow/test_pi_web_api_scouter.py Normal file
View File

@@ -0,0 +1,133 @@
from unittest.mock import ANY, AsyncMock, patch
from pytest import fixture, mark
from scouter.activities.activities import Activities
from scouter.workflow.pi_web_api_scouter import PIWebAPIScouter
@fixture
def pi_web_api_scouter():
return PIWebAPIScouter()
@mark.asyncio
@patch('scouter.workflow.pi_web_api_scouter.workflow', new_callable=AsyncMock)
async def test_pi_web_api_scouter_workflow(mock_workflow, pi_web_api_scouter):
mock_workflow.execute_local_activity_method.return_value = 'test_data'
await pi_web_api_scouter.run(
input_data={
'model_name': 'test_model',
'model_id': 'test_model_id',
'schedule_name': 'test_schedule',
'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
'trigger_laborious': True,
'filters': {'quality': 'good'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'pi_web_api_query': {
'endpoint': '/streamsets/recorded',
'period': '*-1d',
'max_count': 10,
'api_timeout': 30,
},
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'pi_web_api_scouter',
}
}
mock_workflow.execute_local_activity_method.assert_called_once_with(
Activities.get_tag_values,
{
**expected_metadata,
'endpoint': '/streamsets/recorded',
'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'},
'period': '*-1d',
'api_timeout': 30,
'max_count': 10,
},
start_to_close_timeout=ANY,
retry_policy=ANY,
)
mock_workflow.execute_child_workflow.assert_called_once_with(
'subworkflow.core_scouter',
{
'model_name': 'test_model',
'model_id': 'test_model_id',
'schedule_name': 'test_schedule',
'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
'trigger_laborious': True,
'filters': {'quality': 'good'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'workflow_name': 'pi_web_api_scouter',
'data': 'test_data',
'metadata': expected_metadata,
'pi_web_api_query': {
'endpoint': '/streamsets/recorded',
'period': '*-1d',
'max_count': 10,
'api_timeout': 30,
},
},
)
@mark.asyncio
@patch('scouter.workflow.pi_web_api_scouter.workflow', new_callable=AsyncMock)
async def test_pi_web_api_scouter_workflow_empty(mock_workflow, pi_web_api_scouter):
mock_workflow.execute_local_activity_method.return_value = []
await pi_web_api_scouter.run(
input_data={
'model_name': 'test_model',
'model_id': 'test_model_id',
'schedule_name': 'test_schedule',
'model_tags': {'tag1': 'webid1', 'tag2': 'webid2'},
'trigger_laborious': True,
'filters': {'quality': 'good'},
'schema': 'test_schema',
'table_name': 'test_table',
'retention_time': 3600,
'pi_web_api_query': {
'endpoint': '/streamsets/recorded',
'period': '*-1d',
'max_count': 1,
'api_timeout': 30,
},
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'pi_web_api_scouter',
}
}
mock_workflow.execute_local_activity_method.assert_called_once_with(
Activities.get_tag_values,
{
**expected_metadata,
'web_ids': {'tag1': 'webid1', 'tag2': 'webid2'},
'period': '*-1d',
'api_timeout': 30,
'max_count': 1,
'endpoint': '/streamsets/recorded',
},
start_to_close_timeout=ANY,
retry_policy=ANY,
)
mock_workflow.execute_child_workflow.assert_not_called()

129
tests/workflow/test_scouter.py Normal file
View File

@@ -0,0 +1,129 @@
from unittest.mock import ANY, AsyncMock, call, patch
from pytest import fixture, mark
from scouter.activities.activities import Activities
from scouter.workflow.scouter import Scouter
@fixture
def scouter():
return Scouter()
@mark.asyncio
@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
async def test_scouter_workflow(mock_workflow, scouter):
mock_workflow.execute_local_activity_method.side_effect = [
'test_last_data_timestamp',
'test_data',
]
await scouter.run(
input_data={
'topic': 'test_topic',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id',
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter',
}
}
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.get_last_data_timestamp,
{**expected_metadata, 'workflow_name': 'scouter', 'schedule_name': 'test_schedule'},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.load_latest_data,
{
**expected_metadata,
'collection_name': 'raw_test_schedule',
'last_data_timestamp': 'test_last_data_timestamp',
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_called_once_with(
Activities.put_last_data_timestamp,
{
**expected_metadata,
'data': 'test_data',
'workflow_name': 'scouter',
'schedule_name': 'test_schedule',
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
mock_workflow.execute_child_workflow.assert_called_once_with(
'subworkflow.core_scouter',
{
'metadata': expected_metadata,
'topic': 'test_topic',
'data': 'test_data',
'workflow_name': 'scouter',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id',
},
)
@mark.asyncio
@patch('scouter.workflow.scouter.workflow', new_callable=AsyncMock)
async def test_scouter_workflow_empty(mock_workflow, scouter):
mock_workflow.execute_local_activity_method.side_effect = ['test_last_data_timestamp', {}]
await scouter.run(
input_data={
'topic': 'test_topic',
'schedule_name': 'test_schedule',
'model_name': 'test_model',
'model_id': 'test_model_id',
}
)
expected_metadata = {
'metadata': {
'model_id': 'test_model_id',
'model_name': 'test_model',
'schedule_name': 'test_schedule',
'workflow_name': 'scouter',
}
}
mock_workflow.execute_local_activity_method.assert_has_calls(
[
call(
Activities.load_latest_data,
{
**expected_metadata,
'collection_name': 'raw_test_schedule',
'last_data_timestamp': 'test_last_data_timestamp',
},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
mock_workflow.execute_activity_method.assert_not_called()
mock_workflow.execute_child_workflow.assert_not_called()