diff --git a/tests/activities/test_api.py b/tests/activities/test_api.py index 91bd374..77810aa 100644 --- a/tests/activities/test_api.py +++ b/tests/activities/test_api.py @@ -149,8 +149,8 @@ async def test_get_tag_values_success(api_activity): assert result[0]['value'] == 10.5 assert result[1]['name'] == 'tag2' assert result[2]['name'] == 'tag3' - assert result[0]['timestamp'] == '2023-01-01 12:00:00+0000' - assert result[1]['timestamp'] == '2023-01-01 12:01:00+0000' + assert result[0]['timestamp'] == '2023-01-01 12:02:00+0000' + assert result[1]['timestamp'] == '2023-01-01 12:02:00+0000' assert result[2]['timestamp'] == '2023-01-01 12:02:00+0000' diff --git a/tests/workflow/sub_workflows/test_core_scouter.py b/tests/workflow/sub_workflows/test_core_scouter.py index 259b721..608985e 100644 --- a/tests/workflow/sub_workflows/test_core_scouter.py +++ b/tests/workflow/sub_workflows/test_core_scouter.py @@ -20,6 +20,11 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter): 'grouped_data', 'held_data', ] + mock_workflow.execute_activity_method.side_effect = [ + {'affected_rows': 10}, # export_data_to_postgres + None, # write_metrics + None, # store_data_package + ] await core_scouter.run( input_data={ 'metadata': { @@ -113,6 +118,22 @@ async def test_core_scouter_workflow_success(mock_workflow, core_scouter): 'column': 'timestamp', 'format': DATETIME_FORMAT_WITH_TZ, }, + 'on_conflict': 'ignore', + 'unique_columns': ['model_id', 'timestamp', 'variable'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + mock_workflow.execute_activity_method.assert_has_calls( + [ + call( + Activities.write_metrics, + { + **expected_metadata, + 'tag_values': 'held_data', }, retry_policy=ANY, start_to_close_timeout=ANY, @@ -222,3 +243,161 @@ async def test_core_scouter_workflow_with_empty_data(mock_workflow, core_scouter ) assert mock_workflow.execute_local_activity_method.call_count == 3 + mock_workflow.execute_activity_method.assert_not_called() + + +@pytest.mark.asyncio +@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock) +async def test_core_scouter_workflow_with_zero_affected_rows(mock_workflow, core_scouter): + """ + Test that workflow stops after export when no rows are affected + """ + mock_workflow.execute_local_activity_method.side_effect = [ + 'filtered_data', + 'grouped_data', + 'held_data', + ] + mock_workflow.execute_activity_method.return_value = {'affected_rows': 0} + + await core_scouter.run( + input_data={ + 'metadata': { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + }, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'data': 'test_data', + 'trigger_laborious': False, + 'filters': {'test_filter': 'test_value'}, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'retention_time': 3600, + 'model_tags': {}, + 'debug_data_package': True, + 'fill_missing_tags': False, + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + } + + mock_workflow.execute_activity_method.assert_called_once_with( + Activities.export_data_to_postgres, + { + **expected_metadata, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'data': 'held_data', + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + 'on_conflict': 'ignore', + 'unique_columns': ['model_id', 'timestamp', 'variable'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + + +@pytest.mark.asyncio +@patch('scouter.workflow.sub_workflows.core_scouter.workflow', new_callable=AsyncMock) +async def test_core_scouter_workflow_without_debug_data_package(mock_workflow, core_scouter): + """ + Test that store_data_package is not called when debug_data_package is False + """ + mock_workflow.execute_local_activity_method.side_effect = [ + 'filtered_data', + 'grouped_data', + 'held_data', + ] + mock_workflow.execute_activity_method.side_effect = [ + {'affected_rows': 5}, + None, + ] + + await core_scouter.run( + input_data={ + 'metadata': { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + }, + 'workflow_name': 'test_workflow', + 'schedule_name': 'test_schedule', + 'model_name': 'test_model', + 'model_id': 'test_model_id', + 'data': 'test_data', + 'trigger_laborious': False, + 'filters': {'test_filter': 'test_value'}, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'retention_time': 3600, + 'model_tags': {}, + 'debug_data_package': False, + 'fill_missing_tags': False, + } + ) + + expected_metadata = { + 'metadata': { + 'model_id': 'test_model_id', + 'model_name': 'test_model', + 'schedule_name': 'test_schedule', + 'workflow_name': 'test_workflow', + } + } + + mock_workflow.execute_activity_method.assert_has_calls( + [ + call( + Activities.export_data_to_postgres, + { + **expected_metadata, + 'schema': 'test_schema', + 'table_name': 'test_table', + 'data': 'held_data', + 'timestamp_conversion': { + 'column': 'timestamp', + 'format': DATETIME_FORMAT_WITH_TZ, + }, + 'on_conflict': 'ignore', + 'unique_columns': ['model_id', 'timestamp', 'variable'], + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + mock_workflow.execute_activity_method.assert_has_calls( + [ + call( + Activities.write_metrics, + { + **expected_metadata, + 'tag_values': 'held_data', + }, + retry_policy=ANY, + start_to_close_timeout=ANY, + ) + ] + ) + + assert mock_workflow.execute_activity_method.call_count == 2