gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SIENTIAPDE-1445

Browse Source 
SIENTIAPDE-1273 Add base_scouter and pi_web_api_scouter functions to orchestrator utilities for enhanced configuration management. Update tests to validate new functionalities and ensure proper integration with existing workflows.
This commit is contained in:
vitor-aignosi
2025-12-18 10:02:47 -03:00
parent 62e7a7dee9
commit 61d4b23e69
2 changed files with 252 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
tests/orchestrator/utils/test_orchestrator_functions.py
View File

@@ -1,12 +1,14 @@
from unittest.mock import call, patch
from orchestrator.utils.orchestrator_functions import (
base_scouter,
build_tag_config,
common_config,
drift,
gather_read_tags,
minimal_retrain,
overlap_filter_config,
pi_web_api_scouter,
predictions_batch,
process_path_priority,
scouter,
@@ -403,3 +405,196 @@ def test_build_tag_config():
}
assert result == (expected, ['3'])
def test_base_scouter():
config = {
'workflow_type': 'scouter',
'schedule_name': 'test_schedule',
'model_id': 'test_model_id',
'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
'filters': [
{'filter_name': 'test_filter_name', 'policy': 'test_policy'},
{'filter_name': 'test_filter_name2', 'policy': 'test_policy2'},
],
'tag_retention_minutes': 30,
'debug_data_package': True,
'fill_missing_tags': True,
}
result = base_scouter(config)
expected = {
'workflow_type': 'scouter',
'schedule_name': 'test_schedule',
'frequency': '1m',
'offset': '0m',
'max_retry_policy': 1,
'model_id': 'test_model_id',
'model_name': 'test_model_name',
'model_config': {'test_config': 'test_config'},
'trigger_laborious': False,
'filters': {
'test_filter_name': {'policy': 'test_policy'},
'test_filter_name2': {'policy': 'test_policy2'},
},
'schema': 'sientia_data',
'table_name': 'laborious_data',
'retention_time': 30 * 60,
'debug_data_package': True,
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'fill_missing_tags': True,
}
assert result == expected
def test_pi_web_api_scouter():
config = {
'workflow_type': 'scouter',
'schedule_name': 'test_schedule',
'model_id': 'test_model_id',
'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
'filters': [{'filter_name': 'test_filter_name', 'policy': 'test_policy'}],
'read_tags': [
{
'tag_name': 'test_tag_name',
'webid': 'test_webid',
'aggr_func': 'test_aggr_func',
'data_range': [1, 2],
}
],
'tag_retention_minutes': 10,
'pi_web_api_config': {
'endpoint': 'https://test-endpoint.com',
'period': '*-2d',
'max_count': 5,
'api_timeout': 30,
},
'frequency': '1m',
}
result = pi_web_api_scouter(config)
expected = {
'workflow_type': 'scouter',
'schedule_name': 'test_schedule',
'frequency': '1m',
'offset': '0m',
'max_retry_policy': 1,
'model_id': 'test_model_id',
'model_name': 'test_model_name',
'model_config': {'test_config': 'test_config'},
'trigger_laborious': False,
'filters': {'test_filter_name': {'policy': 'test_policy'}},
'schema': 'sientia_data',
'table_name': 'laborious_data',
'retention_time': 10 * 60,
'model_tags': {'test_tag_name': {'webid': 'test_webid', 'aggr_func': 'test_aggr_func', 'data_range': [1, 2]}},
'debug_data_package': False,
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'fill_missing_tags': False,
'pi_web_api_query': {
'endpoint': 'https://test-endpoint.com',
'period': '*-2d',
'max_count': 5,
'api_timeout': 30,
}
}
assert result == expected
def test_pi_web_api_scouter_with_timeout_greater_than_frequency():
config = {
'workflow_type': 'scouter',
'schedule_name': 'test_schedule',
'model_id': 'test_model_id',
'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
'filters': [],
'read_tags': [
{
'tag_name': 'test_tag_name',
'webid': 'test_webid',
}
],
'tag_retention_minutes': 10,
'pi_web_api_config': {
'endpoint': 'https://test-endpoint.com',
'api_timeout': 120,
},
'frequency': '1m',
}
result = pi_web_api_scouter(config)
expected = {
'workflow_type': 'scouter',
'schedule_name': 'test_schedule',
'frequency': '1m',
'offset': '0m',
'max_retry_policy': 1,
'model_id': 'test_model_id',
'model_name': 'test_model_name',
'model_config': {'test_config': 'test_config'},
'trigger_laborious': False,
'filters': {},
'schema': 'sientia_data',
'table_name': 'laborious_data',
'retention_time': 10 * 60,
'model_tags': {'test_tag_name': {'webid': 'test_webid', 'aggr_func': 'lts', 'data_range': [-100, 100]}},
'debug_data_package': False,
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'fill_missing_tags': False,
'pi_web_api_query': {
'endpoint': 'https://test-endpoint.com',
'period': '*-1d',
'max_count': 1,
'api_timeout': 60,
}
}
assert result == expected
def test_pi_web_api_scouter_with_no_timeout():
config = {
'workflow_type': 'scouter',
'schedule_name': 'test_schedule',
'model_id': 'test_model_id',
'model': {'name': 'test_model_name', 'model_config': {'test_config': 'test_config'}},
'filters': [],
'read_tags': [
{
'tag_name': 'test_tag_name',
'webid': 'test_webid',
}
],
'tag_retention_minutes': 10,
'pi_web_api_config': {
'endpoint': 'https://test-endpoint.com',
},
'frequency': '30s',
}
result = pi_web_api_scouter(config)
expected = {
'workflow_type': 'scouter',
'schedule_name': 'test_schedule',
'frequency': '30s',
'offset': '0m',
'max_retry_policy': 1,
'model_id': 'test_model_id',
'model_name': 'test_model_name',
'model_config': {'test_config': 'test_config'},
'trigger_laborious': False,
'filters': {},
'schema': 'sientia_data',
'table_name': 'laborious_data',
'retention_time': 10 * 60,
'model_tags': {'test_tag_name': {'webid': 'test_webid', 'aggr_func': 'lts', 'data_range': [-100, 100]}},
'debug_data_package': False,
'execution_timeout_seconds': 300,
'task_timeout_seconds': 300,
'fill_missing_tags': False,
'pi_web_api_query': {
'endpoint': 'https://test-endpoint.com',
'period': '*-1d',
'max_count': 1,
'api_timeout': 30,
}
}
assert result == expected