gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1974

Browse Source 
Resolve PI Web API pipeline tags through the tag database

- Load active `pi_web_api` connectors and referenced tags behind the
  `connectors-pi-web-api` patch, so replays of older runs are unaffected.
- Resolve `pi_web_api_scouter` read tags and `predictions_batch` output
  tags by `tag_id`; legacy `webid` pipelines produce identical input.
- Add a `connection` block to scouter schedules with the secret reference
  only, plus connector defaults for period, max_count and api_timeout.
- Hold pipelines whose tags span several connectors and notify unresolved
  tags, wrong categories and missing connectors once per run.
- Share the issue notification logic between slots and schedules.
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-28 15:46:09 -03:00
parent 744ca647f7
commit 7e6f1a8782
7 changed files with 1170 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

223
tests/orchestrator/activities/test_formatters.py
View File

@@ -6,6 +6,7 @@ from pytest import fixture
from sientia_do.notifications.models import NotificationLevel
from orchestrator.activities.formatters import Formatters
from orchestrator.utils.orchestrator_functions import pi_web_api_scouter, predictions_batch
@fixture
@@ -1426,3 +1427,225 @@ def test_create_schedule_config_no_chunk_period_warning_when_unchanged(formatter
formatters.create_schedule_config(input_data)
formatters.send_notification.assert_not_called()
# PI Web API through the tag database (connectors-pi-web-api)
PI_SECRET = {'name': 'pi-web-api-auth-token', 'key': 'token'}
PI_CONNECTOR = {
'connector_id': 'pi-usina',
'name': 'PI Usina',
'url': 'https://pi.example/piwebapi',
'pi_web_api': {'auth': {'type': 'bearer', 'secret': PI_SECRET}, 'period': '*-2h'},
}
def _pi_tag(tag_id, address, tag_name, category='Read', connector_id='pi-usina'):
return {
'tag_id': tag_id,
'connector_id': connector_id,
'address': address,
'tag_name': tag_name,
'category': category,
}
def _pi_pipeline(schedule_name, read_tags):
return {
'workflow_type': 'pi_web_api_scouter',
'schedule_name': schedule_name,
'model_id': 'm1',
'model': {'name': 'model'},
'frequency': '1m',
'read_tags': read_tags,
'pi_web_api_config': {'endpoint': '/streamsets/recorded', 'max_count': 5},
'updated_at': '2021-01-01',
}
def _predictions_pipeline(schedule_name, output_config):
return {
'workflow_type': 'predictions_batch',
'schedule_name': schedule_name,
'model_id': 'm2',
'model': {'name': 'model'},
'query': 'SELECT 1',
'pi_web_api_output_config': output_config,
'updated_at': '2021-01-01',
}
def test_process_schedules_legacy_pi_pipelines_unchanged(formatters):
"""Legacy PI Web API pipelines produce byte-identical schedule input"""
scouter = _pi_pipeline('pi', {'TI-101': {'webid': 'F1DP-X', 'aggr_func': 'avg'}})
batch = _predictions_pipeline(
'batch', {'prediction_tags': {'PRED-101': 'F1DP-P'}, 'confidence_tags': {}}
)
with_tag_database = formatters.process_schedules(
{
'pipelines': [scouter, batch],
'pi_connectors': [PI_CONNECTOR],
'pi_connector_tags': [_pi_tag(42, 'F1DP-A', 'TI-101')],
**metadata,
}
)
before_change = {
'scouter': {'pi': {**pi_web_api_scouter(scouter), 'updated_at': '2021-01-01'}},
'laborious': {'batch': {**predictions_batch(batch), 'updated_at': '2021-01-01'}},
}
assert json.dumps(with_tag_database, sort_keys=True) == json.dumps(
before_change, sort_keys=True
)
assert 'connection' not in with_tag_database['scouter']['pi']
assert with_tag_database['laborious']['batch']['pi_web_api_output_config'] == {
'prediction_tags': {'PRED-101': 'F1DP-P'},
'confidence_tags': {},
}
formatters.send_notification.assert_not_called()
def test_process_schedules_resolves_pi_tags_through_tag_database(formatters):
scouter = _pi_pipeline('pi', {'TI-101': {'tag_id': 42, 'aggr_func': 'avg'}})
batch = _predictions_pipeline('batch', {'prediction_tags': [{'tag_id': 77}]})
result = formatters.process_schedules(
{
'pipelines': [scouter, batch],
'pi_connectors': [PI_CONNECTOR],
'pi_connector_tags': [
_pi_tag(42, 'F1DP-A', 'TI-101'),
_pi_tag(77, 'F1DP-P', 'PRED-101', category='Write'),
],
**metadata,
}
)
pi = result['scouter']['pi']
assert pi['model_tags'] == {
'TI-101': {'webid': 'F1DP-A', 'aggr_func': 'avg', 'data_range': [-100, 100]}
}
assert pi['pi_web_api_query'] == {
'endpoint': '/streamsets/recorded',
'period': '*-2h',
'max_count': 5,
'api_timeout': 60,
}
assert pi['connection'] == {
'connector_id': 'pi-usina',
'base_url': 'https://pi.example/piwebapi',
'auth': {'type': 'bearer', 'secret': PI_SECRET},
}
assert result['laborious']['batch']['pi_web_api_output_config'] == {
'prediction_tags': {'PRED-101': 'F1DP-P'}
}
# The builder input is never the pipeline document itself
assert 'pi_web_api_connector' not in scouter
formatters.send_notification.assert_not_called()
def test_process_schedules_secret_value_never_in_schedule_input(formatters):
connector = {
**PI_CONNECTOR,
'pi_web_api': {
'auth': {
'type': 'basic',
'secret': {**PI_SECRET, 'value': 'leaked-credential'},
'password': 'leaked-credential',
}
},
}
result = formatters.process_schedules(
{
'pipelines': [_pi_pipeline('pi', {'TI-101': {'tag_id': 42}})],
'pi_connectors': [connector],
'pi_connector_tags': [_pi_tag(42, 'F1DP-A', 'TI-101')],
**metadata,
}
)
assert result['scouter']['pi']['connection']['auth'] == {'type': 'basic', 'secret': PI_SECRET}
assert 'leaked-credential' not in json.dumps(result)
def test_process_schedules_notifies_unresolved_pi_tags(formatters):
scouter = _pi_pipeline('pi', {'TI-101': {'tag_id': 42}, 'TI-404': {'tag_id': 404}})
batch = _predictions_pipeline('batch', {'prediction_tags': [{'tag_id': 42}]})
result = formatters.process_schedules(
{
'pipelines': [scouter, batch],
'pi_connectors': [PI_CONNECTOR],
'pi_connector_tags': [
_pi_tag(42, 'F1DP-A', 'TI-101'),
_pi_tag(43, 'F1DP-B', 'TI-OFF', connector_id='pi-off'),
],
**metadata,
}
)
# Still scheduled, without the unresolved tags
assert list(result['scouter']['pi']['model_tags']) == ['TI-101']
assert result['laborious']['batch']['pi_web_api_output_config'] == {'prediction_tags': {}}
notified = {
c.kwargs['notification_id']: c.kwargs['message']
for c in formatters.send_notification.call_args_list
}
assert set(notified) == {
'ORCHESTRATOR_PI_TAG_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION',
'ORCHESTRATOR_PI_TAG_WRONG_CATEGORY_DURING_SCHEDULE_CONFIGURATION',
}
assert 'pi/404' in notified['ORCHESTRATOR_PI_TAG_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION']
assert (
'batch/42' in notified['ORCHESTRATOR_PI_TAG_WRONG_CATEGORY_DURING_SCHEDULE_CONFIGURATION']
)
for c in formatters.send_notification.call_args_list:
assert c.kwargs['level'] == NotificationLevel.ERROR
def test_process_schedules_holds_mixed_connector_pipeline(formatters):
second = {**PI_CONNECTOR, 'connector_id': 'pi-other'}
pipeline = _pi_pipeline('pi', {'A': {'tag_id': 1}, 'B': {'tag_id': 2}})
result = formatters.process_schedules(
{
'pipelines': [pipeline],
'pi_connectors': [PI_CONNECTOR, second],
'pi_connector_tags': [
_pi_tag(1, 'F1DP-1', 'A'),
_pi_tag(2, 'F1DP-2', 'B', connector_id='pi-other'),
],
**metadata,
}
)
assert result['scouter'] == {'pi': {'held': True}}
formatters.send_notification.assert_called_once()
kwargs = formatters.send_notification.call_args.kwargs
assert (
kwargs['notification_id']
== 'ORCHESTRATOR_PI_MIXED_CONNECTORS_DURING_SCHEDULE_CONFIGURATION'
)
assert 'pi/pi-other, pi-usina' in kwargs['message']
def test_create_schedule_config_leaves_held_schedule_untouched(formatters):
result = formatters.create_schedule_config(
{
'current_schedule_config': {'scouter': {'pi': '2021-01-01'}, 'laborious': {}},
'schedule_config': {
'scouter': {'pi': {'held': True}, 'new_held': {'held': True}},
'laborious': {},
},
**metadata,
}
)
assert result == {
'to_update': {'scouter': {}, 'laborious': {}},
'to_create': {'scouter': {}, 'laborious': {}},
'to_delete': {'scouter': [], 'laborious': []},
}

285
tests/orchestrator/utils/test_orchestrator_functions.py
View File

@@ -13,12 +13,16 @@ from orchestrator.utils.orchestrator_functions import (
gather_read_tags,
index_connector_tags,
index_connectors,
index_pi_connectors,
minimal_retrain,
overlap_filter_config,
pi_web_api_scouter,
predictions_batch,
process_path_priority,
referenced_pi_tag_ids,
referenced_tag_ids,
resolve_pi_output_tags,
resolve_pi_read_tags,
resolve_tag_names,
scouter,
simple_metrics,
@@ -1230,3 +1234,284 @@ def test_pi_web_api_scouter_with_no_timeout():
)
def test_frequency_alert_warning(frequency_alert, warns):
assert (frequency_alert_warning(frequency_alert) is not None) is warns
# PI Web API through the tag database (connectors-pi-web-api)
PI_SECRET = {'name': 'pi-web-api-auth-token', 'key': 'token'}
def _pi_connector(connector_id='pi-usina', **pi_web_api):
return {
'connector_id': connector_id,
'name': 'PI Usina',
'url': f'https://{connector_id}.example/piwebapi',
'pi_web_api': {'auth': {'type': 'bearer', 'secret': dict(PI_SECRET)}, **pi_web_api},
}
def _pi_tag(tag_id, address, tag_name, category='Read', connector_id='pi-usina'):
return {
'tag_id': tag_id,
'connector_id': connector_id,
'address': address,
'tag_name': tag_name,
'category': category,
}
def _pi_scouter_pipeline(read_tags, **fields):
return {
'workflow_type': 'pi_web_api_scouter',
'schedule_name': 'pi_schedule',
'model_id': 'model_id',
'model': {'name': 'model_name'},
'frequency': '1m',
'read_tags': read_tags,
'pi_web_api_config': {'endpoint': '/streamsets/recorded'},
**fields,
}
def test_index_pi_connectors_leaves_out_incomplete():
complete = _pi_connector()
no_url = {**_pi_connector('pi-a'), 'url': ''}
no_auth = {**_pi_connector('pi-b'), 'pi_web_api': {}}
inline_token = {
**_pi_connector('pi-c'),
'pi_web_api': {'auth': {'type': 'bearer', 'token': 'secret-value'}},
}
no_secret_key = {
**_pi_connector('pi-d'),
'pi_web_api': {'auth': {'type': 'basic', 'secret': {'name': 'x'}}},
}
index = index_pi_connectors([complete, no_url, no_auth, inline_token, no_secret_key])
assert index == {'pi-usina': complete}
def test_referenced_pi_tag_ids():
pipelines = [
_pi_scouter_pipeline({'TI-101': {'tag_id': 42}, 'TI-102': {'webid': 'F1DP-X'}}),
{
'workflow_type': 'predictions_batch',
'pi_web_api_output_config': {
'prediction_tags': [{'tag_id': '77'}],
'confidence_tags': {'CONF-1': 'F1DP-C'},
},
},
{
'workflow_type': 'xgboost_predictions_batch',
'pi_web_api_output_config': {'confidence_tags': [{'tag_id': 78}]},
},
# OPC scouter tag_ids belong to the OPC loads
{'workflow_type': 'scouter', 'read_tags': [{'tag_id': 7}]},
{'workflow_type': 'drift'},
]
assert referenced_pi_tag_ids(pipelines) == [42, 77, 78, '42', '77', '78']
def test_referenced_pi_tag_ids_none():
assert referenced_pi_tag_ids([_pi_scouter_pipeline({'TI-101': {'webid': 'F1DP-X'}})]) == []
def test_resolve_pi_read_tags_by_tag_id():
connectors = index_pi_connectors([_pi_connector()])
tags = index_connector_tags([_pi_tag(42, 'F1DP-A', 'TI-101-db')])
resolved, connector, issues = resolve_pi_read_tags(
{'TI-101': {'tag_id': 42, 'aggr_func': 'avg'}}, tags, connectors
)
# The key stays the model tag name; only the WebID comes from the tag database
assert resolved == {'TI-101': {'tag_id': 42, 'aggr_func': 'avg', 'webid': 'F1DP-A'}}
assert connector == connectors['pi-usina']
assert not any(issues.values())
def test_resolve_pi_read_tags_legacy_untouched():
legacy = {'TI-101': {'webid': 'F1DP-X'}}
resolved, connector, issues = resolve_pi_read_tags(legacy, {}, {})
assert resolved == legacy
assert resolved['TI-101'] is legacy['TI-101']
assert connector is None
assert not any(issues.values())
def test_resolve_pi_read_tags_issues():
connectors = index_pi_connectors([_pi_connector()])
tags = index_connector_tags(
[
_pi_tag(42, 'F1DP-A', 'TI-101'),
_pi_tag(43, 'F1DP-W', 'PRED-1', category='Write'),
_pi_tag(44, 'F1DP-O', 'TI-OFF', connector_id='pi-inactive'),
]
)
resolved, connector, issues = resolve_pi_read_tags(
{
'TI-101': {'tag_id': 42},
'UNKNOWN': {'tag_id': 99},
'WRITE': {'tag_id': 43},
'OFF': {'tag_id': 44},
'LEGACY': {'webid': 'F1DP-X'},
},
tags,
connectors,
)
# Unresolvable tags are left out; the rest of the pipeline is still scheduled
assert set(resolved) == {'TI-101', 'LEGACY'}
assert connector == connectors['pi-usina']
assert issues == {
'tags': {'99'},
'categories': {'43'},
'connectors': {'pi-inactive'},
'mixed': set(),
}
def test_resolve_pi_read_tags_mixed_connectors():
connectors = index_pi_connectors([_pi_connector('pi-a'), _pi_connector('pi-b')])
tags = index_connector_tags(
[
_pi_tag(1, 'F1DP-1', 'A', connector_id='pi-a'),
_pi_tag(2, 'F1DP-2', 'B', connector_id='pi-b'),
]
)
_resolved, connector, issues = resolve_pi_read_tags(
{'A': {'tag_id': 1}, 'B': {'tag_id': 2}}, tags, connectors
)
assert connector is None
assert issues['mixed'] == {'pi-a, pi-b'}
def test_resolve_pi_output_tags_by_tag_id():
connectors = index_pi_connectors([_pi_connector()])
tags = index_connector_tags(
[
_pi_tag(77, 'F1DP-P', 'PRED-101', category='Write'),
_pi_tag(78, 'F1DP-C', 'CONF-101', category='Prediction Confidence'),
]
)
resolved, issues = resolve_pi_output_tags(
{
'prediction_tags': [{'tag_id': 77}],
'confidence_tags': [{'tag_id': 78}],
'other': 'kept',
},
tags,
connectors,
)
assert resolved == {
'prediction_tags': {'PRED-101': 'F1DP-P'},
'confidence_tags': {'CONF-101': 'F1DP-C'},
'other': 'kept',
}
assert not any(issues.values())
def test_resolve_pi_output_tags_legacy_untouched():
legacy = {'prediction_tags': {'PRED-101': 'F1DP-P'}, 'confidence_tags': {}}
resolved, issues = resolve_pi_output_tags(legacy, {}, {})
assert resolved == legacy
assert resolved['prediction_tags'] is legacy['prediction_tags']
assert not any(issues.values())
def test_resolve_pi_output_tags_issues():
connectors = index_pi_connectors([_pi_connector()])
tags = index_connector_tags(
[
_pi_tag(42, 'F1DP-R', 'TI-101', category='Read'),
_pi_tag(77, 'F1DP-P', 'PRED-101', category='Write'),
]
)
resolved, issues = resolve_pi_output_tags(
{
# A Read tag as prediction, a Write tag as confidence, an unknown tag_id
'prediction_tags': [{'tag_id': 42}, {'tag_id': 99}, {'PRED-LEGACY': 'F1DP-L'}],
'confidence_tags': [{'tag_id': 77}],
},
tags,
connectors,
)
assert resolved['prediction_tags'] == {'PRED-LEGACY': 'F1DP-L'}
assert resolved['confidence_tags'] == {}
assert issues['tags'] == {'99'}
assert issues['categories'] == {'42', '77'}
def test_pi_web_api_scouter_legacy_shape():
"""Without a connector the output is exactly the pre-tag-database one"""
config = _pi_scouter_pipeline(
{'TI-101': {'webid': 'F1DP-X'}},
pi_web_api_config={'endpoint': '/streamsets/recorded', 'max_count': 5, 'api_timeout': 10},
)
result = pi_web_api_scouter(config)
assert 'connection' not in result
assert result['model_tags'] == {
'TI-101': {'webid': 'F1DP-X', 'aggr_func': 'lts', 'data_range': [-100, 100]}
}
assert result['pi_web_api_query'] == {
'endpoint': '/streamsets/recorded',
'period': '*-1d',
'max_count': 5,
'api_timeout': 10,
}
def test_pi_web_api_scouter_connector_defaults_with_pipeline_override():
connector = _pi_connector(period='*-2h', max_count=10, api_timeout=600, data_server_webid='DS1')
config = _pi_scouter_pipeline(
{'TI-101': {'tag_id': 42, 'webid': 'F1DP-A', 'aggr_func': 'avg'}},
pi_web_api_config={'endpoint': '/streamsets/recorded', 'max_count': 5},
pi_web_api_connector=connector,
)
result = pi_web_api_scouter(config)
assert result['model_tags'] == {
'TI-101': {'webid': 'F1DP-A', 'aggr_func': 'avg', 'data_range': [-100, 100]}
}
assert result['pi_web_api_query'] == {
'endpoint': '/streamsets/recorded',
'period': '*-2h',
'max_count': 5,
# The connector default is still capped at the 1m frequency
'api_timeout': 60,
}
assert result['connection'] == {
'connector_id': 'pi-usina',
'base_url': 'https://pi-usina.example/piwebapi',
'auth': {'type': 'bearer', 'secret': PI_SECRET},
'data_server_webid': 'DS1',
}
def test_pi_web_api_scouter_connection_carries_only_the_secret_reference():
connector = _pi_connector()
connector['pi_web_api']['auth']['secret']['value'] = 'leaked-credential'
connector['pi_web_api']['auth']['token'] = 'leaked-credential'
config = _pi_scouter_pipeline(
{'TI-101': {'tag_id': 42, 'webid': 'F1DP-A'}}, pi_web_api_connector=connector
)
result = pi_web_api_scouter(config)
assert result['connection']['auth'] == {'type': 'bearer', 'secret': PI_SECRET}
assert 'leaked-credential' not in str(result)

145
tests/orchestrator/workflows/test_orchestrator.py
View File

@@ -6,6 +6,9 @@ from orchestrator.activities.activities import Activities
from orchestrator.workflows.orchestrator import (
DEFAULT_CONNECTOR_TAGS_QUERY,
DEFAULT_CONNECTORS_QUERY,
DEFAULT_PI_CONNECTOR_TAGS_QUERY,
DEFAULT_PI_CONNECTORS_QUERY,
PI_WEB_API_PATCH,
TAG_DATABASE_PATCH,
Orchestrator,
)
@@ -116,7 +119,10 @@ async def test_run(workflow_mock, referenced_tag_ids_mock, orchestrator):
]
)
workflow_mock.patched.assert_called_once_with(TAG_DATABASE_PATCH)
assert workflow_mock.patched.call_args_list == [
call(TAG_DATABASE_PATCH),
call(PI_WEB_API_PATCH),
]
referenced_tag_ids_mock.assert_called_once_with(
workflow_mock.start_local_activity_method.return_value
)
@@ -163,6 +169,9 @@ async def test_run(workflow_mock, referenced_tag_ids_mock, orchestrator):
**metadata,
'pipelines': workflow_mock.start_local_activity_method.return_value,
'connector_tags': workflow_mock.start_local_activity_method.return_value,
# No PI Web API tag with tag_id: nothing loaded
'pi_connectors': [],
'pi_connector_tags': [],
},
retry_policy=ANY,
start_to_close_timeout=ANY,
@@ -466,7 +475,10 @@ async def test_run_without_tag_ids_skips_tag_database(workflow_mock, _ids, orche
await orchestrator.run(input_data)
workflow_mock.patched.assert_called_once_with(TAG_DATABASE_PATCH)
assert workflow_mock.patched.call_args_list == [
call(TAG_DATABASE_PATCH),
call(PI_WEB_API_PATCH),
]
assert _tag_database_calls(workflow_mock) == []
slots_input = _process_slots_input(workflow_mock)
assert slots_input['connectors'] == []
@@ -491,3 +503,132 @@ async def test_run_replay_before_tag_database_patch(workflow_mock, _ids, orchest
slots_input = _process_slots_input(workflow_mock)
assert slots_input['connectors'] == []
assert slots_input['connector_tags'] == []
def _process_schedules_input(workflow_mock):
return next(
c.args[1]
for c in workflow_mock.start_local_activity_method.call_args_list
if c.args[0] is Activities.process_schedules
)
def _pi_tag_database_calls(workflow_mock):
"""The PI Web API loads: the ones whose aggregation is not the OPC one"""
return [
c
for c in _tag_database_calls(workflow_mock)
if c.args[1]['query']['aggregation'][-1]
in (
DEFAULT_PI_CONNECTORS_QUERY['aggregation'][-1],
DEFAULT_PI_CONNECTOR_TAGS_QUERY['aggregation'][-1],
)
]
@mark.asyncio
@patch('orchestrator.workflows.orchestrator.referenced_pi_tag_ids', return_value=[42, '42'])
@patch('orchestrator.workflows.orchestrator.referenced_tag_ids', return_value=[])
@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
async def test_run_loads_pi_tag_database(workflow_mock, _ids, _pi_ids, orchestrator):
"""A new run loads the PI connectors and tags and hands them to process_schedules only"""
workflow_mock.patched = MagicMock(return_value=True)
input_data = {
'pipelines_query': 'SELECT * FROM bucket',
'opc_servers_query': 'SELECT * FROM servers',
'schedule_name': 'test-schedule-name',
}
await orchestrator.run(input_data)
assert workflow_mock.patched.call_args_list == [
call(TAG_DATABASE_PATCH),
call(PI_WEB_API_PATCH),
]
queries = [c.args[1]['query'] for c in _tag_database_calls(workflow_mock)]
assert queries == [
DEFAULT_PI_CONNECTORS_QUERY,
{
**DEFAULT_PI_CONNECTOR_TAGS_QUERY,
'aggregation': [
{'$match': {'tag_id': {'$in': [42, '42']}}},
*DEFAULT_PI_CONNECTOR_TAGS_QUERY['aggregation'],
],
},
]
for c in _tag_database_calls(workflow_mock):
assert c.args[1]['query']['aggregation'] is not DEFAULT_PI_CONNECTORS_QUERY['aggregation']
loaded = workflow_mock.start_local_activity_method.return_value
schedules_input = _process_schedules_input(workflow_mock)
assert schedules_input['pi_connectors'] is loaded
assert schedules_input['pi_connector_tags'] is loaded
# process_slots input unchanged: no PI key, OPC loads skipped (no OPC tag_id)
slots_input = _process_slots_input(workflow_mock)
assert 'pi_connectors' not in slots_input
assert 'pi_connector_tags' not in slots_input
assert slots_input['connectors'] == []
assert slots_input['connector_tags'] == []
@mark.asyncio
@patch('orchestrator.workflows.orchestrator.referenced_pi_tag_ids', return_value=[42])
@patch('orchestrator.workflows.orchestrator.referenced_tag_ids', return_value=[])
@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
async def test_run_with_pi_tag_database_queries_from_input(
workflow_mock, _ids, _pi_ids, orchestrator
):
"""The input overrides the default PI Web API aggregations"""
workflow_mock.patched = MagicMock(return_value=True)
pi_connectors_query = {
'collection': 'connectors',
'aggregation': [{'$match': {'type': 'pi_web_api'}}],
}
pi_connector_tags_query = {
'collection': 'connector_tags',
'aggregation': [{'$match': {'status': 'Active'}}],
}
input_data = {
'pipelines_query': 'SELECT * FROM bucket',
'opc_servers_query': 'SELECT * FROM servers',
'pi_connectors_query': pi_connectors_query,
'pi_connector_tags_query': pi_connector_tags_query,
'schedule_name': 'test-schedule-name',
}
await orchestrator.run(input_data)
queries = [c.args[1]['query'] for c in _tag_database_calls(workflow_mock)]
assert queries == [
pi_connectors_query,
{
'collection': 'connector_tags',
'aggregation': [
{'$match': {'tag_id': {'$in': [42]}}},
{'$match': {'status': 'Active'}},
],
},
]
@mark.asyncio
@patch('orchestrator.workflows.orchestrator.referenced_pi_tag_ids', return_value=[42])
@patch('orchestrator.workflows.orchestrator.referenced_tag_ids', return_value=[7])
@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
async def test_run_replay_before_pi_web_api_patch(workflow_mock, _ids, _pi_ids, orchestrator):
"""A run started after the tag database patch but before the PI one replays without PI loads"""
workflow_mock.patched = MagicMock(side_effect=lambda patch_id: patch_id == TAG_DATABASE_PATCH)
input_data = {
'pipelines_query': 'SELECT * FROM bucket',
'opc_servers_query': 'SELECT * FROM servers',
'schedule_name': 'test-schedule-name',
}
await orchestrator.run(input_data)
assert _pi_tag_database_calls(workflow_mock) == []
assert len(_tag_database_calls(workflow_mock)) == 2
schedules_input = _process_schedules_input(workflow_mock)
assert schedules_input['pi_connectors'] == []
assert schedules_input['pi_connector_tags'] == []