gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1974

Browse Source 
SIENTIAPDE-1974-connectors-pi-web-api: Consolidate OPC server and tag data sourcing to the tag database. This change removes the legacy opc-servers collection and enforces tag_id as the sole reference for all read and write tags via connectors and connector_tags. The Orchestrator workflow now exclusively loads OPC server and tag information from the tag database, simplifying data flow and removing conditional logic for legacy data sources. Improved error reporting for pipeline timestamp updates and updated documentation, examples, and tests are also included.
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-29 10:25:51 -03:00
parent 9ba2fc74c4
commit 0c9eaab3f5
30 changed files with 1082 additions and 3051 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
tests/orchestrator/activities/test_formatters.py
View File

@@ -232,42 +232,65 @@ def _scouter_pipeline(schedule_name, model_id, read_tags):
}
slot_opc_servers = [
{'id': '1', 'server_name': 'kepserver', 'url': 'opc.tcp://kep:49320', 'uri': 'urn:kep'},
slot_connectors = [
{
'connector_id': 'kepserver',
'name': 'kepserver',
'url': 'opc.tcp://kep:49320',
'server_uri': 'urn:kep',
'status': 'Active',
},
]
def _read_tag_doc(tag_id, address, tag_name, connector_id='kepserver'):
return {
'tag_id': tag_id,
'connector_id': connector_id,
'address': address,
'tag_name': tag_name,
'category': 'Read',
'status': 'Active',
}
slot_connector_tags = [
_read_tag_doc(1, 'ns=2;s=A', 'a'),
_read_tag_doc(2, 'ns=2;s=B', 'b'),
_read_tag_doc(3, 'ns=2;s=C', 'c'),
]
def _slot_input(pipelines, active_ingestors, **extra):
return {
'connectors': slot_connectors,
'connector_tags': slot_connector_tags,
'active_ingestors': active_ingestors,
'pipelines': pipelines,
**metadata,
**extra,
}
def test_process_slots(formatters):
"""Slot v2 split by address: the names of an address never land in different slots"""
pipelines = [
_scouter_pipeline(
'sin',
'm1',
[
{'server_id': '1', 'tag_address': 'ns=2;s=A', 'tag_name': 'a', 'frequency': 1000},
{'server_id': '1', 'tag_address': 'ns=2;s=B', 'tag_name': 'b', 'frequency': 1000},
],
'sin', 'm1', [{'tag_id': 1, 'frequency': 1000}, {'tag_id': 2, 'frequency': 1000}]
),
_scouter_pipeline(
'cos',
'm2',
[
{'server_id': '1', 'tag_address': 'ns=2;s=A', 'tag_name': 'x', 'frequency': 500},
{'server_id': '1', 'tag_address': 'ns=2;s=C', 'tag_name': 'c', 'frequency': 4000},
],
[{'tag_id': 1, 'tag_name': 'x', 'frequency': 500}, {'tag_id': 3, 'frequency': 4000}],
),
]
input_data = {
'opc_servers': slot_opc_servers,
'active_ingestors': ['heartbeat:ingestor:1', 'heartbeat:ingestor:2'],
'pipelines': pipelines,
**metadata,
}
result = formatters.process_slots(input_data)
result = formatters.process_slots(
_slot_input(pipelines, ['heartbeat:ingestor:1', 'heartbeat:ingestor:2'])
)
server = {
'server_id': '1',
'server_id': 'kepserver',
'name': 'kepserver',
'url': 'opc.tcp://kep:49320',
'server_uri': 'urn:kep',
@@ -337,21 +360,9 @@ def test_process_slots(formatters):
def test_process_slots_without_ingestors_builds_one_slot(formatters):
pipelines = [
_scouter_pipeline(
'sin',
'm1',
[{'server_id': '1', 'tag_address': 'ns=2;s=A', 'tag_name': 'a', 'frequency': 1000}],
)
]
input_data = {
'opc_servers': slot_opc_servers,
'active_ingestors': [],
'pipelines': pipelines,
**metadata,
}
pipelines = [_scouter_pipeline('sin', 'm1', [{'tag_id': 1, 'frequency': 1000}])]
result = formatters.process_slots(input_data)
result = formatters.process_slots(_slot_input(pipelines, []))
assert list(result) == ['1']
assert list(result['1']['kepserver']['tags']) == ['ns=2;s=A']
@@ -359,21 +370,9 @@ def test_process_slots_without_ingestors_builds_one_slot(formatters):
def test_process_slots_more_ingestors_than_addresses(formatters):
"""Extra ingestors get empty slots, as before"""
pipelines = [
_scouter_pipeline(
'sin',
'm1',
[{'server_id': '1', 'tag_address': 'ns=2;s=A', 'tag_name': 'a', 'frequency': 1000}],
)
]
input_data = {
'opc_servers': slot_opc_servers,
'active_ingestors': ['i1', 'i2', 'i3'],
'pipelines': pipelines,
**metadata,
}
pipelines = [_scouter_pipeline('sin', 'm1', [{'tag_id': 1, 'frequency': 1000}])]
result = formatters.process_slots(input_data)
result = formatters.process_slots(_slot_input(pipelines, ['i1', 'i2', 'i3']))
assert list(result) == ['1', '2', '3']
assert list(result['1']['kepserver']['tags']) == ['ns=2;s=A']
@@ -382,12 +381,11 @@ def test_process_slots_more_ingestors_than_addresses(formatters):
def test_process_slots_with_tag_database(formatters):
"""A read tag with tag_id takes address, name and server from the tag database"""
"""A read tag takes address, name and server from the tag database"""
pipelines = [
_scouter_pipeline('sin', 'm1', [{'tag_id': 7, 'frequency': 1000}]),
]
input_data = {
'opc_servers': slot_opc_servers,
'connectors': [
{
'connector_id': 3,
@@ -445,38 +443,24 @@ def test_process_slots_notifies_unresolved_tags_once(formatters):
'sin',
'm1',
[
{'server_id': '1', 'tag_address': 'ns=2;s=A', 'tag_name': 'a', 'frequency': 1000},
{'server_id': '9', 'tag_address': 'ns=2;s=B', 'tag_name': 'b', 'frequency': 1000},
{'tag_id': 1, 'frequency': 1000},
{'tag_name': 'b', 'frequency': 1000},
{'tag_id': 404, 'frequency': 1000},
{'tag_id': 8, 'frequency': 1000},
{'server_id': '1', 'tag_address': 'ns=2;s=C', 'tag_name': 'c'},
{'tag_id': 3},
],
),
]
input_data = {
'opc_servers': slot_opc_servers,
'connectors': [],
'connector_tags': [
{'tag_id': 8, 'connector_id': 5, 'address': 'ns=3;i=8', 'tag_name': 'Level'}
],
'active_ingestors': ['i1', 'i2'],
'pipelines': pipelines,
**metadata,
}
connector_tags = [*slot_connector_tags, _read_tag_doc(8, 'ns=3;i=8', 'Level', 'other')]
result = formatters.process_slots(input_data)
result = formatters.process_slots(
_slot_input(pipelines, ['i1', 'i2'], connector_tags=connector_tags)
)
assert list(result['1']['kepserver']['tags']) == ['ns=2;s=A']
assert result['2'] == {}
formatters.send_notification.assert_has_calls(
[
call(
metadata=metadata['metadata'],
notification_id='ORCHESTRATOR_SERVER_NOT_FOUND_DURING_SLOT_CONFIGURATION',
message='Servers 9 not found in opc_servers',
block='orchestrator',
level=NotificationLevel.ERROR,
),
call(
metadata=metadata['metadata'],
notification_id='ORCHESTRATOR_TAG_NOT_FOUND_DURING_SLOT_CONFIGURATION',
@@ -487,7 +471,7 @@ def test_process_slots_notifies_unresolved_tags_once(formatters):
call(
metadata=metadata['metadata'],
notification_id='ORCHESTRATOR_CONNECTOR_NOT_FOUND_DURING_SLOT_CONFIGURATION',
message='Connectors 5 not found, not active or incomplete in connectors',
message='Connectors other not found, not active or incomplete in connectors',
block='orchestrator',
level=NotificationLevel.ERROR,
),
@@ -495,16 +479,16 @@ def test_process_slots_notifies_unresolved_tags_once(formatters):
metadata=metadata['metadata'],
notification_id='ORCHESTRATOR_INVALID_READ_TAG_DURING_SLOT_CONFIGURATION',
message=(
'Read tags sin/ns=2;s=C have no integer frequency, an active_read without '
'an integer cycle_multiple, or lack server_id, tag_address or tag_name; or '
'their pipeline lacks schedule_name, model_id or model.name'
'Read tags sin/3, sin/b have no integer tag_id, no integer frequency or an '
'active_read without an integer cycle_multiple; or their pipeline lacks '
'schedule_name, model_id or model.name'
),
block='orchestrator',
level=NotificationLevel.ERROR,
),
]
)
assert formatters.send_notification.call_count == 4
assert formatters.send_notification.call_count == 3
def test_process_slots_survives_malformed_tag_database(formatters):
@@ -514,25 +498,22 @@ def test_process_slots_survives_malformed_tag_database(formatters):
'sin',
'm1',
[
{'server_id': '1', 'tag_address': 'ns=2;s=A', 'tag_name': 'a', 'frequency': 1000},
{'tag_id': 1, 'frequency': 1000},
{'tag_id': 7, 'frequency': 1000},
{'tag_id': 8, 'frequency': 1000},
],
),
]
input_data = {
'opc_servers': slot_opc_servers,
'connectors': [{'connector_id': 3, 'name': 'plant', 'server_uri': 'urn:plant'}],
'connector_tags': [
{'tag_id': 7, 'connector_id': 3, 'address': 'ns=3;i=1', 'tag_name': 'P'},
{'tag_id': 8, 'connector_id': 3, 'tag_name': 'no address'},
],
'active_ingestors': [],
'pipelines': pipelines,
**metadata,
}
connectors = [*slot_connectors, {'connector_id': 3, 'name': 'plant', 'server_uri': 'urn:plant'}]
connector_tags = [
*slot_connector_tags,
{'tag_id': 7, 'connector_id': 3, 'address': 'ns=3;i=1', 'tag_name': 'P'},
{'tag_id': 8, 'connector_id': 3, 'tag_name': 'no address'},
]
result = formatters.process_slots(input_data)
result = formatters.process_slots(
_slot_input(pipelines, [], connectors=connectors, connector_tags=connector_tags)
)
assert list(result['1']) == ['kepserver']
notified = {c.kwargs['notification_id'] for c in formatters.send_notification.call_args_list}
@@ -545,30 +526,16 @@ def test_process_slots_survives_malformed_tag_database(formatters):
def test_process_slots_is_stable_across_pipeline_order(formatters):
"""The same pipelines in another order give the same slots (O6)"""
pipelines = [
_scouter_pipeline(
'sin',
'm1',
[{'server_id': '1', 'tag_address': 'ns=2;s=A', 'tag_name': 'a', 'frequency': 1000}],
),
_scouter_pipeline('sin', 'm1', [{'tag_id': 1, 'frequency': 1000}]),
_scouter_pipeline(
'cos',
'm2',
[
{'server_id': '1', 'tag_address': 'ns=2;s=B', 'tag_name': 'b', 'frequency': 1000},
{'server_id': '1', 'tag_address': 'ns=2;s=A', 'tag_name': 'x', 'frequency': 1000},
],
[{'tag_id': 2, 'frequency': 1000}, {'tag_id': 1, 'tag_name': 'x', 'frequency': 1000}],
),
]
def run(order):
return formatters.process_slots(
{
'opc_servers': slot_opc_servers,
'active_ingestors': ['i1', 'i2'],
'pipelines': order,
**metadata,
}
)
return formatters.process_slots(_slot_input(order, ['i1', 'i2']))
assert json.dumps(run(pipelines)) == json.dumps(run(list(reversed(pipelines))))
@@ -596,7 +563,6 @@ def test_process_schedules_names_scouter_tags_like_the_slot(formatters):
)
slots = formatters.process_slots(
{
'opc_servers': [],
'connectors': connectors,
'connector_tags': connector_tags,
'active_ingestors': [],
@@ -738,25 +704,6 @@ def test_create_schedule_config_updates_on_newer_updated_at(formatters):
assert set(result['to_update']['scouter']) == {'s'}
def test_create_schedule_config_accepts_timestamp_only_current_schedules(formatters):
"""
A run started before config_hash replays format_schedule_config's timestamp-only
result: with no stored hash, the schedule is redeployed once to record it.
"""
schedule = {'frequency': 60, 'updated_at': '2021-01-01'}
result = formatters.create_schedule_config(
{
'current_schedule_config': {'scouter': {'s': '2021-01-02'}},
'schedule_config': {'scouter': {'s': schedule}, 'laborious': {}},
}
)
assert result['to_update']['scouter'] == {
's': {**schedule, 'config_hash': schedule_config_hash(schedule)}
}
def test_create_slot_config(formatters):
input_data = {
'current_slot_config': {
@@ -776,19 +723,19 @@ def test_create_slot_config(formatters):
def test_create_slot_config_skips_unchanged_slots(formatters):
"""Only new or changed slots are written (O6); current slots come keyed by Redis key"""
slot_v2 = {'kepserver': {'server_id': '1', 'tags': {'ns=2;s=A': {'names': []}}}}
slot_v1 = {'kepserver': {'server_id': '1', 'tags': {'ns=2;s=A': {'topics': ['raw_x']}}}}
slot = {'kepserver': {'server_id': '1', 'tags': {'ns=2;s=A': {'names': []}}}}
changed = {'kepserver': {'server_id': '1', 'tags': {'ns=2;s=B': {'names': []}}}}
input_data = {
'current_slot_config': {
'slot:opc_tags:1': slot_v2,
'slot:opc_tags:2': slot_v1,
'slot:opc_tags:1': slot,
'slot:opc_tags:2': changed,
},
'slot_config': {'1': slot_v2, '2': slot_v2, '3': {}},
'slot_config': {'1': slot, '2': slot, '3': {}},
}
result = formatters.create_slot_config(input_data)
assert result == {'to_delete': [], 'to_insert': {'2': slot_v2, '3': {}}}
assert result == {'to_delete': [], 'to_insert': {'2': slot, '3': {}}}
def test_create_slot_config_without_current_slots(formatters):
@@ -1448,7 +1395,9 @@ def test_create_schedule_config_warns_on_frequency_alert_below_floor(formatters)
def test_create_schedule_config_warns_on_frequency_alert_below_floor_on_update(formatters):
input_data = _schedule_config_with_frequency_alert('30s')
input_data['current_schedule_config'] = {'laborious': {'m1-simple_metrics-60m': '2026-10-06'}}
input_data['current_schedule_config'] = {
'laborious': {'m1-simple_metrics-60m': {'updated_at': '2026-10-06', 'config_hash': None}}
}
result = formatters.create_schedule_config(input_data)
@@ -1579,7 +1528,7 @@ def test_process_schedules_resolves_write_tags_through_tag_database(formatters):
def test_process_schedules_notifies_unresolved_write_tags_once(formatters):
pipelines = [
_batch_pipeline('a', [{'tag_id': 404}, {'tag_id': 7}, {'tag_id': 9}]),
_batch_pipeline('b', [{'server_id': '1', 'type': 'prediction', 'addr': 'ns=2;s=L'}]),
_batch_pipeline('b', [{'tag_name': 'L'}]),
]
result = formatters.process_schedules(
@@ -1622,8 +1571,8 @@ def test_process_schedules_notifies_unresolved_write_tags_once(formatters):
metadata=metadata['metadata'],
notification_id='ORCHESTRATOR_INVALID_WRITE_TAG_DURING_SCHEDULE_CONFIGURATION',
message=(
"Write tags b/{'server_id': '1', 'type': 'prediction', 'addr': 'ns=2;s=L'} "
'have no tag_id or a data_type the OPC Gateway does not accept'
"Write tags b/{'tag_name': 'L'} have no integer tag_id or a data_type the OPC "
'Gateway does not accept'
),
block='orchestrator',
level=NotificationLevel.ERROR,

38
tests/orchestrator/activities/test_mongo_db.py
View File

@@ -1,7 +1,7 @@
from datetime import UTC, datetime
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
from pytest import fixture, mark
from pytest import fixture, mark, raises
from sientia_do.notifications.models import NotificationLevel
from sientia_do.temporal.constants import DATETIME_FORMAT_MS_WITH_TZ
@@ -337,15 +337,17 @@ def test_update_pipelines_timestamps_failure(now_mock, mongo_db):
**metadata,
}
mongo_db.mongo_db_repository.update_one.side_effect = Exception('Error')
# The first update fails: the second is still tried, and only the failure is notified
mongo_db.mongo_db_repository.update_one.side_effect = [Exception('Error'), 1]
try:
mongo_db.update_pipelines_timestamps(input_data)
except Exception as e:
assert str(e) == 'Error'
except RuntimeError as e:
assert str(e) == 'Failed to update 1 of 2 pipelines timestamps'
assert mongo_db.mongo_db_repository.update_one.call_count == 2
mongo_db.send_notification.assert_called_once_with(
metadata=metadata['metadata'],
notification_id='MONGODB_UPDATE_PIPELINES_ERROR',
message='Failed to update pipelines timestamps: Error',
message='Failed to update pipelines timestamps: test1/test1: Error',
block='update_pipelines_timestamps',
level=NotificationLevel.ERROR,
attachment_content=ANY,
@@ -355,6 +357,32 @@ def test_update_pipelines_timestamps_failure(now_mock, mongo_db):
raise AssertionError('Expected an exception to be raised')
@patch('orchestrator.activities.mongo_db.now')
def test_update_pipelines_timestamps_failures_notified_together(now_mock, mongo_db):
input_data = {
'updated_pipelines': [
{'schedule_name': 'test1', 'namespace': 'ns', 'success': True},
{'schedule_name': 'test2', 'namespace': 'ns', 'success': True},
{'schedule_name': 'test3', 'namespace': 'ns', 'success': True},
],
**metadata,
}
mongo_db.mongo_db_repository.update_one.side_effect = [
Exception('down'),
1,
Exception('timeout'),
]
with raises(RuntimeError, match='Failed to update 2 of 3 pipelines timestamps'):
mongo_db.update_pipelines_timestamps(input_data)
assert mongo_db.mongo_db_repository.update_one.call_count == 3
mongo_db.send_notification.assert_called_once()
assert mongo_db.send_notification.call_args.kwargs['message'] == (
'Failed to update pipelines timestamps: ns/test1: down; ns/test3: timeout'
)
@patch('orchestrator.activities.mongo_db.now')
def test_create_pipelines_timestamps_success(now_mock, mongo_db):
input_data = {

14
tests/orchestrator/utils/tag_database_fixtures.py
View File

@@ -11,19 +11,9 @@ def scouter_pipeline_doc(schedule_name, model_id, read_tags, workflow_type='scou
}
def legacy_tag(address, name, frequency=1000, server_id='1', **extra):
return {
'server_id': server_id,
'tag_address': address,
'tag_name': name,
'frequency': frequency,
**extra,
}
TAGS_BY_ID = {
'7': {'tag_id': 7, 'connector_id': 3, 'address': 'ns=3;i=1001', 'tag_name': 'Pressure'},
'8': {'tag_id': 8, 'connector_id': 5, 'address': 'ns=3;i=1002', 'tag_name': 'Level'},
7: {'tag_id': 7, 'connector_id': 3, 'address': 'ns=3;i=1001', 'tag_name': 'Pressure'},
8: {'tag_id': 8, 'connector_id': 5, 'address': 'ns=3;i=1002', 'tag_name': 'Level'},
}

391
tests/orchestrator/utils/test_opc_slots.py
View File

@@ -4,7 +4,6 @@ from orchestrator.utils.opc_slots import (
)
from tests.orchestrator.utils.tag_database_fixtures import (
TAGS_BY_ID,
legacy_tag,
scouter_pipeline_doc,
)
@@ -18,44 +17,58 @@ def _name(schedule_name, tag_name, model_id):
}
opc_servers = {
'1': {
'id': '1',
'server_name': 'kepserver',
'url': 'opc.tcp://kep:49320',
'uri': 'urn:kep',
'cert_path': 'certs/client.pem',
},
'2': {'id': '2', 'server_name': 'simulator', 'url': 'opc.tcp://sim:4840', 'uri': 'urn:sim'},
}
def _read(tag_id, frequency=1000, **extra):
return {'tag_id': tag_id, 'frequency': frequency, **extra}
def _connector(connector_id, name, **extra):
return {
'connector_id': connector_id,
'name': name,
'url': f'opc.tcp://{name}:4840',
'server_uri': f'urn:{name}',
**extra,
}
def _db_tag(tag_id, connector_id, address, tag_name):
return {
'tag_id': tag_id,
'connector_id': connector_id,
'address': address,
'tag_name': tag_name,
}
connectors = {
'3': {
'connector_id': 3,
'name': 'plant',
'url': 'opc.tcp://plant:4840',
'server_uri': 'urn:plant',
'use_security': True,
'subscription_period_ms': 1500,
}
'1': _connector(1, 'kepserver', use_security=False),
'3': _connector(3, 'plant', use_security=True, subscription_period_ms=1500),
}
def _gather(pipelines, servers=None, connector_index=None, tag_index=None):
tags_by_id = {
**TAGS_BY_ID,
10: _db_tag(10, 1, 'ns=2;s=A', 'A'),
11: _db_tag(11, 1, 'ns=2;s=B', 'B'),
12: _db_tag(12, 1, 'ns=2;s=A', 'A2'),
13: _db_tag(13, 3, 'ns=2;s=A', 'A on plant'),
14: _db_tag(14, 1, 'ns=2;s=C', 'C'),
}
def _gather(pipelines, connector_index=None, tag_index=None):
return gather_read_tags(
pipelines,
opc_servers if servers is None else servers,
connectors if connector_index is None else connector_index,
TAGS_BY_ID if tag_index is None else tag_index,
tags_by_id if tag_index is None else tag_index,
)
def test_gather_read_tags_multi_name():
"""One address read by two pipelines with different names: one entry, two names"""
pipelines = [
scouter_pipeline_doc('sin', 'm1', [legacy_tag('ns=2;s=A', 'Counter', frequency=15000)]),
scouter_pipeline_doc('cos', 'm2', [legacy_tag('ns=2;s=A', 'Contador', frequency=5000)]),
scouter_pipeline_doc('sin', 'm1', [_read(10, 15000, tag_name='Counter')]),
scouter_pipeline_doc('cos', 'm2', [_read(10, 5000, tag_name='Contador')]),
]
entries, issues = _gather(pipelines)
@@ -64,81 +77,47 @@ def test_gather_read_tags_multi_name():
assert entries[0]['address'] == 'ns=2;s=A'
assert entries[0]['frequency'] == 5000
assert entries[0]['names'] == [_name('cos', 'Contador', 'm2'), _name('sin', 'Counter', 'm1')]
assert issues == {'tags': set(), 'connectors': set(), 'servers': set(), 'invalid': set()}
assert issues == {'tags': set(), 'connectors': set(), 'invalid': set()}
def test_gather_read_tags_same_name_in_two_pipelines():
"""The same name read by two pipelines still goes to both collections"""
pipelines = [
scouter_pipeline_doc('sin', 'm1', [legacy_tag('ns=2;s=A', 'Counter')]),
scouter_pipeline_doc('cos', 'm2', [legacy_tag('ns=2;s=A', 'Counter')]),
scouter_pipeline_doc('sin', 'm1', [_read(10)]),
scouter_pipeline_doc('cos', 'm2', [_read(10)]),
]
entries, _issues = _gather(pipelines)
assert entries[0]['names'] == [_name('cos', 'Counter', 'm2'), _name('sin', 'Counter', 'm1')]
assert entries[0]['names'] == [_name('cos', 'A', 'm2'), _name('sin', 'A', 'm1')]
def test_gather_read_tags_duplicate_tag_in_one_pipeline():
pipelines = [
scouter_pipeline_doc(
'sin', 'm1', [legacy_tag('ns=2;s=A', 'a'), legacy_tag('ns=2;s=A', 'a')]
),
]
entries, _issues = _gather([scouter_pipeline_doc('sin', 'm1', [_read(10), _read(10)])])
entries, _issues = _gather(pipelines)
assert entries[0]['names'] == [_name('sin', 'A', 'm1')]
assert entries[0]['names'] == [_name('sin', 'a', 'm1')]
def test_gather_read_tags_two_tag_ids_on_one_address():
"""Two connector_tags on the same node: one entry, one name per tag"""
entries, _issues = _gather([scouter_pipeline_doc('sin', 'm1', [_read(10), _read(12)])])
assert len(entries) == 1
assert entries[0]['names'] == [_name('sin', 'A', 'm1'), _name('sin', 'A2', 'm1')]
def test_gather_read_tags_same_address_on_two_servers():
pipelines = [
scouter_pipeline_doc(
'sin',
'm1',
[legacy_tag('ns=2;s=A', 'a', server_id='2'), legacy_tag('ns=2;s=A', 'a')],
),
]
entries, _issues = _gather(pipelines)
entries, _issues = _gather([scouter_pipeline_doc('sin', 'm1', [_read(13), _read(10)])])
assert [(e['server']['server_id'], e['address']) for e in entries] == [
('1', 'ns=2;s=A'),
('2', 'ns=2;s=A'),
('3', 'ns=2;s=A'),
]
def test_gather_read_tags_legacy_server():
"""Without tag_id: opc-servers fields, no certificate paths, security from cert_path"""
entries, _issues = _gather([scouter_pipeline_doc('sin', 'm1', [legacy_tag('ns=2;s=A', 'a')])])
assert entries[0]['server'] == {
'server_id': '1',
'name': 'kepserver',
'url': 'opc.tcp://kep:49320',
'server_uri': 'urn:kep',
'use_security': True,
'subscription_period_ms': None,
'from_tag_database': False,
'key': 'kepserver',
}
def test_gather_read_tags_numeric_server_id():
"""A numeric server_id in the pipeline still finds its opc-servers document"""
entries, issues = _gather(
[scouter_pipeline_doc('sin', 'm1', [legacy_tag('ns=2;s=A', 'a', server_id=1)])]
)
assert entries[0]['server']['server_id'] == '1'
assert not issues['servers']
def test_gather_read_tags_with_tag_database():
"""With tag_id: address, name and server come from connector_tags + connectors"""
entries, _issues = _gather(
[scouter_pipeline_doc('sin', 'm1', [{'tag_id': 7, 'frequency': 1000}])]
)
"""Address, name and server come from connector_tags + connectors"""
entries, _issues = _gather([scouter_pipeline_doc('sin', 'm1', [_read(7)])])
assert entries == [
{
@@ -149,7 +128,6 @@ def test_gather_read_tags_with_tag_database():
'server_uri': 'urn:plant',
'use_security': True,
'subscription_period_ms': 1500,
'from_tag_database': True,
'key': 'plant',
},
'address': 'ns=3;i=1001',
@@ -161,97 +139,60 @@ def test_gather_read_tags_with_tag_database():
def test_gather_read_tags_pipeline_name_wins_over_tag_database():
entries, _issues = _gather(
[scouter_pipeline_doc('sin', 'm1', [{'tag_id': 7, 'tag_name': 'P1', 'frequency': 1000}])]
)
entries, _issues = _gather([scouter_pipeline_doc('sin', 'm1', [_read(7, tag_name='P1')])])
assert entries[0]['names'] == [_name('sin', 'P1', 'm1')]
def test_gather_read_tags_unknown_tag_id():
entries, issues = _gather(
[scouter_pipeline_doc('sin', 'm1', [{'tag_id': 404, 'frequency': 1000}])]
)
entries, issues = _gather([scouter_pipeline_doc('sin', 'm1', [_read(404)])])
assert entries == []
assert issues == {'tags': {'404'}, 'connectors': set(), 'servers': set(), 'invalid': set()}
assert issues == {'tags': {'404'}, 'connectors': set(), 'invalid': set()}
def test_gather_read_tags_inactive_connector():
"""Connector 5 is not among the active connectors: its tag is left out"""
entries, issues = _gather(
[scouter_pipeline_doc('sin', 'm1', [{'tag_id': 8, 'frequency': 1000}])]
)
entries, issues = _gather([scouter_pipeline_doc('sin', 'm1', [_read(8)])])
assert entries == []
assert issues == {'tags': set(), 'connectors': {'5'}, 'servers': set(), 'invalid': set()}
assert issues == {'tags': set(), 'connectors': {'5'}, 'invalid': set()}
def test_gather_read_tags_unknown_server():
entries, issues = _gather(
[scouter_pipeline_doc('sin', 'm1', [legacy_tag('ns=2;s=A', 'a', server_id='9')])]
)
assert entries == []
assert issues == {'tags': set(), 'connectors': set(), 'servers': {'9'}, 'invalid': set()}
def test_gather_read_tags_mixed_with_and_without_tag_id():
"""Both paths in one run; the same node via tag_id and via tag_address merges"""
connector_1 = {
'connector_id': 1,
'name': 'kepserver',
'url': 'opc.tcp://kep:49320',
'server_uri': 'urn:kep',
'use_security': True,
'subscription_period_ms': 2000,
}
tag_index = {
'20': {'tag_id': 20, 'connector_id': 1, 'address': 'ns=2;s=A', 'tag_name': 'A from db'},
}
pipelines = [
scouter_pipeline_doc(
'sin', 'm1', [legacy_tag('ns=2;s=A', 'a'), legacy_tag('ns=2;s=B', 'b')]
),
scouter_pipeline_doc(
'cos', 'm2', [{'tag_id': 20, 'frequency': 500}, {'tag_id': 7, 'frequency': 1000}]
),
]
entries, issues = _gather(
pipelines,
connector_index={**connectors, '1': connector_1},
tag_index={**TAGS_BY_ID, **tag_index},
)
assert [(e['server']['server_id'], e['address']) for e in entries] == [
('1', 'ns=2;s=A'),
('1', 'ns=2;s=B'),
('3', 'ns=3;i=1001'),
]
assert entries[0]['names'] == [_name('cos', 'A from db', 'm2'), _name('sin', 'a', 'm1')]
assert entries[0]['frequency'] == 500
# Same server_id in connectors and opc-servers: the connectors fields win
assert entries[0]['server']['subscription_period_ms'] == 2000
assert entries[1]['server'] is entries[0]['server']
assert issues == {'tags': set(), 'connectors': set(), 'servers': set(), 'invalid': set()}
def test_gather_read_tags_server_name_clash():
"""Two servers with the same name get distinct slot keys, stable by server_id"""
servers = {
'1': opc_servers['1'],
'2': {**opc_servers['2'], 'server_name': 'kepserver'},
}
def test_gather_read_tags_without_tag_id():
"""connectors is the only source of the servers: a read tag needs a tag_id"""
pipelines = [
scouter_pipeline_doc(
'sin',
'm1',
[legacy_tag('ns=2;s=A', 'a', server_id='2'), legacy_tag('ns=2;s=B', 'b')],
),
[
_read(10),
{'tag_name': 'b', 'frequency': 1000},
{'tag_id': '', 'tag_name': 'c', 'frequency': 1000},
{'tag_id': None, 'tag_name': 'd', 'frequency': 1000},
# tag_id is int in connector_tags: text is no reference
{'tag_id': '11', 'frequency': 1000},
],
)
]
entries, _issues = _gather(pipelines, servers=servers)
entries, issues = _gather(pipelines)
assert [e['address'] for e in entries] == ['ns=2;s=A']
assert issues == {
'tags': set(),
'connectors': set(),
'invalid': {'sin/b', 'sin/c', 'sin/d', 'sin/11'},
}
def test_gather_read_tags_server_name_clash():
"""Two servers with the same name get distinct slot keys, stable by server_id"""
connector_index = {**connectors, '2': _connector(2, 'kepserver')}
tag_index = {**tags_by_id, 20: _db_tag(20, 2, 'ns=2;s=A', 'A')}
pipelines = [scouter_pipeline_doc('sin', 'm1', [_read(20), _read(11)])]
entries, _issues = _gather(pipelines, connector_index, tag_index)
assert [e['server']['key'] for e in entries] == ['kepserver', 'kepserver:2']
assert [e['server']['name'] for e in entries] == ['kepserver', 'kepserver']
@@ -260,25 +201,17 @@ def test_gather_read_tags_server_name_clash():
def test_gather_read_tags_active_read():
"""Active read: on if any pipeline asks, with the lowest cycle_multiple (D3)"""
pipelines = [
scouter_pipeline_doc('p1', 'm1', [legacy_tag('ns=2;s=A', 'a')]),
scouter_pipeline_doc('p1', 'm1', [_read(10)]),
scouter_pipeline_doc(
'p2',
'm2',
[legacy_tag('ns=2;s=A', 'a', active_read={'enabled': True, 'cycle_multiple': 4})],
'p2', 'm2', [_read(10, active_read={'enabled': True, 'cycle_multiple': 4})]
),
scouter_pipeline_doc(
'p3',
'm3',
[legacy_tag('ns=2;s=A', 'a', active_read={'enabled': True, 'cycle_multiple': 2})],
'p3', 'm3', [_read(10, active_read={'enabled': True, 'cycle_multiple': 2})]
),
scouter_pipeline_doc(
'p4',
'm4',
[legacy_tag('ns=2;s=B', 'b', active_read={'enabled': False, 'cycle_multiple': 1})],
),
scouter_pipeline_doc(
'p5', 'm5', [legacy_tag('ns=2;s=C', 'c', active_read={'enabled': True})]
'p4', 'm4', [_read(11, active_read={'enabled': False, 'cycle_multiple': 1})]
),
scouter_pipeline_doc('p5', 'm5', [_read(14, active_read={'enabled': True})]),
]
entries, _issues = _gather(pipelines)
@@ -293,14 +226,10 @@ def test_gather_read_tags_active_read():
def test_gather_read_tags_filters_non_scouter_workflows():
"""Only scouter pipelines feed the slots"""
pipelines = [
scouter_pipeline_doc('sin', 'm1', [legacy_tag('ns=2;s=A', 'a')]),
scouter_pipeline_doc(
'pred', 'm2', [legacy_tag('ns=2;s=P', 'p')], workflow_type='predictions_batch'
),
scouter_pipeline_doc(
'retrain', 'm3', [legacy_tag('ns=2;s=R', 'r')], workflow_type='minimal_retrain'
),
scouter_pipeline_doc('drift', 'm4', [legacy_tag('ns=2;s=D', 'd')], workflow_type='drift'),
scouter_pipeline_doc('sin', 'm1', [_read(10)]),
scouter_pipeline_doc('pred', 'm2', [_read(11)], workflow_type='predictions_batch'),
scouter_pipeline_doc('retrain', 'm3', [_read(14)], workflow_type='minimal_retrain'),
scouter_pipeline_doc('drift', 'm4', [_read(7)], workflow_type='drift'),
]
entries, _issues = _gather(pipelines)
@@ -312,25 +241,13 @@ def test_gather_read_tags_without_read_tags():
pipeline = scouter_pipeline_doc('sin', 'm1', [])
del pipeline['read_tags']
assert _gather([pipeline]) == (
[],
{'tags': set(), 'connectors': set(), 'servers': set(), 'invalid': set()},
)
assert _gather([pipeline]) == ([], {'tags': set(), 'connectors': set(), 'invalid': set()})
def test_build_tag_config():
"""Slot v2 (contract C1): no certificate paths, subscription period per server"""
pipelines = [
scouter_pipeline_doc(
'sin',
'm1',
[
legacy_tag('ns=2;s=A', 'a', frequency=1000),
legacy_tag('ns=2;s=B', 'b', frequency='300'),
legacy_tag('ns=2;s=C', 'c', frequency=200, server_id='2'),
{'tag_id': 7, 'frequency': 100},
],
)
scouter_pipeline_doc('sin', 'm1', [_read(10, 1000), _read(11, '300'), _read(7, 100)])
]
entries, _issues = _gather(pipelines)
@@ -340,35 +257,21 @@ def test_build_tag_config():
'kepserver': {
'server_id': '1',
'name': 'kepserver',
'url': 'opc.tcp://kep:49320',
'server_uri': 'urn:kep',
'use_security': True,
'url': 'opc.tcp://kepserver:4840',
'server_uri': 'urn:kepserver',
'use_security': False,
# No subscription_period_ms on the connector: half its lowest tag frequency
'subscription_period_ms': 150,
'tags': {
'ns=2;s=A': {
'frequency': 1000,
'active_read': {'enabled': False},
'names': [_name('sin', 'a', 'm1')],
'names': [_name('sin', 'A', 'm1')],
},
'ns=2;s=B': {
'frequency': 300,
'active_read': {'enabled': False},
'names': [_name('sin', 'b', 'm1')],
},
},
},
'simulator': {
'server_id': '2',
'name': 'simulator',
'url': 'opc.tcp://sim:4840',
'server_uri': 'urn:sim',
'use_security': False,
'subscription_period_ms': 100,
'tags': {
'ns=2;s=C': {
'frequency': 200,
'active_read': {'enabled': False},
'names': [_name('sin', 'c', 'm1')],
'names': [_name('sin', 'B', 'm1')],
},
},
},
@@ -394,49 +297,22 @@ def test_build_tag_config_empty():
assert build_tag_config([]) == {}
def test_gather_read_tags_empty_tag_id_keeps_legacy_fields():
"""tag_id '' is no tag database reference: the current fields are used (P4)"""
entries, issues = _gather(
[scouter_pipeline_doc('sin', 'm1', [legacy_tag('ns=2;s=A', 'a', tag_id='')])]
)
assert [(e['server']['server_id'], e['address']) for e in entries] == [('1', 'ns=2;s=A')]
assert issues == {'tags': set(), 'connectors': set(), 'servers': set(), 'invalid': set()}
def test_gather_read_tags_invalid_read_tags():
"""Unusable read tags are reported and skipped; the others still make the slot"""
pipelines = [
scouter_pipeline_doc(
'sin',
'm1',
[
legacy_tag('ns=2;s=A', 'a'),
{'server_id': '1', 'tag_address': 'ns=2;s=B', 'tag_name': 'b'},
legacy_tag('ns=2;s=C', 'c', frequency='fast'),
{'server_id': '1', 'tag_address': 'ns=2;s=D', 'frequency': 1000},
{'tag_address': 'ns=2;s=E', 'tag_name': 'e', 'frequency': 1000},
{'tag_id': 7},
],
)
scouter_pipeline_doc('sin', 'm1', [_read(10), {'tag_id': 11}, _read(7, frequency='fast')])
]
entries, issues = _gather(pipelines)
assert [e['address'] for e in entries] == ['ns=2;s=A']
assert issues['invalid'] == {
'sin/7',
'sin/ns=2;s=B',
'sin/ns=2;s=C',
'sin/ns=2;s=D',
'sin/ns=2;s=E',
}
assert issues['invalid'] == {'sin/11', 'sin/7'}
def test_gather_read_tags_frequency_type_does_not_depend_on_order():
"""15000 and '15000' give the same int frequency in any pipeline order (O6)"""
as_int = scouter_pipeline_doc('sin', 'm1', [legacy_tag('ns=2;s=A', 'a', frequency=15000)])
as_text = scouter_pipeline_doc('cos', 'm2', [legacy_tag('ns=2;s=A', 'a', frequency='15000')])
as_int = scouter_pipeline_doc('sin', 'm1', [_read(10, 15000)])
as_text = scouter_pipeline_doc('cos', 'm2', [_read(10, '15000')])
first, _ = _gather([as_int, as_text])
second, _ = _gather([as_text, as_int])
@@ -448,24 +324,19 @@ def test_gather_read_tags_frequency_type_does_not_depend_on_order():
def test_gather_read_tags_server_key_never_collides():
"""A server literally named 'x:3' does not share its key with server 3"""
servers = {
'1': {**opc_servers['1'], 'server_name': 'x:3'},
'2': {**opc_servers['2'], 'server_name': 'x'},
'3': {'id': '3', 'server_name': 'x', 'url': 'opc.tcp://three:4840', 'uri': 'urn:3'},
connector_index = {
'1': _connector(1, 'x:3'),
'2': _connector(2, 'x'),
'3': _connector(3, 'x'),
}
pipelines = [
scouter_pipeline_doc(
'sin',
'm1',
[
legacy_tag('ns=2;s=A', 'a', server_id='1'),
legacy_tag('ns=2;s=B', 'b', server_id='2'),
legacy_tag('ns=2;s=C', 'c', server_id='3'),
],
),
]
tag_index = {
1: _db_tag(1, 1, 'ns=2;s=A', 'a'),
2: _db_tag(2, 2, 'ns=2;s=B', 'b'),
3: _db_tag(3, 3, 'ns=2;s=C', 'c'),
}
pipelines = [scouter_pipeline_doc('sin', 'm1', [_read(1), _read(2), _read(3)])]
entries, _issues = _gather(pipelines, servers=servers)
entries, _issues = _gather(pipelines, connector_index, tag_index)
slot = build_tag_config(entries)
keys = [e['server']['key'] for e in entries]
@@ -481,33 +352,33 @@ def test_gather_read_tags_invalid_cycle_multiple_does_not_fail_the_slot():
'sin',
'm1',
[
legacy_tag('ns=2;s=A', 'a'),
legacy_tag('ns=2;s=B', 'b', active_read={'enabled': True, 'cycle_multiple': 'x'}),
legacy_tag('ns=2;s=C', 'c', active_read={'enabled': True, 'cycle_multiple': None}),
_read(10),
_read(11, active_read={'enabled': True, 'cycle_multiple': 'x'}),
_read(7, active_read={'enabled': True, 'cycle_multiple': None}),
# Not a dict: treated as no active read
legacy_tag('ns=2;s=D', 'd', active_read=True),
_read(14, active_read=True),
],
)
]
entries, issues = _gather(pipelines)
assert [e['address'] for e in entries] == ['ns=2;s=A', 'ns=2;s=D']
assert [e['address'] for e in entries] == ['ns=2;s=A', 'ns=2;s=C']
assert entries[1]['active_read'] == {'enabled': False}
assert issues['invalid'] == {'sin/ns=2;s=B', 'sin/ns=2;s=C'}
assert issues['invalid'] == {'sin/11', 'sin/7'}
def test_gather_read_tags_skips_incomplete_pipelines():
"""A pipeline without schedule_name, model_id or model.name is reported, not raised"""
no_model = scouter_pipeline_doc('no-model', 'm1', [legacy_tag('ns=2;s=A', 'a')])
no_model = scouter_pipeline_doc('no-model', 'm1', [_read(10)])
del no_model['model']
no_name = scouter_pipeline_doc('no-name', 'm2', [legacy_tag('ns=2;s=B', 'b')])
no_name = scouter_pipeline_doc('no-name', 'm2', [_read(10)])
no_name['model'] = {}
no_model_id = scouter_pipeline_doc('no-id', None, [legacy_tag('ns=2;s=C', 'c')])
no_schedule = scouter_pipeline_doc(None, 'm4', [legacy_tag('ns=2;s=D', 'd')])
ok = scouter_pipeline_doc('ok', 'm5', [legacy_tag('ns=2;s=E', 'e')])
no_model_id = scouter_pipeline_doc('no-id', None, [_read(10)])
no_schedule = scouter_pipeline_doc(None, 'm4', [_read(10)])
ok = scouter_pipeline_doc('ok', 'm5', [_read(11)])
entries, issues = _gather([no_model, no_name, no_model_id, no_schedule, ok])
assert [e['address'] for e in entries] == ['ns=2;s=E']
assert [e['address'] for e in entries] == ['ns=2;s=B']
assert issues['invalid'] == {'no-model', 'no-name', 'no-id', 'None'}

82
tests/orchestrator/utils/test_tag_database.py
View File

@@ -7,13 +7,12 @@ from orchestrator.utils.tag_database import (
referenced_tag_ids,
resolve_tag_names,
resolve_write_tags,
tag_id_text,
tag_id_of,
)
from tests.orchestrator.utils.tag_database_fixtures import (
OPC_CONNECTORS,
TAGS_BY_ID,
batch_pipeline_doc,
legacy_tag,
scouter_pipeline_doc,
write_tag_doc,
)
@@ -22,44 +21,49 @@ from tests.orchestrator.utils.tag_database_fixtures import (
def test_index_connector_tags():
tags = [
{'tag_id': 7, 'connector_id': 3, 'address': 'ns=3;i=1', 'tag_name': 'a'},
# tag_id is int in connector_tags: a text id is no tag
{'tag_id': '8', 'connector_id': 3, 'address': 'ns=3;i=2', 'tag_name': 'b'},
]
assert index_connector_tags(tags) == {'7': tags[0], '8': tags[1]}
assert index_connector_tags(tags) == {7: tags[0]}
def test_resolve_tag_names():
read_tags = [
{'tag_id': 7},
{'tag_id': '7', 'tag_name': 'own name'},
{'tag_id': 7, 'tag_name': 'own name'},
{'tag_id': '7'},
{'tag_id': 404},
{'tag_name': 'legacy'},
{'tag_name': 'no tag_id'},
]
assert resolve_tag_names(read_tags, TAGS_BY_ID) == [
{'tag_id': 7, 'tag_name': 'Pressure'},
{'tag_id': '7', 'tag_name': 'own name'},
{'tag_id': 7, 'tag_name': 'own name'},
{'tag_id': '7'},
{'tag_id': 404},
{'tag_name': 'legacy'},
{'tag_name': 'no tag_id'},
]
def test_referenced_tag_ids():
"""Ids in int and text form, scouter pipelines only, empty or null tag_id ignored"""
"""Int ids only, scouter pipelines only; text, empty or null tag_id ignored"""
pipelines = [
scouter_pipeline_doc(
'sin', 'm1', [{'tag_id': 7}, {'tag_id': '12'}, {'tag_id': ''}, {'tag_id': None}]
),
scouter_pipeline_doc('cos', 'm2', [{'tag_id': 'abc'}, legacy_tag('ns=2;s=A', 'a')]),
scouter_pipeline_doc(
'cos', 'm2', [{'tag_id': 'abc'}, {'tag_name': 'a', 'frequency': 1000}]
),
scouter_pipeline_doc('pred', 'm3', [{'tag_id': 99}], workflow_type='predictions_batch'),
]
# Sorted by text (ints first), so the query is the same on every run
assert referenced_tag_ids(pipelines) == [12, 7, '12', '7', 'abc']
# Sorted, so the query is the same on every run
assert referenced_tag_ids(pipelines) == [7]
def test_referenced_tag_ids_without_tag_ids():
pipeline = scouter_pipeline_doc('sin', 'm1', [legacy_tag('ns=2;s=A', 'a')])
pipeline = scouter_pipeline_doc('sin', 'm1', [{'tag_name': 'a', 'frequency': 1000}])
no_read_tags = scouter_pipeline_doc('cos', 'm2', [])
del no_read_tags['read_tags']
@@ -70,11 +74,11 @@ def test_filter_by_tag_ids():
"""A tag_id $match goes in front, on a new aggregation list"""
query = {'collection': 'connector_tags', 'aggregation': [{'$project': {'_id': 0}}]}
result = filter_by_tag_ids(query, [7, '7'])
result = filter_by_tag_ids(query, [7, 8])
assert result == {
'collection': 'connector_tags',
'aggregation': [{'$match': {'tag_id': {'$in': [7, '7']}}}, {'$project': {'_id': 0}}],
'aggregation': [{'$match': {'tag_id': {'$in': [7, 8]}}}, {'$project': {'_id': 0}}],
}
assert result['aggregation'] is not query['aggregation']
assert query['aggregation'] == [{'$project': {'_id': 0}}]
@@ -93,26 +97,21 @@ def test_index_skips_incomplete_documents():
{'name': 'no id', 'url': 'opc.tcp://x:4840', 'server_uri': 'urn:x'},
]
assert list(index_connector_tags(tags)) == ['7']
assert list(index_connector_tags(tags)) == [7]
assert list(index_connectors(connector_list)) == ['3']
def test_tag_id_text_is_canonical_for_integers():
assert tag_id_text({'tag_id': 7}) == '7'
assert tag_id_text({'tag_id': '007'}) == '7'
assert tag_id_text({'tag_id': ' 12 '}) == '12'
assert tag_id_text({'tag_id': '-3'}) == '-3'
assert tag_id_text({'tag_id': 'abc'}) == 'abc'
assert tag_id_text({'tag_id': ''}) is None
assert tag_id_text({'tag_id': None}) is None
assert tag_id_text({}) is None
def test_referenced_tag_ids_never_raises_on_non_ascii_digits():
"""'²'.isdigit() is True but int('²') raises: it must stay text (runs in the workflow)"""
pipelines = [scouter_pipeline_doc('sin', 'm1', [{'tag_id': '²'}, {'tag_id': '007'}])]
assert referenced_tag_ids(pipelines) == [7, '7', '²']
def test_tag_id_of_accepts_only_int():
"""connector_tags stores tag_id as int: anything else is no reference"""
assert tag_id_of({'tag_id': 7}) == 7
assert tag_id_of({'tag_id': '7'}) is None
assert tag_id_of({'tag_id': 'abc'}) is None
assert tag_id_of({'tag_id': 7.0}) is None
assert tag_id_of({'tag_id': True}) is None
assert tag_id_of({'tag_id': ''}) is None
assert tag_id_of({'tag_id': None}) is None
assert tag_id_of({}) is None
assert tag_id_of('ns=2;s=raw') is None
def test_referenced_tag_ids_includes_write_tags():
@@ -123,7 +122,7 @@ def test_referenced_tag_ids_includes_write_tags():
batch_pipeline_doc('retrain', [{'tag_id': 99}], workflow_type='minimal_retrain'),
]
assert referenced_tag_ids(pipelines) == [7, 77, 78, 79, '7', '77', '78', '79']
assert referenced_tag_ids(pipelines) == [7, 77, 78, 79]
def test_index_connector_tags_filters_status_and_category():
@@ -136,10 +135,10 @@ def test_index_connector_tags_filters_status_and_category():
{'tag_id': 5, 'connector_id': 'plant', 'address': 'ns=2;s=N', 'tag_name': 'n'},
]
assert list(index_connector_tags(tags)) == ['1', '5']
assert list(index_connector_tags(tags, WRITE_CATEGORIES)) == ['2', '3', '5']
# Every document, whatever its status or category, for the updated_at
assert list(index_tag_documents(tags)) == ['1', '2', '3', '4', '5']
assert list(index_connector_tags(tags)) == [1, 5]
assert list(index_connector_tags(tags, WRITE_CATEGORIES)) == [2, 3, 5]
# Every document, whatever its status or category
assert list(index_tag_documents(tags)) == [1, 2, 3, 4, 5]
def test_index_connectors_leaves_out_inactive():
@@ -163,7 +162,7 @@ def test_resolve_write_tags():
connectors = {**OPC_CONNECTORS, 'other': {**OPC_CONNECTORS['plant'], 'connector_id': 'other'}}
output_config, issues = resolve_write_tags(
[{'tag_id': 77}, {'tag_id': '78'}, {'tag_id': 79}], tags, connectors
[{'tag_id': 77}, {'tag_id': 78}, {'tag_id': 79}], tags, connectors
)
assert output_config == {
@@ -196,9 +195,11 @@ def test_resolve_write_tags_issues():
{'tag_id': 4},
{'tag_id': 5},
{'tag_id': 404},
# Legacy entries are no longer accepted
{'server_id': '1', 'type': 'prediction', 'addr': 'ns=2;s=L'},
# Every write tag is a tag_id
{'tag_name': 'L'},
{'tag_id': None},
# tag_id is int: a text id is invalid
{'tag_id': '6'},
'ns=2;s=raw',
{'tag_id': 6},
]
@@ -210,7 +211,8 @@ def test_resolve_write_tags_issues():
assert issues['connectors'] == {'gone'}
assert issues['invalid'] == {
'3',
"{'server_id': '1', 'type': 'prediction', 'addr': 'ns=2;s=L'}",
"{'tag_name': 'L'}",
"{'tag_id': '6'}",
"{'tag_id': None}",
'ns=2;s=raw',
}

1505
tests/orchestrator/workflows/histories/orchestrator_before_tag_database.json
View File

File diff suppressed because it is too large Load Diff

1322
tests/orchestrator/workflows/histories/orchestrator_tag_database.json
View File

File diff suppressed because it is too large Load Diff

53
tests/orchestrator/workflows/test_orchestrator.py
View File

@@ -1,4 +1,4 @@
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
from unittest.mock import ANY, AsyncMock, call, patch
from pytest import fixture, mark
@@ -6,7 +6,6 @@ from orchestrator.activities.activities import Activities
from orchestrator.workflows.orchestrator import (
DEFAULT_CONNECTOR_TAGS_QUERY,
DEFAULT_CONNECTORS_QUERY,
TAG_DATABASE_PATCH,
Orchestrator,
)
@@ -30,10 +29,8 @@ metadata = {
@patch('orchestrator.workflows.orchestrator.referenced_tag_ids', return_value=[7, '7'])
@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
async def test_run(workflow_mock, referenced_tag_ids_mock, orchestrator):
workflow_mock.patched = MagicMock(return_value=True)
input_data = {
'pipelines_query': 'SELECT * FROM bucket',
'opc_servers_query': 'SELECT * FROM servers',
'schedule_name': 'test-schedule-name',
}
@@ -54,16 +51,13 @@ async def test_run(workflow_mock, referenced_tag_ids_mock, orchestrator):
]
)
workflow_mock.start_local_activity_method.assert_has_calls(
[
call(
Activities.find_documents_in_mongodb,
{**metadata, 'query': input_data['opc_servers_query']},
retry_policy=ANY,
start_to_close_timeout=ANY,
)
]
)
# connectors is the only source of the OPC servers: opc-servers is never loaded
assert not [
c
for c in workflow_mock.start_local_activity_method.call_args_list
if c.args[0] is Activities.find_documents_in_mongodb
and c.args[1]['query'] != {'collection': 'orchestrated_schedules'}
]
workflow_mock.start_local_activity_method.assert_has_calls(
[
@@ -116,7 +110,6 @@ async def test_run(workflow_mock, referenced_tag_ids_mock, orchestrator):
]
)
assert workflow_mock.patched.call_args_list == [call(TAG_DATABASE_PATCH)]
referenced_tag_ids_mock.assert_called_once_with(
workflow_mock.start_local_activity_method.return_value
)
@@ -177,7 +170,6 @@ async def test_run(workflow_mock, referenced_tag_ids_mock, orchestrator):
Activities.process_slots,
{
**metadata,
'opc_servers': workflow_mock.start_local_activity_method.return_value,
'connectors': workflow_mock.start_local_activity_method.return_value,
'connector_tags': workflow_mock.start_local_activity_method.return_value,
'active_ingestors': workflow_mock.start_local_activity_method.return_value,
@@ -424,7 +416,6 @@ def _process_slots_input(workflow_mock):
@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
async def test_run_with_tag_database_queries_from_input(workflow_mock, _ids, orchestrator):
"""The input overrides the default tag database aggregations (SIENTIAPDE-1974, D9)"""
workflow_mock.patched = MagicMock(return_value=True)
connectors_query = {'collection': 'connectors', 'aggregation': [{'$match': {'type': 'opc_ua'}}]}
connector_tags_query = {
'collection': 'connector_tags',
@@ -432,7 +423,6 @@ async def test_run_with_tag_database_queries_from_input(workflow_mock, _ids, orc
}
input_data = {
'pipelines_query': 'SELECT * FROM bucket',
'opc_servers_query': 'SELECT * FROM servers',
'connectors_query': connectors_query,
'connector_tags_query': connector_tags_query,
'schedule_name': 'test-schedule-name',
@@ -458,31 +448,8 @@ async def test_run_with_tag_database_queries_from_input(workflow_mock, _ids, orc
@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
async def test_run_without_tag_ids_skips_tag_database(workflow_mock, _ids, orchestrator):
"""No read tag with tag_id: the tag database is not queried at all (P4)"""
workflow_mock.patched = MagicMock(return_value=True)
input_data = {
'pipelines_query': 'SELECT * FROM bucket',
'opc_servers_query': 'SELECT * FROM servers',
'schedule_name': 'test-schedule-name',
}
await orchestrator.run(input_data)
assert workflow_mock.patched.call_args_list == [call(TAG_DATABASE_PATCH)]
assert _tag_database_calls(workflow_mock) == []
slots_input = _process_slots_input(workflow_mock)
assert slots_input['connectors'] == []
assert slots_input['connector_tags'] == []
@mark.asyncio
@patch('orchestrator.workflows.orchestrator.referenced_tag_ids', return_value=[7])
@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
async def test_run_replay_before_tag_database_patch(workflow_mock, _ids, orchestrator):
"""A run started before the patch replays without the tag database loads"""
workflow_mock.patched = MagicMock(return_value=False)
input_data = {
'pipelines_query': 'SELECT * FROM bucket',
'opc_servers_query': 'SELECT * FROM servers',
'schedule_name': 'test-schedule-name',
}
@@ -520,10 +487,8 @@ def test_default_connector_tags_query_loads_every_category_and_status():
@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
async def test_run_hands_the_tag_database_to_process_schedules(workflow_mock, _ids, orchestrator):
"""The write tags of process_schedules need the connectors as well as the tags"""
workflow_mock.patched = MagicMock(return_value=True)
input_data = {
'pipelines_query': 'SELECT * FROM bucket',
'opc_servers_query': 'SELECT * FROM servers',
'schedule_name': 'test-schedule-name',
}
@@ -541,11 +506,9 @@ async def test_run_hands_the_tag_database_to_process_schedules(workflow_mock, _i
@patch('orchestrator.workflows.orchestrator.workflow', new_callable=AsyncMock)
async def test_run_without_changes_skips_reports_and_timestamps(workflow_mock, _ids, orchestrator):
"""Nothing created, updated or deleted: no report and no pipeline timestamp activity"""
workflow_mock.patched = MagicMock(return_value=True)
workflow_mock.start_activity_method = AsyncMock(return_value=[])
input_data = {
'pipelines_query': 'SELECT * FROM bucket',
'opc_servers_query': 'SELECT * FROM servers',
'schedule_name': 'test-schedule-name',
}

22
tests/orchestrator/workflows/test_orchestrator_replay.py
View File

@@ -1,18 +1,15 @@
"""
Replay real Orchestrator histories against the current workflow code.
Replay a real Orchestrator history against the current workflow code.
The histories were recorded on the e2e stack (real Temporal, Mongo and Redis):
- orchestrator_before_tag_database.json: main before SIENTIAPDE-1974 (no patch marker,
no tag database load), as a run in flight during the deploy would replay;
- orchestrator_tag_database.json: this version, with the SIENTIAPDE-1974-tag-database
patch and the connectors / connector_tags loads of read and write tags by tag_id.
orchestrator_tag_database.json was recorded on the e2e stack (real Temporal, Mongo and
Redis), with the connectors / connector_tags loads of read and write tags by tag_id.
A NondeterminismError here means the change breaks runs started on those versions.
A NondeterminismError here means a change to the workflow breaks runs in flight.
"""
from pathlib import Path
from pytest import mark
import pytest
from temporalio.client import WorkflowHistory
from temporalio.worker import Replayer
@@ -21,13 +18,10 @@ from orchestrator.workflows.orchestrator import Orchestrator
HISTORIES = Path(__file__).parent / 'histories'
@mark.asyncio
@mark.parametrize(
'history_file', ['orchestrator_before_tag_database.json', 'orchestrator_tag_database.json']
)
async def test_replay_recorded_history(history_file):
@pytest.mark.asyncio
async def test_replay_recorded_history():
history = WorkflowHistory.from_json(
'orchestrator-history', (HISTORIES / history_file).read_text()
'orchestrator-history', (HISTORIES / 'orchestrator_tag_database.json').read_text()
)
await Replayer(workflows=[Orchestrator]).replay_workflow(history)