gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1974

Browse Source 
SIENTIAPDE-1974  Write OPC tags through the tag database and drop the PI Web API path

- predictions_batch write_tags are now [{tag_id}] only: resolve_write_tags builds
  the opc_output_config from connector_tags + connectors (server = connector_id,
  Write / Prediction Confidence -> prediction_tags / confidence_tags, address and
  data_type from the tag, float by default). The legacy server_id/type/addr write
  tags are no longer read and are notified as invalid.
- A pipeline's effective updated_at is the latest of its own and of the
  connector_tags it references, so renaming, moving or deactivating a tag
  redeploys it (removes the rename limitation).
- The connector_tags query loads the referenced tags of every category and
  status; the code filters them. aggregate_documents_in_mongodb formats only
  datetime timestamp fields.
- Remove the PI Web API tag database path (connectors-pi-web-api patch, PI loads,
  resolution, connection and held schedules): PI pipelines behave as on main.
- tag_id is canonical text and the workflow-side id collection never raises;
  a malformed pipeline no longer fails the whole OPC slot.
- Replay test against histories recorded on the e2e stack (main and this
  version), e2e scenarios for write tags and tag changes, and the runbook
  script scripts/sientiapde_1974_backfill_write_tags.js.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-29 08:25:01 -03:00
parent 9eaec54fc8
commit 2c331b5f16
28 changed files with 4451 additions and 1385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
e2e/scenario_inputs/orchestrator_conflict.json
View File

@@ -22,11 +22,17 @@
"frequency": "1m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
"write_tags": [{"tag_id": 101}]
}
],
"connectors": [
{"connector_id": "srv-1", "type": "opc_ua", "name": "opc-1", "url": "opc.tcp://opc-1:4840",
"server_uri": "urn:opc-1", "use_security": false, "status": "Active"}
],
"connector_tags": [
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],

12
e2e/scenario_inputs/orchestrator_create_only.json
View File

@@ -22,11 +22,17 @@
"frequency": "1m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
"write_tags": [{"tag_id": 101}]
}
],
"connectors": [
{"connector_id": "srv-1", "type": "opc_ua", "name": "opc-1", "url": "opc.tcp://opc-1:4840",
"server_uri": "urn:opc-1", "use_security": false, "status": "Active"}
],
"connector_tags": [
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],

12
e2e/scenario_inputs/orchestrator_happy_path.json
View File

@@ -22,9 +22,7 @@
"frequency": "1m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
"write_tags": [{"tag_id": 101}]
},
{
"schedule_name": "drift-gpu",
@@ -39,6 +37,14 @@
"interval_minutes": 60
}
],
"connectors": [
{"connector_id": "srv-1", "type": "opc_ua", "name": "opc-1", "url": "opc.tcp://opc-1:4840",
"server_uri": "urn:opc-1", "use_security": false, "status": "Active"}
],
"connector_tags": [
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true, "name": "opc-1"}
],

12
e2e/scenario_inputs/orchestrator_noop.json
View File

@@ -22,11 +22,17 @@
"frequency": "1m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
"write_tags": [{"tag_id": 101}]
}
],
"connectors": [
{"connector_id": "srv-1", "type": "opc_ua", "name": "opc-1", "url": "opc.tcp://opc-1:4840",
"server_uri": "urn:opc-1", "use_security": false, "status": "Active"}
],
"connector_tags": [
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],

118
e2e/scenario_inputs/orchestrator_pi_web_api.json
View File

@@ -1,118 +0,0 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e-pi-web-api",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"connectors": [
{
"connector_id": "pi-usina",
"type": "pi_web_api",
"name": "PI Usina",
"url": "https://pi-usina.example/piwebapi",
"status": "Active",
"pi_web_api": {
"auth": {"type": "bearer", "secret": {"name": "pi-web-api-auth-token", "key": "token"}},
"period": "*-2h",
"max_count": 10
}
},
{
"connector_id": "pi-other",
"type": "pi_web_api",
"name": "PI Other",
"url": "https://pi-other.example/piwebapi",
"status": "Active",
"pi_web_api": {
"auth": {"type": "basic", "secret": {"name": "pi-other-auth", "key": "credentials"}}
}
},
{
"connector_id": "plant",
"type": "opc_ua",
"name": "plant",
"url": "opc.tcp://plant:4840",
"server_uri": "urn:plant",
"use_security": false,
"subscription_period_ms": 500,
"status": "Active"
}
],
"connector_tags": [
{"tag_id": 42, "connector_id": "pi-usina", "address": "F1DP-A", "tag_name": "TI-101-db", "category": "Read", "status": "Active"},
{"tag_id": 43, "connector_id": "pi-other", "address": "F1DP-B", "tag_name": "TI-102-db", "category": "Read", "status": "Active"},
{"tag_id": 77, "connector_id": "pi-usina", "address": "F1DP-P", "tag_name": "PRED-101", "category": "Write", "status": "Active"},
{"tag_id": 78, "connector_id": "pi-usina", "address": "F1DP-C", "tag_name": "CONF-101", "category": "Prediction Confidence", "status": "Active"},
{"tag_id": 7, "connector_id": "plant", "address": "ns=3;i=1001", "tag_name": "Pressure", "category": "Read", "status": "Active"}
],
"pipelines": [
{
"schedule_name": "pi-tagdb",
"workflow_type": "pi_web_api_scouter",
"model_id": "model-pi",
"model": {"name": "Model model-pi"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"read_tags": {
"TI-101": {"tag_id": 42, "aggr_func": "avg"},
"TI-404": {"tag_id": 404}
},
"pi_web_api_config": {"endpoint": "/streamsets/recorded", "max_count": 5}
},
{
"schedule_name": "pi-legacy",
"workflow_type": "pi_web_api_scouter",
"model_id": "model-pi-legacy",
"model": {"name": "Model model-pi-legacy"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"read_tags": {"TI-900": {"webid": "F1DP-X"}},
"pi_web_api_config": {"endpoint": "/streamsets/recorded", "api_timeout": 10}
},
{
"schedule_name": "pred-pi",
"workflow_type": "predictions_batch",
"model_id": "model-pi",
"model": {"name": "Model model-pi"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"query": "SELECT 1",
"pi_web_api_output_config": {
"prediction_tags": [{"tag_id": 77}],
"confidence_tags": [{"tag_id": 78}, {"tag_id": 42}]
}
},
{
"schedule_name": "pred-pi-legacy",
"workflow_type": "predictions_batch",
"model_id": "model-pi-legacy",
"model": {"name": "Model model-pi-legacy"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"query": "SELECT 1",
"pi_web_api_output_config": {"prediction_tags": {"PRED-900": "F1DP-L"}}
},
{
"schedule_name": "opc-tagdb",
"workflow_type": "scouter",
"model_id": "model-opc",
"model": {"name": "Model model-opc"},
"active": true,
"updated_at": "@now",
"frequency": "1m",
"read_tags": [{"tag_id": 7, "frequency": 1000, "aggr_func": "avg"}]
}
],
"opc_servers": [],
"active_ingestors": ["heartbeat:ingestor:1"]
}

64
e2e/scenario_inputs/orchestrator_tag_database.json Normal file
View File

@@ -0,0 +1,64 @@
{
"workflow_input": {
"schedule_name": "orchestrator-e2e-tag-database",
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"connectors": [
{
"connector_id": "plant",
"type": "opc_ua",
"name": "plant",
"url": "opc.tcp://plant:4840",
"server_uri": "urn:plant",
"use_security": false,
"subscription_period_ms": 500,
"status": "Active",
"updated_at": "@now-1d"
}
],
"connector_tags": [
{"tag_id": 7, "connector_id": "plant", "address": "ns=3;i=1001", "tag_name": "Pressure", "category": "Read", "status": "Active", "updated_at": "@now-1d"},
{"tag_id": 101, "connector_id": "plant", "address": "ns=3;i=2001", "tag_name": "PRED-101", "category": "Write", "data_type": "double", "status": "Active", "updated_at": "@now-1d"},
{"tag_id": 102, "connector_id": "plant", "address": "ns=3;i=2002", "tag_name": "CONF-101", "category": "Prediction Confidence", "status": "Active", "updated_at": "@now-1d"},
{"tag_id": 103, "connector_id": "plant", "address": "ns=3;i=2003", "tag_name": "PRED-OLD", "category": "Write", "status": "Inactive", "updated_at": "@now-1d"}
],
"pipelines": [
{
"schedule_name": "opc-tagdb",
"workflow_type": "scouter",
"model_id": "model-opc",
"model": {"name": "Model model-opc"},
"active": true,
"updated_at": "@now-2d",
"frequency": "1m",
"read_tags": [{"tag_id": 7, "frequency": 1000, "aggr_func": "avg"}]
},
{
"schedule_name": "pred-tagdb",
"workflow_type": "predictions_batch",
"model_id": "model-opc",
"model": {"name": "Model model-opc"},
"active": true,
"updated_at": "@now-2d",
"frequency": "1m",
"query": "SELECT 1",
"write_tags": [
{"tag_id": 101},
{"tag_id": 102},
{"tag_id": 7},
{"tag_id": 103},
{"tag_id": 404},
{"server_id": "plant", "type": "prediction", "addr": "ns=3;i=9999"}
]
}
],
"opc_servers": [],
"active_ingestors": ["heartbeat:ingestor:1"]
}

12
e2e/scenario_inputs/orchestrator_update_only.json
View File

@@ -22,11 +22,17 @@
"frequency": "5m",
"offset": "0m",
"query": "SELECT 1",
"write_tags": [
{"server_id": "srv-1", "type": "prediction", "addr": "ns=2;s=Tag1"}
]
"write_tags": [{"tag_id": 101}]
}
],
"connectors": [
{"connector_id": "srv-1", "type": "opc_ua", "name": "opc-1", "url": "opc.tcp://opc-1:4840",
"server_uri": "urn:opc-1", "use_security": false, "status": "Active"}
],
"connector_tags": [
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],

45
e2e/scenarios.md
View File

@@ -27,8 +27,8 @@ Source: `e2e/test_orchestrator_main_workflow.py`
### 1.1.1 Happy path
Pipelines in Mongo → both Temporal schedules created in the `laborious` namespace, one
`orchestrated_schedules` entry per pipeline, and no PI Web API output config when no pipeline uses
the tag database.
`orchestrated_schedules` entry per pipeline, and the `predictions_batch` write tag (`tag_id`)
resolved into `opc_output_config` through the tag database.
### 1.3.1 Create-only
@@ -66,37 +66,30 @@ Two `drift` pipelines on `frequency='1m'`: one without `interval_minutes` and wi
`interval_minutes=60` and `chunk_period='s'` → both forwarded unchanged, no warning
(SIENTIAPDE-2212).
### 1.9 PI Web API through the tag database (connectors-pi-web-api)
### 1.9 OPC tag database: read and write tags by tag_id
Scenario input: `orchestrator_pi_web_api.json` (two `pi_web_api` connectors, one `opc_ua`
connector, their `connector_tags`, PI Web API pipelines by `tag_id` and legacy ones, and an OPC
scouter by `tag_id`).
Scenario input: `orchestrator_tag_database.json` (one `opc_ua` connector, its `connector_tags` of
every category, one inactive, an OPC scouter reading by `tag_id` and a `predictions_batch` writing by
`tag_id`, with a Read tag, an inactive tag, an unknown `tag_id` and a legacy entry among its write
tags).
#### 1.9.1 PI scouter through the tag database
#### 1.9.1 Write tags through the tag database
`pi_web_api_scouter` with `tag_id` read tags → `model_tags` WebID from `connector_tags.address`,
`pi_web_api_query` with the connector `period` under the pipeline `max_count` (`api_timeout` capped
at the frequency), and `connection` with base URL and the secret reference only. An unknown
`tag_id` is left out and notified (`ORCHESTRATOR_PI_TAG_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION`).
A legacy pipeline (inline `webid`) gets the pre-change input, without `connection`.
`write_tags: [{tag_id}]` → `opc_output_config` keyed by `connector_id`, address and `data_type`
from `connector_tags` (`float` when absent), `Write` → `prediction_tags`, `Prediction Confidence` →
`confidence_tags`. The Read, inactive and unknown tags are left out and notified once
(`ORCHESTRATOR_WRITE_TAG_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION`); the legacy entry is notified
(`ORCHESTRATOR_INVALID_WRITE_TAG_DURING_SCHEDULE_CONFIGURATION`).
#### 1.9.2 PI output tags through the tag database
#### 1.9.2 A tag change redeploys its pipelines
`predictions_batch` `pi_web_api_output_config` given as `tag_id` lists → `{tag_name: WebID}` from
`Write` / `Prediction Confidence` tags; a `Read` tag given as a confidence tag is left out and
notified (`ORCHESTRATOR_PI_TAG_WRONG_CATEGORY_DURING_SCHEDULE_CONFIGURATION`). A legacy
`{tag_name: webid}` mapping passes through unchanged.
Renaming the read tag and moving the write tag in `connector_tags` (with a newer `updated_at`),
without touching the pipelines → the next run redeploys both: the scouter `model_tags` and the
slot `names` take the new name, the `opc_output_config` the new address.
#### 1.9.3 Mixed PI connectors hold the schedule
#### 1.9.3 OPC slot from the tag database
A pipeline updated to read tags of two PI connectors → its existing scouter schedule is neither
updated nor deleted, and the conflict is notified
(`ORCHESTRATOR_PI_MIXED_CONNECTORS_DURING_SCHEDULE_CONFIGURATION`).
#### 1.9.4 OPC slot unchanged by PI connectors
The OPC slot v2 written to Redis for an OPC scouter by `tag_id` is the expected one with the PI
connectors and tags present, and identical after removing them.
The slot v2 written to Redis holds only the active Read tag, with the connector's server fields.
---

253
e2e/test_orchestrator_main_workflow.py
View File

@@ -100,9 +100,11 @@ async def test_scenario_1_1_1_happy_path(
assert {'pred-legacy', 'drift-gpu'} <= set(laborious_schedules), (
f'Expected Temporal schedules in laborious, got {laborious_schedules}'
)
# No pipeline uses the tag database: the PI Web API loads stay empty
# The write tag comes from the tag database, keyed by the connector the gateway writes to
pred = await _schedule_input(temporal_host, 'pred-legacy')
assert pred['pi_web_api_output_config'] == {}
assert pred['opc_output_config'] == {
'srv-1': {'prediction_tags': {'ns=2;s=Tag1': {'data_type': 'float'}}}
}
@pytest.mark.e2e
@@ -377,17 +379,15 @@ async def test_scenario_1_8_1_drift_window_and_chunk_period(
assert warnings[0]['model_id'] == 'model-dr'
# 1.9 PI Web API through the tag database (connectors-pi-web-api)
PI_SECRET = {'name': 'pi-web-api-auth-token', 'key': 'token'}
# 1.9 OPC tag database: read and write tags by tag_id (SIENTIAPDE-1974)
async def _run_pi_scenario(temporal_env: WorkflowEnvironment, scenario: dict, tag: str) -> None:
async def _run_tag_database_scenario(temporal_env: WorkflowEnvironment, tag: str) -> None:
await start_and_await_workflow(
temporal_env.client,
Orchestrator.run,
scenario['workflow_input'],
make_workflow_id(f'orchestrator-pi-{tag}'),
load_scenario_input('orchestrator_tag_database')['workflow_input'],
make_workflow_id(f'orchestrator-tagdb-{tag}'),
)
@@ -395,132 +395,6 @@ def _notifications(notification_handler, notification_id: str) -> list[dict]:
return list(notification_handler.mongo_collection.find({'notification_id': notification_id}))
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_9_1_pi_scouter_through_tag_database(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
temporal_host: str,
redis_client: Redis,
notification_handler,
):
"""1.9.1: tag_id read tags get WebID + connection from the connector; legacy unchanged."""
scenario = load_scenario_input('orchestrator_pi_web_api')
_apply_orchestrator_seeds(scenario, mongo_uri, redis_client)
await _run_pi_scenario(temporal_env, scenario, 'scouter')
resolved = await _schedule_input(temporal_host, 'pi-tagdb', namespace='scouter')
# The unknown tag_id is left out; the rest of the pipeline is still scheduled
assert resolved['model_tags'] == {
'TI-101': {'webid': 'F1DP-A', 'aggr_func': 'avg', 'data_range': [-100, 100]}
}
assert resolved['pi_web_api_query'] == {
'endpoint': '/streamsets/recorded',
'period': '*-2h', # connector default
'max_count': 5, # pipeline wins over the connector
'api_timeout': 60, # capped at the frequency
}
assert resolved['connection'] == {
'connector_id': 'pi-usina',
'base_url': 'https://pi-usina.example/piwebapi',
'auth': {'type': 'bearer', 'secret': PI_SECRET},
}
legacy = await _schedule_input(temporal_host, 'pi-legacy', namespace='scouter')
assert 'connection' not in legacy
assert legacy['model_tags'] == {
'TI-900': {'webid': 'F1DP-X', 'aggr_func': 'lts', 'data_range': [-100, 100]}
}
assert legacy['pi_web_api_query'] == {
'endpoint': '/streamsets/recorded',
'period': '*-1d',
'max_count': 1,
'api_timeout': 10,
}
[not_found] = _notifications(
notification_handler, 'ORCHESTRATOR_PI_TAG_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION'
)
assert 'pi-tagdb/404' in not_found['message']
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_9_2_pi_output_tags_through_tag_database(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
temporal_host: str,
redis_client: Redis,
notification_handler,
):
"""1.9.2: predictions_batch output tags by tag_id become {tag_name: WebID}; legacy unchanged."""
scenario = load_scenario_input('orchestrator_pi_web_api')
_apply_orchestrator_seeds(scenario, mongo_uri, redis_client)
await _run_pi_scenario(temporal_env, scenario, 'output')
resolved = await _schedule_input(temporal_host, 'pred-pi')
# The Read tag given as a confidence tag is left out
assert resolved['pi_web_api_output_config'] == {
'prediction_tags': {'PRED-101': 'F1DP-P'},
'confidence_tags': {'CONF-101': 'F1DP-C'},
}
legacy = await _schedule_input(temporal_host, 'pred-pi-legacy')
assert legacy['pi_web_api_output_config'] == {'prediction_tags': {'PRED-900': 'F1DP-L'}}
[wrong_category] = _notifications(
notification_handler, 'ORCHESTRATOR_PI_TAG_WRONG_CATEGORY_DURING_SCHEDULE_CONFIGURATION'
)
assert 'pred-pi/42' in wrong_category['message']
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_9_3_pi_mixed_connectors_hold_schedule(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
temporal_host: str,
redis_client: Redis,
notification_handler,
):
"""1.9.3: tags of two PI connectors in one pipeline leave its schedule untouched."""
scenario = load_scenario_input('orchestrator_pi_web_api')
_apply_orchestrator_seeds(scenario, mongo_uri, redis_client)
await _run_pi_scenario(temporal_env, scenario, 'hold-create')
before = await _schedule_input(temporal_host, 'pi-tagdb', namespace='scouter')
mongo = MongoClient(mongo_uri)
try:
mongo[E2E_DATABASE]['pipelines'].update_one(
{'schedule_name': 'pi-tagdb'},
{
'$set': {
'read_tags.TI-102': {'tag_id': 43},
'updated_at': datetime.now(UTC) + timedelta(minutes=1),
}
},
)
finally:
mongo.close()
await _run_pi_scenario(temporal_env, scenario, 'hold-mixed')
assert 'pi-tagdb' in await _schedule_ids(temporal_host, 'scouter')
assert await _schedule_input(temporal_host, 'pi-tagdb', namespace='scouter') == before
[mixed] = _notifications(
notification_handler, 'ORCHESTRATOR_PI_MIXED_CONNECTORS_DURING_SCHEDULE_CONFIGURATION'
)
assert 'pi-tagdb/pi-other, pi-usina' in mixed['message']
def _opc_slots(redis_client: Redis) -> dict[str, dict]:
return {
key if isinstance(key, str) else key.decode(): json.loads(redis_client.get(key))
@@ -530,21 +404,106 @@ def _opc_slots(redis_client: Redis) -> dict[str, dict]:
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_9_4_opc_slot_unchanged_by_pi_connectors(
async def test_scenario_1_9_1_write_tags_through_tag_database(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
temporal_host: str,
redis_client: Redis,
notification_handler,
):
"""1.9.1: write_tags by tag_id become the opc_output_config; the rest is notified."""
_apply_orchestrator_seeds(
load_scenario_input('orchestrator_tag_database'), mongo_uri, redis_client
)
await _run_tag_database_scenario(temporal_env, 'write')
pred = await _schedule_input(temporal_host, 'pred-tagdb')
assert pred['opc_output_config'] == {
'plant': {
'prediction_tags': {'ns=3;i=2001': {'data_type': 'double'}},
'confidence_tags': {'ns=3;i=2002': {'data_type': 'float'}},
}
}
not_found = _notifications(
notification_handler, 'ORCHESTRATOR_WRITE_TAG_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION'
)
assert len(not_found) == 1
# A Read tag, an inactive tag and an unknown tag_id
for value in ('pred-tagdb/7', 'pred-tagdb/103', 'pred-tagdb/404'):
assert value in not_found[0]['message']
invalid = _notifications(
notification_handler, 'ORCHESTRATOR_INVALID_WRITE_TAG_DURING_SCHEDULE_CONFIGURATION'
)
assert len(invalid) == 1
assert 'ns=3;i=9999' in invalid[0]['message']
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_9_2_tag_change_redeploys_pipelines(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
temporal_host: str,
redis_client: Redis,
):
"""1.9.2: changing a tag in connector_tags redeploys the pipelines that reference it."""
_apply_orchestrator_seeds(
load_scenario_input('orchestrator_tag_database'), mongo_uri, redis_client
)
await _run_tag_database_scenario(temporal_env, 'before-change')
scouter = await _schedule_input(temporal_host, 'opc-tagdb', namespace='scouter')
assert set(scouter['model_tags']) == {'Pressure'}
# Rename the read tag and move the write tag, without touching the pipelines
changed_at = datetime.now(UTC) + timedelta(seconds=1)
mongo = MongoClient(mongo_uri)
try:
tags = mongo[E2E_DATABASE]['connector_tags']
tags.update_one({'tag_id': 7}, {'$set': {'tag_name': 'Pressao', 'updated_at': changed_at}})
tags.update_one(
{'tag_id': 101}, {'$set': {'address': 'ns=3;i=3001', 'updated_at': changed_at}}
)
finally:
mongo.close()
await _run_tag_database_scenario(temporal_env, 'after-change')
scouter = await _schedule_input(temporal_host, 'opc-tagdb', namespace='scouter')
assert set(scouter['model_tags']) == {'Pressao'}
pred = await _schedule_input(temporal_host, 'pred-tagdb')
assert pred['opc_output_config']['plant']['prediction_tags'] == {
'ns=3;i=3001': {'data_type': 'double'}
}
# The slot names the read tag like the scouter does (SC1)
[slot] = _opc_slots(redis_client).values()
[name] = slot['plant']['tags']['ns=3;i=1001']['names']
assert name['tag_name'] == 'Pressao'
@pytest.mark.e2e
@pytest.mark.asyncio
async def test_scenario_1_9_3_opc_slot_from_tag_database(
temporal_env: WorkflowEnvironment,
orchestrator_worker,
stub_workers,
mongo_uri: str,
redis_client: Redis,
):
"""1.9.4: the OPC slot built from the tag database ignores the PI connectors and tags."""
scenario = load_scenario_input('orchestrator_pi_web_api')
_apply_orchestrator_seeds(scenario, mongo_uri, redis_client)
"""1.9.3: the slot v2 holds only the active Read tags of the tag database."""
_apply_orchestrator_seeds(
load_scenario_input('orchestrator_tag_database'), mongo_uri, redis_client
)
await _run_pi_scenario(temporal_env, scenario, 'slot-with-pi')
with_pi = _opc_slots(redis_client)
await _run_tag_database_scenario(temporal_env, 'slot')
assert with_pi == {
assert _opc_slots(redis_client) == {
'slot:opc_tags:1': {
'plant': {
'server_id': 'plant',
@@ -570,15 +529,3 @@ async def test_scenario_1_9_4_opc_slot_unchanged_by_pi_connectors(
}
}
}
mongo = MongoClient(mongo_uri)
try:
mongo[E2E_DATABASE]['connectors'].delete_many({'type': 'pi_web_api'})
mongo[E2E_DATABASE]['connector_tags'].delete_many({'connector_id': {'$ne': 'plant'}})
finally:
mongo.close()
redis_client.delete(*redis_client.keys('slot:opc_tags:*'))
await _run_pi_scenario(temporal_env, scenario, 'slot-without-pi')
assert _opc_slots(redis_client) == with_pi