gitea_admin/sientia-dataops-orchestrator_temporal
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

SIENTIAPDE-1974

Browse Source 
SIENTIAPDE-1974-connectors-pi-web-api: Consolidate OPC server and tag data sourcing to the tag database. This change removes the legacy opc-servers collection and enforces tag_id as the sole reference for all read and write tags via connectors and connector_tags. The Orchestrator workflow now exclusively loads OPC server and tag information from the tag database, simplifying data flow and removing conditional logic for legacy data sources. Improved error reporting for pipeline timestamp updates and updated documentation, examples, and tests are also included.
This commit is contained in:
Vitor Pimentel dos Santos
2026-10-29 10:25:51 -03:00
parent 9ba2fc74c4
commit 0c9eaab3f5
30 changed files with 1082 additions and 3051 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
e2e/helpers.py
View File

@@ -25,8 +25,6 @@ MONGO_COLLECTIONS = (
'receiver_groups',
'orchestrated_schedules',
'pipelines',
'opc_servers',
'opc-servers',
'connectors',
'connector_tags',
)
@@ -145,23 +143,6 @@ def seed_pipelines(
client.close()
def seed_opc_servers(
mongo_uri: str,
database: str,
servers: list[dict[str, Any]],
*,
collection: str = 'opc_servers',
) -> None:
"""Insert OPC server documents into the test Mongo database."""
client = MongoClient(mongo_uri)
try:
coll = client[database][collection]
if servers:
coll.insert_many(servers)
finally:
client.close()
def seed_tag_database(
mongo_uri: str,
database: str,

7
e2e/scenario_inputs/orchestrator_conflict.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
@@ -33,8 +29,5 @@
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

7
e2e/scenario_inputs/orchestrator_create_only.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
@@ -33,8 +29,5 @@
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

7
e2e/scenario_inputs/orchestrator_delete_only.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
@@ -24,8 +20,5 @@
"interval_minutes": 60
}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

5
e2e/scenario_inputs/orchestrator_drift_window.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
@@ -33,6 +29,5 @@
"chunk_period": "s"
}
],
"opc_servers": [],
"active_ingestors": ["heartbeat:ingestor:1"]
}

5
e2e/scenario_inputs/orchestrator_empty.json
View File

@@ -4,13 +4,8 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [],
"opc_servers": [],
"active_ingestors": []
}

7
e2e/scenario_inputs/orchestrator_happy_path.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
@@ -45,8 +41,5 @@
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true, "name": "opc-1"}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

7
e2e/scenario_inputs/orchestrator_noop.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
@@ -33,9 +29,6 @@
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"],
"orchestrated_schedules": [
{

5
e2e/scenario_inputs/orchestrator_simple_metrics_windows.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
@@ -40,6 +36,5 @@
"interval_minutes": 1440
}
],
"opc_servers": [],
"active_ingestors": ["heartbeat:ingestor:1"]
}

7
e2e/scenario_inputs/orchestrator_tag_database.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"connectors": [
@@ -55,10 +51,9 @@
{"tag_id": 7},
{"tag_id": 103},
{"tag_id": 404},
{"server_id": "plant", "type": "prediction", "addr": "ns=3;i=9999"}
{"tag_name": "PRED-NO-ID"}
]
}
],
"opc_servers": [],
"active_ingestors": ["heartbeat:ingestor:1"]
}

27
e2e/scenario_inputs/orchestrator_ttl_index.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
@@ -22,24 +18,17 @@
"frequency": "1m",
"offset": "0m",
"read_tags": [
{
"server_id": "srv-1",
"tag_name": "Read1",
"tag_address": "ns=2;s=Read1",
"aggr_func": "lts",
"frequency": 1000
}
{"tag_id": 1, "aggr_func": "lts", "frequency": 1000}
]
}
],
"opc_servers": [
{
"id": "srv-1",
"active": true,
"server_name": "opc-1",
"url": "opc.tcp://localhost:4840",
"uri": "urn:opcfoundation:UA:DemoServer"
}
"connectors": [
{"connector_id": "opc-1", "type": "opc_ua", "name": "opc-1", "url": "opc.tcp://localhost:4840",
"server_uri": "urn:opcfoundation:UA:DemoServer", "use_security": false, "status": "Active"}
],
"connector_tags": [
{"tag_id": 1, "connector_id": "opc-1", "address": "ns=2;s=Read1", "tag_name": "Read1",
"category": "Read", "status": "Active"}
],
"active_ingestors": ["heartbeat:ingestor:1"]
}

7
e2e/scenario_inputs/orchestrator_update_only.json
View File

@@ -4,10 +4,6 @@
"pipelines_query": {
"collection": "pipelines",
"aggregation": [{"$match": {"active": true}}]
},
"opc_servers_query": {
"collection": "opc_servers",
"filters": {"active": true}
}
},
"pipelines": [
@@ -33,9 +29,6 @@
{"tag_id": 101, "connector_id": "srv-1", "address": "ns=2;s=Tag1", "tag_name": "PRED-1",
"category": "Write", "status": "Active", "updated_at": "@now-1d"}
],
"opc_servers": [
{"id": "srv-1", "active": true}
],
"active_ingestors": ["heartbeat:ingestor:1"],
"orchestrated_schedules": [
{

6
e2e/scenarios.md
View File

@@ -70,15 +70,15 @@ Two `drift` pipelines on `frequency='1m'`: one without `interval_minutes` and wi
Scenario input: `orchestrator_tag_database.json` (one `opc_ua` connector, its `connector_tags` of
every category, one inactive, an OPC scouter reading by `tag_id` and a `predictions_batch` writing by
`tag_id`, with a Read tag, an inactive tag, an unknown `tag_id` and a legacy entry among its write
tags).
`tag_id`, with a Read tag, an inactive tag, an unknown `tag_id` and an entry without `tag_id` among its
write tags).
#### 1.9.1 Write tags through the tag database
`write_tags: [{tag_id}]` → `opc_output_config` keyed by `connector_id`, address and `data_type`
from `connector_tags` (`float` when absent), `Write` → `prediction_tags`, `Prediction Confidence` →
`confidence_tags`. The Read, inactive and unknown tags are left out and notified once
(`ORCHESTRATOR_WRITE_TAG_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION`); the legacy entry is notified
(`ORCHESTRATOR_WRITE_TAG_NOT_FOUND_DURING_SCHEDULE_CONFIGURATION`); the entry without `tag_id` is notified
(`ORCHESTRATOR_INVALID_WRITE_TAG_DURING_SCHEDULE_CONFIGURATION`).
#### 1.9.2 A tag change redeploys its pipelines

6
e2e/test_orchestrator_main_workflow.py
View File

@@ -15,7 +15,6 @@ from e2e.helpers import (
load_scenario_input,
make_workflow_id,
seed_active_ingestors,
seed_opc_servers,
seed_orchestrated_schedules,
seed_pipelines,
seed_tag_database,
@@ -43,7 +42,7 @@ def _apply_orchestrator_seeds(
redis_client: Redis,
) -> None:
"""
Seed Mongo and Redis with the pipelines/opc_servers/tag database/ingestors of a scenario.
Seed Mongo and Redis with the pipelines/tag database/ingestors of a scenario.
Args:
scenario: Scenario payload returned by ``load_scenario_input``.
@@ -51,7 +50,6 @@ def _apply_orchestrator_seeds(
redis_client: Redis client connected to the test instance.
"""
seed_pipelines(mongo_uri, E2E_DATABASE, scenario.get('pipelines', []))
seed_opc_servers(mongo_uri, E2E_DATABASE, scenario.get('opc_servers', []))
seed_tag_database(
mongo_uri,
E2E_DATABASE,
@@ -454,7 +452,7 @@ async def test_scenario_1_9_1_write_tags_through_tag_database(
notification_handler, 'ORCHESTRATOR_INVALID_WRITE_TAG_DURING_SCHEDULE_CONFIGURATION'
)
assert len(invalid) == 1
assert 'ns=3;i=9999' in invalid[0]['message']
assert 'PRED-NO-ID' in invalid[0]['message']
@pytest.mark.e2e